From 043496ca3abe864c38c730d4a29123eb5e22f9f0 Mon Sep 17 00:00:00 2001 From: Nazar Leush Date: Mon, 20 Apr 2026 19:59:59 +0300 Subject: [PATCH 01/21] implement aliases for `getVars` method --- lib/core.js | 17 ++++++++++++++--- lib/loader/utils.js | 18 ++++++++++++++++++ 2 files changed, 32 insertions(+), 3 deletions(-) diff --git a/lib/core.js b/lib/core.js index 94ae22480..8086a643f 100644 --- a/lib/core.js +++ b/lib/core.js @@ -943,9 +943,20 @@ } } } - } else if (r.method.name === "getVars") { - for(var key in r.data) { - var v = r.data[key]; + } else if (r.method.name === "getVars" || r.method.name in pluginUtils.GET_VARS_METHODS) { + + var result_data = r.data; + + // Set specific alias as key. + if (r.method.name in pluginUtils.GET_VARS_METHODS) { + var key = pluginUtils.GET_VARS_METHODS[r.method.name]; + result_data = { + [key]: r.data + }; + } + + for(var key in result_data) { + var v = result_data[key]; // TODO: work with arrays? if (v !== '' && v !== null && ((typeof v === 'string' && !/^\s+$/.test(v)) || typeof v === 'number' || typeof v === 'object')) { diff --git a/lib/loader/utils.js b/lib/loader/utils.js index d6decc3e7..064d9bc91 100644 --- a/lib/loader/utils.js +++ b/lib/loader/utils.js @@ -43,6 +43,24 @@ "prepareLink" ]; + const GET_VARS_ALIAS = [ + 'signals', + 'policy', + 'sources' + ]; + + export const GET_VARS_METHODS = { + // getSignals: signals + }; + + // Adds `GET_VARS_ALIAS` to `PLUGIN_METHODS` as `signals` => `getSignals`. + // Fills `GET_VARS_METHODS` with proper key values. + GET_VARS_ALIAS.forEach(alias => { + const methodName = `get${alias[0].toUpperCase()}${alias.substring(1)}`; + PLUGIN_METHODS.push(methodName); + GET_VARS_METHODS[methodName] = alias; + }); + export const PLUGIN_FIELDS = PLUGIN_METHODS.concat([ "mixins" ]); From bff3859918c9c097491d0c18dfecc5cb7387d62c Mon Sep 17 00:00:00 2001 From: Nazar Leush Date: Wed, 22 Apr 2026 19:27:01 +0300 Subject: [PATCH 02/21] `plugins.always` try run plugin every call --- lib/core.js | 25 ++++++++++++++++++------- lib/loader/pluginLoader.js | 4 ++++ 2 files changed, 22 insertions(+), 7 deletions(-) diff --git a/lib/core.js b/lib/core.js index 8086a643f..eaa9b9991 100644 --- a/lib/core.js +++ b/lib/core.js @@ -513,6 +513,15 @@ addAllGeneric(); } + function addOneMorePlugin(pluginId) { + var exists = initialPlugins.find(function(plugin) { + return plugin.id === pluginId; + }); + if (!exists) { + initialPlugins.push(plugins[pluginId]); + } + } + if (options.forceParams) { // TODO: replace forEach options.forceParams.forEach(function(param) { @@ -525,17 +534,19 @@ for(var k = 0; k < paramPlugins.length; k++) { var foundPluginId = paramPlugins[k]; - - var exists = initialPlugins.find(function(plugin) { - return plugin.id === foundPluginId; - }); - if (!exists) { - initialPlugins.push(plugins[foundPluginId]); - } + addOneMorePlugin(foundPluginId); } } }); } + + // Add `always` plugins. + for(var i = 0; i < pluginsList.length; i++) { + var plugin = pluginsList[i]; + if (plugin.always) { + addOneMorePlugin(plugin.id); + } + } } return { diff --git a/lib/loader/pluginLoader.js b/lib/loader/pluginLoader.js index e9b7b807c..70b1d2368 100644 --- a/lib/loader/pluginLoader.js +++ b/lib/loader/pluginLoader.js @@ -390,6 +390,10 @@ pluginDeclaration.listed = plugin.listed; } + if ('always' in plugin) { + pluginDeclaration.always = plugin.always; + } + // Store plugin. plugins[pluginDeclaration.id] = pluginDeclaration; pluginsModules[pluginDeclaration.id] = pluginDeclaration.module; From ab002cef5b953038ff000832ee3739415f37a4ad Mon Sep 17 00:00:00 2001 From: Nazar Leush Date: Fri, 24 Apr 2026 18:45:14 +0300 Subject: [PATCH 03/21] use `options.dataMode` and `/data/` plugins --- lib/core.js | 20 +++++++++----------- lib/loader/pluginLoader.js | 7 +++---- 2 files changed, 12 insertions(+), 15 deletions(-) diff --git a/lib/core.js b/lib/core.js index eaa9b9991..7bbcfc1c5 100644 --- a/lib/core.js +++ b/lib/core.js @@ -458,13 +458,19 @@ // Use all generic plugins. for(var i = 0; i < pluginsList.length; i++) { var plugin = pluginsList[i]; - if (!plugin.domain && !plugin.custom) { + if (!plugin.domain + && !plugin.custom + // dataMode - enables data plugins + && ((!options.dataMode && !plugin.data) || (options.dataMode && plugin.data))) { + var match = plugin.pluginReMatchesUrl(domain, uri); if (match) { pluginsUrlMatches[plugin.id] = match; } - // false - has re, no match. + // match = result; - url matched plugin.re. + // match = undefined; - domain has no plugin.re. + // match = false - plugin has re, and no match. if (match !== false) { initialPlugins.push(plugin); } @@ -473,7 +479,7 @@ } // In domain debug: add all plugins before domain plugins to make them low priority. - if (options.mixAllWithDomainPlugin) { + if (options.mixAllWithDomainPlugin || options.dataMode) { addAllGeneric(); } @@ -539,14 +545,6 @@ } }); } - - // Add `always` plugins. - for(var i = 0; i < pluginsList.length; i++) { - var plugin = pluginsList[i]; - if (plugin.always) { - addOneMorePlugin(plugin.id); - } - } } return { diff --git a/lib/loader/pluginLoader.js b/lib/loader/pluginLoader.js index 70b1d2368..d6379460a 100644 --- a/lib/loader/pluginLoader.js +++ b/lib/loader/pluginLoader.js @@ -370,6 +370,9 @@ // Plugins in folder 'custom' or 'core' will be run only on explicit dependency. pluginDeclaration.custom = (bits.indexOf('custom') > -1 || bits.indexOf('system') > -1) && !plugin.generic; + // Plugins in folder 'data' will be run only in data_mode. + pluginDeclaration.data = bits.indexOf('data') > -1; + var stat = fs.statSync(pluginPath); pluginDeclaration.modified = new Date(stat.mtime); pluginDeclaration.getPluginLastModifiedDate = getPluginLastModifiedDate; @@ -390,10 +393,6 @@ pluginDeclaration.listed = plugin.listed; } - if ('always' in plugin) { - pluginDeclaration.always = plugin.always; - } - // Store plugin. plugins[pluginDeclaration.id] = pluginDeclaration; pluginsModules[pluginDeclaration.id] = pluginDeclaration.module; From 4750e4caf7454fc46c9517988d0e77ef9398ddd0 Mon Sep 17 00:00:00 2001 From: Nazar Leush Date: Fri, 24 Apr 2026 20:19:58 +0300 Subject: [PATCH 04/21] fix loading data plugins --- lib/core.js | 2 +- lib/loader/pluginLoader.js | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/core.js b/lib/core.js index 7bbcfc1c5..b7cdcd57d 100644 --- a/lib/core.js +++ b/lib/core.js @@ -461,7 +461,7 @@ if (!plugin.domain && !plugin.custom // dataMode - enables data plugins - && ((!options.dataMode && !plugin.data) || (options.dataMode && plugin.data))) { + && (options.dataMode || !plugin.data)) { var match = plugin.pluginReMatchesUrl(domain, uri); if (match) { diff --git a/lib/loader/pluginLoader.js b/lib/loader/pluginLoader.js index d6379460a..497f2bf8a 100644 --- a/lib/loader/pluginLoader.js +++ b/lib/loader/pluginLoader.js @@ -668,6 +668,7 @@ await loadPlugins(pluginsRoot, 'custom'); await loadPlugins(pluginsRoot, 'links'); await loadPlugins(pluginsRoot, 'meta'); + await loadPlugins(pluginsRoot, 'data'); await loadPlugins(pluginsRoot, 'templates'); // TODO: if has multiple modules_listing - CUSTOM_PLUGINS_PATH will be loaded multiple times. @@ -677,6 +678,7 @@ await loadPlugins(CONFIG.CUSTOM_PLUGINS_PATH, 'custom'); await loadPlugins(CONFIG.CUSTOM_PLUGINS_PATH, 'links'); await loadPlugins(CONFIG.CUSTOM_PLUGINS_PATH, 'meta'); + await loadPlugins(CONFIG.CUSTOM_PLUGINS_PATH, 'data'); await loadPlugins(CONFIG.CUSTOM_PLUGINS_PATH, 'templates'); } else { console.warn('Custom plugin folder "' + CONFIG.CUSTOM_PLUGINS_PATH + '" not found.'); From c8ee082cb6d742a046379c090f2a4b9212cd4c23 Mon Sep 17 00:00:00 2001 From: Nazar Leush Date: Fri, 24 Apr 2026 20:20:20 +0300 Subject: [PATCH 05/21] debugger: allow data request --- modules/api/views.js | 1 + modules/debug/views.js | 6 ++++-- static/js/debug.js | 1 + static/js/iframely.js | 1 + views/debug.ejs | 7 ++++++- 5 files changed, 13 insertions(+), 3 deletions(-) diff --git a/modules/api/views.js b/modules/api/views.js index 421ab7d85..00bdbe74a 100644 --- a/modules/api/views.js +++ b/modules/api/views.js @@ -137,6 +137,7 @@ export default function(app) { debug: getBooleanParam(req, 'debug'), returnProviderOptionsUsage: getBooleanParam(req, 'debug'), mixAllWithDomainPlugin: getBooleanParam(req, 'mixAllWithDomainPlugin'), + dataMode: getBooleanParam(req, 'dataMode'), forceParams: req.query.meta === "true" ? CONFIG.DEBUG_CONTEXTS : null, whitelist: getBooleanParam(req, 'whitelist'), readability: getBooleanParam(req, 'readability'), diff --git a/modules/debug/views.js b/modules/debug/views.js index 9aae734da..bb68e71e4 100644 --- a/modules/debug/views.js +++ b/modules/debug/views.js @@ -10,12 +10,14 @@ export default function(app) { DEBUG = {"on":1, "true":1}[req.query.debug]; } - res.render('debug',{ + res.render('debug', { uri: req.query.uri, mixAllWithDomainPlugin: !!{"on":1, "true":1}[req.query.mixAllWithDomainPlugin], + dataMode: !!{"on":1, "true":1}[req.query.dataMode], refresh: !!{"on":1, "true":1}[req.query.refresh], DEBUG: DEBUG, + DATA_DEBUG_ENABLED: CONFIG.DATA_DEBUG_ENABLED, QUERY: getProviderOptionsQuery(req.query) }); }); -}; \ No newline at end of file +}; diff --git a/static/js/debug.js b/static/js/debug.js index 457751dac..d21347029 100644 --- a/static/js/debug.js +++ b/static/js/debug.js @@ -326,6 +326,7 @@ function processUrl() { var query = { debug: true, group: false, + dataMode: $('[name="dataMode"]').is(":checked"), mixAllWithDomainPlugin: $('[name="mixAllWithDomainPlugin"]').is(":checked"), refresh: $('[name="refresh"]').is(":checked") }; diff --git a/static/js/iframely.js b/static/js/iframely.js index f13861da6..d7ca4d06e 100644 --- a/static/js/iframely.js +++ b/static/js/iframely.js @@ -151,6 +151,7 @@ uri: !options.url ? uri : undefined, url: options.url ? uri : undefined, debug: options.debug, + dataMode: options.dataMode, mixAllWithDomainPlugin: options.mixAllWithDomainPlugin, refresh: options.refresh, meta: options.meta, diff --git a/views/debug.ejs b/views/debug.ejs index 0022d83eb..065a5b999 100644 --- a/views/debug.ejs +++ b/views/debug.ejs @@ -47,6 +47,11 @@ <% if (DEBUG) { %>
+ <% if (DATA_DEBUG_ENABLED) { %> + + <% } %> @@ -98,4 +103,4 @@
- \ No newline at end of file + From 1a081f5beac3303effac47fdd3451abd16f14d4d Mon Sep 17 00:00:00 2001 From: Nazar Leush Date: Mon, 27 Apr 2026 17:26:51 +0300 Subject: [PATCH 06/21] simplify `getDomainOptions` call --- lib/utils.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/utils.js b/lib/utils.js index 94098c49a..a4a3e8647 100644 --- a/lib/utils.js +++ b/lib/utils.js @@ -105,7 +105,7 @@ export function prepareRequestOptions(request_options, options) { } } - var enable_domain_prerender = options?.getDomainOptions && options.getDomainOptions('meta.prerender'); + var enable_domain_prerender = options.getDomainOptions('meta.prerender'); if (enable_domain_prerender) { setPrerender(enable_domain_prerender); } From bd561effbfd67f1e52711d630b65a16c86bd96c4 Mon Sep 17 00:00:00 2001 From: Nazar Leush Date: Mon, 27 Apr 2026 18:02:34 +0300 Subject: [PATCH 07/21] config: add `GET_VARS_ALIASES` --- config.js | 6 ++++++ lib/loader/utils.js | 8 +++----- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/config.js b/config.js index 4a320562a..965d4e3a5 100644 --- a/config.js +++ b/config.js @@ -367,6 +367,12 @@ 'ENET', 'HPE_INVALID_', 'ERR_SSL_' + ], + + GET_VARS_ALIASES: [ + 'signals', + 'policy', + 'sources' ] }; diff --git a/lib/loader/utils.js b/lib/loader/utils.js index 064d9bc91..428f9bfb9 100644 --- a/lib/loader/utils.js +++ b/lib/loader/utils.js @@ -1,3 +1,5 @@ + import CONFIG from '../../config.loader.js'; + export const DEFAULT_PARAMS = [ "url", "urlMatch", @@ -43,11 +45,7 @@ "prepareLink" ]; - const GET_VARS_ALIAS = [ - 'signals', - 'policy', - 'sources' - ]; + const GET_VARS_ALIAS = CONFIG.GET_VARS_ALIASES || [] export const GET_VARS_METHODS = { // getSignals: signals From e934e31e651b6cc5d8f1e18048f673f779facba0 Mon Sep 17 00:00:00 2001 From: Nazar Leush Date: Mon, 27 Apr 2026 19:48:45 +0300 Subject: [PATCH 08/21] fix undefined --- lib/utils.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/utils.js b/lib/utils.js index a4a3e8647..20e7b8d01 100644 --- a/lib/utils.js +++ b/lib/utils.js @@ -105,7 +105,8 @@ export function prepareRequestOptions(request_options, options) { } } - var enable_domain_prerender = options.getDomainOptions('meta.prerender'); + // Some calls (like oembed) use basic options without `getDomainOptions`. + var enable_domain_prerender = options?.getDomainOptions && options.getDomainOptions('meta.prerender'); if (enable_domain_prerender) { setPrerender(enable_domain_prerender); } From 26034fa5d74f7f9835fe9d860680a007b0eb18f3 Mon Sep 17 00:00:00 2001 From: Nazar Leush Date: Mon, 27 Apr 2026 21:21:20 +0300 Subject: [PATCH 09/21] show `vars` in debugger meta section --- static/js/debug.js | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/static/js/debug.js b/static/js/debug.js index d21347029..aac044dc9 100644 --- a/static/js/debug.js +++ b/static/js/debug.js @@ -252,6 +252,19 @@ function showEmbeds($embeds, data, filterByRel) { $meta.append('' + (DEBUG ? ('' + pluginId + '') : '') + '' + key + '' + linkify(data.meta[key]) + '') }); + // Render "vars". + if (data.vars) { + metaKeys = Object.keys(data.vars); + metaKeys.forEach(function(key) { + if (key == "_sources") { + return; + } + var pluginId = data.vars._sources && data.vars._sources[key] || ''; + usedPlugins[pluginId] = true; + + $meta.append('' + (DEBUG ? ('' + pluginId + '') : '') + 'vars: ' + key + '
' + JSON.stringify(data.vars[key], null, 4) + '
') + }); + } if (DEBUG && data.vary && data.vary.join) { $meta.append('vary' + data.vary.join('
') + '') } From 2c5067dd570da3198a59f4a7d2363a66996be9c0 Mon Sep 17 00:00:00 2001 From: Ivan Paramonau Date: Thu, 30 Apr 2026 10:55:20 -0400 Subject: [PATCH 10/21] remove unnecessary entities --- config.js | 10 +++++----- lib/core.js | 6 +++--- lib/loader/utils.js | 12 ++---------- 3 files changed, 10 insertions(+), 18 deletions(-) diff --git a/config.js b/config.js index 965d4e3a5..c4dd73afa 100644 --- a/config.js +++ b/config.js @@ -369,11 +369,11 @@ 'ERR_SSL_' ], - GET_VARS_ALIASES: [ - 'signals', - 'policy', - 'sources' - ] + GET_VARS_METHODS: { + 'getSignals': 'signals', + 'getPolicy': 'policy', + 'getSources': 'sources' + } }; // Providers config loader. diff --git a/lib/core.js b/lib/core.js index b7cdcd57d..d150c4441 100644 --- a/lib/core.js +++ b/lib/core.js @@ -952,13 +952,13 @@ } } } - } else if (r.method.name === "getVars" || r.method.name in pluginUtils.GET_VARS_METHODS) { + } else if (r.method.name === "getVars" || CONFIG.GET_VARS_METHODS && r.method.name in CONFIG.GET_VARS_METHODS) { var result_data = r.data; // Set specific alias as key. - if (r.method.name in pluginUtils.GET_VARS_METHODS) { - var key = pluginUtils.GET_VARS_METHODS[r.method.name]; + if (CONFIG.GET_VARS_METHODS && r.method.name in CONFIG.GET_VARS_METHODS) { + var key = CONFIG.GET_VARS_METHODS[r.method.name]; result_data = { [key]: r.data }; diff --git a/lib/loader/utils.js b/lib/loader/utils.js index 428f9bfb9..f10a6da66 100644 --- a/lib/loader/utils.js +++ b/lib/loader/utils.js @@ -45,18 +45,10 @@ "prepareLink" ]; - const GET_VARS_ALIAS = CONFIG.GET_VARS_ALIASES || [] - export const GET_VARS_METHODS = { - // getSignals: signals - }; - - // Adds `GET_VARS_ALIAS` to `PLUGIN_METHODS` as `signals` => `getSignals`. - // Fills `GET_VARS_METHODS` with proper key values. - GET_VARS_ALIAS.forEach(alias => { - const methodName = `get${alias[0].toUpperCase()}${alias.substring(1)}`; + // Adds vars methods to PLUGIN_METHODS` as `getSignals`. + CONFIG.GET_VARS_METHODS && Object.keys(CONFIG.GET_VARS_METHODS).forEach(methodName => { PLUGIN_METHODS.push(methodName); - GET_VARS_METHODS[methodName] = alias; }); export const PLUGIN_FIELDS = PLUGIN_METHODS.concat([ From 1804b6cf77b82bd82d5b32574f0021aaf447bc34 Mon Sep 17 00:00:00 2001 From: Nazar Leush Date: Wed, 22 Apr 2026 18:42:22 +0300 Subject: [PATCH 11/21] html escape query params (#647) --- modules/api/utils.js | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/modules/api/utils.js b/modules/api/utils.js index 026c32094..3fcf9e3bc 100644 --- a/modules/api/utils.js +++ b/modules/api/utils.js @@ -14,6 +14,21 @@ export function getProviderOptionsQuery(query) { return providerOptionsQuery; } +const HTML_ESCAPE_MAP = { + '&': '&', + '<': '<', + '>': '>', + '"': '"', + "'": ''' +}; + +function escapeHTML(value) { + if (typeof value !== "string") { + return value; + } + return value.replace(/[&<>"']/g, char => HTML_ESCAPE_MAP[char]); +} + function normalizeValue(value) { if (value === 'true') { return true; @@ -27,7 +42,12 @@ function normalizeValue(value) { if (/^(\d+)?\.\d+$/.test(value)) { return parseFloat(value); } - return value; + if (typeof value === 'string') { + // Escape string value in case it will be used in html. + return escapeHTML(value); + } + // Return nothing if unknown type or array. + return; } export function getProviderOptionsFromQuery(query) { @@ -45,7 +65,9 @@ export function getProviderOptionsFromQuery(query) { for(var key in query) { if (key.length > 1 && _RE.test(key)) { var value = normalizeValue(query[key]); - providerOptions[key] = value; + if (typeof value !== 'undefined') { + providerOptions[key] = value; + } } } From 9214b72233ca55aa306bba1c494dc8027fbcb356 Mon Sep 17 00:00:00 2001 From: Aliaksei <34516115+siniakinaa@users.noreply.github.com> Date: Thu, 23 Apr 2026 18:39:33 +0300 Subject: [PATCH 12/21] [Domains] dailymail.co.uk is dailymail.com now (#648) * replace dailymail.co.uk with dailymail.com * dailymail.video: change aspect-ratio * fix aspect-ratio --- .../dailymail.embeddedvideo.js | 6 +++--- .../dailymail.galleryvideo.js | 5 +++-- .../{dailymail.co.uk => dailymail.com}/dailymail.video.js | 5 +++-- .../domains/{dailymail.co.uk => dailymail.com}/mol.im.js | 0 plugins/links/hosted/23video-hosted.js | 3 ++- 5 files changed, 11 insertions(+), 8 deletions(-) rename plugins/domains/{dailymail.co.uk => dailymail.com}/dailymail.embeddedvideo.js (50%) rename plugins/domains/{dailymail.co.uk => dailymail.com}/dailymail.galleryvideo.js (55%) rename plugins/domains/{dailymail.co.uk => dailymail.com}/dailymail.video.js (82%) rename plugins/domains/{dailymail.co.uk => dailymail.com}/mol.im.js (100%) diff --git a/plugins/domains/dailymail.co.uk/dailymail.embeddedvideo.js b/plugins/domains/dailymail.com/dailymail.embeddedvideo.js similarity index 50% rename from plugins/domains/dailymail.co.uk/dailymail.embeddedvideo.js rename to plugins/domains/dailymail.com/dailymail.embeddedvideo.js index 9c3cb49e3..1fade16f2 100644 --- a/plugins/domains/dailymail.co.uk/dailymail.embeddedvideo.js +++ b/plugins/domains/dailymail.com/dailymail.embeddedvideo.js @@ -2,7 +2,7 @@ export default { // direct "share" links to players from DailyMail articles. They end with #v-1467332342001 re: [ - /^https?:\/\/www\.dailymail\.co\.uk\/[^#]+#(v\-\d+)$/i + /^https?:\/\/www\.dailymail\.com\/[^#]+#(v\-\d+)$/i ], provides: 'dailymailVideoID', @@ -19,7 +19,7 @@ export default { tests: [{ noFeeds: true }, - "https://www.dailymail.co.uk/lifestyle/article-11847441/Watch-new-Duke-Edinburgh-introduces-Sophie-duchess-time.html#v-6755232965847660921", - "https://www.dailymail.co.uk/news/article-3556177/Was-MH17-shot-Ukrainian-fighter-jet-BBC-documentary-claims-Boeing-777-targeted-plane.html#v-8296301435444282732" + "https://www.dailymail.com/lifestyle/article-11847441/Watch-new-Duke-Edinburgh-introduces-Sophie-duchess-time.html#v-6755232965847660921", + "https://www.dailymail.com/news/article-3556177/Was-MH17-shot-Ukrainian-fighter-jet-BBC-documentary-claims-Boeing-777-targeted-plane.html#v-8296301435444282732" ] }; \ No newline at end of file diff --git a/plugins/domains/dailymail.co.uk/dailymail.galleryvideo.js b/plugins/domains/dailymail.com/dailymail.galleryvideo.js similarity index 55% rename from plugins/domains/dailymail.co.uk/dailymail.galleryvideo.js rename to plugins/domains/dailymail.com/dailymail.galleryvideo.js index 8c5fdfb66..dc5e1f6ac 100644 --- a/plugins/domains/dailymail.co.uk/dailymail.galleryvideo.js +++ b/plugins/domains/dailymail.com/dailymail.galleryvideo.js @@ -1,7 +1,7 @@ export default { re: [ - /^https?:\/\/www\.dailymail\.co\.uk\/video\/\w+\/video\-(\d+)\//i + /^https?:\/\/www\.dailymail\.com\/video\/\w+\/video\-(\d+)\//i ], provides: 'dailymailVideoID', @@ -20,6 +20,7 @@ export default { tests: [{ noFeeds: true }, - "https://www.dailymail.co.uk/video/news/video-1284607/Heart-rending-scenes-child-marriage-ceremonies-India.html" + "https://www.dailymail.com/video/news/video-1284607/Heart-rending-scenes-child-marriage-ceremonies-India.html", + "https://www.dailymail.com/video/royals/video-3639009/Video-Celebrations-late-Queens-100th-birthday.html" ] }; \ No newline at end of file diff --git a/plugins/domains/dailymail.co.uk/dailymail.video.js b/plugins/domains/dailymail.com/dailymail.video.js similarity index 82% rename from plugins/domains/dailymail.co.uk/dailymail.video.js rename to plugins/domains/dailymail.com/dailymail.video.js index 0bcb22eaf..c474da77e 100644 --- a/plugins/domains/dailymail.co.uk/dailymail.video.js +++ b/plugins/domains/dailymail.com/dailymail.video.js @@ -36,7 +36,7 @@ export default { href: dailymailVideo.plugins['social-share'].embedUrl, // no SSL rel: CONFIG.R.player, accept: CONFIG.T.text_html, - "aspect-ratio": 484 / 282, // taken from mp4 aspect + "aspect-ratio": 698 / 423, // taken from native embed "padding-bottom": 50 + 50 + 50, scrolling: 'no' }]; @@ -45,6 +45,7 @@ export default { tests: [{ noFeeds: true }, - "https://www.dailymail.co.uk/video/news/video-2895505/SVB-members-sets-bank-culture-apart-others.html" + "https://www.dailymail.com/video/news/video-2895505/SVB-members-sets-bank-culture-apart-others.html", + "https://www.dailymail.com/video/news/video-3639123/Trump-says-expects-bomb-Iran-ceasefire-deal-ends.html", ] }; \ No newline at end of file diff --git a/plugins/domains/dailymail.co.uk/mol.im.js b/plugins/domains/dailymail.com/mol.im.js similarity index 100% rename from plugins/domains/dailymail.co.uk/mol.im.js rename to plugins/domains/dailymail.com/mol.im.js diff --git a/plugins/links/hosted/23video-hosted.js b/plugins/links/hosted/23video-hosted.js index bfbf353cc..7f880cb7e 100644 --- a/plugins/links/hosted/23video-hosted.js +++ b/plugins/links/hosted/23video-hosted.js @@ -20,6 +20,7 @@ export default { "https://video.itu.dk/live/13796543", "https://video.ku.dk/video/11827941/visual-social-media-lab-farida-vis-anne", "https://video.nextconf.eu/video/1880845/data-without-limits", - "https://www.fftv.no/skipatruljen-s3e3-voss-resort", + "https://video.itu.dk/video/91172759/kandidat-i-softwaredesign-los", + // "https://www.fftv.no/skipatruljen-s3e3-voss-resort", No oEmbed discovery tags ] }; \ No newline at end of file From c515cfed57488c689b5ea78999fad21c27e57d7a Mon Sep 17 00:00:00 2001 From: Nazar Leush Date: Thu, 30 Apr 2026 18:02:31 +0300 Subject: [PATCH 13/21] Meta: use `decodeHTMLStrict` instead of `decodeHTML5` (#652) * meta: use `decodeHTMLStrict` instead of `decodeHTML5` * replace all `decodeHTML5` with `decodeHTMLStrict` --- lib/plugins/system/meta/HTMLMetaHandler.js | 10 +++++----- plugins/domains/dailymail.com/dailymail.video.js | 10 +++++----- plugins/domains/google.com/docs.google.com.js | 4 ++-- plugins/links/embedURL/embedURL.js | 4 ++-- plugins/meta/description-from-p-tag.js | 4 ++-- 5 files changed, 16 insertions(+), 16 deletions(-) diff --git a/lib/plugins/system/meta/HTMLMetaHandler.js b/lib/plugins/system/meta/HTMLMetaHandler.js index 8b4328f63..fd467538b 100644 --- a/lib/plugins/system/meta/HTMLMetaHandler.js +++ b/lib/plugins/system/meta/HTMLMetaHandler.js @@ -1,4 +1,4 @@ -import { decodeHTML5 } from 'entities'; +import { decodeHTMLStrict } from 'entities'; import * as url from 'url'; import * as utils from '../../../utils.js'; import { ldParser } from './ld-json.js'; @@ -57,7 +57,7 @@ HTMLMetaHandler.prototype.onopentag = function(name, attributes) { if (tagName) { if (/&[^;]+;/i.test(tagName)) { - tagName = decodeHTML5(tagName); + tagName = decodeHTMLStrict(tagName); } var propertyParts = tagName.split(/(?::|\.)/); @@ -98,7 +98,7 @@ HTMLMetaHandler.prototype.onopentag = function(name, attributes) { var refresh = metaTag.content && metaTag.content.match(/url=(?:'|")?([^'"]+)(?:'|")?/i); if (refresh) { - this._refresh = decodeHTML5(refresh[1]); + this._refresh = decodeHTMLStrict(refresh[1]); } } else if (metaTag.name == "description") { @@ -265,11 +265,11 @@ HTMLMetaHandler.prototype._finalMerge = function(tag) { if (typeof(obj[k]) == 'string'){ // decode HTML entities after decoding the charset // otherwise we would end up with a string with mixed encoding - obj[k] = decodeHTML5(encodeText(that._charset, obj[k])); + obj[k] = decodeHTMLStrict(encodeText(that._charset, obj[k])); if (k === 'href') { try { - // Process 'href' after decodeHTML5. + // Process 'href' after decodeHTMLStrict. var href = obj[k]; href = href.replace(/(\r\n|\n|\r)/gm,""); href = url.resolve(that._uri, href); diff --git a/plugins/domains/dailymail.com/dailymail.video.js b/plugins/domains/dailymail.com/dailymail.video.js index c474da77e..ca3272baf 100644 --- a/plugins/domains/dailymail.com/dailymail.video.js +++ b/plugins/domains/dailymail.com/dailymail.video.js @@ -1,4 +1,4 @@ -import { decodeHTML5 } from 'entities'; +import { decodeHTMLStrict } from 'entities'; export default { @@ -13,15 +13,15 @@ export default { if ($player.length == 1) { return { - dailymailVideo: JSON.parse(decodeHTML5($player.attr('data-opts'))) + dailymailVideo: JSON.parse(decodeHTMLStrict($player.attr('data-opts'))) } } }, getMeta: function(dailymailVideo, decode) { return { - title: decodeHTML5(decode(dailymailVideo.title)), - description: decodeHTML5(decode(dailymailVideo.descr)) + title: decodeHTMLStrict(decode(dailymailVideo.title)), + description: decodeHTMLStrict(decode(dailymailVideo.descr)) } }, @@ -48,4 +48,4 @@ export default { "https://www.dailymail.com/video/news/video-2895505/SVB-members-sets-bank-culture-apart-others.html", "https://www.dailymail.com/video/news/video-3639123/Trump-says-expects-bomb-Iran-ceasefire-deal-ends.html", ] -}; \ No newline at end of file +}; diff --git a/plugins/domains/google.com/docs.google.com.js b/plugins/domains/google.com/docs.google.com.js index b03d9c6b4..58d9639b3 100644 --- a/plugins/domains/google.com/docs.google.com.js +++ b/plugins/domains/google.com/docs.google.com.js @@ -1,4 +1,4 @@ -import { decodeHTML5 } from 'entities'; +import { decodeHTMLStrict } from 'entities'; export default { @@ -132,7 +132,7 @@ export default { var key = $el.attr('itemprop'); if (key) { - var value = decodeHTML5(decode($el.attr('content') || $el.attr('href'))); + var value = decodeHTMLStrict(decode($el.attr('content') || $el.attr('href'))); result[key] = value; } }); diff --git a/plugins/links/embedURL/embedURL.js b/plugins/links/embedURL/embedURL.js index f0f58e1c4..a263714ed 100644 --- a/plugins/links/embedURL/embedURL.js +++ b/plugins/links/embedURL/embedURL.js @@ -1,4 +1,4 @@ -import { decodeHTML5 } from 'entities'; +import { decodeHTMLStrict } from 'entities'; export default { @@ -55,7 +55,7 @@ export default { var key = $el.attr('itemprop'); if (key) { - var value = decodeHTML5(decode($el.attr('content') || $el.attr('href'))); + var value = decodeHTMLStrict(decode($el.attr('content') || $el.attr('href'))); result[key] = value; } }); diff --git a/plugins/meta/description-from-p-tag.js b/plugins/meta/description-from-p-tag.js index ca3e11b2d..345d819d0 100644 --- a/plugins/meta/description-from-p-tag.js +++ b/plugins/meta/description-from-p-tag.js @@ -1,4 +1,4 @@ -import { decodeHTML5 } from 'entities'; +import { decodeHTMLStrict } from 'entities'; export default { @@ -11,7 +11,7 @@ export default { cheerio("body p").each(function() { var $p = cheerio(this); if ($p.children("label, input, button, div, script, span").length === 0 && !$p.parents("noscript, header,#header,[role='banner']").length) { - var someText = decodeHTML5(decode($p.text())); + var someText = decodeHTMLStrict(decode($p.text())); var requiredLimit = Number.isInteger(__allowPTagDescription) ? __allowPTagDescription : 64; if (someText.length > requiredLimit) { description = someText; From 0e5c0e30962fab5374c2738bd2da8621e21bbf9e Mon Sep 17 00:00:00 2001 From: Nazar Leush Date: Mon, 4 May 2026 18:04:53 +0300 Subject: [PATCH 14/21] `htmlparser` can return `plainresponse` --- lib/plugins/system/htmlparser/htmlparser.js | 8 +++++++- lib/utils.js | 4 ++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/lib/plugins/system/htmlparser/htmlparser.js b/lib/plugins/system/htmlparser/htmlparser.js index 0633fceae..da915eac7 100644 --- a/lib/plugins/system/htmlparser/htmlparser.js +++ b/lib/plugins/system/htmlparser/htmlparser.js @@ -159,7 +159,13 @@ export default { } } - if('content-type' in headers && !/text\/html|application\/xhtml\+xml/gi.test(headers['content-type'])){ + if ('content-type' in headers && /text\/markdown/gi.test(headers['content-type'])){ + return cb(null, { + plainresponse: resp + }); + } + + if ('content-type' in headers && !/text\/html|application\/xhtml\+xml/gi.test(headers['content-type'])){ abortController.abort(); return cacheAndRespond(null, { diff --git a/lib/utils.js b/lib/utils.js index 20e7b8d01..8c730cbf2 100644 --- a/lib/utils.js +++ b/lib/utils.js @@ -242,7 +242,7 @@ export function prepareRequestOptions(request_options, options) { follow = 0; } - // Custom redirect logit for cookies. + // Custom redirect logic for cookies. if (options.followRedirect && options.reuseCookies) { redirect = 'manual'; follow = 0; @@ -255,7 +255,7 @@ export function prepareRequestOptions(request_options, options) { uri: url, method: 'GET', headers: { - 'Accept': '*/*' + 'Accept': options.acceptHeader || '*/*' }, timeout: options.timeout || CONFIG.RESPONSE_TIMEOUT, redirect: redirect, From 76eb9f96179250279a1034d8ca4ccd87443bd52f Mon Sep 17 00:00:00 2001 From: Nazar Leush Date: Mon, 4 May 2026 19:53:13 +0300 Subject: [PATCH 15/21] detect `BIG_CONTEXT` by `abortController`; use `__nonHtmlContentResponse` attr --- lib/core.js | 7 ++++++- lib/plugins/system/htmlparser/htmlparser.js | 6 ++++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/lib/core.js b/lib/core.js index d150c4441..ee3cca066 100644 --- a/lib/core.js +++ b/lib/core.js @@ -1154,7 +1154,7 @@ return hasDomainData; } - const BIG_CONTEXT = ['htmlparser', 'readability', 'decode', 'cheerio']; + const BIG_CONTEXT = ['readability', 'decode', 'cheerio']; function prepareResultData(uri, result, options) { @@ -1332,6 +1332,11 @@ if (allData) { for(var i = 0; i < allData.length; i++) { var r = allData[i]; + for(var key in r.data) { + if (r.data[key].abortController) { + r.data[key] = 'BIG_CONTEXT'; + } + } for(var j = 0; j < BIG_CONTEXT.length; j++) { var d = BIG_CONTEXT[j]; if (r.data && r.data[d]) { diff --git a/lib/plugins/system/htmlparser/htmlparser.js b/lib/plugins/system/htmlparser/htmlparser.js index da915eac7..9bf23351e 100644 --- a/lib/plugins/system/htmlparser/htmlparser.js +++ b/lib/plugins/system/htmlparser/htmlparser.js @@ -159,9 +159,11 @@ export default { } } - if ('content-type' in headers && /text\/markdown/gi.test(headers['content-type'])){ + if ('content-type' in headers + && headers['content-type'].indexOf('text/markdown') > -1 + && options.acceptHeader.indexOf('text/markdown') > -1){ return cb(null, { - plainresponse: resp + __nonHtmlContentResponse: resp }); } From 77a6a3bbdd895a0850562e075c3d15530d0423d0 Mon Sep 17 00:00:00 2001 From: Nazar Leush Date: Tue, 5 May 2026 17:15:50 +0300 Subject: [PATCH 16/21] move `__allowHtmlparser` to `cachedMeta` --- lib/plugins/system/htmlparser/htmlparser.js | 4 ++-- lib/plugins/system/meta/cachedMeta.js | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/plugins/system/htmlparser/htmlparser.js b/lib/plugins/system/htmlparser/htmlparser.js index 9bf23351e..e2dfd8647 100644 --- a/lib/plugins/system/htmlparser/htmlparser.js +++ b/lib/plugins/system/htmlparser/htmlparser.js @@ -16,7 +16,7 @@ export default { '__statusCode' ], - getData: function(url, whitelistRecord, options, __noCachedHtmlparserFallback, __allowHtmlparser, cb) { + getData: function(url, whitelistRecord, options, __noCachedHtmlparserFallback, cb) { var options2 = {...options, ...{ followRedirect: !!options.followHTTPRedirect, @@ -161,7 +161,7 @@ export default { if ('content-type' in headers && headers['content-type'].indexOf('text/markdown') > -1 - && options.acceptHeader.indexOf('text/markdown') > -1){ + && options.acceptHeader?.indexOf('text/markdown') > -1){ return cb(null, { __nonHtmlContentResponse: resp }); diff --git a/lib/plugins/system/meta/cachedMeta.js b/lib/plugins/system/meta/cachedMeta.js index 1a1497968..0df1dcb61 100644 --- a/lib/plugins/system/meta/cachedMeta.js +++ b/lib/plugins/system/meta/cachedMeta.js @@ -19,7 +19,7 @@ export default { '__statusCode' ], - getData: function(url, options, whitelistRecord, cb) { + getData: function(url, __allowHtmlparser, options, whitelistRecord, cb) { // Ignore proxy.cache_ttl, if options.cache_ttl === 0 - do not read from cache. if (options.refresh || options.cache_ttl === 0) { From 060c23855de3b3bd07f3fffba3be8439658abc64 Mon Sep 17 00:00:00 2001 From: Nazar Leush Date: Tue, 5 May 2026 17:41:53 +0300 Subject: [PATCH 17/21] use `options.requestHeaders` to customize main request headers --- lib/plugins/system/htmlparser/htmlparser.js | 31 ++++++++++++++++++--- lib/utils.js | 5 +++- 2 files changed, 31 insertions(+), 5 deletions(-) diff --git a/lib/plugins/system/htmlparser/htmlparser.js b/lib/plugins/system/htmlparser/htmlparser.js index e2dfd8647..c03263248 100644 --- a/lib/plugins/system/htmlparser/htmlparser.js +++ b/lib/plugins/system/htmlparser/htmlparser.js @@ -8,6 +8,31 @@ import { extendCookiesJar } from '../../../fetch.js'; var getUrlFunctional = utils.getUrlFunctional; import { CollectingHandlerForMutliTarget } from './CollectingHandlerForMutliTarget.js'; +// Check if first request `Accept` header matches response `content-type`. +function matchContentTypeHeaders(requestHeaders, responseHeaders) { + var requestAcceptHeaders = requestHeaders?.['Accept']; + if (!requestAcceptHeaders) { + return; + } + var responseContentType = responseHeaders['content-type']; + if (!responseContentType) { + return; + } + + var requestContentType = requestAcceptHeaders.split(/[;, ]/)[0]; + + if (requestContentType) { + if (/html|\*/gi.test(requestContentType)) { + // Skip "any" or "html" type. + return; + } + if (responseContentType.indexOf(requestContentType) > - 1 ) { + return true; + } + } + +} + export default { provides: [ @@ -159,15 +184,13 @@ export default { } } - if ('content-type' in headers - && headers['content-type'].indexOf('text/markdown') > -1 - && options.acceptHeader?.indexOf('text/markdown') > -1){ + if (matchContentTypeHeaders(options.requestHeaders, headers)) { return cb(null, { __nonHtmlContentResponse: resp }); } - if ('content-type' in headers && !/text\/html|application\/xhtml\+xml/gi.test(headers['content-type'])){ + if ('content-type' in headers && !/text\/html|application\/xhtml\+xml/gi.test(headers['content-type'])) { abortController.abort(); return cacheAndRespond(null, { diff --git a/lib/utils.js b/lib/utils.js index 8c730cbf2..fd6f36706 100644 --- a/lib/utils.js +++ b/lib/utils.js @@ -255,7 +255,10 @@ export function prepareRequestOptions(request_options, options) { uri: url, method: 'GET', headers: { - 'Accept': options.acceptHeader || '*/*' + ...{ + 'Accept': '*/*' + }, + ...options.requestHeaders }, timeout: options.timeout || CONFIG.RESPONSE_TIMEOUT, redirect: redirect, From 309d7259e62ba58040dc18fc072002901b22ec40 Mon Sep 17 00:00:00 2001 From: Nazar Leush Date: Tue, 5 May 2026 18:15:25 +0300 Subject: [PATCH 18/21] fix meta for __allowHtmlparser --- lib/plugins/system/htmlparser/__allowHtmlparser.js | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/plugins/system/htmlparser/__allowHtmlparser.js b/lib/plugins/system/htmlparser/__allowHtmlparser.js index 0f4a79e45..d65d35b7a 100644 --- a/lib/plugins/system/htmlparser/__allowHtmlparser.js +++ b/lib/plugins/system/htmlparser/__allowHtmlparser.js @@ -4,6 +4,7 @@ export default { provides: [ // Run for all who requests htmlparser or meta. 'htmlparser', + 'meta', '__allowHtmlparser' ], From 49817c0f8b531b0b777c746f3e998bba4e71c0cb Mon Sep 17 00:00:00 2001 From: Nazar Leush Date: Wed, 6 May 2026 15:19:31 +0300 Subject: [PATCH 19/21] meta: allow customize cache prefix --- lib/plugins/system/meta/utils.js | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/lib/plugins/system/meta/utils.js b/lib/plugins/system/meta/utils.js index ff5c1fa70..7187ab536 100644 --- a/lib/plugins/system/meta/utils.js +++ b/lib/plugins/system/meta/utils.js @@ -1,6 +1,12 @@ export function getMetaCacheKey(url, whitelistRecord, options) { - var meta_key = 'meta:' + url; + var meta_key = 'meta'; + + if (options.metaKeyPrefix) { + meta_key += ':' + options.metaKeyPrefix; + } + + meta_key += ':' + url; var whitelistHash = whitelistRecord && whitelistRecord.getRecordHash(); if (whitelistHash) { @@ -15,4 +21,4 @@ export function getMetaCacheKey(url, whitelistRecord, options) { return meta_key; }; -export const notPlugin = true; \ No newline at end of file +export const notPlugin = true; From 3cf7e0191e61ff7817e4e02e38c6d29590ce1587 Mon Sep 17 00:00:00 2001 From: Nazar Leush Date: Wed, 6 May 2026 15:39:09 +0300 Subject: [PATCH 20/21] Update audit.log --- audit.log | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/audit.log b/audit.log index 41614f097..157cb1c12 100644 --- a/audit.log +++ b/audit.log @@ -13,6 +13,20 @@ │ More info │ https://github.com/advisories/GHSA-j3q9-mxjg-w52f │ └─────────────────────┴────────────────────────────────────────────────────────┘ ┌─────────────────────┬────────────────────────────────────────────────────────┐ +│ high │ Mongoose's Improper Sanitization of $nor in │ +│ │ sanitizeFilter May Allow NoSQL Injection │ +├─────────────────────┼────────────────────────────────────────────────────────┤ +│ Package │ mongoose │ +├─────────────────────┼────────────────────────────────────────────────────────┤ +│ Vulnerable versions │ >=8.0.0 <=8.22.0 │ +├─────────────────────┼────────────────────────────────────────────────────────┤ +│ Patched versions │ >=8.22.1 │ +├─────────────────────┼────────────────────────────────────────────────────────┤ +│ Paths │ .>mongoose │ +├─────────────────────┼────────────────────────────────────────────────────────┤ +│ More info │ https://github.com/advisories/GHSA-wpg9-53fq-2r8h │ +└─────────────────────┴────────────────────────────────────────────────────────┘ +┌─────────────────────┬────────────────────────────────────────────────────────┐ │ moderate │ brace-expansion: Zero-step sequence causes process │ │ │ hang and memory exhaustion │ ├─────────────────────┼────────────────────────────────────────────────────────┤ @@ -68,5 +82,5 @@ ├─────────────────────┼────────────────────────────────────────────────────────┤ │ More info │ https://github.com/advisories/GHSA-qj8w-gfj5-8c6v │ └─────────────────────┴────────────────────────────────────────────────────────┘ -5 vulnerabilities found -Severity: 4 moderate | 1 high +6 vulnerabilities found +Severity: 4 moderate | 2 high From 74cddea227577905cce60412f14dc70740737dab Mon Sep 17 00:00:00 2001 From: Nazar Leush Date: Wed, 6 May 2026 17:58:22 +0300 Subject: [PATCH 21/21] htmlparser: add `provides` `__nonHtmlContentResponse` --- lib/plugins/system/htmlparser/htmlparser.js | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/plugins/system/htmlparser/htmlparser.js b/lib/plugins/system/htmlparser/htmlparser.js index c03263248..2e1e0c000 100644 --- a/lib/plugins/system/htmlparser/htmlparser.js +++ b/lib/plugins/system/htmlparser/htmlparser.js @@ -38,6 +38,7 @@ export default { provides: [ 'self', '__nonHtmlContentData', + '__nonHtmlContentResponse', '__statusCode' ],