diff --git a/audit.log b/audit.log index 41614f097..157cb1c12 100644 --- a/audit.log +++ b/audit.log @@ -13,6 +13,20 @@ │ More info │ https://github.com/advisories/GHSA-j3q9-mxjg-w52f │ └─────────────────────┴────────────────────────────────────────────────────────┘ ┌─────────────────────┬────────────────────────────────────────────────────────┐ +│ high │ Mongoose's Improper Sanitization of $nor in │ +│ │ sanitizeFilter May Allow NoSQL Injection │ +├─────────────────────┼────────────────────────────────────────────────────────┤ +│ Package │ mongoose │ +├─────────────────────┼────────────────────────────────────────────────────────┤ +│ Vulnerable versions │ >=8.0.0 <=8.22.0 │ +├─────────────────────┼────────────────────────────────────────────────────────┤ +│ Patched versions │ >=8.22.1 │ +├─────────────────────┼────────────────────────────────────────────────────────┤ +│ Paths │ .>mongoose │ +├─────────────────────┼────────────────────────────────────────────────────────┤ +│ More info │ https://github.com/advisories/GHSA-wpg9-53fq-2r8h │ +└─────────────────────┴────────────────────────────────────────────────────────┘ +┌─────────────────────┬────────────────────────────────────────────────────────┐ │ moderate │ brace-expansion: Zero-step sequence causes process │ │ │ hang and memory exhaustion │ ├─────────────────────┼────────────────────────────────────────────────────────┤ @@ -68,5 +82,5 @@ ├─────────────────────┼────────────────────────────────────────────────────────┤ │ More info │ https://github.com/advisories/GHSA-qj8w-gfj5-8c6v │ └─────────────────────┴────────────────────────────────────────────────────────┘ -5 vulnerabilities found -Severity: 4 moderate | 1 high +6 vulnerabilities found +Severity: 4 moderate | 2 high diff --git a/config.js b/config.js index 4a320562a..c4dd73afa 100644 --- a/config.js +++ b/config.js @@ -367,7 +367,13 @@ 'ENET', 'HPE_INVALID_', 'ERR_SSL_' - ] + ], + + GET_VARS_METHODS: { + 'getSignals': 'signals', + 'getPolicy': 'policy', + 'getSources': 'sources' + } }; // Providers config loader. diff --git a/lib/core.js b/lib/core.js index 94ae22480..e60030ad0 100644 --- a/lib/core.js +++ b/lib/core.js @@ -458,13 +458,19 @@ // Use all generic plugins. for(var i = 0; i < pluginsList.length; i++) { var plugin = pluginsList[i]; - if (!plugin.domain && !plugin.custom) { + if (!plugin.domain + && !plugin.custom + // dataMode - enables data plugins + && (options.dataMode || !plugin.data)) { + var match = plugin.pluginReMatchesUrl(domain, uri); if (match) { pluginsUrlMatches[plugin.id] = match; } - // false - has re, no match. + // match = result; - url matched plugin.re. + // match = undefined; - domain has no plugin.re. + // match = false - plugin has re, and no match. if (match !== false) { initialPlugins.push(plugin); } @@ -473,7 +479,7 @@ } // In domain debug: add all plugins before domain plugins to make them low priority. - if (options.mixAllWithDomainPlugin) { + if (options.mixAllWithDomainPlugin || options.dataMode) { addAllGeneric(); } @@ -513,6 +519,15 @@ addAllGeneric(); } + function addOneMorePlugin(pluginId) { + var exists = initialPlugins.find(function(plugin) { + return plugin.id === pluginId; + }); + if (!exists) { + initialPlugins.push(plugins[pluginId]); + } + } + if (options.forceParams) { // TODO: replace forEach options.forceParams.forEach(function(param) { @@ -525,13 +540,7 @@ for(var k = 0; k < paramPlugins.length; k++) { var foundPluginId = paramPlugins[k]; - - var exists = initialPlugins.find(function(plugin) { - return plugin.id === foundPluginId; - }); - if (!exists) { - initialPlugins.push(plugins[foundPluginId]); - } + addOneMorePlugin(foundPluginId); } } }); @@ -943,9 +952,20 @@ } } } - } else if (r.method.name === "getVars") { - for(var key in r.data) { - var v = r.data[key]; + } else if (r.method.name === "getVars" || CONFIG.GET_VARS_METHODS && r.method.name in CONFIG.GET_VARS_METHODS) { + + var result_data = r.data; + + // Set specific alias as key. + if (CONFIG.GET_VARS_METHODS && r.method.name in CONFIG.GET_VARS_METHODS) { + var key = CONFIG.GET_VARS_METHODS[r.method.name]; + result_data = { + [key]: r.data + }; + } + + for(var key in result_data) { + var v = result_data[key]; // TODO: work with arrays? if (v !== '' && v !== null && ((typeof v === 'string' && !/^\s+$/.test(v)) || typeof v === 'number' || typeof v === 'object')) { @@ -1134,7 +1154,7 @@ return hasDomainData; } - const BIG_CONTEXT = ['htmlparser', 'readability', 'decode', 'cheerio']; + const BIG_CONTEXT = ['readability', 'decode', 'cheerio']; function prepareResultData(uri, result, options) { @@ -1312,10 +1332,9 @@ if (allData) { for(var i = 0; i < allData.length; i++) { var r = allData[i]; - for(var j = 0; j < BIG_CONTEXT.length; j++) { - var d = BIG_CONTEXT[j]; - if (r.data && r.data[d]) { - r.data[d] = 'BIG_CONTEXT'; + for(var key in r.data) { + if (r.data[key].abortController || BIG_CONTEXT.indexOf(key) > -1) { + r.data[key] = 'BIG_CONTEXT'; } } } diff --git a/lib/loader/pluginLoader.js b/lib/loader/pluginLoader.js index e9b7b807c..497f2bf8a 100644 --- a/lib/loader/pluginLoader.js +++ b/lib/loader/pluginLoader.js @@ -370,6 +370,9 @@ // Plugins in folder 'custom' or 'core' will be run only on explicit dependency. pluginDeclaration.custom = (bits.indexOf('custom') > -1 || bits.indexOf('system') > -1) && !plugin.generic; + // Plugins in folder 'data' will be run only in data_mode. + pluginDeclaration.data = bits.indexOf('data') > -1; + var stat = fs.statSync(pluginPath); pluginDeclaration.modified = new Date(stat.mtime); pluginDeclaration.getPluginLastModifiedDate = getPluginLastModifiedDate; @@ -665,6 +668,7 @@ await loadPlugins(pluginsRoot, 'custom'); await loadPlugins(pluginsRoot, 'links'); await loadPlugins(pluginsRoot, 'meta'); + await loadPlugins(pluginsRoot, 'data'); await loadPlugins(pluginsRoot, 'templates'); // TODO: if has multiple modules_listing - CUSTOM_PLUGINS_PATH will be loaded multiple times. @@ -674,6 +678,7 @@ await loadPlugins(CONFIG.CUSTOM_PLUGINS_PATH, 'custom'); await loadPlugins(CONFIG.CUSTOM_PLUGINS_PATH, 'links'); await loadPlugins(CONFIG.CUSTOM_PLUGINS_PATH, 'meta'); + await loadPlugins(CONFIG.CUSTOM_PLUGINS_PATH, 'data'); await loadPlugins(CONFIG.CUSTOM_PLUGINS_PATH, 'templates'); } else { console.warn('Custom plugin folder "' + CONFIG.CUSTOM_PLUGINS_PATH + '" not found.'); diff --git a/lib/loader/utils.js b/lib/loader/utils.js index d6decc3e7..f10a6da66 100644 --- a/lib/loader/utils.js +++ b/lib/loader/utils.js @@ -1,3 +1,5 @@ + import CONFIG from '../../config.loader.js'; + export const DEFAULT_PARAMS = [ "url", "urlMatch", @@ -43,6 +45,12 @@ "prepareLink" ]; + + // Adds vars methods to PLUGIN_METHODS` as `getSignals`. + CONFIG.GET_VARS_METHODS && Object.keys(CONFIG.GET_VARS_METHODS).forEach(methodName => { + PLUGIN_METHODS.push(methodName); + }); + export const PLUGIN_FIELDS = PLUGIN_METHODS.concat([ "mixins" ]); diff --git a/lib/plugins/system/htmlparser/__allowHtmlparser.js b/lib/plugins/system/htmlparser/__allowHtmlparser.js index 0f4a79e45..d65d35b7a 100644 --- a/lib/plugins/system/htmlparser/__allowHtmlparser.js +++ b/lib/plugins/system/htmlparser/__allowHtmlparser.js @@ -4,6 +4,7 @@ export default { provides: [ // Run for all who requests htmlparser or meta. 'htmlparser', + 'meta', '__allowHtmlparser' ], diff --git a/lib/plugins/system/htmlparser/htmlparser.js b/lib/plugins/system/htmlparser/htmlparser.js index 0633fceae..0cedbf49a 100644 --- a/lib/plugins/system/htmlparser/htmlparser.js +++ b/lib/plugins/system/htmlparser/htmlparser.js @@ -13,10 +13,11 @@ export default { provides: [ 'self', '__nonHtmlContentData', + '__nonHtmlContentResponse', '__statusCode' ], - getData: function(url, whitelistRecord, options, __noCachedHtmlparserFallback, __allowHtmlparser, cb) { + getData: function(url, whitelistRecord, options, __noCachedHtmlparserFallback, cb) { var options2 = {...options, ...{ followRedirect: !!options.followHTTPRedirect, @@ -159,17 +160,25 @@ export default { } } - if('content-type' in headers && !/text\/html|application\/xhtml\+xml/gi.test(headers['content-type'])){ - abortController.abort(); + if ('content-type' in headers && !/text\/html|application\/xhtml\+xml/gi.test(headers['content-type'])) { - return cacheAndRespond(null, { - __nonHtmlContentData: { - type: headers['content-type'], - content_length: headers['content-length'], - 'set-cookie': headers['set-cookie'] - }, - headers: headers - }); + if (options.requestHeaders?.['Accept']?.indexOf(headers['content-type'].split(';')[0]) > -1) { + return cb(null, { + __nonHtmlContentResponse: resp + }); + + } else { + abortController.abort(); + + return cacheAndRespond(null, { + __nonHtmlContentData: { + type: headers['content-type'], + content_length: headers['content-length'], + 'set-cookie': headers['set-cookie'] + }, + headers: headers + }); + } } // Init htmlparser handler. diff --git a/lib/plugins/system/meta/cachedMeta.js b/lib/plugins/system/meta/cachedMeta.js index 1a1497968..0df1dcb61 100644 --- a/lib/plugins/system/meta/cachedMeta.js +++ b/lib/plugins/system/meta/cachedMeta.js @@ -19,7 +19,7 @@ export default { '__statusCode' ], - getData: function(url, options, whitelistRecord, cb) { + getData: function(url, __allowHtmlparser, options, whitelistRecord, cb) { // Ignore proxy.cache_ttl, if options.cache_ttl === 0 - do not read from cache. if (options.refresh || options.cache_ttl === 0) { diff --git a/lib/plugins/system/meta/utils.js b/lib/plugins/system/meta/utils.js index ff5c1fa70..7187ab536 100644 --- a/lib/plugins/system/meta/utils.js +++ b/lib/plugins/system/meta/utils.js @@ -1,6 +1,12 @@ export function getMetaCacheKey(url, whitelistRecord, options) { - var meta_key = 'meta:' + url; + var meta_key = 'meta'; + + if (options.metaKeyPrefix) { + meta_key += ':' + options.metaKeyPrefix; + } + + meta_key += ':' + url; var whitelistHash = whitelistRecord && whitelistRecord.getRecordHash(); if (whitelistHash) { @@ -15,4 +21,4 @@ export function getMetaCacheKey(url, whitelistRecord, options) { return meta_key; }; -export const notPlugin = true; \ No newline at end of file +export const notPlugin = true; diff --git a/lib/utils.js b/lib/utils.js index 94098c49a..fd6f36706 100644 --- a/lib/utils.js +++ b/lib/utils.js @@ -105,6 +105,7 @@ export function prepareRequestOptions(request_options, options) { } } + // Some calls (like oembed) use basic options without `getDomainOptions`. var enable_domain_prerender = options?.getDomainOptions && options.getDomainOptions('meta.prerender'); if (enable_domain_prerender) { setPrerender(enable_domain_prerender); @@ -241,7 +242,7 @@ export function prepareRequestOptions(request_options, options) { follow = 0; } - // Custom redirect logit for cookies. + // Custom redirect logic for cookies. if (options.followRedirect && options.reuseCookies) { redirect = 'manual'; follow = 0; @@ -254,7 +255,10 @@ export function prepareRequestOptions(request_options, options) { uri: url, method: 'GET', headers: { - 'Accept': '*/*' + ...{ + 'Accept': '*/*' + }, + ...options.requestHeaders }, timeout: options.timeout || CONFIG.RESPONSE_TIMEOUT, redirect: redirect, diff --git a/modules/api/views.js b/modules/api/views.js index 421ab7d85..00bdbe74a 100644 --- a/modules/api/views.js +++ b/modules/api/views.js @@ -137,6 +137,7 @@ export default function(app) { debug: getBooleanParam(req, 'debug'), returnProviderOptionsUsage: getBooleanParam(req, 'debug'), mixAllWithDomainPlugin: getBooleanParam(req, 'mixAllWithDomainPlugin'), + dataMode: getBooleanParam(req, 'dataMode'), forceParams: req.query.meta === "true" ? CONFIG.DEBUG_CONTEXTS : null, whitelist: getBooleanParam(req, 'whitelist'), readability: getBooleanParam(req, 'readability'), diff --git a/modules/debug/views.js b/modules/debug/views.js index 9aae734da..bb68e71e4 100644 --- a/modules/debug/views.js +++ b/modules/debug/views.js @@ -10,12 +10,14 @@ export default function(app) { DEBUG = {"on":1, "true":1}[req.query.debug]; } - res.render('debug',{ + res.render('debug', { uri: req.query.uri, mixAllWithDomainPlugin: !!{"on":1, "true":1}[req.query.mixAllWithDomainPlugin], + dataMode: !!{"on":1, "true":1}[req.query.dataMode], refresh: !!{"on":1, "true":1}[req.query.refresh], DEBUG: DEBUG, + DATA_DEBUG_ENABLED: CONFIG.DATA_DEBUG_ENABLED, QUERY: getProviderOptionsQuery(req.query) }); }); -}; \ No newline at end of file +}; diff --git a/static/js/debug.js b/static/js/debug.js index 457751dac..9b3d8ca01 100644 --- a/static/js/debug.js +++ b/static/js/debug.js @@ -252,6 +252,19 @@ function showEmbeds($embeds, data, filterByRel) { $meta.append('' + (DEBUG ? ('' + pluginId + '') : '') + '' + key + '' + linkify(data.meta[key]) + '') }); + // Render "vars". + if (data.vars) { + metaKeys = Object.keys(data.vars); + metaKeys.forEach(function(key) { + if (key == "_sources") { + return; + } + var pluginId = data.vars._sources && data.vars._sources[key] || ''; + usedPlugins[pluginId] = true; + + $meta.append('' + (DEBUG ? ('' + pluginId + '') : '') + 'vars: ' + key + '
' + JSON.stringify(data.vars[key], null, 4) + '
') + }); + } if (DEBUG && data.vary && data.vary.join) { $meta.append('vary' + data.vary.join('
') + '') } @@ -293,6 +306,16 @@ function findAllRels(data) { return _.intersection(result, REL_GROUPS); } +function getTimingData(data) { + var result = {}; + data.allData.forEach(function(pluginResult) { + if (pluginResult.time) { + result[pluginResult.method.pluginId] = pluginResult.time + } + }); + return {timings: result}; +} + function processUrl() { var uri = $.trim($('.s-uri').focus().val()); @@ -305,6 +328,7 @@ function processUrl() { var $resultTabs = $('.s-result-div').hide(); var $result = $('.s-debug-result'); + var $timingResult = $('.s-timing-result'); var $context = $('.s-debug-context'); var $response = $('.s-json'); var $embeds = $('.s-embeds'); @@ -326,6 +350,7 @@ function processUrl() { var query = { debug: true, group: false, + dataMode: $('[name="dataMode"]').is(":checked"), mixAllWithDomainPlugin: $('[name="mixAllWithDomainPlugin"]').is(":checked"), refresh: $('[name="refresh"]').is(":checked") }; @@ -336,7 +361,6 @@ function processUrl() { if (error) { $status.attr('class', 'alert alert-error').show().text(jqXHR.status + ' - ' + error + ' - ' +jqXHR.responseText); - $result.renderObject(data); return; } @@ -370,6 +394,9 @@ function processUrl() { // Render all debug data. $result.renderObject(data); + // Render timing data. + $timingResult.renderObject(getTimingData(data)); + var clearData = $.extend(true, {}, data); delete clearData.allData; delete clearData.time; diff --git a/static/js/iframely.js b/static/js/iframely.js index f13861da6..d7ca4d06e 100644 --- a/static/js/iframely.js +++ b/static/js/iframely.js @@ -151,6 +151,7 @@ uri: !options.url ? uri : undefined, url: options.url ? uri : undefined, debug: options.debug, + dataMode: options.dataMode, mixAllWithDomainPlugin: options.mixAllWithDomainPlugin, refresh: options.refresh, meta: options.meta, diff --git a/views/debug.ejs b/views/debug.ejs index 0022d83eb..c635a1bb7 100644 --- a/views/debug.ejs +++ b/views/debug.ejs @@ -47,6 +47,11 @@ <% if (DEBUG) { %>
+ <% if (DATA_DEBUG_ENABLED) { %> + + <% } %> @@ -89,6 +94,7 @@
+

                         

                     
@@ -98,4 +104,4 @@ - \ No newline at end of file +