diff --git a/defaultmodules/newsfeed/feeditem.js b/defaultmodules/newsfeed/feeditem.js index a570c61041..29dc159518 100644 --- a/defaultmodules/newsfeed/feeditem.js +++ b/defaultmodules/newsfeed/feeditem.js @@ -63,11 +63,10 @@ const sanitizeBasicHtml = (html, allowedTags = []) => { * @param {object} item - The parsed feed item from feedparser. * @param {object} [options] - Normalization options. * @param {string[]} [options.allowedBasicHtmlTags] - Inline tags allowed to survive sanitization. - * @param {boolean} [options.useCorsProxy] - Whether the item's article should use the CORS proxy. * @param {boolean} [options.logFeedWarnings] - Whether to log a warning for items missing a title or publication date. - * @returns {{title:string, description:string, pubdate:string, url:string, useCorsProxy:boolean, hash:string}|null} The normalized item, or null when title or pubdate is missing. + * @returns {{title:string, description:string, pubdate:string, url:string, hash:string}|null} The normalized item, or null when title or pubdate is missing. */ -const normalizeFeedItem = (item, { allowedBasicHtmlTags = [], useCorsProxy = false, logFeedWarnings = false } = {}) => { +const normalizeFeedItem = (item, { allowedBasicHtmlTags = [], logFeedWarnings = false } = {}) => { // feedparser strips HTML from item.title; recover the raw title so inline // formatting tags (e.g. ) in titles survive sanitizeBasicHtml below. const title = (item["rss:title"] && item["rss:title"]["#"]) || (item["atom:title"] && item["atom:title"]["#"]) || item.title; @@ -111,7 +110,6 @@ const normalizeFeedItem = (item, { allowedBasicHtmlTags = [], useCorsProxy = fal description, pubdate, url, - useCorsProxy, // Hash on the original title so the dedup identity is stable regardless of allowedBasicHtmlTags hash: crypto.createHash("sha256").update(`${pubdate} :: ${title} :: ${url}`).digest("hex") }; diff --git a/defaultmodules/newsfeed/newsfeedfetcher.js b/defaultmodules/newsfeed/newsfeedfetcher.js index db6bc27b7e..83d3ce166d 100644 --- a/defaultmodules/newsfeed/newsfeedfetcher.js +++ b/defaultmodules/newsfeed/newsfeedfetcher.js @@ -24,14 +24,12 @@ class NewsfeedFetcher { * @param {number} reloadInterval - Time in ms between fetches * @param {string} encoding - Encoding of the feed (e.g., 'UTF-8') * @param {boolean} logFeedWarnings - If true log warnings when there is an error parsing a news article - * @param {boolean} useCorsProxy - If true cors proxy is used for article url's * @param {string[]} allowedBasicHtmlTags - Basic formatting tags to keep in title and description. Only tags from the safe list are honored; anything else is ignored. */ - constructor (url, reloadInterval, encoding, logFeedWarnings, useCorsProxy, allowedBasicHtmlTags = []) { + constructor (url, reloadInterval, encoding, logFeedWarnings, allowedBasicHtmlTags = []) { this.url = url; this.encoding = encoding; this.logFeedWarnings = logFeedWarnings; - this.useCorsProxy = useCorsProxy; // Keep only tags from the hardcoded safe list; warn about (and ignore) anything else. const requestedTags = (Array.isArray(allowedBasicHtmlTags) ? allowedBasicHtmlTags : []).map((tag) => String(tag).trim().toLowerCase()); @@ -96,7 +94,6 @@ class NewsfeedFetcher { parser.on("data", (item) => { const normalizedItem = normalizeFeedItem(item, { allowedBasicHtmlTags: this.allowedBasicHtmlTags, - useCorsProxy: this.useCorsProxy, logFeedWarnings: this.logFeedWarnings }); if (normalizedItem) { diff --git a/defaultmodules/newsfeed/node_helper.js b/defaultmodules/newsfeed/node_helper.js index 9d24f4f75b..7e2f49b92d 100644 --- a/defaultmodules/newsfeed/node_helper.js +++ b/defaultmodules/newsfeed/node_helper.js @@ -86,7 +86,6 @@ module.exports = NodeHelper.create({ const url = feed.url || ""; const encoding = feed.encoding || "UTF-8"; const reloadInterval = feed.reloadInterval || config.reloadInterval || 5 * 60 * 1000; - const useCorsProxy = feed.useCorsProxy ?? true; try { new URL(url); @@ -99,7 +98,7 @@ module.exports = NodeHelper.create({ let fetcher; if (typeof this.fetchers[url] === "undefined") { Log.log(`Create new newsfetcher for url: ${url} - Interval: ${reloadInterval}`); - fetcher = new NewsfeedFetcher(url, reloadInterval, encoding, config.logFeedWarnings, useCorsProxy, config.allowedBasicHtmlTags); + fetcher = new NewsfeedFetcher(url, reloadInterval, encoding, config.logFeedWarnings, config.allowedBasicHtmlTags); fetcher.onReceive(() => { this.broadcastFeeds(); diff --git a/tests/unit/modules/default/newsfeed/newsfeedfetcher_spec.js b/tests/unit/modules/default/newsfeed/newsfeedfetcher_spec.js index cef7f2b680..baa9b2c437 100644 --- a/tests/unit/modules/default/newsfeed/newsfeedfetcher_spec.js +++ b/tests/unit/modules/default/newsfeed/newsfeedfetcher_spec.js @@ -14,32 +14,34 @@ const feedResponse = (fetcher, xml = xmlContent) => new Promise((resolve) => { describe("NewsfeedFetcher", () => { it("parses all items from the test fixture", async () => { - const fetcher = new NewsfeedFetcher("http://test.example/feed", 60000, "UTF-8", false, false); + const fetcher = new NewsfeedFetcher("http://test.example/feed", 60000, "UTF-8", false); const items = await feedResponse(fetcher); expect(items).toHaveLength(10); }); it("parses title, URL, publication date and description", async () => { - const fetcher = new NewsfeedFetcher("http://test.example/feed", 60000, "UTF-8", false, false); + const fetcher = new NewsfeedFetcher("http://test.example/feed", 60000, "UTF-8", false); const items = await feedResponse(fetcher); const item = items[0]; - expect(item.title).toBeTruthy(); - expect(item.url).toMatch(/^https?:\/\//); - expect(item.pubdate).toBeTruthy(); + expect(item.title).toBe("QPanel 0.13.0"); + expect(item.url).toBe("https://rodrigoramirez.com/qpanel-0-13-0/"); + expect(item.pubdate).toBe("2016-09-20T11:16:08.000Z"); expect(item.description).toBeTruthy(); }); it("strips HTML tags from descriptions", async () => { - const fetcher = new NewsfeedFetcher("http://test.example/feed", 60000, "UTF-8", false, false); + const fetcher = new NewsfeedFetcher("http://test.example/feed", 60000, "UTF-8", false); const items = await feedResponse(fetcher); - expect(items[0].description).not.toContain("

"); + for (const item of items) { + expect(item.description).not.toMatch(/<[^>]+>/); + } }); it("defaults to an empty description when an Atom entry has no summary content", async () => { - const fetcher = new NewsfeedFetcher("http://test.example/feed", 60000, "UTF-8", false, false); + const fetcher = new NewsfeedFetcher("http://test.example/feed", 60000, "UTF-8", false); const feed = ` @@ -55,7 +57,7 @@ describe("NewsfeedFetcher", () => { }); it("generates a stable SHA-256 hash for each item", async () => { - const fetcher = new NewsfeedFetcher("http://test.example/feed", 60000, "UTF-8", false, false); + const fetcher = new NewsfeedFetcher("http://test.example/feed", 60000, "UTF-8", false); const items = await feedResponse(fetcher); for (const item of items) { @@ -65,7 +67,7 @@ describe("NewsfeedFetcher", () => { it("discards items without a title", async () => { const xml = xmlContent.replace("QPanel 0.13.0", ""); - const fetcher = new NewsfeedFetcher("http://test.example/feed", 60000, "UTF-8", false, false); + const fetcher = new NewsfeedFetcher("http://test.example/feed", 60000, "UTF-8", false); const items = await feedResponse(fetcher, xml); expect(items).toHaveLength(9); @@ -73,14 +75,14 @@ describe("NewsfeedFetcher", () => { it("discards items without a publication date", async () => { const xml = xmlContent.replace("Tue, 20 Sep 2016 11:16:08 +0000", ""); - const fetcher = new NewsfeedFetcher("http://test.example/feed", 60000, "UTF-8", false, false); + const fetcher = new NewsfeedFetcher("http://test.example/feed", 60000, "UTF-8", false); const items = await feedResponse(fetcher, xml); expect(items).toHaveLength(9); }); it("calls onError when feed XML is malformed", async () => { - const fetcher = new NewsfeedFetcher("http://test.example/feed", 60000, "UTF-8", false, false); + const fetcher = new NewsfeedFetcher("http://test.example/feed", 60000, "UTF-8", false); const error = new Promise((resolve) => fetcher.onError((_fetcher, info) => resolve(info))); fetcher.httpFetcher.emit("response", { body: Readable.from(["this is not xml at all <<<"]) }); @@ -88,7 +90,7 @@ describe("NewsfeedFetcher", () => { }); it("calls onError when an HTTP error is emitted", async () => { - const fetcher = new NewsfeedFetcher("http://test.example/feed", 60000, "UTF-8", false, false); + const fetcher = new NewsfeedFetcher("http://test.example/feed", 60000, "UTF-8", false); const error = new Promise((resolve) => fetcher.onError((_fetcher, info) => resolve(info))); fetcher.httpFetcher.emit("error", { message: "404", translationKey: "MODULE_ERROR_CLIENT_ERROR" }); @@ -98,22 +100,30 @@ describe("NewsfeedFetcher", () => { it("increases reloadInterval when feed TTL is larger", async () => { const ttlMinutes = 30; const xml = xmlContent.replace(" ", ` ${ttlMinutes}\n `); - const fetcher = new NewsfeedFetcher("http://test.example/feed", 60000, "UTF-8", false, false); + const fetcher = new NewsfeedFetcher("http://test.example/feed", 60000, "UTF-8", false); await feedResponse(fetcher, xml); expect(fetcher.httpFetcher.reloadInterval).toBe(ttlMinutes * 60 * 1000); }); + it("does not decrease reloadInterval when feed TTL is smaller", async () => { + const xml = xmlContent.replace(" ", " 1\n "); + const fetcher = new NewsfeedFetcher("http://test.example/feed", 600000, "UTF-8", false); + await feedResponse(fetcher, xml); + + expect(fetcher.httpFetcher.reloadInterval).toBe(600000); + }); + it("caps TTL at 24 hours", async () => { const xml = xmlContent.replace(" ", " 2880\n "); - const fetcher = new NewsfeedFetcher("http://test.example/feed", 60000, "UTF-8", false, false); + const fetcher = new NewsfeedFetcher("http://test.example/feed", 60000, "UTF-8", false); await feedResponse(fetcher, xml); expect(fetcher.httpFetcher.reloadInterval).toBe(24 * 60 * 60 * 1000); }); it("updates reloadInterval only when the configured value is smaller", () => { - const fetcher = new NewsfeedFetcher("http://test.example/feed", 60000, "UTF-8", false, false); + const fetcher = new NewsfeedFetcher("http://test.example/feed", 60000, "UTF-8", false); fetcher.setReloadInterval(30000); expect(fetcher.httpFetcher.reloadInterval).toBe(30000); @@ -123,13 +133,23 @@ describe("NewsfeedFetcher", () => { expect(fetcher.httpFetcher.reloadInterval).toBe(30000); }); - it("attaches useCorsProxy without changing item URLs", async () => { - const fetcher = new NewsfeedFetcher("http://test.example/feed", 60000, "UTF-8", false, true); + it("uses allowedBasicHtmlTags from the fifth constructor argument", async () => { + const xml = xmlContent.replace("QPanel 0.13.0", "<![CDATA[<em>QPanel</em> 0.13.0]]>"); + const fetcher = new NewsfeedFetcher("http://test.example/feed", 60000, "UTF-8", false, ["em"]); + const items = await feedResponse(fetcher, xml); + + expect(items[0].title).toBe("QPanel 0.13.0"); + }); + + it("keeps article URLs direct and omits the removed proxy metadata", async () => { + const fetcher = new NewsfeedFetcher("http://test.example/feed", 60000, "UTF-8", false); const items = await feedResponse(fetcher); + expect(items).toHaveLength(10); for (const item of items) { - expect(item.useCorsProxy).toBe(true); + expect(item.url).toMatch(/^https?:\/\//); expect(item.url).not.toContain("/cors?url="); + expect(item).not.toHaveProperty("useCorsProxy"); } }); }); diff --git a/tests/unit/modules/default/newsfeed/node_helper_spec.js b/tests/unit/modules/default/newsfeed/node_helper_spec.js index 3375e0407e..4a6d4cd180 100644 --- a/tests/unit/modules/default/newsfeed/node_helper_spec.js +++ b/tests/unit/modules/default/newsfeed/node_helper_spec.js @@ -91,7 +91,6 @@ describe("newsfeed node_helper", () => { helper.createFetcher({ url: "https://example.com/feed.xml", encoding: "ISO-8859-1", - useCorsProxy: false, reloadInterval: 120000 }, config);