Skip to content

Commit c8d4d30

Browse files
theRealPadstertheRealPadster
committed
Add support for custom JS in more APIs
1 parent 360199d commit c8d4d30

7 files changed

Lines changed: 210 additions & 7 deletions

File tree

‎src/diffbot.js‎

Lines changed: 31 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ class Diffbot {
2626
* @param {string} [options.proxy] Used to specify the IP address of a custom proxy that will be used to fetch the target page, instead of Diffbot's default IPs/proxies. (Ex: &proxy=168.212.226.204)
2727
* @param {string} [options.proxyAuth] Used to specify the authentication parameters that will be used with the proxy specified in the &proxy parameter. (Ex: &proxyAuth=username:password)
2828
* @param {string} [options.body] Optional HTML markup to pass as POST body
29+
* @param {string} [options.customJS] This functionality is currently in beta. See docs for details: https://docs.diffbot.com/docs/en/api-analyze#custom-javascript
2930
* @returns {Object} The analyze query results
3031
*/
3132
analyze(options) {
@@ -43,7 +44,7 @@ class Diffbot {
4344

4445
if (options.fields)
4546
diffbot_url += `&fields=${options.fields.join(',')}`;
46-
47+
4748
// Use of `paging` for the analyze endpoint is undocumented, but does work and was recommended by Diffbot support
4849
// See: https://github.com/theRealPadster/diffbot-api-node/issues/16
4950
if (options.paging != undefined)
@@ -66,7 +67,10 @@ class Diffbot {
6667

6768
const method = options.body ? 'POST' : 'GET';
6869

69-
let req = request.generate(diffbot_url, method, options.body);
70+
let req = request.generate(diffbot_url, method, options.body,
71+
options.customJS ? { // strip out newlines and whitespace
72+
'X-Forward-X-Evaluate': options.customJS.replace(/(\r?\n|\r)\s+/g, ''),
73+
} : null);
7074
let ret = this.test ? req : request.exec(req);
7175

7276
return ret;
@@ -87,6 +91,7 @@ class Diffbot {
8791
* @param {string} [options.proxyAuth] Used to specify the authentication parameters that will be used with the proxy specified in the &proxy parameter. (Ex: &proxyAuth=username:password)
8892
* @param {string[]} [options.naturalLanguage] Used to request the output of the Diffbot Natural Language API in the field naturalLanguage. Example: &naturalLanguage=entities,facts,categories,sentiment.
8993
* @param {string} [options.body] Optional HTML markup to pass as POST body
94+
* @param {string} [options.customJS] This functionality is currently in beta. See docs for details: https://docs.diffbot.com/docs/en/api-article#custom-javascript
9095
* @returns {Object} The article query results
9196
*/
9297
article(options) {
@@ -132,7 +137,10 @@ class Diffbot {
132137

133138
const method = options.body ? 'POST' : 'GET';
134139

135-
let req = request.generate(diffbot_url, method, options.body);
140+
let req = request.generate(diffbot_url, method, options.body,
141+
options.customJS ? { // strip out newlines and whitespace
142+
'X-Forward-X-Evaluate': options.customJS.replace(/(\r?\n|\r)\s+/g, ''),
143+
} : null);
136144
let ret = this.test ? req : request.exec(req);
137145

138146
return ret;
@@ -149,6 +157,7 @@ class Diffbot {
149157
* @param {string} [options.proxy] Used to specify the IP address of a custom proxy that will be used to fetch the target page, instead of Diffbot's default IPs/proxies. (Ex: &proxy=168.212.226.204)
150158
* @param {string} [options.proxyAuth] Used to specify the authentication parameters that will be used with the proxy specified in the &proxy parameter. (Ex: &proxyAuth=username:password)
151159
* @param {string} [options.body] Optional HTML markup to pass as POST body
160+
* @param {string} [options.customJS] This functionality is currently in beta. See docs for details: https://docs.diffbot.com/docs/en/api-discussion#custom-javascript
152161
* @returns {Object} The discussion query results
153162
*/
154163
discussion(options) {
@@ -178,7 +187,10 @@ class Diffbot {
178187

179188
const method = options.body ? 'POST' : 'GET';
180189

181-
let req = request.generate(diffbot_url, method, options.body);
190+
let req = request.generate(diffbot_url, method, options.body,
191+
options.customJS ? { // strip out newlines and whitespace
192+
'X-Forward-X-Evaluate': options.customJS.replace(/(\r?\n|\r)\s+/g, ''),
193+
} : null);
182194
let ret = this.test ? req : request.exec(req);
183195

184196
return ret;
@@ -240,6 +252,7 @@ class Diffbot {
240252
* @param {string} [options.proxy] Used to specify the IP address of a custom proxy that will be used to fetch the target page, instead of Diffbot's default IPs/proxies. (Ex: &proxy=168.212.226.204)
241253
* @param {string} [options.proxyAuth] Used to specify the authentication parameters that will be used with the proxy specified in the &proxy parameter. (Ex: &proxyAuth=username:password)
242254
* @param {string} [options.body] Optional HTML markup to pass as POST body
255+
* @param {string} [options.customJS] This functionality is currently in beta. See docs for details: https://docs.diffbot.com/docs/en/api-image#custom-javascript
243256
* @returns {Object} The image query results
244257
*/
245258
image(options) {
@@ -266,7 +279,10 @@ class Diffbot {
266279

267280
const method = options.body ? 'POST' : 'GET';
268281

269-
let req = request.generate(diffbot_url, method, options.body);
282+
let req = request.generate(diffbot_url, method, options.body,
283+
options.customJS ? { // strip out newlines and whitespace
284+
'X-Forward-X-Evaluate': options.customJS.replace(/(\r?\n|\r)\s+/g, ''),
285+
} : null);
270286
let ret = this.test ? req : request.exec(req);
271287

272288
return ret;
@@ -283,6 +299,7 @@ class Diffbot {
283299
* @param {string} [options.proxy] Used to specify the IP address of a custom proxy that will be used to fetch the target page, instead of Diffbot's default IPs/proxies. (Ex: &proxy=168.212.226.204)
284300
* @param {string} [options.proxyAuth] Used to specify the authentication parameters that will be used with the proxy specified in the &proxy parameter. (Ex: &proxyAuth=username:password)
285301
* @param {string} [options.body] Optional HTML markup to pass as POST body
302+
* @param {string} [options.customJS] This functionality is currently in beta. See docs for details: https://docs.diffbot.com/docs/en/api-product#custom-javascript
286303
* @returns {Object} The product query results
287304
*/
288305
product(options) {
@@ -312,7 +329,10 @@ class Diffbot {
312329

313330
const method = options.body ? 'POST' : 'GET';
314331

315-
let req = request.generate(diffbot_url, method, options.body);
332+
let req = request.generate(diffbot_url, method, options.body,
333+
options.customJS ? { // strip out newlines and whitespace
334+
'X-Forward-X-Evaluate': options.customJS.replace(/(\r?\n|\r)\s+/g, ''),
335+
} : null);
316336
let ret = this.test ? req : request.exec(req);
317337

318338
return ret;
@@ -328,6 +348,7 @@ class Diffbot {
328348
* @param {string} [options.proxy] Used to specify the IP address of a custom proxy that will be used to fetch the target page, instead of Diffbot's default IPs/proxies. (Ex: &proxy=168.212.226.204)
329349
* @param {string} [options.proxyAuth] Used to specify the authentication parameters that will be used with the proxy specified in the &proxy parameter. (Ex: &proxyAuth=username:password)
330350
* @param {string} [options.body] Optional HTML markup to pass as POST body
351+
* @param {string} [options.customJS] This functionality is currently in beta. See docs for details: https://docs.diffbot.com/docs/en/api-video#custom-javascript
331352
* @returns {Object} The video query results
332353
*/
333354
video(options) {
@@ -354,7 +375,10 @@ class Diffbot {
354375

355376
const method = options.body ? 'POST' : 'GET';
356377

357-
let req = request.generate(diffbot_url, method, options.body);
378+
let req = request.generate(diffbot_url, method, options.body,
379+
options.customJS ? { // strip out newlines and whitespace
380+
'X-Forward-X-Evaluate': options.customJS.replace(/(\r?\n|\r)\s+/g, ''),
381+
} : null);
358382
let ret = this.test ? req : request.exec(req);
359383

360384
return ret;

‎test/analyze.test.js‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,36 @@ describe('Analyze Tests', function() {
2424
return Promise.resolve(true);
2525
});
2626

27+
it('should generate the analyze GET request with custom JS', async () => {
28+
const url = 'https://www.theverge.com/2020/8/25/21400240/epic-apple-ruling-unreal-engine-fortnite-temporary-restraining-order';
29+
function start(){};
30+
function end(){};
31+
const customJS = function() {
32+
start();
33+
setTimeout(function() {
34+
var loadMoreNode = document.querySelector('a.loadMore');
35+
if (loadMoreNode != null) {
36+
loadMoreNode.click();
37+
setTimeout(function() {
38+
end();
39+
}, 800);
40+
} else {
41+
end();
42+
}
43+
}, 500);
44+
}.toString();
45+
46+
let request = await diffbot.analyze({ url, customJS });
47+
48+
expect(request.url).to.equal(`https://api.diffbot.com/v3/analyze?token=${diffbot.token}&url=${encodeURIComponent(url)}`);
49+
expect(request.method).to.equal('GET');
50+
expect(request.body).to.be.undefined;
51+
expect(request.headers).to.be.an('object');
52+
expect(request.headers['X-Forward-X-Evaluate']).to.equal(customJS.replace(/(\r?\n|\r)\s+/g, ''));
53+
54+
return Promise.resolve(true);
55+
});
56+
2757
it('should error on no url', async () => {
2858
expect(() => {
2959
return diffbot.analyze({});

‎test/article.test.js‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,36 @@ describe('Article Tests', function() {
2525
return Promise.resolve(true);
2626
});
2727

28+
it('should generate the article GET request with custom JS', async () => {
29+
const url = 'https://www.theverge.com/2020/8/25/21400240/epic-apple-ruling-unreal-engine-fortnite-temporary-restraining-order';
30+
function start(){};
31+
function end(){};
32+
const customJS = function() {
33+
start();
34+
setTimeout(function() {
35+
var loadMoreNode = document.querySelector('a.loadMore');
36+
if (loadMoreNode != null) {
37+
loadMoreNode.click();
38+
setTimeout(function() {
39+
end();
40+
}, 800);
41+
} else {
42+
end();
43+
}
44+
}, 500);
45+
}.toString();
46+
47+
let request = await diffbot.article({ url, customJS });
48+
49+
expect(request.url).to.equal(`https://api.diffbot.com/v3/article?token=${diffbot.token}&url=${encodeURIComponent(url)}`);
50+
expect(request.method).to.equal('GET');
51+
expect(request.body).to.be.undefined;
52+
expect(request.headers).to.be.an('object');
53+
expect(request.headers['X-Forward-X-Evaluate']).to.equal(customJS.replace(/(\r?\n|\r)\s+/g, ''));
54+
55+
return Promise.resolve(true);
56+
});
57+
2858
it('should generate the article POST request', async () => {
2959
const body = 'Now is the time for all good robots to come to the aid of their-- oh never mind, run!';
3060

‎test/discussion.test.js‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,36 @@ describe('Discussion Tests', function() {
2121
return Promise.resolve(true);
2222
});
2323

24+
it('should generate the discussion GET request with custom JS', async () => {
25+
const url = 'https://www.theverge.com/2020/8/25/21400240/epic-apple-ruling-unreal-engine-fortnite-temporary-restraining-order';
26+
function start(){};
27+
function end(){};
28+
const customJS = function() {
29+
start();
30+
setTimeout(function() {
31+
var loadMoreNode = document.querySelector('a.loadMore');
32+
if (loadMoreNode != null) {
33+
loadMoreNode.click();
34+
setTimeout(function() {
35+
end();
36+
}, 800);
37+
} else {
38+
end();
39+
}
40+
}, 500);
41+
}.toString();
42+
43+
let request = await diffbot.discussion({ url, customJS });
44+
45+
expect(request.url).to.equal(`https://api.diffbot.com/v3/discussion?token=${diffbot.token}&url=${encodeURIComponent(url)}`);
46+
expect(request.method).to.equal('GET');
47+
expect(request.body).to.be.undefined;
48+
expect(request.headers).to.be.an('object');
49+
expect(request.headers['X-Forward-X-Evaluate']).to.equal(customJS.replace(/(\r?\n|\r)\s+/g, ''));
50+
51+
return Promise.resolve(true);
52+
});
53+
2454
it('should generate the discussion POST request', async () => {
2555
const url = 'https://www.theverge.com/2020/8/25/21400240/epic-apple-ruling-unreal-engine-fortnite-temporary-restraining-order';
2656
const body = '<html><body><h1>Article title</h1><div><h2>Person 1</h2><p>Blah blah</p></div><div><h2>Person 2</h2><p>Hah hah</p></div></body></html>';

‎test/image.test.js‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,36 @@ describe('Image Tests', function() {
2020
return Promise.resolve(true);
2121
});
2222

23+
it('should generate the image GET request with custom JS', async () => {
24+
const url = 'https://www.deviantart.com/up-tchi/art/Coral-village-852927725';
25+
function start(){};
26+
function end(){};
27+
const customJS = function() {
28+
start();
29+
setTimeout(function() {
30+
var loadMoreNode = document.querySelector('a.loadMore');
31+
if (loadMoreNode != null) {
32+
loadMoreNode.click();
33+
setTimeout(function() {
34+
end();
35+
}, 800);
36+
} else {
37+
end();
38+
}
39+
}, 500);
40+
}.toString();
41+
42+
let request = await diffbot.image({ url, customJS });
43+
44+
expect(request.url).to.equal(`https://api.diffbot.com/v3/image?token=${diffbot.token}&url=${encodeURIComponent(url)}`);
45+
expect(request.method).to.equal('GET');
46+
expect(request.body).to.be.undefined;
47+
expect(request.headers).to.be.an('object');
48+
expect(request.headers['X-Forward-X-Evaluate']).to.equal(customJS.replace(/(\r?\n|\r)\s+/g, ''));
49+
50+
return Promise.resolve(true);
51+
});
52+
2353
it('should generate the image POST request', async () => {
2454
const url = 'https://www.google.com/';
2555
const body = '<html><body><h2>This is the Google logo</h2><div><img src="images/branding/googlelogo/1x/googlelogo_color_272x92dp.png"></div></body></html>';

‎test/product.test.js‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,35 @@ describe('Product Tests', function() {
3535
return Promise.resolve(true);
3636
});
3737

38+
it('should generate the product GET request with custom JS', async () => {
39+
function start(){};
40+
function end(){};
41+
const customJS = function() {
42+
start();
43+
setTimeout(function() {
44+
var loadMoreNode = document.querySelector('a.loadMore');
45+
if (loadMoreNode != null) {
46+
loadMoreNode.click();
47+
setTimeout(function() {
48+
end();
49+
}, 800);
50+
} else {
51+
end();
52+
}
53+
}, 500);
54+
}.toString();
55+
56+
let request = await diffbot.product({ url, customJS });
57+
58+
expect(request.url).to.equal(`https://api.diffbot.com/v3/product?token=${diffbot.token}&url=${encodeURIComponent(url)}`);
59+
expect(request.method).to.equal('GET');
60+
expect(request.body).to.be.undefined;
61+
expect(request.headers).to.be.an('object');
62+
expect(request.headers['X-Forward-X-Evaluate']).to.equal(customJS.replace(/(\r?\n|\r)\s+/g, ''));
63+
64+
return Promise.resolve(true);
65+
});
66+
3867
it('should error on no url', async () => {
3968
expect(() => {
4069
return diffbot.product({});

‎test/video.test.js‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,36 @@ describe('Video Tests', function() {
2020
return Promise.resolve(true);
2121
});
2222

23+
it('should generate the video GET request with custom JS', async () => {
24+
const url = 'https://www.youtube.com/watch?v=HeiPdaTQTfo';
25+
function start(){};
26+
function end(){};
27+
const customJS = function() {
28+
start();
29+
setTimeout(function() {
30+
var loadMoreNode = document.querySelector('a.loadMore');
31+
if (loadMoreNode != null) {
32+
loadMoreNode.click();
33+
setTimeout(function() {
34+
end();
35+
}, 800);
36+
} else {
37+
end();
38+
}
39+
}, 500);
40+
}.toString();
41+
42+
let request = await diffbot.video({ url, customJS });
43+
44+
expect(request.url).to.equal(`https://api.diffbot.com/v3/video?token=${diffbot.token}&url=${encodeURIComponent(url)}`);
45+
expect(request.method).to.equal('GET');
46+
expect(request.body).to.be.undefined;
47+
expect(request.headers).to.be.an('object');
48+
expect(request.headers['X-Forward-X-Evaluate']).to.equal(customJS.replace(/(\r?\n|\r)\s+/g, ''));
49+
50+
return Promise.resolve(true);
51+
});
52+
2353
it('should generate the video POST request', async () => {
2454
const url = 'https://www.youtube.com/watch?v=HeiPdaTQTfo';
2555
const body = '<html><body><video><source src="movie.mp4" type="video/mp4"></video></body></html>';

0 commit comments

Comments
 (0)