532 lines
19 KiB
JavaScript
532 lines
19 KiB
JavaScript
import { dirname } from 'path';
|
|
import { fileURLToPath } from 'url';
|
|
import fs from 'fs';
|
|
import FormData from 'form-data';
|
|
import assert from 'assert';
|
|
import rimraf from 'rimraf';
|
|
import path from 'path';
|
|
import { URLSearchParams } from 'url';
|
|
import standardFetch from 'node-fetch';
|
|
import FetchCache, { MemoryCache, FileSystemCache, getCacheKey } from '../src/index.js';
|
|
import { Agent } from 'http';
|
|
|
|
const __dirname = dirname(fileURLToPath(import.meta.url));
|
|
|
|
const CACHE_PATH = path.join(__dirname, '..', '.cache');
|
|
const expectedPngBuffer = fs.readFileSync(path.join(__dirname, 'expected_png.png'));
|
|
|
|
const TWO_HUNDRED_URL = 'https://httpbin.org/status/200';
|
|
const FOUR_HUNDRED_URL = 'https://httpbin.org/status/400';
|
|
const THREE_HUNDRED_TWO_URL = 'https://httpbin.org/status/302';
|
|
const TEXT_BODY_URL = 'https://httpbin.org/robots.txt';
|
|
const JSON_BODY_URL = 'https://httpbin.org/json';
|
|
const PNG_BODY_URL = 'https://httpbin.org/image/png';
|
|
|
|
const TEXT_BODY_EXPECTED = 'User-agent: *\nDisallow: /deny\n';
|
|
|
|
let cachedFetch;
|
|
let body;
|
|
|
|
function post(body) {
|
|
return { method: 'POST', body };
|
|
}
|
|
|
|
function removeDates(arrOrObj) {
|
|
if (arrOrObj.date) {
|
|
const copy = { ...arrOrObj };
|
|
delete copy.date;
|
|
return copy;
|
|
}
|
|
|
|
if (Array.isArray(arrOrObj)) {
|
|
if (Array.isArray(arrOrObj[0])) {
|
|
return arrOrObj.filter(e => e[0] !== 'date');
|
|
}
|
|
|
|
return arrOrObj.filter(e => !Date.parse(e));
|
|
}
|
|
|
|
return arrOrObj;
|
|
}
|
|
|
|
function wait(ms) {
|
|
return new Promise((fulfill) => setTimeout(fulfill, ms));
|
|
}
|
|
|
|
async function dualFetch(...args) {
|
|
const [cachedFetchResponse, standardFetchResponse] = await Promise.all([
|
|
cachedFetch(...args),
|
|
standardFetch(...args),
|
|
]);
|
|
|
|
return { cachedFetchResponse, standardFetchResponse };
|
|
}
|
|
|
|
beforeEach(async function() {
|
|
rimraf.sync(CACHE_PATH);
|
|
cachedFetch = FetchCache.withCache(new MemoryCache());
|
|
});
|
|
|
|
let res;
|
|
|
|
describe('Basic property tests', function() {
|
|
it('Has a status property', async function() {
|
|
let { cachedFetchResponse, standardFetchResponse } = await dualFetch(TWO_HUNDRED_URL);
|
|
assert.strictEqual(cachedFetchResponse.status, standardFetchResponse.status);
|
|
|
|
cachedFetchResponse = await cachedFetch(TWO_HUNDRED_URL);
|
|
assert.strictEqual(cachedFetchResponse.status, standardFetchResponse.status);
|
|
});
|
|
|
|
it('Has a statusText property', async function() {
|
|
let { cachedFetchResponse, standardFetchResponse } = await dualFetch(TWO_HUNDRED_URL);
|
|
assert.strictEqual(cachedFetchResponse.statusText, standardFetchResponse.statusText);
|
|
|
|
cachedFetchResponse = await cachedFetch(TWO_HUNDRED_URL);
|
|
assert.strictEqual(cachedFetchResponse.statusText, standardFetchResponse.statusText);
|
|
});
|
|
|
|
it('Has a url property', async function() {
|
|
let { cachedFetchResponse, standardFetchResponse } = await dualFetch(TWO_HUNDRED_URL);
|
|
assert.strictEqual(cachedFetchResponse.url, standardFetchResponse.url);
|
|
|
|
cachedFetchResponse = await cachedFetch(TWO_HUNDRED_URL);
|
|
assert.strictEqual(cachedFetchResponse.url, standardFetchResponse.url);
|
|
});
|
|
|
|
it('Has an ok property', async function() {
|
|
let { cachedFetchResponse, standardFetchResponse } = await dualFetch(FOUR_HUNDRED_URL);
|
|
assert.strictEqual(cachedFetchResponse.ok, standardFetchResponse.ok);
|
|
assert.strictEqual(cachedFetchResponse.status, standardFetchResponse.status);
|
|
|
|
cachedFetchResponse = await cachedFetch(FOUR_HUNDRED_URL);
|
|
assert.strictEqual(cachedFetchResponse.ok, standardFetchResponse.ok);
|
|
assert.strictEqual(cachedFetchResponse.status, standardFetchResponse.status);
|
|
});
|
|
|
|
it('Has a redirected property', async function() {
|
|
let { cachedFetchResponse, standardFetchResponse } = await dualFetch(THREE_HUNDRED_TWO_URL);
|
|
assert.strictEqual(cachedFetchResponse.redirected, standardFetchResponse.redirected);
|
|
|
|
cachedFetchResponse = await cachedFetch(THREE_HUNDRED_TWO_URL);
|
|
assert.strictEqual(cachedFetchResponse.redirected, standardFetchResponse.redirected);
|
|
});
|
|
}).timeout(10000);
|
|
|
|
describe('Header tests', function() {
|
|
it('Gets correct raw headers', async function() {
|
|
let { cachedFetchResponse, standardFetchResponse } = await dualFetch(TWO_HUNDRED_URL);
|
|
assert.deepStrictEqual(
|
|
removeDates(cachedFetchResponse.headers.raw()),
|
|
removeDates(standardFetchResponse.headers.raw()),
|
|
);
|
|
|
|
cachedFetchResponse = await cachedFetch(TWO_HUNDRED_URL);
|
|
assert.deepStrictEqual(
|
|
removeDates(cachedFetchResponse.headers.raw()),
|
|
removeDates(standardFetchResponse.headers.raw()),
|
|
);
|
|
});
|
|
|
|
it('Gets correct header keys', async function() {
|
|
let { cachedFetchResponse, standardFetchResponse } = await dualFetch(TWO_HUNDRED_URL);
|
|
assert.deepStrictEqual([...cachedFetchResponse.headers.keys()], [...standardFetchResponse.headers.keys()]);
|
|
|
|
cachedFetchResponse = await cachedFetch(TWO_HUNDRED_URL);
|
|
assert.deepStrictEqual([...cachedFetchResponse.headers.keys()], [...standardFetchResponse.headers.keys()]);
|
|
});
|
|
|
|
it('Gets correct header values', async function() {
|
|
let { cachedFetchResponse, standardFetchResponse } = await dualFetch(TWO_HUNDRED_URL);
|
|
assert.deepStrictEqual(
|
|
removeDates([...cachedFetchResponse.headers.values()]),
|
|
removeDates([...standardFetchResponse.headers.values()]),
|
|
);
|
|
|
|
cachedFetchResponse = await cachedFetch(TWO_HUNDRED_URL);
|
|
assert.deepStrictEqual(
|
|
removeDates([...cachedFetchResponse.headers.values()]),
|
|
removeDates([...standardFetchResponse.headers.values()]),
|
|
);
|
|
});
|
|
|
|
it('Gets correct header entries', async function() {
|
|
let { cachedFetchResponse, standardFetchResponse } = await dualFetch(TWO_HUNDRED_URL);
|
|
assert.deepStrictEqual(
|
|
removeDates([...cachedFetchResponse.headers.entries()]),
|
|
removeDates([...standardFetchResponse.headers.entries()]),
|
|
);
|
|
|
|
cachedFetchResponse = await cachedFetch(TWO_HUNDRED_URL);
|
|
assert.deepStrictEqual(
|
|
removeDates([...cachedFetchResponse.headers.entries()]),
|
|
removeDates([...standardFetchResponse.headers.entries()]),
|
|
);
|
|
});
|
|
|
|
it('Can get a header by value', async function() {
|
|
let { cachedFetchResponse, standardFetchResponse } = await dualFetch(TWO_HUNDRED_URL);
|
|
assert(standardFetchResponse.headers.get('content-length'));
|
|
assert.deepStrictEqual(cachedFetchResponse.headers.get('content-length'), standardFetchResponse.headers.get('content-length'));
|
|
|
|
cachedFetchResponse = await cachedFetch(TWO_HUNDRED_URL);
|
|
assert.deepStrictEqual(cachedFetchResponse.headers.get('content-length'), standardFetchResponse.headers.get('content-length'));
|
|
});
|
|
|
|
it('Returns undefined for non-existent header', async function() {
|
|
const headerName = 'zzzz';
|
|
let { cachedFetchResponse, standardFetchResponse } = await dualFetch(TWO_HUNDRED_URL);
|
|
assert(!standardFetchResponse.headers.get(headerName));
|
|
assert.deepStrictEqual(cachedFetchResponse.headers.get(headerName), standardFetchResponse.headers.get(headerName));
|
|
|
|
cachedFetchResponse = await cachedFetch(TWO_HUNDRED_URL);
|
|
assert.deepStrictEqual(cachedFetchResponse.headers.get(headerName), standardFetchResponse.headers.get(headerName));
|
|
});
|
|
|
|
it('Can get whether a header is present', async function() {
|
|
let { cachedFetchResponse, standardFetchResponse } = await dualFetch(TWO_HUNDRED_URL);
|
|
assert(standardFetchResponse.headers.has('content-length'));
|
|
assert.deepStrictEqual(cachedFetchResponse.headers.has('content-length'), standardFetchResponse.headers.has('content-length'));
|
|
|
|
cachedFetchResponse = await cachedFetch(TWO_HUNDRED_URL);
|
|
assert.deepStrictEqual(cachedFetchResponse.headers.has('content-length'), standardFetchResponse.headers.has('content-length'));
|
|
});
|
|
}).timeout(10000);
|
|
|
|
describe('Cache tests', function() {
|
|
it('Uses cache', async function() {
|
|
res = await cachedFetch(TWO_HUNDRED_URL);
|
|
assert.strictEqual(res.fromCache, false);
|
|
|
|
res = await cachedFetch(TWO_HUNDRED_URL);
|
|
assert.strictEqual(res.fromCache, true);
|
|
});
|
|
|
|
it('Can eject from cache', async function() {
|
|
res = await cachedFetch(TWO_HUNDRED_URL);
|
|
assert.strictEqual(res.fromCache, false);
|
|
|
|
res = await cachedFetch(TWO_HUNDRED_URL);
|
|
assert.strictEqual(res.fromCache, true);
|
|
|
|
await res.ejectFromCache();
|
|
|
|
res = await cachedFetch(TWO_HUNDRED_URL);
|
|
assert.strictEqual(res.fromCache, false);
|
|
|
|
res = await cachedFetch(TWO_HUNDRED_URL);
|
|
assert.strictEqual(res.fromCache, true);
|
|
});
|
|
|
|
it('Does not error if ejecting from cache twice', async function() {
|
|
res = await cachedFetch(TWO_HUNDRED_URL);
|
|
assert.strictEqual(res.fromCache, false);
|
|
|
|
await res.ejectFromCache();
|
|
await res.ejectFromCache();
|
|
});
|
|
|
|
it('Gives different string bodies different cache keys', async function() {
|
|
res = await cachedFetch(TWO_HUNDRED_URL, post('a'));
|
|
assert.strictEqual(res.fromCache, false);
|
|
|
|
res = await cachedFetch(TWO_HUNDRED_URL, post('b'));
|
|
assert.strictEqual(res.fromCache, false);
|
|
});
|
|
|
|
it('Gives same string bodies same cache keys', async function() {
|
|
res = await cachedFetch(TWO_HUNDRED_URL, post('a'));
|
|
assert.strictEqual(res.fromCache, false);
|
|
|
|
res = await cachedFetch(TWO_HUNDRED_URL, post('a'));
|
|
assert.strictEqual(res.fromCache, true);
|
|
});
|
|
|
|
it('Gives different URLSearchParams different cache keys', async function() {
|
|
res = await cachedFetch(TWO_HUNDRED_URL, post(new URLSearchParams('a=a')));
|
|
assert.strictEqual(res.fromCache, false);
|
|
|
|
res = await cachedFetch(TWO_HUNDRED_URL, post(new URLSearchParams('a=b')));
|
|
assert.strictEqual(res.fromCache, false);
|
|
});
|
|
|
|
it('Gives same URLSearchParams same cache keys', async function() {
|
|
res = await cachedFetch(TWO_HUNDRED_URL, post(new URLSearchParams('a=a')));
|
|
assert.strictEqual(res.fromCache, false);
|
|
|
|
res = await cachedFetch(TWO_HUNDRED_URL, post(new URLSearchParams('a=a')));
|
|
assert.strictEqual(res.fromCache, true);
|
|
});
|
|
|
|
it('Gives different read streams different cache keys', async function() {
|
|
const s1 = fs.createReadStream(path.join(__dirname, 'expected_png.png'));
|
|
const s2 = fs.createReadStream(path.join(__dirname, '..', 'src', 'index.js'));
|
|
|
|
res = await cachedFetch(TWO_HUNDRED_URL, post(s1));
|
|
assert.strictEqual(res.fromCache, false);
|
|
|
|
res = await cachedFetch(TWO_HUNDRED_URL, post(s2));
|
|
assert.strictEqual(res.fromCache, false);
|
|
});
|
|
|
|
it('Gives the same read streams the same cache key', async function() {
|
|
const s1 = fs.createReadStream(path.join(__dirname, 'expected_png.png'));
|
|
|
|
res = await cachedFetch(TWO_HUNDRED_URL, post(s1));
|
|
assert.strictEqual(res.fromCache, false);
|
|
|
|
res = await cachedFetch(TWO_HUNDRED_URL, post(s1));
|
|
assert.strictEqual(res.fromCache, true);
|
|
});
|
|
|
|
it('Gives different form data different cache keys', async function() {
|
|
const data1 = new FormData();
|
|
data1.append('a', 'a');
|
|
|
|
const data2 = new FormData();
|
|
data2.append('b', 'b');
|
|
|
|
res = await cachedFetch(TWO_HUNDRED_URL, post(data1));
|
|
assert.strictEqual(res.fromCache, false);
|
|
|
|
res = await cachedFetch(TWO_HUNDRED_URL, post(data2));
|
|
assert.strictEqual(res.fromCache, false);
|
|
});
|
|
|
|
it('Gives same form data same cache keys', async function() {
|
|
const data1 = new FormData();
|
|
data1.append('a', 'a');
|
|
|
|
const data2 = new FormData();
|
|
data2.append('a', 'a');
|
|
|
|
res = await cachedFetch(TWO_HUNDRED_URL, post(data1));
|
|
assert.strictEqual(res.fromCache, false);
|
|
|
|
res = await cachedFetch(TWO_HUNDRED_URL, post(data2));
|
|
assert.strictEqual(res.fromCache, true);
|
|
});
|
|
|
|
it('Does not error with custom agent with circular properties', async function() {
|
|
const agent = new Agent();
|
|
agent.agent = agent;
|
|
|
|
await cachedFetch('http://httpbin.org/status/200', { agent });
|
|
})
|
|
}).timeout(10000);
|
|
|
|
describe('Data tests', function() {
|
|
it('Supports request objects', async function() {
|
|
let request = new standardFetch.Request('https://google.com', { body: 'test', method: 'POST' });
|
|
res = await cachedFetch(request);
|
|
assert.strictEqual(res.fromCache, false);
|
|
|
|
request = new standardFetch.Request('https://google.com', { body: 'test', method: 'POST' });
|
|
res = await cachedFetch(request);
|
|
assert.strictEqual(res.fromCache, true);
|
|
});
|
|
|
|
it('Supports request objects with custom headers', async function() {
|
|
const request1 = new standardFetch.Request(TWO_HUNDRED_URL, { headers: { 'XXX': 'YYY' } });
|
|
const request2 = new standardFetch.Request(TWO_HUNDRED_URL, { headers: { 'XXX': 'ZZZ' } });
|
|
|
|
res = await cachedFetch(request1);
|
|
assert.strictEqual(res.fromCache, false);
|
|
|
|
res = await cachedFetch(request2);
|
|
assert.strictEqual(res.fromCache, false);
|
|
});
|
|
|
|
it('Refuses to consume body twice', async function() {
|
|
res = await cachedFetch(TEXT_BODY_URL);
|
|
await res.text();
|
|
|
|
try {
|
|
await res.text();
|
|
throw new Error('The above line should have thrown.');
|
|
} catch (err) {
|
|
assert(err.message.includes('body used already for:'));
|
|
}
|
|
});
|
|
|
|
it('Can get text body', async function() {
|
|
res = await cachedFetch(TEXT_BODY_URL);
|
|
body = await res.text();
|
|
assert.strictEqual(body, TEXT_BODY_EXPECTED);
|
|
assert.strictEqual(res.fromCache, false);
|
|
|
|
res = await cachedFetch(TEXT_BODY_URL);
|
|
body = await res.text();
|
|
assert.strictEqual(body, TEXT_BODY_EXPECTED);
|
|
assert.strictEqual(res.fromCache, true);
|
|
});
|
|
|
|
it('Can get JSON body', async function() {
|
|
res = await cachedFetch(JSON_BODY_URL);
|
|
body = await res.json();
|
|
assert(body.slideshow);
|
|
assert.strictEqual(res.fromCache, false);
|
|
|
|
res = await cachedFetch(JSON_BODY_URL);
|
|
body = await res.json();
|
|
assert(body.slideshow);
|
|
assert.strictEqual(res.fromCache, true);
|
|
});
|
|
|
|
it('Can get PNG buffer body', async function() {
|
|
res = await cachedFetch(PNG_BODY_URL);
|
|
body = await res.buffer();
|
|
assert.strictEqual(expectedPngBuffer.equals(body), true);
|
|
assert.strictEqual(res.fromCache, false);
|
|
|
|
res = await cachedFetch(PNG_BODY_URL);
|
|
body = await res.buffer();
|
|
assert.strictEqual(expectedPngBuffer.equals(body), true);
|
|
assert.strictEqual(res.fromCache, true);
|
|
});
|
|
|
|
it('Can stream a body', async function() {
|
|
res = await cachedFetch(TEXT_BODY_URL);
|
|
body = '';
|
|
|
|
for await (const chunk of res.body) {
|
|
body += chunk.toString();
|
|
}
|
|
|
|
assert.strictEqual(TEXT_BODY_EXPECTED, body);
|
|
assert.strictEqual(res.fromCache, false);
|
|
|
|
res = await cachedFetch(TEXT_BODY_URL);
|
|
body = '';
|
|
|
|
for await (const chunk of res.body) {
|
|
body += chunk.toString();
|
|
}
|
|
|
|
assert.strictEqual(TEXT_BODY_EXPECTED, body);
|
|
assert.strictEqual(res.fromCache, true);
|
|
});
|
|
|
|
it('Errors if the body type is not supported', async function() {
|
|
try {
|
|
await cachedFetch(TEXT_BODY_URL, { body: {} });
|
|
throw new Error('It was supposed to throw');
|
|
} catch (err) {
|
|
assert(err.message.includes('Unsupported body type'));
|
|
}
|
|
});
|
|
|
|
it('Uses cache even if you make multiple requests at the same time', async function() {
|
|
const [res1, res2] = await Promise.all([
|
|
cachedFetch('http://httpbin.org/status/200'),
|
|
cachedFetch('http://httpbin.org/status/200'),
|
|
]);
|
|
|
|
// One should be false, the other should be true
|
|
assert(res1.fromCache !== res2.fromCache);
|
|
});
|
|
}).timeout(10000);
|
|
|
|
describe('Memory cache tests', function() {
|
|
it('Supports TTL', async function() {
|
|
cachedFetch = FetchCache.withCache(new MemoryCache({ ttl: 100 }));
|
|
let res = await cachedFetch(TWO_HUNDRED_URL);
|
|
assert.strictEqual(res.fromCache, false);
|
|
res = await cachedFetch(TWO_HUNDRED_URL);
|
|
assert.strictEqual(res.fromCache, true);
|
|
|
|
await wait(200);
|
|
|
|
res = await cachedFetch(TWO_HUNDRED_URL);
|
|
assert.strictEqual(res.fromCache, false);
|
|
});
|
|
}).timeout(10000);
|
|
|
|
describe('File system cache tests', function() {
|
|
it('Supports TTL', async function() {
|
|
cachedFetch = FetchCache.withCache(new FileSystemCache({ ttl: 100 }));
|
|
let res = await cachedFetch(TWO_HUNDRED_URL);
|
|
assert.strictEqual(res.fromCache, false);
|
|
res = await cachedFetch(TWO_HUNDRED_URL);
|
|
assert.strictEqual(res.fromCache, true);
|
|
|
|
await wait(200);
|
|
|
|
res = await cachedFetch(TWO_HUNDRED_URL);
|
|
assert.strictEqual(res.fromCache, false);
|
|
});
|
|
|
|
it('Can get PNG buffer body', async function() {
|
|
cachedFetch = FetchCache.withCache(new FileSystemCache());
|
|
res = await cachedFetch(PNG_BODY_URL);
|
|
body = await res.buffer();
|
|
assert.strictEqual(expectedPngBuffer.equals(body), true);
|
|
assert.strictEqual(res.fromCache, false);
|
|
|
|
res = await cachedFetch(PNG_BODY_URL);
|
|
body = await res.buffer();
|
|
assert.strictEqual(expectedPngBuffer.equals(body), true);
|
|
assert.strictEqual(res.fromCache, true);
|
|
});
|
|
|
|
it('Can eject from cache', async function() {
|
|
cachedFetch = FetchCache.withCache(new FileSystemCache());
|
|
|
|
res = await cachedFetch(TWO_HUNDRED_URL);
|
|
assert.strictEqual(res.fromCache, false);
|
|
|
|
res = await cachedFetch(TWO_HUNDRED_URL);
|
|
assert.strictEqual(res.fromCache, true);
|
|
|
|
await res.ejectFromCache();
|
|
|
|
res = await cachedFetch(TWO_HUNDRED_URL);
|
|
assert.strictEqual(res.fromCache, false);
|
|
|
|
res = await cachedFetch(TWO_HUNDRED_URL);
|
|
assert.strictEqual(res.fromCache, true);
|
|
});
|
|
});
|
|
|
|
describe('Cache mode tests', function() {
|
|
it('Can use the only-if-cached cache control setting via init', async function() {
|
|
res = await cachedFetch(TWO_HUNDRED_URL, { headers: { 'Cache-Control': 'only-if-cached' } });
|
|
assert(!res);
|
|
res = await cachedFetch(TWO_HUNDRED_URL, { headers: { 'Cache-Control': 'only-if-cached' } });
|
|
assert(!res);
|
|
res = await cachedFetch(TWO_HUNDRED_URL);
|
|
assert(res && !res.fromCache);
|
|
res = await cachedFetch(TWO_HUNDRED_URL, { headers: { 'Cache-Control': 'only-if-cached' } });
|
|
assert(res && res.fromCache);
|
|
await res.ejectFromCache();
|
|
res = await cachedFetch(TWO_HUNDRED_URL, { headers: { 'Cache-Control': 'only-if-cached' } });
|
|
assert(!res);
|
|
});
|
|
|
|
it('Can use the only-if-cached cache control setting via resource', async function() {
|
|
res = await cachedFetch(new standardFetch.Request(TWO_HUNDRED_URL, { headers: { 'Cache-Control': 'only-if-cached' } }));
|
|
assert(!res);
|
|
res = await cachedFetch(new standardFetch.Request(TWO_HUNDRED_URL));
|
|
assert(res && !res.fromCache);
|
|
res = await cachedFetch(new standardFetch.Request(TWO_HUNDRED_URL, { headers: { 'Cache-Control': 'only-if-cached' } }));
|
|
assert(res && res.fromCache);
|
|
});
|
|
});
|
|
|
|
describe('Cache key tests', function() {
|
|
it('Can calculate a cache key and check that it exists', async function() {
|
|
const cache = new MemoryCache();
|
|
cachedFetch = FetchCache.withCache(cache);
|
|
await cachedFetch(TWO_HUNDRED_URL);
|
|
|
|
const cacheKey = getCacheKey(TWO_HUNDRED_URL);
|
|
const nonExistentCacheKey = getCacheKey(TEXT_BODY_URL);
|
|
|
|
const cacheKeyResult = await cache.get(cacheKey);
|
|
const nonExistentCacheKeyResult = await cache.get(nonExistentCacheKey);
|
|
|
|
assert(cacheKeyResult);
|
|
assert(!nonExistentCacheKeyResult);
|
|
});
|
|
});
|