overhaul to use node-fetch internals

This commit is contained in:
Randall Schmidt 2021-07-05 18:14:42 -04:00
parent 45ca35f057
commit e8ad8da0bb
6 changed files with 87 additions and 68 deletions

@ -1,5 +1,15 @@
import { Readable } from 'stream';
import { KeyTimeout } from './key_timeout.js'; import { KeyTimeout } from './key_timeout.js';
function streamToBuffer(stream) {
const chunks = [];
return new Promise((resolve, reject) => {
stream.on('data', (chunk) => chunks.push(Buffer.from(chunk)));
stream.on('error', (err) => reject(err));
stream.on('end', () => resolve(Buffer.concat(chunks)));
});
}
export class MemoryCache { export class MemoryCache {
constructor(options = {}) { constructor(options = {}) {
this.ttl = options.ttl; this.ttl = options.ttl;
@ -8,7 +18,15 @@ export class MemoryCache {
} }
get(key) { get(key) {
return this.cache[key]; const cachedValue = this.cache[key];
if (cachedValue) {
return {
bodyStream: Readable.from(cachedValue.bodyBuffer),
metaData: cachedValue.metaData,
};
}
return undefined;
} }
remove(key) { remove(key) {
@ -16,8 +34,9 @@ export class MemoryCache {
delete this.cache[key]; delete this.cache[key];
} }
set(key, value) { async set(key, bodyStream, metaData) {
this.cache[key] = value; const bodyBuffer = await streamToBuffer(bodyStream);
this.cache[key] = { bodyBuffer, metaData };
if (typeof this.ttl === 'number') { if (typeof this.ttl === 'number') {
this.keyTimeout.updateTimeout(key, this.ttl, () => this.remove(key)); this.keyTimeout.updateTimeout(key, this.ttl, () => this.remove(key));

@ -1,42 +1,54 @@
import stream from 'stream'; import { Response } from 'node-fetch';
import { Headers } from './headers.js'; import { PassThrough } from 'stream';
export class Response { const responseInternalSymbol = Object.getOwnPropertySymbols(new Response())[1];
constructor(raw, ejectSelfFromCache, fromCache) {
Object.assign(this, raw); export class NFCResponse extends Response {
this.ejectSelfFromCache = ejectSelfFromCache; constructor(bodyStream, metaData, ejectFromCache, fromCache) {
this.headers = new Headers(raw.headers); const stream1 = new PassThrough();
const stream2 = new PassThrough();
bodyStream.pipe(stream1);
bodyStream.pipe(stream2);
super(stream1, metaData);
this.ejectFromCache = ejectFromCache;
this.fromCache = fromCache; this.fromCache = fromCache;
this.bodyUsed = false; this.serializationStream = stream2;
if (this.bodyBuffer.type === 'Buffer') {
this.bodyBuffer = Buffer.from(this.bodyBuffer);
}
} }
get body() { static fromNodeFetchResponse(res, ejectFromCache) {
return stream.Readable.from(this.bodyBuffer); const bodyStream = res.body;
const metaData = {
url: res.url,
status: res.status,
statusText: res.statusText,
headers: res.headers.raw(),
size: res.size,
timeout: res.timeout,
counter: res[responseInternalSymbol].counter,
};
return new NFCResponse(bodyStream, metaData, ejectFromCache, false);
} }
consumeBody() { static fromCachedResponse(bodyStream, rawMetaData, ejectSelfFromCache) {
if (this.bodyUsed) { return new NFCResponse(bodyStream, rawMetaData, ejectSelfFromCache, true);
throw new Error('Error: body used already');
} }
this.bodyUsed = true; serialize() {
return this.bodyBuffer; return {
} bodyStream: this.serializationStream,
metaData: {
async text() { url: this.url,
return this.consumeBody().toString(); status: this.status,
} statusText: this.statusText,
headers: this.headers.raw(),
async json() { size: this.size,
return JSON.parse(this.consumeBody().toString()); timeout: this.timeout,
} counter: this[responseInternalSymbol].counter,
},
async buffer() { };
return this.consumeBody();
} }
ejectFromCache() { ejectFromCache() {

@ -2,10 +2,10 @@ import fetch from 'node-fetch';
import fs from 'fs'; import fs from 'fs';
import { URLSearchParams } from 'url'; import { URLSearchParams } from 'url';
import crypto from 'crypto'; import crypto from 'crypto';
import { Response } from './classes/response.js'; import { NFCResponse } from './classes/response.js';
import { MemoryCache } from './classes/caching/memory_cache.js'; import { MemoryCache } from './classes/caching/memory_cache.js';
const CACHE_VERSION = 2; const CACHE_VERSION = 3;
function md5(str) { function md5(str) {
return crypto.createHash('md5').update(str).digest('hex'); return crypto.createHash('md5').update(str).digest('hex');
@ -71,21 +71,6 @@ function getCacheKey(requestArguments) {
return md5(JSON.stringify([resourceCacheKeyJson, initCacheKeyJson, CACHE_VERSION])); return md5(JSON.stringify([resourceCacheKeyJson, initCacheKeyJson, CACHE_VERSION]));
} }
async function createRawResponse(fetchRes) {
const buffer = await fetchRes.buffer();
return {
status: fetchRes.status,
statusText: fetchRes.statusText,
type: fetchRes.type,
url: fetchRes.url,
ok: fetchRes.ok,
headers: fetchRes.headers.raw(),
redirected: fetchRes.redirected,
bodyBuffer: buffer,
};
}
async function getResponse(cache, requestArguments) { async function getResponse(cache, requestArguments) {
const cacheKey = getCacheKey(requestArguments); const cacheKey = getCacheKey(requestArguments);
const cachedValue = await cache.get(cacheKey); const cachedValue = await cache.get(cacheKey);
@ -93,13 +78,22 @@ async function getResponse(cache, requestArguments) {
const ejectSelfFromCache = () => cache.remove(cacheKey); const ejectSelfFromCache = () => cache.remove(cacheKey);
if (cachedValue) { if (cachedValue) {
return new Response(cachedValue, ejectSelfFromCache, true); if (cachedValue.bodyStream.readableEnded) {
throw new Error('Cache returned a body stream that has already been read to end.');
}
return NFCResponse.fromCachedResponse(
cachedValue.bodyStream,
cachedValue.metaData,
ejectSelfFromCache,
);
} }
const fetchResponse = await fetch(...requestArguments); const fetchResponse = await fetch(...requestArguments);
const rawResponse = await createRawResponse(fetchResponse); const nfcResponse = NFCResponse.fromNodeFetchResponse(fetchResponse, ejectSelfFromCache);
await cache.set(cacheKey, rawResponse); const nfcResponseSerialized = nfcResponse.serialize();
return new Response(rawResponse, ejectSelfFromCache, false); await cache.set(cacheKey, nfcResponseSerialized.bodyStream, nfcResponseSerialized.metaData);
return nfcResponse;
} }
function createFetchWithCache(cache) { function createFetchWithCache(cache) {

5
package-lock.json generated

@ -1508,11 +1508,6 @@
"mime-types": "^2.1.12" "mime-types": "^2.1.12"
} }
}, },
"fpersist": {
"version": "1.0.5",
"resolved": "https://registry.npmjs.org/fpersist/-/fpersist-1.0.5.tgz",
"integrity": "sha512-WXY+zZXlOo1dU+wS8rqigz5PFu7WHBDd0vcaaWcnu319bPJi/IeWipOmi1PNaHAUqFVSzp1mLpNkgX6g2uLGbQ=="
},
"fromentries": { "fromentries": {
"version": "1.3.2", "version": "1.3.2",
"resolved": "https://registry.npmjs.org/fromentries/-/fromentries-1.3.2.tgz", "resolved": "https://registry.npmjs.org/fromentries/-/fromentries-1.3.2.tgz",

@ -38,7 +38,6 @@
"rimraf": "^3.0.2" "rimraf": "^3.0.2"
}, },
"dependencies": { "dependencies": {
"fpersist": "^1.0.5",
"node-fetch": "2.6.1" "node-fetch": "2.6.1"
}, },
"husky": { "husky": {

@ -130,22 +130,22 @@ describe('Header tests', function() {
it('Gets correct header keys', async function() { it('Gets correct header keys', async function() {
let { cachedFetchResponse, standardFetchResponse } = await dualFetch(TWO_HUNDRED_URL); let { cachedFetchResponse, standardFetchResponse } = await dualFetch(TWO_HUNDRED_URL);
assert.deepStrictEqual(cachedFetchResponse.headers.keys(), [...standardFetchResponse.headers.keys()]); assert.deepStrictEqual([...cachedFetchResponse.headers.keys()], [...standardFetchResponse.headers.keys()]);
cachedFetchResponse = await cachedFetch(TWO_HUNDRED_URL); cachedFetchResponse = await cachedFetch(TWO_HUNDRED_URL);
assert.deepStrictEqual(cachedFetchResponse.headers.keys(), [...standardFetchResponse.headers.keys()]); assert.deepStrictEqual([...cachedFetchResponse.headers.keys()], [...standardFetchResponse.headers.keys()]);
}); });
it('Gets correct header values', async function() { it('Gets correct header values', async function() {
let { cachedFetchResponse, standardFetchResponse } = await dualFetch(TWO_HUNDRED_URL); let { cachedFetchResponse, standardFetchResponse } = await dualFetch(TWO_HUNDRED_URL);
assert.deepStrictEqual( assert.deepStrictEqual(
removeDates(cachedFetchResponse.headers.values()), removeDates([...cachedFetchResponse.headers.values()]),
removeDates([...standardFetchResponse.headers.values()]), removeDates([...standardFetchResponse.headers.values()]),
); );
cachedFetchResponse = await cachedFetch(TWO_HUNDRED_URL); cachedFetchResponse = await cachedFetch(TWO_HUNDRED_URL);
assert.deepStrictEqual( assert.deepStrictEqual(
removeDates(cachedFetchResponse.headers.values()), removeDates([...cachedFetchResponse.headers.values()]),
removeDates([...standardFetchResponse.headers.values()]), removeDates([...standardFetchResponse.headers.values()]),
); );
}); });
@ -153,13 +153,13 @@ describe('Header tests', function() {
it('Gets correct header entries', async function() { it('Gets correct header entries', async function() {
let { cachedFetchResponse, standardFetchResponse } = await dualFetch(TWO_HUNDRED_URL); let { cachedFetchResponse, standardFetchResponse } = await dualFetch(TWO_HUNDRED_URL);
assert.deepStrictEqual( assert.deepStrictEqual(
removeDates(cachedFetchResponse.headers.entries()), removeDates([...cachedFetchResponse.headers.entries()]),
removeDates([...standardFetchResponse.headers.entries()]), removeDates([...standardFetchResponse.headers.entries()]),
); );
cachedFetchResponse = await cachedFetch(TWO_HUNDRED_URL); cachedFetchResponse = await cachedFetch(TWO_HUNDRED_URL);
assert.deepStrictEqual( assert.deepStrictEqual(
removeDates(cachedFetchResponse.headers.entries()), removeDates([...cachedFetchResponse.headers.entries()]),
removeDates([...standardFetchResponse.headers.entries()]), removeDates([...standardFetchResponse.headers.entries()]),
); );
}); });
@ -334,7 +334,7 @@ describe('Data tests', function() {
await res.text(); await res.text();
throw new Error('The above line should have thrown.'); throw new Error('The above line should have thrown.');
} catch (err) { } catch (err) {
assert(err.message.includes('Error: body used already')); assert(err.message.includes('body used already for:'));
} }
}); });