move source into src dir

src/classes/caching/file_system_cache.js

@@ -0,0 +1,75 @@
+import cacache from 'cacache';
+import { Readable } from 'stream';
+import { KeyTimeout } from './key_timeout.js';
+
+function getBodyAndMetaKeys(key) {
+  return [`${key}body`, `${key}meta`];
+}
+
+export class FileSystemCache {
+  constructor(options = {}) {
+    this.ttl = options.ttl;
+    this.keyTimeout = new KeyTimeout();
+    this.cacheDirectory = options.cacheDirectory || '.cache';
+  }
+
+  async get(key) {
+    const [, metaKey] = getBodyAndMetaKeys(key);
+
+    const metaInfo = await cacache.get.info(this.cacheDirectory, metaKey);
+
+    if (!metaInfo) {
+      return undefined;
+    }
+
+    const metaBuffer = await cacache.get.byDigest(this.cacheDirectory, metaInfo.integrity);
+    const metaData = JSON.parse(metaBuffer);
+    const { bodyStreamIntegrity, bodyStreamLength } = metaData;
+    delete metaData.bodyStreamIntegrity;
+    delete metaData.bodyStreamLength;
+
+    const bodyStream = bodyStreamLength > 0
+      ? cacache.get.stream.byDigest(this.cacheDirectory, bodyStreamIntegrity)
+      : Readable.from(Buffer.alloc(0));
+
+    return {
+      bodyStream,
+      metaData,
+    };
+  }
+
+  remove(key) {
+    const [bodyKey, metaKey] = getBodyAndMetaKeys(key);
+
+    this.keyTimeout.clearTimeout(key);
+
+    return Promise.all([
+      cacache.rm.entry(this.cacheDirectory, bodyKey),
+      cacache.rm.entry(this.cacheDirectory, metaKey),
+    ]);
+  }
+
+  async set(key, bodyStream, metaData, bodyStreamLength) {
+    const [bodyKey, metaKey] = getBodyAndMetaKeys(key);
+    const metaCopy = { ...metaData, bodyStreamLength };
+
+    this.keyTimeout.clearTimeout(key);
+
+    if (bodyStreamLength > 0) {
+      metaCopy.bodyStreamIntegrity = await new Promise((fulfill, reject) => {
+        bodyStream.pipe(cacache.put.stream(this.cacheDirectory, bodyKey))
+          .on('integrity', (i) => fulfill(i))
+          .on('error', (e) => {
+            reject(e);
+          });
+      });
+    }
+
+    const metaBuffer = Buffer.from(JSON.stringify(metaCopy));
+    await cacache.put(this.cacheDirectory, metaKey, metaBuffer);
+
+    if (typeof this.ttl === 'number') {
+      this.keyTimeout.updateTimeout(key, this.ttl, () => this.remove(key));
+    }
+  }
+}

src/classes/caching/key_timeout.js

@@ -0,0 +1,16 @@
+export class KeyTimeout {
+  constructor() {
+    this.timeoutHandleForKey = {};
+  }
+
+  clearTimeout(key) {
+    clearTimeout(this.timeoutHandleForKey[key]);
+  }
+
+  updateTimeout(key, durationMs, callback) {
+    this.clearTimeout(key);
+    this.timeoutHandleForKey[key] = setTimeout(() => {
+      callback();
+    }, durationMs);
+  }
+}

src/classes/caching/memory_cache.js

@@ -0,0 +1,45 @@
+import { Readable } from 'stream';
+import { KeyTimeout } from './key_timeout.js';
+
+function streamToBuffer(stream) {
+  const chunks = [];
+  return new Promise((resolve, reject) => {
+    stream.on('data', (chunk) => chunks.push(Buffer.from(chunk)));
+    stream.on('error', (err) => reject(err));
+    stream.on('end', () => resolve(Buffer.concat(chunks)));
+  });
+}
+
+export class MemoryCache {
+  constructor(options = {}) {
+    this.ttl = options.ttl;
+    this.keyTimeout = new KeyTimeout();
+    this.cache = {};
+  }
+
+  get(key) {
+    const cachedValue = this.cache[key];
+    if (cachedValue) {
+      return {
+        bodyStream: Readable.from(cachedValue.bodyBuffer),
+        metaData: cachedValue.metaData,
+      };
+    }
+
+    return undefined;
+  }
+
+  remove(key) {
+    this.keyTimeout.clearTimeout(key);
+    delete this.cache[key];
+  }
+
+  async set(key, bodyStream, metaData) {
+    const bodyBuffer = await streamToBuffer(bodyStream);
+    this.cache[key] = { bodyBuffer, metaData };
+
+    if (typeof this.ttl === 'number') {
+      this.keyTimeout.updateTimeout(key, this.ttl, () => this.remove(key));
+    }
+  }
+}

src/classes/headers.js

@@ -0,0 +1,31 @@
+export class Headers {
+  constructor(rawHeaders) {
+    this.rawHeaders = rawHeaders;
+  }
+
+  entries() {
+    return Object.entries(this.rawHeaders)
+      .sort((e1, e2) => e1[0].localeCompare(e2[0]))
+      .map(([key, val]) => [key, val[0]]);
+  }
+
+  keys() {
+    return this.entries().map((e) => e[0]);
+  }
+
+  values() {
+    return this.entries().map((e) => e[1]);
+  }
+
+  get(name) {
+    return (this.rawHeaders[name.toLowerCase()] || [])[0] || null;
+  }
+
+  has(name) {
+    return !!this.get(name);
+  }
+
+  raw() {
+    return this.rawHeaders;
+  }
+}

src/classes/response.js

@@ -0,0 +1,61 @@
+import { Response } from 'node-fetch';
+import { PassThrough } from 'stream';
+
+const responseInternalSymbol = Object.getOwnPropertySymbols(new Response())[1];
+
+export class NFCResponse extends Response {
+  constructor(bodyStream, metaData, ejectFromCache, fromCache) {
+    const stream1 = new PassThrough();
+    const stream2 = new PassThrough();
+
+    bodyStream.pipe(stream1);
+    bodyStream.pipe(stream2);
+
+    super(stream1, metaData);
+    this.ejectFromCache = ejectFromCache;
+    this.fromCache = fromCache;
+    this.serializationStream = stream2;
+  }
+
+  static fromNodeFetchResponse(res, ejectFromCache) {
+    const bodyStream = res.body;
+    const metaData = {
+      url: res.url,
+      status: res.status,
+      statusText: res.statusText,
+      headers: res.headers.raw(),
+      size: res.size,
+      timeout: res.timeout,
+      counter: res[responseInternalSymbol].counter,
+    };
+
+    return new NFCResponse(bodyStream, metaData, ejectFromCache, false);
+  }
+
+  static fromCachedResponse(bodyStream, rawMetaData, ejectSelfFromCache) {
+    if (bodyStream.readableEnded) {
+      throw new Error('Cache returned a body stream that has already been read to end.');
+    }
+
+    return new NFCResponse(bodyStream, rawMetaData, ejectSelfFromCache, true);
+  }
+
+  serialize() {
+    return {
+      bodyStream: this.serializationStream,
+      metaData: {
+        url: this.url,
+        status: this.status,
+        statusText: this.statusText,
+        headers: this.headers.raw(),
+        size: this.size,
+        timeout: this.timeout,
+        counter: this[responseInternalSymbol].counter,
+      },
+    };
+  }
+
+  ejectFromCache() {
+    return this.ejectSelfFromCache();
+  }
+}

src/index.js

@@ -0,0 +1,146 @@
+import fetch, { Request } from 'node-fetch';
+import fs from 'fs';
+import { URLSearchParams } from 'url';
+import crypto from 'crypto';
+import locko from 'locko';
+import { NFCResponse } from './classes/response.js';
+import { MemoryCache } from './classes/caching/memory_cache.js';
+
+const CACHE_VERSION = 3;
+
+function md5(str) {
+  return crypto.createHash('md5').update(str).digest('hex');
+}
+
+// Since the bounday in FormData is random,
+// we ignore it for purposes of calculating
+// the cache key.
+function getFormDataCacheKey(formData) {
+  const cacheKey = { ...formData };
+  const boundary = formData.getBoundary();
+
+  // eslint-disable-next-line no-underscore-dangle
+  delete cacheKey._boundary;
+
+  const boundaryReplaceRegex = new RegExp(boundary, 'g');
+
+  // eslint-disable-next-line no-underscore-dangle
+  cacheKey._streams = cacheKey._streams.map((s) => {
+    if (typeof s === 'string') {
+      return s.replace(boundaryReplaceRegex, '');
+    }
+
+    return s;
+  });
+
+  return cacheKey;
+}
+
+function getBodyCacheKeyJson(body) {
+  if (!body) {
+    return body;
+  } if (typeof body === 'string') {
+    return body;
+  } if (body instanceof URLSearchParams) {
+    return body.toString();
+  } if (body instanceof fs.ReadStream) {
+    return body.path;
+  } if (body.toString && body.toString() === '[object FormData]') {
+    return getFormDataCacheKey(body);
+  } if (body instanceof Buffer) {
+    return body.toString();
+  }
+
+  throw new Error('Unsupported body type. Supported body types are: string, number, undefined, null, url.URLSearchParams, fs.ReadStream, FormData');
+}
+
+function getRequestCacheKey(req) {
+  return {
+    cache: req.cache,
+    credentials: req.credentials,
+    destination: req.destination,
+    headers: req.headers,
+    integrity: req.integrity,
+    method: req.method,
+    redirect: req.redirect,
+    referrer: req.referrer,
+    referrerPolicy: req.referrerPolicy,
+    url: req.url,
+    body: getBodyCacheKeyJson(req.body),
+  };
+}
+
+function getCacheKey(requestArguments) {
+  const resource = requestArguments[0];
+  const init = requestArguments[1] || {};
+
+  const resourceCacheKeyJson = resource instanceof Request
+    ? getRequestCacheKey(resource)
+    : { url: resource };
+
+  const initCacheKeyJson = { ...init };
+
+  resourceCacheKeyJson.body = getBodyCacheKeyJson(resourceCacheKeyJson.body);
+  initCacheKeyJson.body = getBodyCacheKeyJson(initCacheKeyJson.body);
+
+  delete initCacheKeyJson.agent;
+
+  return md5(JSON.stringify([resourceCacheKeyJson, initCacheKeyJson, CACHE_VERSION]));
+}
+
+async function getResponse(cache, requestArguments) {
+  const cacheKey = getCacheKey(requestArguments);
+  let cachedValue = await cache.get(cacheKey);
+
+  const ejectSelfFromCache = () => cache.remove(cacheKey);
+
+  if (cachedValue) {
+    return NFCResponse.fromCachedResponse(
+      cachedValue.bodyStream,
+      cachedValue.metaData,
+      ejectSelfFromCache,
+    );
+  }
+
+  await locko.lock(cacheKey);
+  try {
+    cachedValue = await cache.get(cacheKey);
+    if (cachedValue) {
+      return NFCResponse.fromCachedResponse(
+        cachedValue.bodyStream,
+        cachedValue.metaData,
+        ejectSelfFromCache,
+      );
+    }
+
+    const fetchResponse = await fetch(...requestArguments);
+    const nfcResponse = NFCResponse.fromNodeFetchResponse(fetchResponse, ejectSelfFromCache);
+    const contentLength = Number.parseInt(nfcResponse.headers.get('content-length'), 10) || 0;
+    const nfcResponseSerialized = nfcResponse.serialize();
+
+    await cache.set(
+      cacheKey,
+      nfcResponseSerialized.bodyStream,
+      nfcResponseSerialized.metaData,
+      contentLength,
+    );
+
+    return nfcResponse;
+  } finally {
+    locko.unlock(cacheKey);
+  }
+}
+
+function createFetchWithCache(cache) {
+  const fetchCache = (...args) => getResponse(cache, args);
+  fetchCache.withCache = createFetchWithCache;
+
+  return fetchCache;
+}
+
+const defaultFetch = createFetchWithCache(new MemoryCache());
+
+export default defaultFetch;
+export const fetchBuilder = defaultFetch;
+export { MemoryCache } from './classes/caching/memory_cache.js';
+export { FileSystemCache } from './classes/caching/file_system_cache.js';