node-fetch-cache/test/tests.js

import { dirname } from 'path';
import { fileURLToPath } from 'url';
import fs from 'fs';
import FormData from 'form-data';
import assert from 'assert';
import rimraf from 'rimraf';
import path from 'path';
import { URLSearchParams } from 'url';
import standardFetch from 'node-fetch';
import FetchCache, { MemoryCache, FileSystemCache, getCacheKey } from '../src/index.js';
import { Agent } from 'http';

const __dirname = dirname(fileURLToPath(import.meta.url));

const CACHE_PATH = path.join(__dirname, '..', '.cache');
const expectedPngBuffer = fs.readFileSync(path.join(__dirname, 'expected_png.png'));

const TWO_HUNDRED_URL = 'https://httpbin.org/status/200';
const FOUR_HUNDRED_URL = 'https://httpbin.org/status/400';
const THREE_HUNDRED_TWO_URL = 'https://httpbin.org/status/302';
const TEXT_BODY_URL = 'https://httpbin.org/robots.txt';
const JSON_BODY_URL = 'https://httpbin.org/json';
const PNG_BODY_URL = 'https://httpbin.org/image/png';

const TEXT_BODY_EXPECTED = 'User-agent: *\nDisallow: /deny\n';

let cachedFetch;
let body;

function post(body) {
  return { method: 'POST', body };
}

function removeDates(arrOrObj) {
  if (arrOrObj.date) {
    const copy = { ...arrOrObj };
    delete copy.date;
    return copy;
  }

  if (Array.isArray(arrOrObj)) {
    if (Array.isArray(arrOrObj[0])) {
      return arrOrObj.filter(e => e[0] !== 'date');
    }

    return arrOrObj.filter(e => !Date.parse(e));
  }

  return arrOrObj;
}

function wait(ms) {
  return new Promise((fulfill) => setTimeout(fulfill, ms));
}

async function dualFetch(...args) {
  const [cachedFetchResponse, standardFetchResponse] = await Promise.all([
    cachedFetch(...args),
    standardFetch(...args),
  ]);

  return { cachedFetchResponse, standardFetchResponse };
}

beforeEach(async function() {
  rimraf.sync(CACHE_PATH);
  cachedFetch = FetchCache.withCache(new MemoryCache());
});

let res;

describe('Basic property tests', function() {
  it('Has a status property', async function() {
    let { cachedFetchResponse, standardFetchResponse } = await dualFetch(TWO_HUNDRED_URL);
    assert.strictEqual(cachedFetchResponse.status, standardFetchResponse.status);

    cachedFetchResponse = await cachedFetch(TWO_HUNDRED_URL);
    assert.strictEqual(cachedFetchResponse.status, standardFetchResponse.status);
  });

  it('Has a statusText property', async function() {
    let { cachedFetchResponse, standardFetchResponse } = await dualFetch(TWO_HUNDRED_URL);
    assert.strictEqual(cachedFetchResponse.statusText, standardFetchResponse.statusText);

    cachedFetchResponse = await cachedFetch(TWO_HUNDRED_URL);
    assert.strictEqual(cachedFetchResponse.statusText, standardFetchResponse.statusText);
  });

  it('Has a url property', async function() {
    let { cachedFetchResponse, standardFetchResponse } = await dualFetch(TWO_HUNDRED_URL);
    assert.strictEqual(cachedFetchResponse.url, standardFetchResponse.url);

    cachedFetchResponse = await cachedFetch(TWO_HUNDRED_URL);
    assert.strictEqual(cachedFetchResponse.url, standardFetchResponse.url);
  });

  it('Has an ok property', async function() {
    let { cachedFetchResponse, standardFetchResponse } = await dualFetch(FOUR_HUNDRED_URL);
    assert.strictEqual(cachedFetchResponse.ok, standardFetchResponse.ok);
    assert.strictEqual(cachedFetchResponse.status, standardFetchResponse.status);

    cachedFetchResponse = await cachedFetch(FOUR_HUNDRED_URL);
    assert.strictEqual(cachedFetchResponse.ok, standardFetchResponse.ok);
    assert.strictEqual(cachedFetchResponse.status, standardFetchResponse.status);
  });

  it('Has a redirected property', async function() {
    let { cachedFetchResponse, standardFetchResponse } = await dualFetch(THREE_HUNDRED_TWO_URL);
    assert.strictEqual(cachedFetchResponse.redirected, standardFetchResponse.redirected);

    cachedFetchResponse = await cachedFetch(THREE_HUNDRED_TWO_URL);
    assert.strictEqual(cachedFetchResponse.redirected, standardFetchResponse.redirected);
  });
}).timeout(10000);

describe('Header tests', function() {
  it('Gets correct raw headers', async function() {
    let { cachedFetchResponse, standardFetchResponse } = await dualFetch(TWO_HUNDRED_URL);
    assert.deepStrictEqual(
      removeDates(cachedFetchResponse.headers.raw()),
      removeDates(standardFetchResponse.headers.raw()),
    );

    cachedFetchResponse = await cachedFetch(TWO_HUNDRED_URL);
    assert.deepStrictEqual(
      removeDates(cachedFetchResponse.headers.raw()),
      removeDates(standardFetchResponse.headers.raw()),
    );
  });

  it('Gets correct header keys', async function() {
    let { cachedFetchResponse, standardFetchResponse } = await dualFetch(TWO_HUNDRED_URL);
    assert.deepStrictEqual([...cachedFetchResponse.headers.keys()], [...standardFetchResponse.headers.keys()]);

    cachedFetchResponse = await cachedFetch(TWO_HUNDRED_URL);
    assert.deepStrictEqual([...cachedFetchResponse.headers.keys()], [...standardFetchResponse.headers.keys()]);
  });

  it('Gets correct header values', async function() {
    let { cachedFetchResponse, standardFetchResponse } = await dualFetch(TWO_HUNDRED_URL);
    assert.deepStrictEqual(
      removeDates([...cachedFetchResponse.headers.values()]),
      removeDates([...standardFetchResponse.headers.values()]),
    );

    cachedFetchResponse = await cachedFetch(TWO_HUNDRED_URL);
    assert.deepStrictEqual(
      removeDates([...cachedFetchResponse.headers.values()]),
      removeDates([...standardFetchResponse.headers.values()]),
    );
  });

  it('Gets correct header entries', async function() {
    let { cachedFetchResponse, standardFetchResponse } = await dualFetch(TWO_HUNDRED_URL);
    assert.deepStrictEqual(
      removeDates([...cachedFetchResponse.headers.entries()]),
      removeDates([...standardFetchResponse.headers.entries()]),
    );

    cachedFetchResponse = await cachedFetch(TWO_HUNDRED_URL);
    assert.deepStrictEqual(
      removeDates([...cachedFetchResponse.headers.entries()]),
      removeDates([...standardFetchResponse.headers.entries()]),
    );
  });

  it('Can get a header by value', async function() {
    let { cachedFetchResponse, standardFetchResponse } = await dualFetch(TWO_HUNDRED_URL);
    assert(standardFetchResponse.headers.get('content-length'));
    assert.deepStrictEqual(cachedFetchResponse.headers.get('content-length'), standardFetchResponse.headers.get('content-length'));

    cachedFetchResponse = await cachedFetch(TWO_HUNDRED_URL);
    assert.deepStrictEqual(cachedFetchResponse.headers.get('content-length'), standardFetchResponse.headers.get('content-length'));
  });

  it('Returns undefined for non-existent header', async function() {
    const headerName = 'zzzz';
    let { cachedFetchResponse, standardFetchResponse } = await dualFetch(TWO_HUNDRED_URL);
    assert(!standardFetchResponse.headers.get(headerName));
    assert.deepStrictEqual(cachedFetchResponse.headers.get(headerName), standardFetchResponse.headers.get(headerName));

    cachedFetchResponse = await cachedFetch(TWO_HUNDRED_URL);
    assert.deepStrictEqual(cachedFetchResponse.headers.get(headerName), standardFetchResponse.headers.get(headerName));
  });

  it('Can get whether a header is present', async function() {
    let { cachedFetchResponse, standardFetchResponse } = await dualFetch(TWO_HUNDRED_URL);
    assert(standardFetchResponse.headers.has('content-length'));
    assert.deepStrictEqual(cachedFetchResponse.headers.has('content-length'), standardFetchResponse.headers.has('content-length'));

    cachedFetchResponse = await cachedFetch(TWO_HUNDRED_URL);
    assert.deepStrictEqual(cachedFetchResponse.headers.has('content-length'), standardFetchResponse.headers.has('content-length'));
  });
}).timeout(10000);

describe('Cache tests', function() {
  it('Uses cache', async function() {
    res = await cachedFetch(TWO_HUNDRED_URL);
    assert.strictEqual(res.fromCache, false);

    res = await cachedFetch(TWO_HUNDRED_URL);
    assert.strictEqual(res.fromCache, true);
  });

  it('Can eject from cache', async function() {
    res = await cachedFetch(TWO_HUNDRED_URL);
    assert.strictEqual(res.fromCache, false);

    res = await cachedFetch(TWO_HUNDRED_URL);
    assert.strictEqual(res.fromCache, true);

    await res.ejectFromCache();

    res = await cachedFetch(TWO_HUNDRED_URL);
    assert.strictEqual(res.fromCache, false);

    res = await cachedFetch(TWO_HUNDRED_URL);
    assert.strictEqual(res.fromCache, true);
  });

  it('Does not error if ejecting from cache twice', async function() {
    res = await cachedFetch(TWO_HUNDRED_URL);
    assert.strictEqual(res.fromCache, false);

    await res.ejectFromCache();
    await res.ejectFromCache();
  });

  it('Gives different string bodies different cache keys', async function() {
    res = await cachedFetch(TWO_HUNDRED_URL, post('a'));
    assert.strictEqual(res.fromCache, false);

    res = await cachedFetch(TWO_HUNDRED_URL, post('b'));
    assert.strictEqual(res.fromCache, false);
  });

  it('Gives same string bodies same cache keys', async function() {
    res = await cachedFetch(TWO_HUNDRED_URL, post('a'));
    assert.strictEqual(res.fromCache, false);

    res = await cachedFetch(TWO_HUNDRED_URL, post('a'));
    assert.strictEqual(res.fromCache, true);
  });

  it('Gives different URLSearchParams different cache keys', async function() {
    res = await cachedFetch(TWO_HUNDRED_URL, post(new URLSearchParams('a=a')));
    assert.strictEqual(res.fromCache, false);

    res = await cachedFetch(TWO_HUNDRED_URL, post(new URLSearchParams('a=b')));
    assert.strictEqual(res.fromCache, false);
  });

  it('Gives same URLSearchParams same cache keys', async function() {
    res = await cachedFetch(TWO_HUNDRED_URL, post(new URLSearchParams('a=a')));
    assert.strictEqual(res.fromCache, false);

    res = await cachedFetch(TWO_HUNDRED_URL, post(new URLSearchParams('a=a')));
    assert.strictEqual(res.fromCache, true);
  });

  it('Gives different read streams different cache keys', async function() {
    const s1 = fs.createReadStream(path.join(__dirname, 'expected_png.png'));
    const s2 = fs.createReadStream(path.join(__dirname, '..', 'src', 'index.js'));

    res = await cachedFetch(TWO_HUNDRED_URL, post(s1));
    assert.strictEqual(res.fromCache, false);

    res = await cachedFetch(TWO_HUNDRED_URL, post(s2));
    assert.strictEqual(res.fromCache, false);
  });

  it('Gives the same read streams the same cache key', async function() {
    const s1 = fs.createReadStream(path.join(__dirname, 'expected_png.png'));

    res = await cachedFetch(TWO_HUNDRED_URL, post(s1));
    assert.strictEqual(res.fromCache, false);

    res = await cachedFetch(TWO_HUNDRED_URL, post(s1));
    assert.strictEqual(res.fromCache, true);
  });

  it('Gives different form data different cache keys', async function() {
    const data1 = new FormData();
    data1.append('a', 'a');

    const data2 = new FormData();
    data2.append('b', 'b');

    res = await cachedFetch(TWO_HUNDRED_URL, post(data1));
    assert.strictEqual(res.fromCache, false);

    res = await cachedFetch(TWO_HUNDRED_URL, post(data2));
    assert.strictEqual(res.fromCache, false);
  });

  it('Gives same form data same cache keys', async function() {
    const data1 = new FormData();
    data1.append('a', 'a');

    const data2 = new FormData();
    data2.append('a', 'a');

    res = await cachedFetch(TWO_HUNDRED_URL, post(data1));
    assert.strictEqual(res.fromCache, false);

    res = await cachedFetch(TWO_HUNDRED_URL, post(data2));
    assert.strictEqual(res.fromCache, true);
  });

  it('Does not error with custom agent with circular properties', async function() {
    const agent = new Agent();
    agent.agent = agent;

    await cachedFetch('http://httpbin.org/status/200', { agent });
  })
}).timeout(10000);

describe('Data tests', function() {
  it('Supports request objects', async function() {
    let request = new standardFetch.Request('https://google.com', { body: 'test', method: 'POST' });
    res = await cachedFetch(request);
    assert.strictEqual(res.fromCache, false);

    request = new standardFetch.Request('https://google.com', { body: 'test', method: 'POST' });
    res = await cachedFetch(request);
    assert.strictEqual(res.fromCache, true);
  });

  it('Supports request objects with custom headers', async function() {
    const request1 = new standardFetch.Request(TWO_HUNDRED_URL, { headers: { 'XXX': 'YYY' } });
    const request2 = new standardFetch.Request(TWO_HUNDRED_URL, { headers: { 'XXX': 'ZZZ' } });

    res = await cachedFetch(request1);
    assert.strictEqual(res.fromCache, false);

    res = await cachedFetch(request2);
    assert.strictEqual(res.fromCache, false);
  });

  it('Refuses to consume body twice', async function() {
    res = await cachedFetch(TEXT_BODY_URL);
    await res.text();

    try {
      await res.text();
      throw new Error('The above line should have thrown.');
    } catch (err) {
      assert(err.message.includes('body used already for:'));
    }
  });

  it('Can get text body', async function() {
    res = await cachedFetch(TEXT_BODY_URL);
    body = await res.text();
    assert.strictEqual(body, TEXT_BODY_EXPECTED);
    assert.strictEqual(res.fromCache, false);

    res = await cachedFetch(TEXT_BODY_URL);
    body = await res.text();
    assert.strictEqual(body, TEXT_BODY_EXPECTED);
    assert.strictEqual(res.fromCache, true);
  });

  it('Can get JSON body', async function() {
    res = await cachedFetch(JSON_BODY_URL);
    body = await res.json();
    assert(body.slideshow);
    assert.strictEqual(res.fromCache, false);

    res = await cachedFetch(JSON_BODY_URL);
    body = await res.json();
    assert(body.slideshow);
    assert.strictEqual(res.fromCache, true);
  });

  it('Can get PNG buffer body', async function() {
    res = await cachedFetch(PNG_BODY_URL);
    body = await res.buffer();
    assert.strictEqual(expectedPngBuffer.equals(body), true);
    assert.strictEqual(res.fromCache, false);

    res = await cachedFetch(PNG_BODY_URL);
    body = await res.buffer();
    assert.strictEqual(expectedPngBuffer.equals(body), true);
    assert.strictEqual(res.fromCache, true);
  });

  it('Can stream a body', async function() {
    res = await cachedFetch(TEXT_BODY_URL);
    body = '';

    for await (const chunk of res.body) {
      body += chunk.toString();
    }

    assert.strictEqual(TEXT_BODY_EXPECTED, body);
    assert.strictEqual(res.fromCache, false);

    res = await cachedFetch(TEXT_BODY_URL);
    body = '';

    for await (const chunk of res.body) {
      body += chunk.toString();
    }

    assert.strictEqual(TEXT_BODY_EXPECTED, body);
    assert.strictEqual(res.fromCache, true);
  });

  it('Errors if the body type is not supported', async function() {
    try {
      await cachedFetch(TEXT_BODY_URL, { body: {} });
      throw new Error('It was supposed to throw');
    } catch (err) {
      assert(err.message.includes('Unsupported body type'));
    }
  });

  it('Uses cache even if you make multiple requests at the same time', async function() {
    const [res1, res2] = await Promise.all([
      cachedFetch('http://httpbin.org/status/200'),
      cachedFetch('http://httpbin.org/status/200'),
    ]);

    // One should be false, the other should be true
    assert(res1.fromCache !== res2.fromCache);
  });
}).timeout(10000);

describe('Memory cache tests', function() {
  it('Supports TTL', async function() {
    cachedFetch = FetchCache.withCache(new MemoryCache({ ttl: 100 }));
    let res = await cachedFetch(TWO_HUNDRED_URL);
    assert.strictEqual(res.fromCache, false);
    res = await cachedFetch(TWO_HUNDRED_URL);
    assert.strictEqual(res.fromCache, true);

    await wait(200);

    res = await cachedFetch(TWO_HUNDRED_URL);
    assert.strictEqual(res.fromCache, false);
  });
}).timeout(10000);

describe('File system cache tests', function() {
  it('Supports TTL', async function() {
    cachedFetch = FetchCache.withCache(new FileSystemCache({ ttl: 100 }));
    let res = await cachedFetch(TWO_HUNDRED_URL);
    assert.strictEqual(res.fromCache, false);
    res = await cachedFetch(TWO_HUNDRED_URL);
    assert.strictEqual(res.fromCache, true);

    await wait(200);

    res = await cachedFetch(TWO_HUNDRED_URL);
    assert.strictEqual(res.fromCache, false);
  });

  it('Can get PNG buffer body', async function() {
    cachedFetch = FetchCache.withCache(new FileSystemCache());
    res = await cachedFetch(PNG_BODY_URL);
    body = await res.buffer();
    assert.strictEqual(expectedPngBuffer.equals(body), true);
    assert.strictEqual(res.fromCache, false);

    res = await cachedFetch(PNG_BODY_URL);
    body = await res.buffer();
    assert.strictEqual(expectedPngBuffer.equals(body), true);
    assert.strictEqual(res.fromCache, true);
  });

  it('Can eject from cache', async function() {
    cachedFetch = FetchCache.withCache(new FileSystemCache());

    res = await cachedFetch(TWO_HUNDRED_URL);
    assert.strictEqual(res.fromCache, false);

    res = await cachedFetch(TWO_HUNDRED_URL);
    assert.strictEqual(res.fromCache, true);

    await res.ejectFromCache();

    res = await cachedFetch(TWO_HUNDRED_URL);
    assert.strictEqual(res.fromCache, false);

    res = await cachedFetch(TWO_HUNDRED_URL);
    assert.strictEqual(res.fromCache, true);
  });
});

describe('Cache mode tests', function() {
  it('Can use the only-if-cached cache control setting via init', async function() {
    res = await cachedFetch(TWO_HUNDRED_URL, { headers: { 'Cache-Control': 'only-if-cached' } });
    assert(!res);
    res = await cachedFetch(TWO_HUNDRED_URL, { headers: { 'Cache-Control': 'only-if-cached' } });
    assert(!res);
    res = await cachedFetch(TWO_HUNDRED_URL);
    assert(res && !res.fromCache);
    res = await cachedFetch(TWO_HUNDRED_URL, { headers: { 'Cache-Control': 'only-if-cached' } });
    assert(res && res.fromCache);
    await res.ejectFromCache();
    res = await cachedFetch(TWO_HUNDRED_URL, { headers: { 'Cache-Control': 'only-if-cached' } });
    assert(!res);
  });

  it('Can use the only-if-cached cache control setting via resource', async function() {
    res = await cachedFetch(new standardFetch.Request(TWO_HUNDRED_URL, { headers: { 'Cache-Control': 'only-if-cached' } }));
    assert(!res);
    res = await cachedFetch(new standardFetch.Request(TWO_HUNDRED_URL));
    assert(res && !res.fromCache);
    res = await cachedFetch(new standardFetch.Request(TWO_HUNDRED_URL, { headers: { 'Cache-Control': 'only-if-cached' } }));
    assert(res && res.fromCache);
  });
});

describe('Cache key tests', function() {
  it('Can calculate a cache key and check that it exists', async function() {
    const cache = new MemoryCache();
    cachedFetch = FetchCache.withCache(cache);
    await cachedFetch(TWO_HUNDRED_URL);

    const cacheKey = getCacheKey(TWO_HUNDRED_URL);
    const nonExistentCacheKey = getCacheKey(TEXT_BODY_URL);

    const cacheKeyResult = await cache.get(cacheKey);
    const nonExistentCacheKeyResult = await cache.get(nonExistentCacheKey);

    assert(cacheKeyResult);
    assert(!nonExistentCacheKeyResult);
  });
});