40 Commits
3.0 ... 2.x

Author SHA1 Message Date
Randall Schmidt
da39daf753 npm ignore nyc_output 2021-06-11 14:04:44 -04:00
Randall Schmidt
87b96ec5ce npm ignore nyc_output 2021-06-11 14:04:25 -04:00
Randall Schmidt
ea89aefcd8 add CI file for github actions 2021-06-11 14:03:48 -04:00
Randall Schmidt
1f25cf946b add coverage script 2021-06-11 14:03:47 -04:00
Randall Schmidt
df86b6acea update readme 2021-06-11 14:03:47 -04:00
Randall Schmidt
43cbd56153 update readme 2021-06-11 14:03:47 -04:00
Randall Schmidt
ba067e65fd update readme 2021-06-11 14:03:46 -04:00
Randall Schmidt
4cae72dce2 custom caching 2021-06-11 14:03:46 -04:00
Randall Schmidt
1ae909985e npm audit fix 2021-06-11 14:03:46 -04:00
Randall Schmidt
6dcc8c7c6f add fpersist as dependency 2021-06-11 14:03:46 -04:00
Randall Schmidt
14061ba617 update readme 2021-06-11 14:03:45 -04:00
Randall Schmidt
95973cb2ce update readme 2021-06-11 14:03:45 -04:00
Randall Schmidt
275e9b66f4 update readme 2021-06-11 14:03:45 -04:00
Randall Schmidt
ce5a142084 update readme 2021-06-11 14:03:45 -04:00
Randall Schmidt
ee8418b15c update readme 2021-06-11 14:03:44 -04:00
Randall Schmidt
e3522e95a7 update readme 2021-06-11 14:03:44 -04:00
Randall Schmidt
075dad56ab update readme 2021-06-11 14:03:44 -04:00
Randall Schmidt
08d209349c update readme 2021-06-11 14:03:44 -04:00
Randall Schmidt
817119c09e update readme 2021-06-11 14:03:43 -04:00
Randall Schmidt
3cbe81bc87 update readme 2021-06-11 14:03:43 -04:00
Randall Schmidt
48de677ac7 update readme 2021-06-11 14:03:43 -04:00
Randall Schmidt
08060adfe6 update readme 2021-06-11 14:03:42 -04:00
Randall Schmidt
0bf89a9644 update readme 2021-06-11 14:03:42 -04:00
Randall Schmidt
f49d0939d6 support streaming response 2021-06-11 14:03:42 -04:00
Randall Schmidt
be468439af only allow body to be consumed once, add some tests 2021-06-11 14:03:42 -04:00
Randall Schmidt
4969a22ed4 give tests longer timeout 2021-06-11 14:03:41 -04:00
Randall Schmidt
6e782f49db add a CACHE_VERSION to invalid caches when versions become incompatible 2021-06-11 14:03:41 -04:00
Randall Schmidt
7e5c71ffe7 add support for FormData 2021-06-11 14:03:41 -04:00
Randall Schmidt
206d6784d1 handle fs read streams 2021-06-11 14:03:41 -04:00
Randall Schmidt
b94babec12 refactor cache key calculation to have more space for dealing with different types of bodies 2021-06-11 14:03:35 -04:00
Randall Schmidt
2e79048563 add raw headers function 2021-06-11 14:03:27 -04:00
Randall Schmidt
077b92ebaa add tests for string body and URLSearchParams bodies 2021-06-11 14:03:26 -04:00
Randall Schmidt
1df05c2584 dont error if ejectFromCache is called when the response is already not cached 2021-06-11 14:03:26 -04:00
Randall Schmidt
76978176d5 add husky to enforce lint rules and tests 2021-06-11 14:03:26 -04:00
Randall Schmidt
022626e7f9 fix lint warnings 2021-06-11 14:03:26 -04:00
Randall Schmidt
49741e9608 add test for cache ejection 2021-06-11 14:03:25 -04:00
Randall Schmidt
8833c2276e add tests, fix flawed logic 2021-06-11 14:03:25 -04:00
Randall Schmidt
37daa38c7c implement new expanded response caching logic 2021-06-11 14:03:19 -04:00
Randall Schmidt
e6ac859d1c add test boilerplate 2021-06-11 14:03:02 -04:00
Randall Schmidt
03dc8d9204 add mocha for tests 2021-06-11 14:02:40 -04:00
21 changed files with 474 additions and 886 deletions

View File

@ -1 +1 @@
test
test

View File

@ -1,4 +1,4 @@
{
module.exports = {
"env": {
"commonjs": true,
"es6": true,
@ -15,7 +15,5 @@
"ecmaVersion": 2018
},
"rules": {
"import/extensions": "off",
"import/prefer-default-export": "off"
}
}
};

2
.gitignore vendored
View File

@ -105,5 +105,3 @@ dist
# Other
.cache
commonjs

View File

@ -1,8 +1,4 @@
.eslintrc.json
.eslintrc.js
test
.cache
.nyc_output
.github
.eslintignore
.vscode
coverage
.nyc_output

132
README.md
View File

@ -22,35 +22,86 @@ The next time you `fetch('http://google.com')`, the response will be returned fr
## API
This module's fetch function has almost the exact same API as node-fetch, and you should consult [the node-fetch documentation](https://www.npmjs.com/package/node-fetch) for how to use it.
This module aims to expose the same API as `node-fetch` does for the most common use cases, but may not support some of the less common functions, properties, and use cases.
This module just adds one extra function to the response object:
### const fetch = require('node-fetch-cache');
### res.ejectFromCache(): Promise\<void\>
Load the module.
This function can be used to eject the response from the cache, so that the next request will perform a true HTTP request rather than returning a cached response.
### async fetch(resource [, init])
This module caches ALL responses, even those with 4xx and 5xx response statuses. You can use this function to uncache such responses if desired. For example:
Same arguments as [node-fetch](https://www.npmjs.com/package/node-fetch).
```js
const fetch = require('node-fetch-cache');
Returns a **CachedResponse**.
fetch('http://google.com')
.then(async response => {
if (!response.ok) {
await response.ejectFromCache();
throw new Error('Non-okay response from google.com');
} else {
return response.text();
}
}).then(text => console.log(text));
```
### async CachedResponse.text()
Returns the body as a string, same as [node-fetch](https://www.npmjs.com/package/node-fetch).
### async CachedResponse.json()
Returns the body as a JavaScript object, parsed from JSON, same as [node-fetch](https://www.npmjs.com/package/node-fetch).
### async CachedResponse.buffer()
Returns the body as a Buffer, same as [node-fetch](https://www.npmjs.com/package/node-fetch).
### CachedResponse.status
Returns the HTTP status code of the response, same as [node-fetch](https://www.npmjs.com/package/node-fetch).
### CachedResponse.statusText
Returns a text represention of the response status, same as [node-fetch](https://www.npmjs.com/package/node-fetch).
### CachedResponse.ok
Returns true if the request returned a successful response status, false otherwise, same as [node-fetch](https://www.npmjs.com/package/node-fetch).
### CachedResponse.redirected
Returns true if the request was redirected, false otherwise, same as [node-fetch](https://www.npmjs.com/package/node-fetch).
### CachedResponse.headers
Returns a **ResponseHeaders** object representing the headers of the response, same as [node-fetch](https://www.npmjs.com/package/node-fetch).
### async CachedResponse.ejectFromCache()
Eject the response from the cache, so that the next request will perform a true HTTP request rather than returning a cached response.
Keep in mind that this module caches **all** responses, even if they return error status codes. You might want to use this function when `!response.ok`, so that you can retry requests.
### ResponseHeaders.entries()
Returns the raw headers as an array of `[key, value]` pairs, same as [node-fetch](https://www.npmjs.com/package/node-fetch).
### ResponseHeaders.keys()
Returns an array of all header keys, same as [node-fetch](https://www.npmjs.com/package/node-fetch).
### ResponseHeaders.values()
Returns an array of all header values, same as [node-fetch](https://www.npmjs.com/package/node-fetch).
### ResponseHeaders.get(key)
Returns the value of the header with the given key, same as [node-fetch](https://www.npmjs.com/package/node-fetch).
### ResponseHeaders.has(key)
Returns true if the headers has a value for the given key, same as [node-fetch](https://www.npmjs.com/package/node-fetch).
### ResponseHeaders.raw
Returns the headers as an object of `{ "key": "value" }` pairs, same as [node-fetch](https://www.npmjs.com/package/node-fetch).
## Streaming
This module does not support Stream request bodies, except for fs.ReadStream. And when using fs.ReadStream, the cache key is generated based only on the path of the stream, not its content. That means if you stream `/my/desktop/image.png` twice, you will get a cached response the second time, **even if the content of image.png has changed**.
This module supports streams like [node-fetch](https://www.npmjs.com/package/node-fetch) does, but with a couple of caveats you should be aware of if you want to use streams.
Streams don't quite play nice with the concept of caching based on request characteristics, because we would have to read the stream to the end to find out what's in it and hash it into a proper cache key.
1. Response bodies are always read into memory even if you stream them to disk. That means if you need to stream large responses that don't fit into RAM, this module may be unsuitable.
2. When streaming a request body with fs.ReadStream, the cache key is generated based only on the path of the stream, not its content. That means if you stream `/my/desktop/image.png` twice, you will get a cached response the second time, **even if the content of image.png has changed**. This module may be unsuitable if you need to stream files in requests and the content of those files can change.
## Cache Customization
@ -63,7 +114,7 @@ This is the default cache delegate. It caches responses in-process in a POJO.
Usage:
```js
const { fetchBuilder, MemoryCache } = require('node-fetch-cache');
const fetchBuilder, { MemoryCache } = require('node-fetch-cache');
const fetch = fetchBuilder.withCache(new MemoryCache(options));
```
@ -84,12 +135,10 @@ Cache to a directory on disk. This allows the cache to survive the process exiti
Usage:
```js
const { fetchBuilder, FileSystemCache } = require('node-fetch-cache');
const fetchBuilder, { FileSystemCache } = require('node-fetch-cache');
const fetch = fetchBuilder.withCache(new FileSystemCache(options));
```
Options:
```js
{
cacheDirectory: '/my/cache/directory/path', // Specify where to keep the cache. If undefined, '.cache' is used by default. If this directory does not exist, it will be created.
@ -99,17 +148,42 @@ Options:
### Provide Your Own
You can implement a caching delegate yourself. The cache simply needs to be an object that has `set(key, bodyStream, bodyMeta)`, `get(key)`, and `remove(key)` functions.
You can implement a caching delegate yourself. The cache simply needs to be an object that has `set(key, value)`, `get(key)`, and `remove(key)` functions.
Check the built-in [MemoryCache](https://github.com/mistval/node-fetch-cache/blob/master/src/classes/caching/memory_cache.js) and [FileSystemCache](https://github.com/mistval/node-fetch-cache/blob/master/src/classes/caching/file_system_cache.js) for examples.
The set function must accept a key (which will be a string) and a value (which will be a JSON-serializable JS object) and store them.
The set function must accept a key (which will be a string), a body stream, and a metadata object (which will be a JSON-serializable JS object). It must store these, and then return an object with a `bodyStream` property, containing a fresh, unread stream of the body content, as well as a `metaData` property, containing the same metaData that was passed in.
The get function should accept a key and return whatever value was set for that key (or `undefined`/`null` if there is no value for that key).
The get function should accept a key and return undefined if no cached value is found, or else an object with a `bodyStream` property, containing a stream of the body content, as well as a `metaData` property, containing the metadata that was stored via the `set(key, bodyStream, bodyMeta)` function.
The remove function should accept a key and remove the cached value associated with that key, if any.
The remove function should accept a key and remove the cached value associated with that key, if any. It is also safe for your caching delegate to remove values from the cache arbitrarily if desired (for example if you want to implement a TTL in the caching delegate).
Both functions can be async.
All three functions may be async.
It is safe to remove values from the cache arbitrarily (for example if you implement a TTL in the caching delegate).
Example: you could make and use your own simple memory cache like this:
```js
class MyMemoryCache {
set(key, value) {
this[key] = value;
}
get(key) {
return this[key];
}
remove(key) {
delete this[key];
}
}
const fetchBuilder = require('node-fetch-cache');
fetch = fetchBuilder.withCache(new MyMemoryCache());
fetch('http://google.com')
.then(response => response.text())
.then(text => console.log(text));
```
## Bugs / Help / Feature Requests / Contributing

View File

@ -0,0 +1,29 @@
const FPersist = require('fpersist');
const KeyTimeout = require('./key_timeout.js');
module.exports = class FileSystemCache {
constructor(options = {}) {
this.ttl = options.ttl;
this.keyTimeout = new KeyTimeout();
const cacheDirectory = options.cacheDirectory || '.cache';
this.cache = new FPersist(cacheDirectory);
}
get(key) {
return this.cache.getItem(key);
}
remove(key) {
this.keyTimeout.clearTimeout(key);
return this.cache.deleteItem(key);
}
async set(key, value) {
await this.cache.setItem(key, value);
if (typeof this.ttl === 'number') {
this.keyTimeout.updateTimeout(key, this.ttl, () => this.remove(key));
}
}
};

View File

@ -1,4 +1,4 @@
export class KeyTimeout {
module.exports = class KeyTimeout {
constructor() {
this.timeoutHandleForKey = {};
}
@ -13,4 +13,4 @@ export class KeyTimeout {
callback();
}, durationMs);
}
}
};

View File

@ -0,0 +1,26 @@
const KeyTimeout = require('./key_timeout.js');
module.exports = class MemoryCache {
constructor(options = {}) {
this.ttl = options.ttl;
this.keyTimeout = new KeyTimeout();
this.cache = {};
}
get(key) {
return this.cache[key];
}
remove(key) {
this.keyTimeout.clearTimeout(key);
delete this.cache[key];
}
set(key, value) {
this.cache[key] = value;
if (typeof this.ttl === 'number') {
this.keyTimeout.updateTimeout(key, this.ttl, () => this.remove(key));
}
}
};

31
classes/headers.js Normal file
View File

@ -0,0 +1,31 @@
class Headers {
constructor(rawHeaders) {
this.rawHeaders = rawHeaders;
}
entries() {
return Object.entries(this.rawHeaders);
}
keys() {
return Object.keys(this.rawHeaders);
}
values() {
return Object.values(this.rawHeaders);
}
get(name) {
return this.rawHeaders[name.toLowerCase()] || null;
}
has(name) {
return !!this.get(name);
}
raw() {
return this.rawHeaders;
}
}
module.exports = Headers;

47
classes/response.js Normal file
View File

@ -0,0 +1,47 @@
const stream = require('stream');
const Headers = require('./headers.js');
class Response {
constructor(raw, ejectSelfFromCache, fromCache) {
Object.assign(this, raw);
this.ejectSelfFromCache = ejectSelfFromCache;
this.headers = new Headers(raw.headers);
this.fromCache = fromCache;
this.bodyUsed = false;
if (this.bodyBuffer.type === 'Buffer') {
this.bodyBuffer = Buffer.from(this.bodyBuffer);
}
}
get body() {
return stream.Readable.from(this.bodyBuffer);
}
consumeBody() {
if (this.bodyUsed) {
throw new Error('Error: body used already');
}
this.bodyUsed = true;
return this.bodyBuffer;
}
text() {
return this.consumeBody().toString();
}
json() {
return JSON.parse(this.consumeBody().toString());
}
buffer() {
return this.consumeBody();
}
ejectFromCache() {
return this.ejectSelfFromCache();
}
}
module.exports = Response;

View File

@ -1,4 +0,0 @@
const mod = require('./index.cjs');
module.exports = mod.default;
Object.assign(module.exports, mod);

114
index.js Normal file
View File

@ -0,0 +1,114 @@
const fetch = require('node-fetch');
const fs = require('fs');
const { URLSearchParams } = require('url');
const crypto = require('crypto');
const Response = require('./classes/response.js');
const MemoryCache = require('./classes/caching/memory_cache.js');
const CACHE_VERSION = 2;
function md5(str) {
return crypto.createHash('md5').update(str).digest('hex');
}
// Since the bounday in FormData is random,
// we ignore it for purposes of calculating
// the cache key.
function getFormDataCacheKey(formData) {
const cacheKey = { ...formData };
if (typeof formData.getBoundary === 'function') {
const boundary = formData.getBoundary();
// eslint-disable-next-line no-underscore-dangle
delete cacheKey._boundary;
// eslint-disable-next-line no-underscore-dangle
if (Array.isArray(cacheKey._streams)) {
const boundaryReplaceRegex = new RegExp(boundary, 'g');
// eslint-disable-next-line no-underscore-dangle
cacheKey._streams = cacheKey._streams.map((s) => {
if (typeof s === 'string') {
return s.replace(boundaryReplaceRegex, '');
}
return s;
});
}
}
return cacheKey;
}
function getBodyCacheKeyJson(body) {
if (!body) {
return body;
} if (typeof body === 'string') {
return body;
} if (body instanceof URLSearchParams) {
return body.toString();
} if (body instanceof fs.ReadStream) {
return body.path;
} if (body.toString && body.toString() === '[object FormData]') {
return getFormDataCacheKey(body);
}
throw new Error('Unsupported body type');
}
function getCacheKey(requestArguments) {
const resource = requestArguments[0];
const init = requestArguments[1] || {};
const resourceCacheKeyJson = typeof resource === 'string' ? { url: resource } : { ...resource };
const initCacheKeyJson = { ...init };
resourceCacheKeyJson.body = getBodyCacheKeyJson(resourceCacheKeyJson.body);
initCacheKeyJson.body = getBodyCacheKeyJson(initCacheKeyJson.body);
return md5(JSON.stringify([resourceCacheKeyJson, initCacheKeyJson, CACHE_VERSION]));
}
async function createRawResponse(fetchRes) {
const buffer = await fetchRes.buffer();
const rawHeaders = Array.from(fetchRes.headers.entries())
.reduce((aggregate, entry) => ({ ...aggregate, [entry[0]]: entry[1] }), {});
return {
status: fetchRes.status,
statusText: fetchRes.statusText,
type: fetchRes.type,
url: fetchRes.url,
ok: fetchRes.ok,
headers: rawHeaders,
redirected: fetchRes.redirected,
bodyBuffer: buffer,
};
}
async function getResponse(cache, requestArguments) {
const cacheKey = getCacheKey(requestArguments);
const cachedValue = await cache.get(cacheKey);
const ejectSelfFromCache = () => cache.remove(cacheKey);
if (cachedValue) {
return new Response(cachedValue, ejectSelfFromCache, true);
}
const fetchResponse = await fetch(...requestArguments);
const rawResponse = await createRawResponse(fetchResponse);
await cache.set(cacheKey, rawResponse);
return new Response(rawResponse, ejectSelfFromCache, false);
}
function createFetchWithCache(cache) {
const fetchCache = (...args) => getResponse(cache, args);
fetchCache.withCache = createFetchWithCache;
return fetchCache;
}
module.exports = createFetchWithCache(new MemoryCache());

240
package-lock.json generated
View File

@ -1,6 +1,6 @@
{
"name": "node-fetch-cache",
"version": "3.0.0",
"version": "1.0.6",
"lockfileVersion": 1,
"requires": true,
"dependencies": {
@ -427,22 +427,6 @@
"integrity": "sha512-ZXRY4jNvVgSVQ8DL3LTcakaAtXwTVUxE81hslsyD2AtoXW/wVob10HkOJ1X/pAlcI7D+2YoZKg5do8G/w6RYgA==",
"dev": true
},
"@npmcli/move-file": {
"version": "1.1.2",
"resolved": "https://registry.npmjs.org/@npmcli/move-file/-/move-file-1.1.2.tgz",
"integrity": "sha512-1SUf/Cg2GzGDyaf15aR9St9TWlb+XvbZXWpDx8YKs7MLzMH/BCeopv+y9vzrzgkfykCGuWOlSu3mZhj2+FQcrg==",
"requires": {
"mkdirp": "^1.0.4",
"rimraf": "^3.0.2"
},
"dependencies": {
"mkdirp": {
"version": "1.0.4",
"resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-1.0.4.tgz",
"integrity": "sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw=="
}
}
},
"@types/color-name": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/@types/color-name/-/color-name-1.1.1.tgz",
@ -477,6 +461,7 @@
"version": "3.1.0",
"resolved": "https://registry.npmjs.org/aggregate-error/-/aggregate-error-3.1.0.tgz",
"integrity": "sha512-4I7Td01quW/RpocfNayFdFVk1qSuoh0E7JrbRJ16nH01HhKFQ88INq9Sd+nd72zqRySlr9BmDA8xlEJ6vJMrYA==",
"dev": true,
"requires": {
"clean-stack": "^2.0.0",
"indent-string": "^4.0.0"
@ -602,7 +587,8 @@
"balanced-match": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz",
"integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c="
"integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=",
"dev": true
},
"binary-extensions": {
"version": "2.1.0",
@ -614,6 +600,7 @@
"version": "1.1.11",
"resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz",
"integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==",
"dev": true,
"requires": {
"balanced-match": "^1.0.0",
"concat-map": "0.0.1"
@ -647,45 +634,6 @@
"node-releases": "^1.1.71"
}
},
"cacache": {
"version": "15.2.0",
"resolved": "https://registry.npmjs.org/cacache/-/cacache-15.2.0.tgz",
"integrity": "sha512-uKoJSHmnrqXgthDFx/IU6ED/5xd+NNGe+Bb+kLZy7Ku4P+BaiWEUflAKPZ7eAzsYGcsAGASJZsybXp+quEcHTw==",
"requires": {
"@npmcli/move-file": "^1.0.1",
"chownr": "^2.0.0",
"fs-minipass": "^2.0.0",
"glob": "^7.1.4",
"infer-owner": "^1.0.4",
"lru-cache": "^6.0.0",
"minipass": "^3.1.1",
"minipass-collect": "^1.0.2",
"minipass-flush": "^1.0.5",
"minipass-pipeline": "^1.2.2",
"mkdirp": "^1.0.3",
"p-map": "^4.0.0",
"promise-inflight": "^1.0.1",
"rimraf": "^3.0.2",
"ssri": "^8.0.1",
"tar": "^6.0.2",
"unique-filename": "^1.1.1"
},
"dependencies": {
"mkdirp": {
"version": "1.0.4",
"resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-1.0.4.tgz",
"integrity": "sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw=="
},
"p-map": {
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/p-map/-/p-map-4.0.0.tgz",
"integrity": "sha512-/bjOqmgETBYB5BoEeGVea8dmvHb2m9GLy1E9W43yeyfP6QQCZGFNa+XRceJEuDB6zqr+gKpIAmlLebMpykw/MQ==",
"requires": {
"aggregate-error": "^3.0.0"
}
}
}
},
"caching-transform": {
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/caching-transform/-/caching-transform-4.0.0.tgz",
@ -749,11 +697,6 @@
"readdirp": "~3.5.0"
}
},
"chownr": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/chownr/-/chownr-2.0.0.tgz",
"integrity": "sha512-bIomtDF5KGpdogkLd9VspvFzk9KfpyyGlS8YFVZl7TGPBHL5snIOnxeshwVgPteQ9b4Eydl+pVbIyE1DcvCWgQ=="
},
"ci-info": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/ci-info/-/ci-info-2.0.0.tgz",
@ -763,7 +706,8 @@
"clean-stack": {
"version": "2.2.0",
"resolved": "https://registry.npmjs.org/clean-stack/-/clean-stack-2.2.0.tgz",
"integrity": "sha512-4diC9HaTE+KRAMWhDhrGOECgWZxoevMc5TlkObMqNSsVU62PYzXZ/SMTjzyGAFF1YusgxGcSWTEXBhp0CPwQ1A=="
"integrity": "sha512-4diC9HaTE+KRAMWhDhrGOECgWZxoevMc5TlkObMqNSsVU62PYzXZ/SMTjzyGAFF1YusgxGcSWTEXBhp0CPwQ1A==",
"dev": true
},
"cli-cursor": {
"version": "3.1.0",
@ -861,7 +805,8 @@
"concat-map": {
"version": "0.0.1",
"resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz",
"integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s="
"integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=",
"dev": true
},
"confusing-browser-globals": {
"version": "1.0.9",
@ -1563,24 +1508,22 @@
"mime-types": "^2.1.12"
}
},
"fpersist": {
"version": "1.0.5",
"resolved": "https://registry.npmjs.org/fpersist/-/fpersist-1.0.5.tgz",
"integrity": "sha512-WXY+zZXlOo1dU+wS8rqigz5PFu7WHBDd0vcaaWcnu319bPJi/IeWipOmi1PNaHAUqFVSzp1mLpNkgX6g2uLGbQ=="
},
"fromentries": {
"version": "1.3.2",
"resolved": "https://registry.npmjs.org/fromentries/-/fromentries-1.3.2.tgz",
"integrity": "sha512-cHEpEQHUg0f8XdtZCc2ZAhrHzKzT0MrFUTcvx+hfxYu7rGMDc5SKoXFh+n4YigxsHXRzc6OrCshdR1bWH6HHyg==",
"dev": true
},
"fs-minipass": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/fs-minipass/-/fs-minipass-2.1.0.tgz",
"integrity": "sha512-V/JgOLFCS+R6Vcq0slCuaeWEdNC3ouDlJMNIsacH2VtALiu9mV4LPrHc5cDl8k5aw6J8jwgWWpiTo5RYhmIzvg==",
"requires": {
"minipass": "^3.0.0"
}
},
"fs.realpath": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz",
"integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8="
"integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8=",
"dev": true
},
"fsevents": {
"version": "2.1.3",
@ -1623,6 +1566,7 @@
"version": "7.1.6",
"resolved": "https://registry.npmjs.org/glob/-/glob-7.1.6.tgz",
"integrity": "sha512-LwaxwyZ72Lk7vZINtNNrywX0ZuLyStrdDtabefZKAY5ZGJhVtgdznluResxNmPitE0SAO+O26sWTHeKSI2wMBA==",
"dev": true,
"requires": {
"fs.realpath": "^1.0.0",
"inflight": "^1.0.4",
@ -1866,22 +1810,20 @@
"imurmurhash": {
"version": "0.1.4",
"resolved": "https://registry.npmjs.org/imurmurhash/-/imurmurhash-0.1.4.tgz",
"integrity": "sha1-khi5srkoojixPcT7a21XbyMUU+o="
"integrity": "sha1-khi5srkoojixPcT7a21XbyMUU+o=",
"dev": true
},
"indent-string": {
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/indent-string/-/indent-string-4.0.0.tgz",
"integrity": "sha512-EdDDZu4A2OyIK7Lr/2zG+w5jmbuk1DVBnEwREQvBzspBJkCEbRa8GxU1lghYcaGJCnRWibjDXlq779X1/y5xwg=="
},
"infer-owner": {
"version": "1.0.4",
"resolved": "https://registry.npmjs.org/infer-owner/-/infer-owner-1.0.4.tgz",
"integrity": "sha512-IClj+Xz94+d7irH5qRyfJonOdfTzuDaifE6ZPWfx0N0+/ATZCbuTPq2prFl526urkQd90WyUKIh1DfBQ2hMz9A=="
"integrity": "sha512-EdDDZu4A2OyIK7Lr/2zG+w5jmbuk1DVBnEwREQvBzspBJkCEbRa8GxU1lghYcaGJCnRWibjDXlq779X1/y5xwg==",
"dev": true
},
"inflight": {
"version": "1.0.6",
"resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz",
"integrity": "sha1-Sb1jMdfQLQwJvJEKEHW6gWW1bfk=",
"dev": true,
"requires": {
"once": "^1.3.0",
"wrappy": "1"
@ -1890,7 +1832,8 @@
"inherits": {
"version": "2.0.4",
"resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz",
"integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ=="
"integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==",
"dev": true
},
"inquirer": {
"version": "7.1.0",
@ -2323,11 +2266,6 @@
"path-exists": "^3.0.0"
}
},
"locko": {
"version": "0.0.3",
"resolved": "https://registry.npmjs.org/locko/-/locko-0.0.3.tgz",
"integrity": "sha512-ekhPWcejAum9WHN2ClkFA8RAUTDyYDlRRb4dSq1wCEPhIS6IMsdSKoWHl1qineCrlMEMbeD1/o2uautG4QEc7w=="
},
"lodash": {
"version": "4.17.21",
"resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz",
@ -2400,14 +2338,6 @@
}
}
},
"lru-cache": {
"version": "6.0.0",
"resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz",
"integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==",
"requires": {
"yallist": "^4.0.0"
}
},
"make-dir": {
"version": "3.1.0",
"resolved": "https://registry.npmjs.org/make-dir/-/make-dir-3.1.0.tgz",
@ -2442,6 +2372,7 @@
"version": "3.0.4",
"resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz",
"integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==",
"dev": true,
"requires": {
"brace-expansion": "^1.1.7"
}
@ -2452,47 +2383,6 @@
"integrity": "sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw==",
"dev": true
},
"minipass": {
"version": "3.1.3",
"resolved": "https://registry.npmjs.org/minipass/-/minipass-3.1.3.tgz",
"integrity": "sha512-Mgd2GdMVzY+x3IJ+oHnVM+KG3lA5c8tnabyJKmHSaG2kAGpudxuOf8ToDkhumF7UzME7DecbQE9uOZhNm7PuJg==",
"requires": {
"yallist": "^4.0.0"
}
},
"minipass-collect": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/minipass-collect/-/minipass-collect-1.0.2.tgz",
"integrity": "sha512-6T6lH0H8OG9kITm/Jm6tdooIbogG9e0tLgpY6mphXSm/A9u8Nq1ryBG+Qspiub9LjWlBPsPS3tWQ/Botq4FdxA==",
"requires": {
"minipass": "^3.0.0"
}
},
"minipass-flush": {
"version": "1.0.5",
"resolved": "https://registry.npmjs.org/minipass-flush/-/minipass-flush-1.0.5.tgz",
"integrity": "sha512-JmQSYYpPUqX5Jyn1mXaRwOda1uQ8HP5KAT/oDSLCzt1BYRhQU0/hDtsB1ufZfEEzMZ9aAVmsBw8+FWsIXlClWw==",
"requires": {
"minipass": "^3.0.0"
}
},
"minipass-pipeline": {
"version": "1.2.4",
"resolved": "https://registry.npmjs.org/minipass-pipeline/-/minipass-pipeline-1.2.4.tgz",
"integrity": "sha512-xuIq7cIOt09RPRJ19gdi4b+RiNvDFYe5JH+ggNvBqGqpQXcru3PcRmOZuHBKWK1Txf9+cQ+HMVN4d6z46LZP7A==",
"requires": {
"minipass": "^3.0.0"
}
},
"minizlib": {
"version": "2.1.2",
"resolved": "https://registry.npmjs.org/minizlib/-/minizlib-2.1.2.tgz",
"integrity": "sha512-bAxsR8BVfj60DWXHE3u30oHzfl4G7khkSuPW+qvpd7jFRHm7dLxOjUk1EHACJ/hxLY8phGJ0YhYHZo7jil7Qdg==",
"requires": {
"minipass": "^3.0.0",
"yallist": "^4.0.0"
}
},
"mkdirp": {
"version": "0.5.5",
"resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.5.tgz",
@ -2939,6 +2829,7 @@
"version": "1.4.0",
"resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz",
"integrity": "sha1-WDsap3WWHUsROsF9nFC6753Xa9E=",
"dev": true,
"requires": {
"wrappy": "1"
}
@ -3050,7 +2941,8 @@
"path-is-absolute": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz",
"integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18="
"integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18=",
"dev": true
},
"path-key": {
"version": "2.0.1",
@ -3124,11 +3016,6 @@
"integrity": "sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA==",
"dev": true
},
"promise-inflight": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/promise-inflight/-/promise-inflight-1.0.1.tgz",
"integrity": "sha1-mEcocL8igTL8vdhoEputEsPAKeM="
},
"punycode": {
"version": "2.1.1",
"resolved": "https://registry.npmjs.org/punycode/-/punycode-2.1.1.tgz",
@ -3230,26 +3117,9 @@
"version": "3.0.2",
"resolved": "https://registry.npmjs.org/rimraf/-/rimraf-3.0.2.tgz",
"integrity": "sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA==",
"requires": {
"glob": "^7.1.3"
}
},
"rollup": {
"version": "2.53.0",
"resolved": "https://registry.npmjs.org/rollup/-/rollup-2.53.0.tgz",
"integrity": "sha512-spgrY78Toh+m0+zaOoeaayJKuzFuWy6o1PdFIBMVwRcuxT0xCOX9A5rChyKe+2ruL4lePKWUMImS4mMW1QAkmQ==",
"dev": true,
"requires": {
"fsevents": "~2.3.2"
},
"dependencies": {
"fsevents": {
"version": "2.3.2",
"resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz",
"integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==",
"dev": true,
"optional": true
}
"glob": "^7.1.3"
}
},
"run-async": {
@ -3430,14 +3300,6 @@
"integrity": "sha1-BOaSb2YolTVPPdAVIDYzuFcpfiw=",
"dev": true
},
"ssri": {
"version": "8.0.1",
"resolved": "https://registry.npmjs.org/ssri/-/ssri-8.0.1.tgz",
"integrity": "sha512-97qShzy1AiyxvPNIkLWoGua7xoQzzPjQ0HAH4B0rWKo7SZ6USuPcrUiAFrws0UH8RrbWmgq3LMTObhPIHbbBeQ==",
"requires": {
"minipass": "^3.1.1"
}
},
"string-width": {
"version": "4.2.0",
"resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.0.tgz",
@ -3577,26 +3439,6 @@
}
}
},
"tar": {
"version": "6.1.0",
"resolved": "https://registry.npmjs.org/tar/-/tar-6.1.0.tgz",
"integrity": "sha512-DUCttfhsnLCjwoDoFcI+B2iJgYa93vBnDUATYEeRx6sntCTdN01VnqsIuTlALXla/LWooNg0yEGeB+Y8WdFxGA==",
"requires": {
"chownr": "^2.0.0",
"fs-minipass": "^2.0.0",
"minipass": "^3.0.0",
"minizlib": "^2.1.1",
"mkdirp": "^1.0.3",
"yallist": "^4.0.0"
},
"dependencies": {
"mkdirp": {
"version": "1.0.4",
"resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-1.0.4.tgz",
"integrity": "sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw=="
}
}
},
"test-exclude": {
"version": "6.0.0",
"resolved": "https://registry.npmjs.org/test-exclude/-/test-exclude-6.0.0.tgz",
@ -3674,22 +3516,6 @@
"is-typedarray": "^1.0.0"
}
},
"unique-filename": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/unique-filename/-/unique-filename-1.1.1.tgz",
"integrity": "sha512-Vmp0jIp2ln35UTXuryvjzkjGdRyf9b2lTXuSYUiPmzRcl3FDtYqAwOnTJkAngD9SWhnoJzDbTKwaOrZ+STtxNQ==",
"requires": {
"unique-slug": "^2.0.0"
}
},
"unique-slug": {
"version": "2.0.2",
"resolved": "https://registry.npmjs.org/unique-slug/-/unique-slug-2.0.2.tgz",
"integrity": "sha512-zoWr9ObaxALD3DOPfjPSqxt4fnZiWblxHIgeWqW8x7UqDzEtHEQLzji2cuJYQFCU6KmoJikOYAZlrTHHebjx2w==",
"requires": {
"imurmurhash": "^0.1.4"
}
},
"uri-js": {
"version": "4.2.2",
"resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.2.2.tgz",
@ -3835,7 +3661,8 @@
"wrappy": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz",
"integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8="
"integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=",
"dev": true
},
"write": {
"version": "1.0.3",
@ -3864,11 +3691,6 @@
"integrity": "sha512-JKhqTOwSrqNA1NY5lSztJ1GrBiUodLMmIZuLiDaMRJ+itFd+ABVE8XBjOvIWL+rSqNDC74LCSFmlb/U4UZ4hJQ==",
"dev": true
},
"yallist": {
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz",
"integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A=="
},
"yaml": {
"version": "1.10.0",
"resolved": "https://registry.npmjs.org/yaml/-/yaml-1.10.0.tgz",

View File

@ -1,20 +1,13 @@
{
"name": "node-fetch-cache",
"version": "3.0.0",
"version": "1.0.6",
"description": "node-fetch with a persistent cache.",
"main": "src/index.js",
"type": "module",
"exports": {
"import": "./src/index.js",
"require": "./commonjs/wrapper.cjs"
},
"main": "index.js",
"scripts": {
"buildcjs": "rollup src/index.js --file commonjs/index.cjs --format cjs",
"test": "npm run buildcjs && mocha --timeout 10000 --exit",
"coverage": "nyc --reporter=lcov --reporter=text npm test",
"test": "mocha --timeout 10000 --exit",
"coverage": "nyc --reporter=lcov npm test",
"lint": "./node_modules/.bin/eslint .",
"lintfix": "./node_modules/.bin/eslint . --fix",
"prepublishOnly": "npm test"
"lintfix": "./node_modules/.bin/eslint . --fix"
},
"repository": {
"type": "git",
@ -41,13 +34,11 @@
"husky": "^4.3.0",
"mocha": "^8.2.1",
"nyc": "^15.1.0",
"rimraf": "^3.0.2",
"rollup": "^2.53.0"
"rimraf": "^3.0.2"
},
"dependencies": {
"cacache": "^15.2.0",
"locko": "0.0.3",
"node-fetch": "2.6.1"
"fpersist": "^1.0.5",
"node-fetch": "*"
},
"husky": {
"hooks": {

View File

@ -1,84 +0,0 @@
import cacache from 'cacache';
import { Readable } from 'stream';
import { KeyTimeout } from './key_timeout.js';
function getBodyAndMetaKeys(key) {
return [`${key}body`, `${key}meta`];
}
export class FileSystemCache {
constructor(options = {}) {
this.ttl = options.ttl;
this.keyTimeout = new KeyTimeout();
this.cacheDirectory = options.cacheDirectory || '.cache';
}
async get(key) {
const [, metaKey] = getBodyAndMetaKeys(key);
const metaInfo = await cacache.get.info(this.cacheDirectory, metaKey);
if (!metaInfo) {
return undefined;
}
const metaBuffer = await cacache.get.byDigest(this.cacheDirectory, metaInfo.integrity);
const metaData = JSON.parse(metaBuffer);
const { bodyStreamIntegrity, empty } = metaData;
delete metaData.bodyStreamIntegrity;
delete metaData.empty;
const bodyStream = empty
? Readable.from(Buffer.alloc(0))
: cacache.get.stream.byDigest(this.cacheDirectory, bodyStreamIntegrity);
return {
bodyStream,
metaData,
};
}
remove(key) {
const [bodyKey, metaKey] = getBodyAndMetaKeys(key);
this.keyTimeout.clearTimeout(key);
return Promise.all([
cacache.rm.entry(this.cacheDirectory, bodyKey),
cacache.rm.entry(this.cacheDirectory, metaKey),
]);
}
async set(key, bodyStream, metaData) {
const [bodyKey, metaKey] = getBodyAndMetaKeys(key);
const metaCopy = { ...metaData };
this.keyTimeout.clearTimeout(key);
try {
metaCopy.bodyStreamIntegrity = await new Promise((fulfill, reject) => {
bodyStream.pipe(cacache.put.stream(this.cacheDirectory, bodyKey))
.on('integrity', (i) => fulfill(i))
.on('error', (e) => {
reject(e);
});
});
} catch (err) {
if (err.code !== 'ENODATA') {
throw err;
}
metaCopy.empty = true;
}
const metaBuffer = Buffer.from(JSON.stringify(metaCopy));
await cacache.put(this.cacheDirectory, metaKey, metaBuffer);
const cachedData = await this.get(key);
if (typeof this.ttl === 'number') {
this.keyTimeout.updateTimeout(key, this.ttl, () => this.remove(key));
}
return cachedData;
}
}

View File

@ -1,47 +0,0 @@
import { Readable } from 'stream';
import { KeyTimeout } from './key_timeout.js';
function streamToBuffer(stream) {
const chunks = [];
return new Promise((resolve, reject) => {
stream.on('data', (chunk) => chunks.push(Buffer.from(chunk)));
stream.on('error', (err) => reject(err));
stream.on('end', () => resolve(Buffer.concat(chunks)));
});
}
export class MemoryCache {
constructor(options = {}) {
this.ttl = options.ttl;
this.keyTimeout = new KeyTimeout();
this.cache = {};
}
get(key) {
const cachedValue = this.cache[key];
if (cachedValue) {
return {
bodyStream: Readable.from(cachedValue.bodyBuffer),
metaData: cachedValue.metaData,
};
}
return undefined;
}
remove(key) {
this.keyTimeout.clearTimeout(key);
delete this.cache[key];
}
async set(key, bodyStream, metaData) {
const bodyBuffer = await streamToBuffer(bodyStream);
this.cache[key] = { bodyBuffer, metaData };
if (typeof this.ttl === 'number') {
this.keyTimeout.updateTimeout(key, this.ttl, () => this.remove(key));
}
return this.get(key);
}
}

View File

@ -1,29 +0,0 @@
import { Response } from 'node-fetch';
const responseInternalSymbol = Object.getOwnPropertySymbols(new Response())[1];
export class NFCResponse extends Response {
constructor(bodyStream, metaData, ejectFromCache, fromCache) {
super(bodyStream, metaData);
this.ejectFromCache = ejectFromCache;
this.fromCache = fromCache;
}
static serializeMetaFromNodeFetchResponse(res) {
const metaData = {
url: res.url,
status: res.status,
statusText: res.statusText,
headers: res.headers.raw(),
size: res.size,
timeout: res.timeout,
counter: res[responseInternalSymbol].counter,
};
return metaData;
}
ejectFromCache() {
return this.ejectSelfFromCache();
}
}

View File

@ -1,150 +0,0 @@
import fetch, { Request } from 'node-fetch';
import fs from 'fs';
import { URLSearchParams } from 'url';
import crypto from 'crypto';
import locko from 'locko';
import { NFCResponse } from './classes/response.js';
import { MemoryCache } from './classes/caching/memory_cache.js';
const CACHE_VERSION = 3;
function md5(str) {
return crypto.createHash('md5').update(str).digest('hex');
}
// Since the bounday in FormData is random,
// we ignore it for purposes of calculating
// the cache key.
function getFormDataCacheKey(formData) {
const cacheKey = { ...formData };
const boundary = formData.getBoundary();
// eslint-disable-next-line no-underscore-dangle
delete cacheKey._boundary;
const boundaryReplaceRegex = new RegExp(boundary, 'g');
// eslint-disable-next-line no-underscore-dangle
cacheKey._streams = cacheKey._streams.map((s) => {
if (typeof s === 'string') {
return s.replace(boundaryReplaceRegex, '');
}
return s;
});
return cacheKey;
}
function getBodyCacheKeyJson(body) {
if (!body) {
return body;
} if (typeof body === 'string') {
return body;
} if (body instanceof URLSearchParams) {
return body.toString();
} if (body instanceof fs.ReadStream) {
return body.path;
} if (body.toString && body.toString() === '[object FormData]') {
return getFormDataCacheKey(body);
} if (body instanceof Buffer) {
return body.toString();
}
throw new Error('Unsupported body type. Supported body types are: string, number, undefined, null, url.URLSearchParams, fs.ReadStream, FormData');
}
function getRequestCacheKey(req) {
return {
cache: req.cache,
credentials: req.credentials,
destination: req.destination,
headers: req.headers,
integrity: req.integrity,
method: req.method,
redirect: req.redirect,
referrer: req.referrer,
referrerPolicy: req.referrerPolicy,
url: req.url,
body: getBodyCacheKeyJson(req.body),
};
}
function getCacheKey(requestArguments) {
const resource = requestArguments[0];
const init = requestArguments[1] || {};
const resourceCacheKeyJson = resource instanceof Request
? getRequestCacheKey(resource)
: { url: resource };
const initCacheKeyJson = { ...init };
resourceCacheKeyJson.body = getBodyCacheKeyJson(resourceCacheKeyJson.body);
initCacheKeyJson.body = getBodyCacheKeyJson(initCacheKeyJson.body);
delete initCacheKeyJson.agent;
return md5(JSON.stringify([resourceCacheKeyJson, initCacheKeyJson, CACHE_VERSION]));
}
async function getResponse(cache, requestArguments) {
const cacheKey = getCacheKey(requestArguments);
let cachedValue = await cache.get(cacheKey);
const ejectSelfFromCache = () => cache.remove(cacheKey);
if (cachedValue) {
return new NFCResponse(
cachedValue.bodyStream,
cachedValue.metaData,
ejectSelfFromCache,
true,
);
}
await locko.lock(cacheKey);
try {
cachedValue = await cache.get(cacheKey);
if (cachedValue) {
return new NFCResponse(
cachedValue.bodyStream,
cachedValue.metaData,
ejectSelfFromCache,
true,
);
}
const fetchResponse = await fetch(...requestArguments);
const serializedMeta = NFCResponse.serializeMetaFromNodeFetchResponse(fetchResponse);
const newlyCachedData = await cache.set(
cacheKey,
fetchResponse.body,
serializedMeta,
);
return new NFCResponse(
newlyCachedData.bodyStream,
newlyCachedData.metaData,
ejectSelfFromCache,
false,
);
} finally {
locko.unlock(cacheKey);
}
}
function createFetchWithCache(cache) {
const fetchCache = (...args) => getResponse(cache, args);
fetchCache.withCache = createFetchWithCache;
return fetchCache;
}
const defaultFetch = createFetchWithCache(new MemoryCache());
export default defaultFetch;
export const fetchBuilder = defaultFetch;
export { MemoryCache } from './classes/caching/memory_cache.js';
export { FileSystemCache } from './classes/caching/file_system_cache.js';

View File

@ -1,18 +0,0 @@
const assert = require('assert');
const fetch = require('../commonjs/wrapper.cjs');
const TWO_HUNDRED_URL = 'https://httpbin.org/status/200';
describe('Commonjs module tests', function() {
it('Can make a request', async function() {
const res = await fetch(TWO_HUNDRED_URL);
assert.strictEqual(res.status, 200);
});
it('Has expected properties', function() {
assert(typeof fetch === 'function');
assert(fetch.MemoryCache);
assert(fetch.FileSystemCache);
assert(fetch.fetchBuilder);
});
});

View File

@ -1,16 +1,11 @@
import { dirname } from 'path';
import { fileURLToPath } from 'url';
import fs from 'fs';
import FormData from 'form-data';
import assert from 'assert';
import rimraf from 'rimraf';
import path from 'path';
import { URLSearchParams } from 'url';
import standardFetch from 'node-fetch';
import FetchCache, { MemoryCache, FileSystemCache } from '../src/index.js';
import { Agent } from 'http';
const __dirname = dirname(fileURLToPath(import.meta.url));
const fs = require('fs');
const FormData = require('form-data');
const assert = require('assert');
const rimraf = require('rimraf');
const path = require('path');
const FetchCache = require('../index.js');
const { URLSearchParams } = require('url');
const MemoryCache = require('../classes/caching/memory_cache.js');
const CACHE_PATH = path.join(__dirname, '..', '.cache');
const expectedPngBuffer = fs.readFileSync(path.join(__dirname, 'expected_png.png'));
@ -24,202 +19,98 @@ const PNG_BODY_URL = 'https://httpbin.org/image/png';
const TEXT_BODY_EXPECTED = 'User-agent: *\nDisallow: /deny\n';
let cachedFetch;
let fetch;
let res;
let body;
function post(body) {
return { method: 'POST', body };
}
function removeDates(arrOrObj) {
if (arrOrObj.date) {
const copy = { ...arrOrObj };
delete copy.date;
return copy;
}
if (Array.isArray(arrOrObj)) {
if (Array.isArray(arrOrObj[0])) {
return arrOrObj.filter(e => e[0] !== 'date');
}
return arrOrObj.filter(e => !Date.parse(e));
}
return arrOrObj;
}
function wait(ms) {
return new Promise((fulfill) => setTimeout(fulfill, ms));
}
async function dualFetch(...args) {
const [cachedFetchResponse, standardFetchResponse] = await Promise.all([
cachedFetch(...args),
standardFetch(...args),
]);
return { cachedFetchResponse, standardFetchResponse };
}
beforeEach(async function() {
rimraf.sync(CACHE_PATH);
cachedFetch = FetchCache.withCache(new MemoryCache());
fetch = FetchCache.withCache(new MemoryCache());
});
let res;
describe('Basic property tests', function() {
it('Has a status property', async function() {
let { cachedFetchResponse, standardFetchResponse } = await dualFetch(TWO_HUNDRED_URL);
assert.strictEqual(cachedFetchResponse.status, standardFetchResponse.status);
res = await fetch(TWO_HUNDRED_URL);
assert.strictEqual(res.status, 200);
cachedFetchResponse = await cachedFetch(TWO_HUNDRED_URL);
assert.strictEqual(cachedFetchResponse.status, standardFetchResponse.status);
res = await fetch(TWO_HUNDRED_URL);
assert.strictEqual(res.status, 200);
});
it('Has a statusText property', async function() {
let { cachedFetchResponse, standardFetchResponse } = await dualFetch(TWO_HUNDRED_URL);
assert.strictEqual(cachedFetchResponse.statusText, standardFetchResponse.statusText);
res = await fetch(TWO_HUNDRED_URL);
assert.strictEqual(res.statusText, 'OK');
cachedFetchResponse = await cachedFetch(TWO_HUNDRED_URL);
assert.strictEqual(cachedFetchResponse.statusText, standardFetchResponse.statusText);
res = await fetch(TWO_HUNDRED_URL);
assert.strictEqual(res.statusText, 'OK');
});
it('Has a url property', async function() {
let { cachedFetchResponse, standardFetchResponse } = await dualFetch(TWO_HUNDRED_URL);
assert.strictEqual(cachedFetchResponse.url, standardFetchResponse.url);
res = await fetch(TWO_HUNDRED_URL);
assert.strictEqual(res.url, TWO_HUNDRED_URL);
cachedFetchResponse = await cachedFetch(TWO_HUNDRED_URL);
assert.strictEqual(cachedFetchResponse.url, standardFetchResponse.url);
res = await fetch(TWO_HUNDRED_URL);
assert.strictEqual(res.url, TWO_HUNDRED_URL);
});
it('Has an ok property', async function() {
let { cachedFetchResponse, standardFetchResponse } = await dualFetch(FOUR_HUNDRED_URL);
assert.strictEqual(cachedFetchResponse.ok, standardFetchResponse.ok);
assert.strictEqual(cachedFetchResponse.status, standardFetchResponse.status);
res = await fetch(FOUR_HUNDRED_URL);
assert.strictEqual(res.ok, false);
assert.strictEqual(res.status, 400);
cachedFetchResponse = await cachedFetch(FOUR_HUNDRED_URL);
assert.strictEqual(cachedFetchResponse.ok, standardFetchResponse.ok);
assert.strictEqual(cachedFetchResponse.status, standardFetchResponse.status);
res = await fetch(FOUR_HUNDRED_URL);
assert.strictEqual(res.ok, false);
assert.strictEqual(res.status, 400);
});
it('Has a headers property', async function() {
res = await fetch(TWO_HUNDRED_URL);
assert.notStrictEqual(res.headers, undefined);
res = await fetch(TWO_HUNDRED_URL);
assert.notStrictEqual(res.headers, undefined);
});
it('Has a redirected property', async function() {
let { cachedFetchResponse, standardFetchResponse } = await dualFetch(THREE_HUNDRED_TWO_URL);
assert.strictEqual(cachedFetchResponse.redirected, standardFetchResponse.redirected);
res = await fetch(THREE_HUNDRED_TWO_URL);
assert.strictEqual(res.redirected, true);
cachedFetchResponse = await cachedFetch(THREE_HUNDRED_TWO_URL);
assert.strictEqual(cachedFetchResponse.redirected, standardFetchResponse.redirected);
});
}).timeout(10000);
describe('Header tests', function() {
it('Gets correct raw headers', async function() {
let { cachedFetchResponse, standardFetchResponse } = await dualFetch(TWO_HUNDRED_URL);
assert.deepStrictEqual(
removeDates(cachedFetchResponse.headers.raw()),
removeDates(standardFetchResponse.headers.raw()),
);
cachedFetchResponse = await cachedFetch(TWO_HUNDRED_URL);
assert.deepStrictEqual(
removeDates(cachedFetchResponse.headers.raw()),
removeDates(standardFetchResponse.headers.raw()),
);
});
it('Gets correct header keys', async function() {
let { cachedFetchResponse, standardFetchResponse } = await dualFetch(TWO_HUNDRED_URL);
assert.deepStrictEqual([...cachedFetchResponse.headers.keys()], [...standardFetchResponse.headers.keys()]);
cachedFetchResponse = await cachedFetch(TWO_HUNDRED_URL);
assert.deepStrictEqual([...cachedFetchResponse.headers.keys()], [...standardFetchResponse.headers.keys()]);
});
it('Gets correct header values', async function() {
let { cachedFetchResponse, standardFetchResponse } = await dualFetch(TWO_HUNDRED_URL);
assert.deepStrictEqual(
removeDates([...cachedFetchResponse.headers.values()]),
removeDates([...standardFetchResponse.headers.values()]),
);
cachedFetchResponse = await cachedFetch(TWO_HUNDRED_URL);
assert.deepStrictEqual(
removeDates([...cachedFetchResponse.headers.values()]),
removeDates([...standardFetchResponse.headers.values()]),
);
});
it('Gets correct header entries', async function() {
let { cachedFetchResponse, standardFetchResponse } = await dualFetch(TWO_HUNDRED_URL);
assert.deepStrictEqual(
removeDates([...cachedFetchResponse.headers.entries()]),
removeDates([...standardFetchResponse.headers.entries()]),
);
cachedFetchResponse = await cachedFetch(TWO_HUNDRED_URL);
assert.deepStrictEqual(
removeDates([...cachedFetchResponse.headers.entries()]),
removeDates([...standardFetchResponse.headers.entries()]),
);
});
it('Can get a header by value', async function() {
let { cachedFetchResponse, standardFetchResponse } = await dualFetch(TWO_HUNDRED_URL);
assert(standardFetchResponse.headers.get('content-length'));
assert.deepStrictEqual(cachedFetchResponse.headers.get('content-length'), standardFetchResponse.headers.get('content-length'));
cachedFetchResponse = await cachedFetch(TWO_HUNDRED_URL);
assert.deepStrictEqual(cachedFetchResponse.headers.get('content-length'), standardFetchResponse.headers.get('content-length'));
});
it('Returns undefined for non-existent header', async function() {
const headerName = 'zzzz';
let { cachedFetchResponse, standardFetchResponse } = await dualFetch(TWO_HUNDRED_URL);
assert(!standardFetchResponse.headers.get(headerName));
assert.deepStrictEqual(cachedFetchResponse.headers.get(headerName), standardFetchResponse.headers.get(headerName));
cachedFetchResponse = await cachedFetch(TWO_HUNDRED_URL);
assert.deepStrictEqual(cachedFetchResponse.headers.get(headerName), standardFetchResponse.headers.get(headerName));
});
it('Can get whether a header is present', async function() {
let { cachedFetchResponse, standardFetchResponse } = await dualFetch(TWO_HUNDRED_URL);
assert(standardFetchResponse.headers.has('content-length'));
assert.deepStrictEqual(cachedFetchResponse.headers.has('content-length'), standardFetchResponse.headers.has('content-length'));
cachedFetchResponse = await cachedFetch(TWO_HUNDRED_URL);
assert.deepStrictEqual(cachedFetchResponse.headers.has('content-length'), standardFetchResponse.headers.has('content-length'));
res = await fetch(THREE_HUNDRED_TWO_URL);
assert.strictEqual(res.redirected, true);
});
}).timeout(10000);
describe('Cache tests', function() {
it('Uses cache', async function() {
res = await cachedFetch(TWO_HUNDRED_URL);
res = await fetch(TWO_HUNDRED_URL);
assert.strictEqual(res.fromCache, false);
res = await cachedFetch(TWO_HUNDRED_URL);
res = await fetch(TWO_HUNDRED_URL);
assert.strictEqual(res.fromCache, true);
});
it('Can eject from cache', async function() {
res = await cachedFetch(TWO_HUNDRED_URL);
res = await fetch(TWO_HUNDRED_URL);
assert.strictEqual(res.fromCache, false);
res = await cachedFetch(TWO_HUNDRED_URL);
res = await fetch(TWO_HUNDRED_URL);
assert.strictEqual(res.fromCache, true);
await res.ejectFromCache();
res = await cachedFetch(TWO_HUNDRED_URL);
res = await fetch(TWO_HUNDRED_URL);
assert.strictEqual(res.fromCache, false);
res = await cachedFetch(TWO_HUNDRED_URL);
res = await fetch(TWO_HUNDRED_URL);
assert.strictEqual(res.fromCache, true);
});
it('Does not error if ejecting from cache twice', async function() {
res = await cachedFetch(TWO_HUNDRED_URL);
it('Does not error if rejecting from cache twice', async function() {
res = await fetch(TWO_HUNDRED_URL);
assert.strictEqual(res.fromCache, false);
await res.ejectFromCache();
@ -227,55 +118,55 @@ describe('Cache tests', function() {
});
it('Gives different string bodies different cache keys', async function() {
res = await cachedFetch(TWO_HUNDRED_URL, post('a'));
res = await fetch(TWO_HUNDRED_URL, post('a'));
assert.strictEqual(res.fromCache, false);
res = await cachedFetch(TWO_HUNDRED_URL, post('b'));
res = await fetch(TWO_HUNDRED_URL, post('b'));
assert.strictEqual(res.fromCache, false);
});
it('Gives same string bodies same cache keys', async function() {
res = await cachedFetch(TWO_HUNDRED_URL, post('a'));
res = await fetch(TWO_HUNDRED_URL, post('a'));
assert.strictEqual(res.fromCache, false);
res = await cachedFetch(TWO_HUNDRED_URL, post('a'));
res = await fetch(TWO_HUNDRED_URL, post('a'));
assert.strictEqual(res.fromCache, true);
});
it('Gives different URLSearchParams different cache keys', async function() {
res = await cachedFetch(TWO_HUNDRED_URL, post(new URLSearchParams('a=a')));
res = await fetch(TWO_HUNDRED_URL, post(new URLSearchParams('a=a')));
assert.strictEqual(res.fromCache, false);
res = await cachedFetch(TWO_HUNDRED_URL, post(new URLSearchParams('a=b')));
res = await fetch(TWO_HUNDRED_URL, post(new URLSearchParams('a=b')));
assert.strictEqual(res.fromCache, false);
});
it('Gives same URLSearchParams same cache keys', async function() {
res = await cachedFetch(TWO_HUNDRED_URL, post(new URLSearchParams('a=a')));
res = await fetch(TWO_HUNDRED_URL, post(new URLSearchParams('a=a')));
assert.strictEqual(res.fromCache, false);
res = await cachedFetch(TWO_HUNDRED_URL, post(new URLSearchParams('a=a')));
res = await fetch(TWO_HUNDRED_URL, post(new URLSearchParams('a=a')));
assert.strictEqual(res.fromCache, true);
});
it('Gives different read streams different cache keys', async function() {
const s1 = fs.createReadStream(path.join(__dirname, 'expected_png.png'));
const s2 = fs.createReadStream(path.join(__dirname, '..', 'src', 'index.js'));
const s1 = fs.createReadStream(__filename);
const s2 = fs.createReadStream(path.join(__dirname, '..', 'index.js'));
res = await cachedFetch(TWO_HUNDRED_URL, post(s1));
res = await fetch(TWO_HUNDRED_URL, post(s1));
assert.strictEqual(res.fromCache, false);
res = await cachedFetch(TWO_HUNDRED_URL, post(s2));
res = await fetch(TWO_HUNDRED_URL, post(s2));
assert.strictEqual(res.fromCache, false);
});
it('Gives the same read streams the same cache key', async function() {
const s1 = fs.createReadStream(path.join(__dirname, 'expected_png.png'));
const s1 = fs.createReadStream(__filename);
res = await cachedFetch(TWO_HUNDRED_URL, post(s1));
res = await fetch(TWO_HUNDRED_URL, post(s1));
assert.strictEqual(res.fromCache, false);
res = await cachedFetch(TWO_HUNDRED_URL, post(s1));
res = await fetch(TWO_HUNDRED_URL, post(s1));
assert.strictEqual(res.fromCache, true);
});
@ -286,10 +177,10 @@ describe('Cache tests', function() {
const data2 = new FormData();
data2.append('b', 'b');
res = await cachedFetch(TWO_HUNDRED_URL, post(data1));
res = await fetch(TWO_HUNDRED_URL, post(data1));
assert.strictEqual(res.fromCache, false);
res = await cachedFetch(TWO_HUNDRED_URL, post(data2));
res = await fetch(TWO_HUNDRED_URL, post(data2));
assert.strictEqual(res.fromCache, false);
});
@ -300,82 +191,65 @@ describe('Cache tests', function() {
const data2 = new FormData();
data2.append('a', 'a');
res = await cachedFetch(TWO_HUNDRED_URL, post(data1));
res = await fetch(TWO_HUNDRED_URL, post(data1));
assert.strictEqual(res.fromCache, false);
res = await cachedFetch(TWO_HUNDRED_URL, post(data2));
res = await fetch(TWO_HUNDRED_URL, post(data2));
assert.strictEqual(res.fromCache, true);
});
it('Does not error with custom agent with circular properties', async function() {
const agent = new Agent();
agent.agent = agent;
await cachedFetch('http://httpbin.org/status/200', { agent });
})
}).timeout(10000);
describe('Data tests', function() {
it('Supports request objects', async function() {
let request = new standardFetch.Request('https://google.com', { body: 'test', method: 'POST' });
res = await cachedFetch(request);
assert.strictEqual(res.fromCache, false);
request = new standardFetch.Request('https://google.com', { body: 'test', method: 'POST' });
res = await cachedFetch(request);
assert.strictEqual(res.fromCache, true);
});
it('Refuses to consume body twice', async function() {
res = await cachedFetch(TEXT_BODY_URL);
res = await fetch(TEXT_BODY_URL);
await res.text();
try {
await res.text();
throw new Error('The above line should have thrown.');
} catch (err) {
assert(err.message.includes('body used already for:'));
// It threw
}
});
it('Can get text body', async function() {
res = await cachedFetch(TEXT_BODY_URL);
res = await fetch(TEXT_BODY_URL);
body = await res.text();
assert.strictEqual(body, TEXT_BODY_EXPECTED);
assert.strictEqual(res.fromCache, false);
res = await cachedFetch(TEXT_BODY_URL);
res = await fetch(TEXT_BODY_URL);
body = await res.text();
assert.strictEqual(body, TEXT_BODY_EXPECTED);
assert.strictEqual(res.fromCache, true);
});
it('Can get JSON body', async function() {
res = await cachedFetch(JSON_BODY_URL);
res = await fetch(JSON_BODY_URL);
body = await res.json();
assert(body.slideshow);
assert.strictEqual(res.fromCache, false);
res = await cachedFetch(JSON_BODY_URL);
res = await fetch(JSON_BODY_URL);
body = await res.json();
assert(body.slideshow);
assert.strictEqual(res.fromCache, true);
});
it('Can get PNG buffer body', async function() {
res = await cachedFetch(PNG_BODY_URL);
res = await fetch(PNG_BODY_URL);
body = await res.buffer();
assert.strictEqual(expectedPngBuffer.equals(body), true);
assert.strictEqual(res.fromCache, false);
res = await cachedFetch(PNG_BODY_URL);
res = await fetch(PNG_BODY_URL);
body = await res.buffer();
assert.strictEqual(expectedPngBuffer.equals(body), true);
assert.strictEqual(res.fromCache, true);
});
it('Can stream a body', async function() {
res = await cachedFetch(TEXT_BODY_URL);
res = await fetch(TEXT_BODY_URL);
body = '';
for await (const chunk of res.body) {
@ -385,7 +259,7 @@ describe('Data tests', function() {
assert.strictEqual(TEXT_BODY_EXPECTED, body);
assert.strictEqual(res.fromCache, false);
res = await cachedFetch(TEXT_BODY_URL);
res = await fetch(TEXT_BODY_URL);
body = '';
for await (const chunk of res.body) {
@ -395,84 +269,4 @@ describe('Data tests', function() {
assert.strictEqual(TEXT_BODY_EXPECTED, body);
assert.strictEqual(res.fromCache, true);
});
it('Errors if the body type is not supported', async function() {
try {
await cachedFetch(TEXT_BODY_URL, { body: {} });
throw new Error('It was supposed to throw');
} catch (err) {
assert(err.message.includes('Unsupported body type'));
}
});
it('Uses cache even if you make multiple requests at the same time', async function() {
const [res1, res2] = await Promise.all([
cachedFetch('http://httpbin.org/status/200'),
cachedFetch('http://httpbin.org/status/200'),
]);
// One should be false, the other should be true
assert(res1.fromCache !== res2.fromCache);
});
}).timeout(10000);
describe('Memory cache tests', function() {
it('Supports TTL', async function() {
cachedFetch = FetchCache.withCache(new MemoryCache({ ttl: 100 }));
let res = await cachedFetch(TWO_HUNDRED_URL);
assert.strictEqual(res.fromCache, false);
res = await cachedFetch(TWO_HUNDRED_URL);
assert.strictEqual(res.fromCache, true);
await wait(200);
res = await cachedFetch(TWO_HUNDRED_URL);
assert.strictEqual(res.fromCache, false);
});
}).timeout(10000);
describe('File system cache tests', function() {
it('Supports TTL', async function() {
cachedFetch = FetchCache.withCache(new FileSystemCache({ ttl: 100 }));
let res = await cachedFetch(TWO_HUNDRED_URL);
assert.strictEqual(res.fromCache, false);
res = await cachedFetch(TWO_HUNDRED_URL);
assert.strictEqual(res.fromCache, true);
await wait(200);
res = await cachedFetch(TWO_HUNDRED_URL);
assert.strictEqual(res.fromCache, false);
});
it('Can get PNG buffer body', async function() {
cachedFetch = FetchCache.withCache(new FileSystemCache());
res = await cachedFetch(PNG_BODY_URL);
body = await res.buffer();
assert.strictEqual(expectedPngBuffer.equals(body), true);
assert.strictEqual(res.fromCache, false);
res = await cachedFetch(PNG_BODY_URL);
body = await res.buffer();
assert.strictEqual(expectedPngBuffer.equals(body), true);
assert.strictEqual(res.fromCache, true);
});
it('Can eject from cache', async function() {
cachedFetch = FetchCache.withCache(new FileSystemCache());
res = await cachedFetch(TWO_HUNDRED_URL);
assert.strictEqual(res.fromCache, false);
res = await cachedFetch(TWO_HUNDRED_URL);
assert.strictEqual(res.fromCache, true);
await res.ejectFromCache();
res = await cachedFetch(TWO_HUNDRED_URL);
assert.strictEqual(res.fromCache, false);
res = await cachedFetch(TWO_HUNDRED_URL);
assert.strictEqual(res.fromCache, true);
});
});