custom caching

This commit is contained in:
Randall Schmidt 2021-06-11 11:25:24 -04:00
parent ad256800f8
commit 831440152a
8 changed files with 101 additions and 31 deletions

@ -118,7 +118,7 @@ const fetchBuilder, { MemoryCache } = require('node-fetch-cache');
const fetch = fetchBuilder.withCache(new MemoryCache(options));
```
Supported options:
Options:
```js
{
@ -140,6 +140,7 @@ const fetch = fetchBuilder.withCache(new FileSystemCache(options));
```js
{
cacheDirectory: '/my/cache/directory/path', // Specify where to keep the cache. If undefined, '.cache' is used by default. If this directory does not exist, it will be created.
ttl: 1000, // Time to live. How long (in ms) responses remain cached before being automatically ejected. If undefined, responses are never automatically ejected from the cache.
}
```

@ -0,0 +1,29 @@
const FPersist = require('fpersist');
const KeyTimeout = require('./key_timeout.js');
module.exports = class FileSystemCache {
constructor(options = {}) {
this.ttl = options.ttl;
this.keyTimeout = new KeyTimeout();
const cacheDirectory = options.cacheDirectory || '.cache';
this.cache = new FPersist(cacheDirectory);
}
get(key) {
return this.cache.getItem(key);
}
remove(key) {
this.keyTimeout.clearTimeout(key);
return this.cache.deleteItem(key);
}
async set(key, value) {
await this.cache.setItem(key, value);
if (typeof this.ttl === 'number') {
this.keyTimeout.updateTimeout(key, this.ttl, () => this.remove(key));
}
}
}

@ -0,0 +1,16 @@
module.exports = class KeyTimeout {
constructor() {
this.timeoutHandleForKey = {};
}
clearTimeout(key) {
clearTimeout(this.timeoutHandleForKey[key]);
}
updateTimeout(key, durationMs, callback) {
this.clearTimeout(key);
this.timeoutHandleForKey[key] = setTimeout(() => {
callback();
}, durationMs);
}
}

@ -0,0 +1,33 @@
const KeyTimeout = require('./key_timeout.js');
module.exports = class MemoryCache {
constructor(options = {}) {
this.ttl = options.ttl;
this.keyTimeout = new KeyTimeout();
if (options.global && !globalThis.nodeFetchCache) {
globalThis.nodeFetchCache = {};
}
this.cache = options.global
? globalThis.nodeFetchCache
: {};
}
get(key) {
return this.cache[key];
}
remove(key) {
this.keyTimeout.clearTimeout(key);
delete this.cache[key];
}
set(key, value) {
this.cache[key] = value;
if (typeof this.ttl === 'number') {
this.keyTimeout.updateTimeout(key, this.ttl, () => this.remove(key));
}
}
}

@ -3,9 +3,9 @@ const stream = require('stream');
const Headers = require('./headers.js');
class Response {
constructor(raw, cacheFilePath, fromCache) {
constructor(raw, ejectSelfFromCache, fromCache) {
Object.assign(this, raw);
this.cacheFilePath = cacheFilePath;
this.ejectSelfFromCache = ejectSelfFromCache;
this.headers = new Headers(raw.headers);
this.fromCache = fromCache;
this.bodyUsed = false;
@ -40,14 +40,8 @@ class Response {
return this.consumeBody();
}
async ejectFromCache() {
try {
await fs.promises.unlink(this.cacheFilePath);
} catch (err) {
if (err.code !== 'ENOENT') {
throw err;
}
}
ejectFromCache() {
return this.ejectSelfFromCache();
}
}

@ -4,6 +4,7 @@ const { URLSearchParams } = require('url');
const crypto = require('crypto');
const path = require('path');
const Response = require('./classes/response.js');
const MemoryCache = require('./classes/caching/memory_cache.js');
const CACHE_VERSION = 2;
@ -88,32 +89,27 @@ async function createRawResponse(fetchRes) {
};
}
async function getResponse(cacheDirPath, requestArguments) {
async function getResponse(cache, requestArguments) {
const cacheKey = getCacheKey(requestArguments);
const cachedFilePath = path.join(cacheDirPath, `${cacheKey}.json`);
const cachedValue = await cache.get(cacheKey);
try {
const rawResponse = JSON.parse(await fs.promises.readFile(cachedFilePath));
return new Response(rawResponse, cachedFilePath, true);
} catch (err) {
const ejectSelfFromCache = () => cache.remove(cacheKey);
if (cachedValue) {
return new Response(cachedValue, ejectSelfFromCache, true);
} else {
const fetchResponse = await fetch(...requestArguments);
const rawResponse = await createRawResponse(fetchResponse);
await fs.promises.writeFile(cachedFilePath, JSON.stringify(rawResponse));
return new Response(rawResponse, cachedFilePath, false);
await cache.set(cacheKey, rawResponse);
return new Response(rawResponse, ejectSelfFromCache, false);
}
}
function createFetch(cacheDirPath) {
let madeDir = false;
function createFetchWithCache(cache) {
const fetch = (...args) => getResponse(cache, args);
fetch.withCache = createFetchWithCache;
return async (...args) => {
if (!madeDir) {
await fs.promises.mkdir(cacheDirPath, { recursive: true });
madeDir = true;
}
return getResponse(cacheDirPath, args);
};
return fetch;
}
module.exports = createFetch;
module.exports = createFetchWithCache(new MemoryCache());

BIN
test/expected_png.png Normal file

Binary file not shown.

After

Width:  |  Height:  |  Size: 7.9 KiB

@ -5,6 +5,7 @@ const rimraf = require('rimraf');
const path = require('path');
const FetchCache = require('../index.js');
const { URLSearchParams } = require('url');
const MemoryCache = require('../classes/caching/memory_cache.js');
const CACHE_PATH = path.join(__dirname, '..', '.cache');
const expectedPngBuffer = fs.readFileSync(path.join(__dirname, 'expected_png.png'));
@ -28,7 +29,7 @@ function post(body) {
beforeEach(async function() {
rimraf.sync(CACHE_PATH);
fetch = FetchCache(CACHE_PATH);
fetch = FetchCache.withCache(new MemoryCache());
});
describe('Basic property tests', function() {