From 831440152a4c57a3f4d8699e6920f8bf81c901f4 Mon Sep 17 00:00:00 2001 From: Randall Schmidt Date: Fri, 11 Jun 2021 11:25:24 -0400 Subject: [PATCH] custom caching --- README.md | 3 ++- classes/caching/file_system_cache.js | 29 +++++++++++++++++++++++ classes/caching/key_timeout.js | 16 +++++++++++++ classes/caching/memory_cache.js | 33 ++++++++++++++++++++++++++ classes/response.js | 14 ++++------- index.js | 34 ++++++++++++--------------- test/expected_png.png | Bin 0 -> 8090 bytes test/tests.js | 3 ++- 8 files changed, 101 insertions(+), 31 deletions(-) create mode 100644 classes/caching/file_system_cache.js create mode 100644 classes/caching/key_timeout.js create mode 100644 classes/caching/memory_cache.js create mode 100644 test/expected_png.png diff --git a/README.md b/README.md index f03c01a..31e3883 100644 --- a/README.md +++ b/README.md @@ -118,7 +118,7 @@ const fetchBuilder, { MemoryCache } = require('node-fetch-cache'); const fetch = fetchBuilder.withCache(new MemoryCache(options)); ``` -Supported options: +Options: ```js { @@ -140,6 +140,7 @@ const fetch = fetchBuilder.withCache(new FileSystemCache(options)); ```js { + cacheDirectory: '/my/cache/directory/path', // Specify where to keep the cache. If undefined, '.cache' is used by default. If this directory does not exist, it will be created. ttl: 1000, // Time to live. How long (in ms) responses remain cached before being automatically ejected. If undefined, responses are never automatically ejected from the cache. } ``` diff --git a/classes/caching/file_system_cache.js b/classes/caching/file_system_cache.js new file mode 100644 index 0000000..074c2d1 --- /dev/null +++ b/classes/caching/file_system_cache.js @@ -0,0 +1,29 @@ +const FPersist = require('fpersist'); +const KeyTimeout = require('./key_timeout.js'); + +module.exports = class FileSystemCache { + constructor(options = {}) { + this.ttl = options.ttl; + this.keyTimeout = new KeyTimeout(); + + const cacheDirectory = options.cacheDirectory || '.cache'; + this.cache = new FPersist(cacheDirectory); + } + + get(key) { + return this.cache.getItem(key); + } + + remove(key) { + this.keyTimeout.clearTimeout(key); + return this.cache.deleteItem(key); + } + + async set(key, value) { + await this.cache.setItem(key, value); + + if (typeof this.ttl === 'number') { + this.keyTimeout.updateTimeout(key, this.ttl, () => this.remove(key)); + } + } +} diff --git a/classes/caching/key_timeout.js b/classes/caching/key_timeout.js new file mode 100644 index 0000000..635c117 --- /dev/null +++ b/classes/caching/key_timeout.js @@ -0,0 +1,16 @@ +module.exports = class KeyTimeout { + constructor() { + this.timeoutHandleForKey = {}; + } + + clearTimeout(key) { + clearTimeout(this.timeoutHandleForKey[key]); + } + + updateTimeout(key, durationMs, callback) { + this.clearTimeout(key); + this.timeoutHandleForKey[key] = setTimeout(() => { + callback(); + }, durationMs); + } +} \ No newline at end of file diff --git a/classes/caching/memory_cache.js b/classes/caching/memory_cache.js new file mode 100644 index 0000000..9428f5d --- /dev/null +++ b/classes/caching/memory_cache.js @@ -0,0 +1,33 @@ +const KeyTimeout = require('./key_timeout.js'); + +module.exports = class MemoryCache { + constructor(options = {}) { + this.ttl = options.ttl; + this.keyTimeout = new KeyTimeout(); + + if (options.global && !globalThis.nodeFetchCache) { + globalThis.nodeFetchCache = {}; + } + + this.cache = options.global + ? globalThis.nodeFetchCache + : {}; + } + + get(key) { + return this.cache[key]; + } + + remove(key) { + this.keyTimeout.clearTimeout(key); + delete this.cache[key]; + } + + set(key, value) { + this.cache[key] = value; + + if (typeof this.ttl === 'number') { + this.keyTimeout.updateTimeout(key, this.ttl, () => this.remove(key)); + } + } +} diff --git a/classes/response.js b/classes/response.js index b22443f..2f2c3e4 100644 --- a/classes/response.js +++ b/classes/response.js @@ -3,9 +3,9 @@ const stream = require('stream'); const Headers = require('./headers.js'); class Response { - constructor(raw, cacheFilePath, fromCache) { + constructor(raw, ejectSelfFromCache, fromCache) { Object.assign(this, raw); - this.cacheFilePath = cacheFilePath; + this.ejectSelfFromCache = ejectSelfFromCache; this.headers = new Headers(raw.headers); this.fromCache = fromCache; this.bodyUsed = false; @@ -40,14 +40,8 @@ class Response { return this.consumeBody(); } - async ejectFromCache() { - try { - await fs.promises.unlink(this.cacheFilePath); - } catch (err) { - if (err.code !== 'ENOENT') { - throw err; - } - } + ejectFromCache() { + return this.ejectSelfFromCache(); } } diff --git a/index.js b/index.js index 6927106..2787119 100644 --- a/index.js +++ b/index.js @@ -4,6 +4,7 @@ const { URLSearchParams } = require('url'); const crypto = require('crypto'); const path = require('path'); const Response = require('./classes/response.js'); +const MemoryCache = require('./classes/caching/memory_cache.js'); const CACHE_VERSION = 2; @@ -88,32 +89,27 @@ async function createRawResponse(fetchRes) { }; } -async function getResponse(cacheDirPath, requestArguments) { +async function getResponse(cache, requestArguments) { const cacheKey = getCacheKey(requestArguments); - const cachedFilePath = path.join(cacheDirPath, `${cacheKey}.json`); + const cachedValue = await cache.get(cacheKey); - try { - const rawResponse = JSON.parse(await fs.promises.readFile(cachedFilePath)); - return new Response(rawResponse, cachedFilePath, true); - } catch (err) { + const ejectSelfFromCache = () => cache.remove(cacheKey); + + if (cachedValue) { + return new Response(cachedValue, ejectSelfFromCache, true); + } else { const fetchResponse = await fetch(...requestArguments); const rawResponse = await createRawResponse(fetchResponse); - await fs.promises.writeFile(cachedFilePath, JSON.stringify(rawResponse)); - return new Response(rawResponse, cachedFilePath, false); + await cache.set(cacheKey, rawResponse); + return new Response(rawResponse, ejectSelfFromCache, false); } } -function createFetch(cacheDirPath) { - let madeDir = false; +function createFetchWithCache(cache) { + const fetch = (...args) => getResponse(cache, args); + fetch.withCache = createFetchWithCache; - return async (...args) => { - if (!madeDir) { - await fs.promises.mkdir(cacheDirPath, { recursive: true }); - madeDir = true; - } - - return getResponse(cacheDirPath, args); - }; + return fetch; } -module.exports = createFetch; +module.exports = createFetchWithCache(new MemoryCache()); diff --git a/test/expected_png.png b/test/expected_png.png new file mode 100644 index 0000000000000000000000000000000000000000..cc0c128257cace9da1d84bb92bbcc4f3a7ed9257 GIT binary patch literal 8090 zcmW+*1y~f%8(u)Eqoo_9LqfV6q`MVRy5o>M0hN|kx{>aZM!FjTNj+&e`sn)a??2Dn z?mhd=?99$r@ArO@+M3F^*p%1+0N}n-QP2hN*8lx5QNinDFe(}V&{@4wkk$9iT!r}< z>n+Yds6Njl>!wG|!|Yb2AXFrvCL|`ff;zM9{Z>*|*00Dh-~WPV_^WosY6JTRA(f&m zv5xFlmXG;zfTF+KD|&*&v)gy!1Ppyl3PC=dmvFeKsGVJ<_gej>R6updJRV*zFJM`} zI2cQ#rm8wJJiLoQFeMEWV~4Jw85kH)#@ISKI=+4T*3mHvPhVf(lAw#XqP@MnrY64I zl9&9;>8U46((v2bRCVySady4O$ji%Hg9wR?#0rri|FU;+aWOM96Gtcg;qHo%Ch!Is zId1C{72FGtAGQ`37Z(<8%FU$**9n;}EiGkXVd3BHT|asM{=KZMtk2P!i;GMDcO^f+ z>$I14MOryk6%|~6DmmEMMa9MG#NStZ|E}QQQCV3Dj>N^ruB)#H=khxU@bSIU(CFvI z8MLjhtf&|n9YyJy{{`jGsw&OzfqRO#tZ2R33|psXXNT3*g+)b)h34nwe?+tN6mdB?*a>hm zM3kl=hf#TO;1C*GE%o6E@cenf$B(Ehov_;r#3CBN5KZcGc1{!j1_12e)~fFBOV7`B zuyY2SUby({=&+`6wnq)F+f8J)_@6uS@bI99bgiF&=M2gTY^tiN3Y+BndlNOa6z{C9 z1p}Msj^SZqVj|RuCA|JT3ZuUm}uj` z1MQH0ynD&O5Hh(9iB5R+J(85Psz5nTP)LYI?3aR;ma>LM-C8dn-*UA%N-(%-c#(Qq zVqzkHtkS4tAi0V8^zSPp&(m3-qhk?j+Hg=V){N7j5E2UuKZS-0Id%X*7u++4j8y8w ztq`BEu-a!lG&Hn;fZG%v3qyVV2#y~`(}4#E_lN@*4GrduD@9^`!vSkMWjkA2VzL5P zGFc;|g|#&)P<>ikO?_w9R#tikZ1;AiW1(W93aG)hLgHE7Dm)zi-Ims*WKBPsAdn25 z($QkgRbH>6);fN`lH9i5WqDsK2i*3@`kERx4i4;8erC^me_-eq>2McIv-;1G3;63U z(i+m;!1r!`JeAM$%6(eRZDiy@U9~kceW}MbI-8XUbHdQ@!!wqvFLZAC#lkAFHw8t7 z%tGSgVw{|!(iRQrXu-*4dvEQw=UiRi#l%cgNQHqqA(|B*kKBr3>%Xt{TxzJv`5-r_ z_HebXs~Qy=`jk)-1;)({wbIfje(U%pAt5%MGH!DFoiQZiwXs3}ATLfYHU}q!gL6{K z_pg-eimjWQkCPL!zjk2Y&HMN8I6BBV1s=s|wW!yYW;&B81qmLEyAoWze03$2hK6r( z5yN%O8u^PIFD2TP`o>pO(EIXypKgEe^B={@wdt7|6(h1N_{p{K>@2tj4-@NSt0jto zo}P-Px3^$zTN@RCDanILn}ZLCKW+!%y)b8U)H737HuUiD*O^{#a!Wcp?S;Ufbv33yx3sab(P?#cvdSC_;KJM1gWa>q%VSBp>Uf{0_4P$-W!H#ghCkjcj3bfK zbW(_|GZdLf74!YW2k^9Rs7|=M|GCHZ_tV5gzw;0QKp%CNmASOEthT>z)Yii78F+6zmqM_c1H-#goj3#e$10ja3LdFknPh84a6cg;B+k^r%Rk=9BhCwz|6WHZ;Nu&Agg zm}}anSfKf8WB3>B*3rxi#j?KXa$PcDAR}Ycv9BqlzsAbvhpQwJLGU2lba}ZW=<(j( zJX4*>e;9NesKG3xY9nr^S9hSS=q9rd_6`jhiFy~RB!5_A$qr!kH4P@4)c<>rn9?@f zdiQ%?EhRZybk;7D0VRT(o10rAJvXqV@|(n=jJ=E7r5JSQYPAiJnfn#kZZ{dHSB&?M zw3A)J#l_<8f@+VAF-OXKwXLbBa5MT(4=+OAg&QEJqFM)CHIQ3SUS3!F>U2TX(TV2a zGU(aaDbO|0$FzU>hh`Vq{inC7k4?UJ>=VXo%mw5XA0?O1=k9CzykC&#(y6yKTcOxOGiBEi5bsO-q}a%IxHy>PfLSg^NNe{ zM@Q#{gOk-Wgm8gYzxLvzwMbHKH|Im>_?g7%EduC2M2lB(@d$X z%iW}%IdCWg9Zo(bKA@?oY0XaoB(Aj+COaJ=km7;lG1HUPrsr`~%qsZh@ZQI@=7iwE zdU0`>oZOzNS!fCYgEjx+@GusE$;ik^Y-}uWtn#^{GKHtW?~kq9QP9+op43=Pt!TS1 zAMjOBpaKi>EL-`rh?rRZCP}xzQ-k`uQ%?a??y{5r5_JEa1?xeTJa8w=)E^g0T3Ueb8)M(~{^*EeEtIa| zVbCIoNV>4LC&3`jX;=XOg4m(`a5looBdyG<>QgNkT7x$Y3u)pMs zGg(IDT89Q9;f}F}MkN@^D(zedS@+VePX_6D-n3%^plq0nweV-+CN@t#bii&E0^376 zJ7Z#SfsL&#!?|D1r%0S^$qa!A-={de6Rgakef_l}nMckpD1SH0jU0(bE==m@n_F-_ zW2fglR<%OkT(u09l*H*ePVHb>p;?361^-sM(uH*Kma z{0n&sLDkGb2Tr$NCa;lpub;el-TDLwlMj31?>|zB12jvlt>_geSDBHe#A_QEG`hp( zy#8xvp%5}d1A~5MCa%{_u7{<1D=5jH%)xP1XPQ}cm&d-@X~UbIip*XuE8A&l7qqnC z<9}-Kp|3y?E2|SFE>LChbB*yIJ;Ei7ot@nt9eFfY4mxgJC6nXxeK#?&w}-3Z9{>7& zxGYc2Frg->NCaLT7inY!cC-(0a6XC!b*d-Et#(jZn5H8)UfJ817!OkkyF$IZ&jt9f zPA_Q3xchAFZfT`(0761SwPY??)_`GbY;3U)cOnTgCAa(4-4zTz-ah+Pru}R$UT|`H zL7@n`HaU8NZ0R6ZXJ=>8w+iycsYD5EhMGagS9c7YMxFlmt(~23K3YpkN-le>Q$F4} z@wySch!cixj*tokfIzUxpPH!S9|-AyuEi0@`0@h~eyGR9oS&b|%gOaAT_5|l$Hx=E zU@*OR>%ulRGio2q-niygyE!^?n1CL!G^F!k`PB3J2qjp7-gjvURp#wm#_(QQ)~D8+ z+uH>t^gpqlp%luy2?uR#D3IDp}IxH z_wP(lB^g5~R6y*Qf}TU&|M)79c!2E!4jDQ21!U+6d>Om-ES^O7>gXAzDAlF6ij^0O z4aHCh30V_iDGU+Dhc4v`#t2=N7R}P*h9|Nr#Y%w#s({S68YJ z4~zZ%HXusBhalEEmcjqMlhW0mUh*Ekx^kiF+2Rf}7u;q{YHOV~qbskHi2tEnQb^NE zlKeV7o|xei8cR(?jt*D@YSZx#L<2$)M`nfRwUwe4CJ$m-+8TfuGLc)X- z#G>2T)-cDlf{e^cmiWoZ3EHdJxVX4BuWbdHU^!O>3$;2{ui|oZ zcfCakN0Xo>z?8`gPN%eY`ZToiI~@e`Q2XukeSzl%9tFxl!d9q2``B3E{lx(o)wgKH zn0#KQsTcpm!n(fxr}&a`95k;ZYHHxGjncpw5^5)Qub~o=YElUmiVo!+smsj;F8_9; z`S|z%GAMBtH4R1RlvF4kM|?QEIPM}cFm77>*4C-X$Noda*!8jOj!NyzmwEZv5@tFeq^qwkXQ1>&}J}CY~#gD0~*+ot}L;J)0r!kl5ow8cOj^wQJok z@Pn)@N^MOI!D?b&Y%J4k{QTkJo65jW8BSXKKq&&u;6bzu+|5n;cXq$#r%yh1vw>FT z=Hz6_PnD_;Oc@w0fPHp~cVQvN+M$#|`6i0d83a-qNF<18AP`6-TUAa`G3&6Mq2BYU zrBz&sY#HHT89smj6Ll5nvX*=IJkUXrw6|vC?#_&Y0!$9CZEO;=IDDyHzto;si@{<) z^@66Mvbm<0IW8dCk+`(f+A7AFH2eg3H5oG+TErr&p~Ml|s`E|hJWU3m*3>TgK?;Rh z2Ah?+Io(dK4_d|1l3YR*L7%)tN!hxz1WZk>tgOt3g-rzF1l;e=f*H|-q%W_WMbKP> zgS2KA$8cPQb_SuSI!k>x7uK~qH#e${MD|~HEbp`H;6X?OV{YfO@K3RLJFH2A+c}yl zziCLAamuvww5eTOSeKW?mX?-`O?Dj}B8!|)uJ6K>MaGlI?IZtCQQo>5@615139skqH*GO59mX4@|78N?NNUN#& zd1a6HuZGKl9?i%dg#G&>1kbj4PEMxDMdmQ$2Jz>nJ$#^0ALx@{AyQ6$p^H1v{YOWq z(a>ZtGBRdM;ok1D^6h(L*HNG(^jh#Wq-0gxjoodae0~2r5|KpQd)U=IT;rjDIK)-Q3)-0Wkz>vQn z6;E+dRhxTAoYh`g+2uqQmFczn9y%JQ@s5GJ`(xAtDi0E5em$K0} z@C$tSIHgS*{;3zvh)3H{Tc^vVhbd%eNQi;|&s~Hxyq;b_|F8eI1>0H<>rxXjv5(>O zWsxdRmwSJn8hLUM0U8vUB5GJ7c=O;cOmY)6-9i1aHSL#ZTS6f+6 zuSDN9qd&jBmJon?d!Mjn7gZJ%w6^%km?6()O-zoTDC9(zNd1BkVDbiv#H)ZYSNG3P zj9>%lyIe*!&zwL|y$$b=(SYxrK|IE9l2&bN6WjbGi2vF-I7E`6(~;n03IHP1sp%i1 zun8hRN`s%o5ra2%atzO{#07f3m5_LCvBeW(5pWQdB!iRLDyx;P&ySuxJh6XG?h4q_`0;9csR|?~x z_8jwt*OJhWhRldB?MYve(yp!<$y^jON#{JwlxC)KFU*_okKu^j*(Pp_s$0J^*}c9e z+bVYT#<^oglrcik9c@Mce%)xb<tZ@WdP|8qsAO}z>+!>7fJt1i%y!{=~x5*MKc-|+Wv zG=;4fRp&DX@bYCn-+!saGgtDc6-cX@>FVx5rGRZM30A2LnSf^KRisrMKMY>+-_$D^ z)`}ZeneXhD7UkDX&3YM4Ua$gP2ZvH#2b93?)Kn2;W46EOfQY&4Th|wWub&@bqyjxXiHqj(-uz8sos|xnOZ*;t z@hn=ZHv!q+4n9KZ`l71H&Ww$+_jSnn^b5tyC;p)e+Tl5wIe(=0wWUD(^i$ZCi4b#D zVy&mrY?OJ+ax~^NJw4Q{>HV266Ds^DhRP;NAOVmH%d7MTF*eJWPe;rUZ?i1(D%3xb z7h78x_xC8d4RxN0KOgr|5^XjIoY3DT{tR-kVFsv?mrX|Ir(^4PcQ-$T2q-v$*-Wac zs=(YCz*}h-oq90wq#s%iD38Mj^ylZrFrJtvdoz+~ zjqz}JzW%{wIT$NM3JLGcGH%tcbvWcRF=2~#?G3|~562_Pe&nJF9o>wj6fA3QPAw}n z3A|}gzh!|D)-EoBezPn+1LXj32hnc(5U6Emx62wAC?zc=cn-!PtkP2ZtbQ4@+m$kd zPjR0}p2khmQ4u3M>vYY`ZhOP=O&Oo@o9DbX$j{H;ngmI75@S;*Ye~){fsP?6kHb7qaU4j6gUGVi#^8cR^|?O7JPLW}tuP@%ZrV*DtAC zZFW62d2cARvC+G+0m#iOEX-?ZsVXg9KRZXkwb@jhnAo=)BcDrh_$~v5dYd@^)?WOj zP%w=Rf!uB!BAFpTVp5V${{AE_El65vH_KuoCMJH)&bZoGw_EbwAWGRtP0PxXxVX1P zM7UBVOmqm}+@AzMkrU-cDD?dN{J9nth2(XB<-iw;$jC@gqYh5iPd8mry1)TBB*af9 z)7BXh+ve!V!#XNUR5;n#N-Ie|Q#UrE1XqQH(QibkW;xpkn*N*c2i$pB?LdSfwLW;c zSgGwVP;0#P^KPcc*38Uodt(E6q~4$OEhWXy$tl_@HK*e~tI}+9)`fyRg7h@GrIpsA z6}f-FzBp!!@vtqOqN@vll>`A+CIu-`P|g$5ftbMM(VxrX;~fi`yNscwCI9JsMSvjd z8}q)Dt=tMWwj6wtEJuQ((IE+yRX0Ckc!aXi;n~}f?w)_??VPVbw)S}uh`;*ypQ6oi z0rsx0@7>)&D!OOg-9!9+ZUvpm^!PeWWX!718Osv-_-)&f94WW1{-C+GNz3})Txtau zkQ=XZyFK{_almcC^1J@yM~B@t6!^;dB_cYt_wp4 ziMD-HvsX+rU=bh_1J5);ffQh3V)6|LIJ*vf$su_5SGL6~uQkTYFI{K35Zmh)ingFF$gzAP}<_&%)0= z7GMaBrIl>;-Zuo?X(VrC-H$Day2V>t<&IV%k&%S1Z$w9?rv3~&bZ-nN!n$MBw;CH7 zhKjX9M-wPVM#S&#g#7(*ZIfQ(^nMWeKwWu-fsP&)7N+{*(FW9~-I+o*to!2KS&+wK zZt}p?;F_;(ZXWvjHEKW!{>qEP`*UX}#WzRI!u0+9ay_VXylUP@&AMO|EgYJdU@F-9zPOJVq+tG z4i(f}+S-bXi%0v2hg3-cHcrmR`^!HVc+_3zi2W}Y>32NJSL=hb7e~rwzn-F_4-5{f z%@$mSIFD-+`y8%dVPb;SO5pF&22_CFlK0bTbvO&D(?X+@{hBgk!bG+tr{4xU%4yP5 z0IcRoNlDq**Z{e5EN)2T%HcMv*3JBatg_L<@YPj(D)|P_?Cflt_kQx{#W6BIwmKlU zs>*dd9pn{PWa+|7$}uLh@9V?Ekr_pSxFBWoDDk2FasU_JIqd4{YGq|r_uy_ohm*|p zd3!R)ZTSbtgX6*krf0=#EJJ=KCCMqp)iJVgI&IIDA8~R{D?2&u$H&Kqw7=k&<+El` zP*y%#X{iTOVpv1b^u=V3^z8h5-q}~X-GeoQ*Vka^n&Kb@rtBNgFz~oc8&q_r!=soW zfUm?FNRnL~u5#YVGIJXkXqlNw2i$@L3${fh3V`>2)z}#u1FJ35l>C6blhe^sqtpG( zxpk=jK8ZRhYR0$}n4`mc!_yf346TWFZ~5NcAr4>zaSJK;jjI^r&KIj7U0YvA(WUIyo_uEOG_HQ8}HpPI;7k|$?n z(NIxQJ$rVhWKCyh0!k!=BcY&RT^K?W?xvp125K)k4*E^nbWcxDR1`LQknOLTnbUni cf#Iv5)@`4jR6XYcu;vcDQq)walC%8yKfnsX9smFU literal 0 HcmV?d00001 diff --git a/test/tests.js b/test/tests.js index b1df47a..5223218 100644 --- a/test/tests.js +++ b/test/tests.js @@ -5,6 +5,7 @@ const rimraf = require('rimraf'); const path = require('path'); const FetchCache = require('../index.js'); const { URLSearchParams } = require('url'); +const MemoryCache = require('../classes/caching/memory_cache.js'); const CACHE_PATH = path.join(__dirname, '..', '.cache'); const expectedPngBuffer = fs.readFileSync(path.join(__dirname, 'expected_png.png')); @@ -28,7 +29,7 @@ function post(body) { beforeEach(async function() { rimraf.sync(CACHE_PATH); - fetch = FetchCache(CACHE_PATH); + fetch = FetchCache.withCache(new MemoryCache()); }); describe('Basic property tests', function() {