import { fetchBuilder, MemoryCache } from "node-fetch-cache"; export class FetchQueue { private _fetch; private _queue: string[]; private _rateLimitReset: number; constructor(ttl: number) { this._fetch = fetchBuilder.withCache( new MemoryCache({ ttl: ttl, }), ); this._queue = []; this._rateLimitReset = Date.now(); } /** * Fetches the given url, and handles rate limiting * re-requesting if the rate limit is exceeded. * * @param url the url to fetch * @returns the response */ public async fetch(url: string): Promise { const now = Date.now(); if (now < this._rateLimitReset) { this._queue.push(url); await new Promise((resolve) => setTimeout(resolve, this._rateLimitReset - now), ); } const response = await this._fetch(url); if (response.status === 429) { const retryAfter = Number(response.headers.get("retry-after")) * 1000; this._queue.push(url); await new Promise((resolve) => setTimeout(resolve, retryAfter)); return this.fetch(this._queue.shift() as string); } if (response.headers.has("x-ratelimit-remaining")) { const remaining = Number(response.headers.get("x-ratelimit-remaining")); if (remaining === 0) { const reset = Number(response.headers.get("x-ratelimit-reset")) * 1000; this._queue.push(url); await new Promise((resolve) => setTimeout(resolve, reset - now)); return this.fetch(this._queue.shift() as string); } } if (this._queue.length > 0) { const nextUrl = this._queue.shift(); return this.fetch(nextUrl as string); } return response; } }