many many many many changes

src/utils/fetchWithQueue.ts

@@ -0,0 +1,59 @@
+import { fetchBuilder, MemoryCache } from "node-fetch-cache";
+
+export class FetchQueue {
+  private _fetch;
+  private _queue: string[];
+  private _rateLimitReset: number;
+
+  constructor(ttl: number) {
+    this._fetch = fetchBuilder.withCache(
+      new MemoryCache({
+        ttl: ttl,
+      }),
+    );
+    this._queue = [];
+    this._rateLimitReset = Date.now();
+  }
+
+  /**
+   * Fetches the given url, and handles rate limiting
+   * re-requesting if the rate limit is exceeded.
+   *
+   * @param url the url to fetch
+   * @returns the response
+   */
+  public async fetch(url: string): Promise<any> {
+    const now = Date.now();
+    if (now < this._rateLimitReset) {
+      this._queue.push(url);
+      await new Promise<void>((resolve) =>
+        setTimeout(resolve, this._rateLimitReset - now),
+      );
+    }
+
+    const response = await this._fetch(url);
+    if (response.status === 429) {
+      const retryAfter = Number(response.headers.get("retry-after")) * 1000;
+      this._queue.push(url);
+      await new Promise<void>((resolve) => setTimeout(resolve, retryAfter));
+      return this.fetch(this._queue.shift() as string);
+    }
+
+    if (response.headers.has("x-ratelimit-remaining")) {
+      const remaining = Number(response.headers.get("x-ratelimit-remaining"));
+      if (remaining === 0) {
+        const reset = Number(response.headers.get("x-ratelimit-reset")) * 1000;
+        this._queue.push(url);
+        await new Promise<void>((resolve) => setTimeout(resolve, reset - now));
+        return this.fetch(this._queue.shift() as string);
+      }
+    }
+
+    if (this._queue.length > 0) {
+      const nextUrl = this._queue.shift();
+      return this.fetch(nextUrl as string);
+    }
+
+    return response;
+  }
+}