This commit is contained in:
@ -1,63 +1,64 @@
|
||||
export class FetchQueue {
|
||||
private _queue: string[];
|
||||
private _rateLimitReset: number;
|
||||
private queue: string[];
|
||||
private rateLimitReset: number;
|
||||
|
||||
constructor() {
|
||||
this._queue = [];
|
||||
this._rateLimitReset = Date.now();
|
||||
this.queue = [];
|
||||
this.rateLimitReset = Date.now();
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetches the given url, and handles rate limiting
|
||||
* re-requesting if the rate limit is exceeded.
|
||||
* Fetches the given URL and handles rate limiting, re-requesting if the rate limit is exceeded.
|
||||
*
|
||||
* @param url the url to fetch
|
||||
* @returns the response
|
||||
* @param url - The URL to fetch.
|
||||
* @returns The response.
|
||||
*/
|
||||
public async fetch(url: string): Promise<any> {
|
||||
public async fetch(url: string, options?: any): Promise<any> {
|
||||
const now = Date.now();
|
||||
if (now < this._rateLimitReset) {
|
||||
this._queue.push(url);
|
||||
|
||||
if (now < this.rateLimitReset) {
|
||||
this.queue.push(url);
|
||||
await new Promise<void>((resolve) =>
|
||||
setTimeout(resolve, this._rateLimitReset - now),
|
||||
setTimeout(resolve, this.rateLimitReset - now),
|
||||
);
|
||||
}
|
||||
|
||||
const response = await fetch(url, {
|
||||
next: {
|
||||
revalidate: 60 * 5, // Keep the data for 5 minutes
|
||||
},
|
||||
});
|
||||
const response = await fetch(url, options);
|
||||
|
||||
if (response.status === 429) {
|
||||
const hasRetryAfter = response.headers.has("retry-after");
|
||||
let retryAfter =
|
||||
Number(
|
||||
hasRetryAfter
|
||||
? response.headers.get("retry-after")
|
||||
: new Date(
|
||||
response.headers.get("X-Rate-Limit-Reset") as string,
|
||||
).getTime() / 1000,
|
||||
) * 1000;
|
||||
const hasRetryAfterHeader = response.headers.has("retry-after");
|
||||
let retryAfter = hasRetryAfterHeader
|
||||
? Number(response.headers.get("retry-after"))
|
||||
: Number(
|
||||
new Date(
|
||||
response.headers.get("X-Rate-Limit-Reset") as string,
|
||||
).getTime() / 1000,
|
||||
) * 1000;
|
||||
|
||||
// Check if we couldn't get the reset time from the headers.
|
||||
// Default to 3 seconds
|
||||
if (!retryAfter) {
|
||||
retryAfter = 3_000; // default to 3 seconds if we can't get the reset time
|
||||
retryAfter = 3000;
|
||||
}
|
||||
this._queue.push(url);
|
||||
|
||||
this.queue.push(url);
|
||||
await new Promise<void>((resolve) => setTimeout(resolve, retryAfter));
|
||||
return this.fetch(this._queue.shift() as string);
|
||||
return this.fetch(this.queue.shift() as string);
|
||||
}
|
||||
|
||||
if (response.headers.has("x-ratelimit-remaining")) {
|
||||
const remaining = Number(response.headers.get("x-ratelimit-remaining"));
|
||||
|
||||
if (remaining === 0) {
|
||||
const reset = Number(response.headers.get("x-ratelimit-reset")) * 1000;
|
||||
this._queue.push(url);
|
||||
this.queue.push(url);
|
||||
await new Promise<void>((resolve) => setTimeout(resolve, reset - now));
|
||||
return this.fetch(this._queue.shift() as string);
|
||||
return this.fetch(this.queue.shift() as string);
|
||||
}
|
||||
}
|
||||
|
||||
if (this._queue.length > 0) {
|
||||
const nextUrl = this._queue.shift();
|
||||
if (this.queue.length > 0) {
|
||||
const nextUrl = this.queue.shift();
|
||||
return this.fetch(nextUrl as string);
|
||||
}
|
||||
|
||||
|
Reference in New Issue
Block a user