update the curve
All checks were successful
deploy / deploy (push) Successful in 47s

This commit is contained in:
Lee
2023-10-24 12:07:48 +01:00
parent 54c9d866b8
commit 49497a9fef
2 changed files with 71 additions and 65 deletions

View File

@ -1,63 +1,64 @@
export class FetchQueue {
private _queue: string[];
private _rateLimitReset: number;
private queue: string[];
private rateLimitReset: number;
constructor() {
this._queue = [];
this._rateLimitReset = Date.now();
this.queue = [];
this.rateLimitReset = Date.now();
}
/**
* Fetches the given url, and handles rate limiting
* re-requesting if the rate limit is exceeded.
* Fetches the given URL and handles rate limiting, re-requesting if the rate limit is exceeded.
*
* @param url the url to fetch
* @returns the response
* @param url - The URL to fetch.
* @returns The response.
*/
public async fetch(url: string): Promise<any> {
public async fetch(url: string, options?: any): Promise<any> {
const now = Date.now();
if (now < this._rateLimitReset) {
this._queue.push(url);
if (now < this.rateLimitReset) {
this.queue.push(url);
await new Promise<void>((resolve) =>
setTimeout(resolve, this._rateLimitReset - now),
setTimeout(resolve, this.rateLimitReset - now),
);
}
const response = await fetch(url, {
next: {
revalidate: 60 * 5, // Keep the data for 5 minutes
},
});
const response = await fetch(url, options);
if (response.status === 429) {
const hasRetryAfter = response.headers.has("retry-after");
let retryAfter =
Number(
hasRetryAfter
? response.headers.get("retry-after")
: new Date(
response.headers.get("X-Rate-Limit-Reset") as string,
).getTime() / 1000,
) * 1000;
const hasRetryAfterHeader = response.headers.has("retry-after");
let retryAfter = hasRetryAfterHeader
? Number(response.headers.get("retry-after"))
: Number(
new Date(
response.headers.get("X-Rate-Limit-Reset") as string,
).getTime() / 1000,
) * 1000;
// Check if we couldn't get the reset time from the headers.
// Default to 3 seconds
if (!retryAfter) {
retryAfter = 3_000; // default to 3 seconds if we can't get the reset time
retryAfter = 3000;
}
this._queue.push(url);
this.queue.push(url);
await new Promise<void>((resolve) => setTimeout(resolve, retryAfter));
return this.fetch(this._queue.shift() as string);
return this.fetch(this.queue.shift() as string);
}
if (response.headers.has("x-ratelimit-remaining")) {
const remaining = Number(response.headers.get("x-ratelimit-remaining"));
if (remaining === 0) {
const reset = Number(response.headers.get("x-ratelimit-reset")) * 1000;
this._queue.push(url);
this.queue.push(url);
await new Promise<void>((resolve) => setTimeout(resolve, reset - now));
return this.fetch(this._queue.shift() as string);
return this.fetch(this.queue.shift() as string);
}
}
if (this._queue.length > 0) {
const nextUrl = this._queue.shift();
if (this.queue.length > 0) {
const nextUrl = this.queue.shift();
return this.fetch(nextUrl as string);
}