From e6b169f3fc40ff9fb84a395df40f27a1a3221db0 Mon Sep 17 00:00:00 2001 From: Liam Date: Wed, 8 Nov 2023 23:17:32 +0000 Subject: [PATCH] move beatsaver map caching to redis --- .env | 4 ++ .env-example | 5 +- package.json | 1 + pnpm-lock.yaml | 73 ++++++++++++++++++++++++++ src/app/api/beatsaver/mapdata/route.ts | 42 ++++++++++----- src/components/icons/BeatSaverLogo.tsx | 10 ++-- src/db/redis.ts | 31 +++++++++++ src/utils/scoresaber/api.ts | 2 +- tsconfig.json | 23 ++------ 9 files changed, 154 insertions(+), 37 deletions(-) create mode 100644 .env create mode 100644 src/db/redis.ts diff --git a/.env b/.env new file mode 100644 index 0000000..2d90141 --- /dev/null +++ b/.env @@ -0,0 +1,4 @@ +SENTRY_AUTH_TOKEN=hi + +# Redis +REDIS_URL=redis://:bigtitsyes7@10.0.0.203:30004 \ No newline at end of file diff --git a/.env-example b/.env-example index 97b3c1f..9d765cf 100644 --- a/.env-example +++ b/.env-example @@ -1 +1,4 @@ -SENTRY_AUTH_TOKEN=hi \ No newline at end of file +SENTRY_AUTH_TOKEN=hi + +# Redis +REDIS_URL=redis://localhost:6379/0 \ No newline at end of file diff --git a/package.json b/package.json index 485e67a..898de67 100644 --- a/package.json +++ b/package.json @@ -39,6 +39,7 @@ "react-chartjs-2": "^5.2.0", "react-dom": "^18", "react-toastify": "^9.1.3", + "redis": "^4.6.10", "sharp": "^0.32.6", "tailwind-merge": "^2.0.0", "tailwindcss-animate": "^1.0.7", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index ca9eb13..9c79554 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -89,6 +89,9 @@ dependencies: react-toastify: specifier: ^9.1.3 version: 9.1.3(react-dom@18.2.0)(react@18.2.0) + redis: + specifier: ^4.6.10 + version: 4.6.10 sharp: specifier: ^0.32.6 version: 0.32.6 @@ -1062,6 +1065,55 @@ packages: '@babel/runtime': 7.23.2 dev: false + /@redis/bloom@1.2.0(@redis/client@1.5.11): + resolution: {integrity: sha512-HG2DFjYKbpNmVXsa0keLHp/3leGJz1mjh09f2RLGGLQZzSHpkmZWuwJbAvo3QcRY8p80m5+ZdXZdYOSBLlp7Cg==} + peerDependencies: + '@redis/client': ^1.0.0 + dependencies: + '@redis/client': 1.5.11 + dev: false + + /@redis/client@1.5.11: + resolution: {integrity: sha512-cV7yHcOAtNQ5x/yQl7Yw1xf53kO0FNDTdDU6bFIMbW6ljB7U7ns0YRM+QIkpoqTAt6zK5k9Fq0QWlUbLcq9AvA==} + engines: {node: '>=14'} + dependencies: + cluster-key-slot: 1.1.2 + generic-pool: 3.9.0 + yallist: 4.0.0 + dev: false + + /@redis/graph@1.1.0(@redis/client@1.5.11): + resolution: {integrity: sha512-16yZWngxyXPd+MJxeSr0dqh2AIOi8j9yXKcKCwVaKDbH3HTuETpDVPcLujhFYVPtYrngSco31BUcSa9TH31Gqg==} + peerDependencies: + '@redis/client': ^1.0.0 + dependencies: + '@redis/client': 1.5.11 + dev: false + + /@redis/json@1.0.6(@redis/client@1.5.11): + resolution: {integrity: sha512-rcZO3bfQbm2zPRpqo82XbW8zg4G/w4W3tI7X8Mqleq9goQjAGLL7q/1n1ZX4dXEAmORVZ4s1+uKLaUOg7LrUhw==} + peerDependencies: + '@redis/client': ^1.0.0 + dependencies: + '@redis/client': 1.5.11 + dev: false + + /@redis/search@1.1.5(@redis/client@1.5.11): + resolution: {integrity: sha512-hPP8w7GfGsbtYEJdn4n7nXa6xt6hVZnnDktKW4ArMaFQ/m/aR7eFvsLQmG/mn1Upq99btPJk+F27IQ2dYpCoUg==} + peerDependencies: + '@redis/client': ^1.0.0 + dependencies: + '@redis/client': 1.5.11 + dev: false + + /@redis/time-series@1.0.5(@redis/client@1.5.11): + resolution: {integrity: sha512-IFjIgTusQym2B5IZJG3XKr5llka7ey84fw/NOYqESP5WUfQs9zz1ww/9+qoz4ka/S6KcGBodzlCeZ5UImKbscg==} + peerDependencies: + '@redis/client': ^1.0.0 + dependencies: + '@redis/client': 1.5.11 + dev: false + /@rollup/plugin-commonjs@24.0.0(rollup@2.78.0): resolution: {integrity: sha512-0w0wyykzdyRRPHOb0cQt14mIBLujfAv6GgP6g8nvg/iBxEm112t3YPPq+Buqe2+imvElTka+bjNlJ/gB56TD8g==} engines: {node: '>=14.0.0'} @@ -1719,6 +1771,11 @@ packages: engines: {node: '>=6'} dev: false + /cluster-key-slot@1.1.2: + resolution: {integrity: sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA==} + engines: {node: '>=0.10.0'} + dev: false + /color-convert@2.0.1: resolution: {integrity: sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==} engines: {node: '>=7.0.0'} @@ -2524,6 +2581,11 @@ packages: resolution: {integrity: sha512-xckBUXyTIqT97tq2x2AMb+g163b5JFysYk0x4qxNFwbfQkmNZoiRHb6sPzI9/QV33WeuvVYBUIiD4NzNIyqaRQ==} dev: true + /generic-pool@3.9.0: + resolution: {integrity: sha512-hymDOu5B53XvN4QT9dBmZxPX4CWhBPPLguTZ9MMFeFa/Kg0xWVfylOVNlJji/E7yTZWFd/q9GO5TxDLq156D7g==} + engines: {node: '>= 4'} + dev: false + /get-intrinsic@1.2.2: resolution: {integrity: sha512-0gSo4ml/0j98Y3lngkFEot/zhiCeWsbYIlZ+uZOVgzLyLaUw7wxUL+nCTP0XJvJg1AXulJRI3UJi8GsbDuxdGA==} dependencies: @@ -3766,6 +3828,17 @@ packages: dependencies: picomatch: 2.3.1 + /redis@4.6.10: + resolution: {integrity: sha512-mmbyhuKgDiJ5TWUhiKhBssz+mjsuSI/lSZNPI9QvZOYzWvYGejtb+W3RlDDf8LD6Bdl5/mZeG8O1feUGhXTxEg==} + dependencies: + '@redis/bloom': 1.2.0(@redis/client@1.5.11) + '@redis/client': 1.5.11 + '@redis/graph': 1.1.0(@redis/client@1.5.11) + '@redis/json': 1.0.6(@redis/client@1.5.11) + '@redis/search': 1.1.5(@redis/client@1.5.11) + '@redis/time-series': 1.0.5(@redis/client@1.5.11) + dev: false + /reflect.getprototypeof@1.0.4: resolution: {integrity: sha512-ECkTw8TmJwW60lOTR+ZkODISW6RQ8+2CL3COqtiJKLd6MmB45hN51HprHFziKLGkAuTGQhBb91V8cy+KHlaCjw==} engines: {node: '>= 0.4'} diff --git a/src/app/api/beatsaver/mapdata/route.ts b/src/app/api/beatsaver/mapdata/route.ts index 8e32e58..821beea 100644 --- a/src/app/api/beatsaver/mapdata/route.ts +++ b/src/app/api/beatsaver/mapdata/route.ts @@ -1,7 +1,8 @@ +import { Redis } from "@/db/redis"; import { BeatsaverMap } from "@/schemas/beatsaver/BeatsaverMap"; import { BeatsaverAPI } from "@/utils/beatsaver/api"; -const mapCache = new Map(); +await Redis.connectRedis(); export async function GET(request: Request) { const { searchParams } = new URL(request.url); @@ -10,24 +11,41 @@ export async function GET(request: Request) { return new Response("mapHashes parameter is required", { status: 400 }); } const idOnly = searchParams.get("idonly") === "true"; + let totalInCache = 0; const maps: Record = {}; for (const mapHash of mapHashes) { - if (mapCache.has(mapHash)) { - maps[mapHash] = mapCache.get(mapHash)!; + const cachedMap = await ( + await Redis.client + ).get(`beatsaver:map:${mapHash}`); + if (cachedMap) { + maps[mapHash] = JSON.parse(cachedMap); + totalInCache++; } else { const map = await BeatsaverAPI.fetchMapByHash(mapHash); - if (map) { - maps[mapHash] = map; - mapCache.set(mapHash, map); - } - if (map && idOnly) { - maps[mapHash] = { id: map.id }; + if (!map) { + continue; } + maps[mapHash] = map; + await ( + await Redis.client + ).set("beatsaver:map:" + mapHash, JSON.stringify(map), { + EX: 60 * 60 * 24 * 7, // 7 days + }); + } + + if (idOnly) { + maps[mapHash] = { id: (maps[mapHash] as BeatsaverMap).id }; } } - return new Response(JSON.stringify(maps), { - headers: { "content-type": "application/json;charset=UTF-8" }, - }); + return new Response( + JSON.stringify({ + maps, + totalInCache, + }), + { + headers: { "content-type": "application/json;charset=UTF-8" }, + }, + ); } diff --git a/src/components/icons/BeatSaverLogo.tsx b/src/components/icons/BeatSaverLogo.tsx index 9de0812..d504543 100644 --- a/src/components/icons/BeatSaverLogo.tsx +++ b/src/components/icons/BeatSaverLogo.tsx @@ -16,14 +16,14 @@ export default function BeatSaverLogo({ version="1.1" className={className} > - - + + - - + + ); diff --git a/src/db/redis.ts b/src/db/redis.ts new file mode 100644 index 0000000..23c96e9 --- /dev/null +++ b/src/db/redis.ts @@ -0,0 +1,31 @@ +import { createClient } from "redis"; + +let redisClient = connectRedis(); + +async function connectRedis() { + console.log("Connecting to redis"); + const client = createClient({ + url: process.env.REDIS_URL, + }); + await client.connect(); + + client.on("connect", () => { + console.log("Connected to redis"); + }); + + client.on("error", (error) => { + console.error("There was an error connecting to redis: " + error); + setTimeout(() => { + redisClient = connectRedis(); + }, 5_000); // 5 seconds + }); + + return client; +} + +// todo: add disconnect handler + +export const Redis = { + client: redisClient, + connectRedis, +}; diff --git a/src/utils/scoresaber/api.ts b/src/utils/scoresaber/api.ts index f3fb960..68f98a6 100644 --- a/src/utils/scoresaber/api.ts +++ b/src/utils/scoresaber/api.ts @@ -176,7 +176,7 @@ async function fetchScoresWithBeatsaverData( }); const mapJson = await mapResponse.json(); for (const score of scores) { - const mapData = mapJson[score.leaderboard.songHash]; + const mapData = mapJson.maps[score.leaderboard.songHash]; if (mapData) { scoresWithBeatsaverData[score.leaderboard.songHash] = { score: score.score, diff --git a/tsconfig.json b/tsconfig.json index 3c494fd..9a6b153 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -1,11 +1,7 @@ { "compilerOptions": { - "target": "ES2016", - "lib": [ - "dom", - "dom.iterable", - "esnext" - ], + "target": "ES2022", + "lib": ["dom", "dom.iterable", "esnext"], "allowJs": true, "skipLibCheck": true, "strict": true, @@ -24,18 +20,9 @@ } ], "paths": { - "@/*": [ - "./src/*" - ] + "@/*": ["./src/*"] } }, - "include": [ - "next-env.d.ts", - "**/*.ts", - "**/*.tsx", - ".next/types/**/*.ts" - ], - "exclude": [ - "node_modules" - ] + "include": ["next-env.d.ts", "**/*.ts", "**/*.tsx", ".next/types/**/*.ts"], + "exclude": ["node_modules"] }