From 756138f8a0e61749537d086af7835d938e9a18d9 Mon Sep 17 00:00:00 2001 From: Liam <67254223+RealFascinated@users.noreply.github.com> Date: Thu, 20 Oct 2022 18:00:27 +0100 Subject: [PATCH] Moved caches to redis --- .env.local-example | 4 +- package.json | 1 + pages/api/beatleader/stars.js | 31 +++++++++----- pages/api/beatsaver/art/[hash].js | 53 +++++++++++++++-------- pages/api/beatsaver/map.js | 4 +- pages/api/steamavatar.js | 70 ++++++++++++++++++++----------- pages/api/validateid.js | 12 +++--- src/caches/BLMapStarCache.js | 3 -- src/caches/SongArtCacheDir.js | 10 ----- src/caches/SteamAvatarCacheDir.js | 10 ----- src/caches/SteamIdCache.js | 3 -- src/helpers/validateSteamId.js | 40 ++++++++++++++++++ src/utils/redisUtils.js | 47 +++++++++++++++++++++ src/utils/utils.js | 39 ++--------------- yarn.lock | 57 +++++++++++++++++++++++++ 15 files changed, 261 insertions(+), 123 deletions(-) delete mode 100644 src/caches/BLMapStarCache.js delete mode 100644 src/caches/SongArtCacheDir.js delete mode 100644 src/caches/SteamAvatarCacheDir.js delete mode 100644 src/caches/SteamIdCache.js create mode 100644 src/helpers/validateSteamId.js create mode 100644 src/utils/redisUtils.js diff --git a/.env.local-example b/.env.local-example index 52cd16b..7ff861f 100644 --- a/.env.local-example +++ b/.env.local-example @@ -5,7 +5,7 @@ SITE_COLOR=#0EBFE9 SITE_URL=https://bs-overlay.fascinated.cc HTTP_PROXY=https://proxy.fascinated.cc -REDIS_IP=127.0.0.1 -REDIS_USERNAME=yes +REDIS_PORT=6379 +REDIS_HOST=127.0.0.1 REDIS_PASSWORD=yes REDIS_DATABASE=0 \ No newline at end of file diff --git a/package.json b/package.json index 8494cd3..10310bd 100644 --- a/package.json +++ b/package.json @@ -11,6 +11,7 @@ "@emotion/cache": "^11.10.3", "@emotion/server": "^11.10.0", "@nextui-org/react": "^1.0.0-beta.10", + "ioredis": "^5.2.3", "next": "12", "next-seo": "^5.6.0", "next-themes": "^0.2.1", diff --git a/pages/api/beatleader/stars.js b/pages/api/beatleader/stars.js index 95e94ab..dea90e1 100644 --- a/pages/api/beatleader/stars.js +++ b/pages/api/beatleader/stars.js @@ -1,17 +1,28 @@ import fetch from "node-fetch"; -import BLMapStarCache from "../../../src/caches/BLMapStarCache"; import WebsiteTypes from "../../../src/consts/LeaderboardType"; +import RedisUtils from "../../../src/utils/redisUtils"; +const KEY = "BL_MAP_STAR_"; + +/** + * + * @param {Request} req + * @param {Response} res + * @returns + */ export default async function handler(req, res) { const mapHash = req.query.hash.replace("custom_level_", "").toLowerCase(); const difficulty = req.query.difficulty; const characteristic = req.query.characteristic; - if (BLMapStarCache.has(mapHash)) { - return res.json({ + const exists = await RedisUtils.exists(`${KEY}${mapHash}`); + if (exists) { + const data = await RedisUtils.getValue(`${KEY}${mapHash}`); + res.setHeader("Cache-Status", "hit"); + + return res.status(200).json({ status: "OK", - message: "Cache hit for " + mapHash, - stars: BLMapStarCache.get(mapHash), + stars: Number.parseFloat(data), }); } @@ -26,16 +37,16 @@ export default async function handler(req, res) { } ); if (data.status === 404) { - return res.json({ + return res.status(404).json({ status: 404, - message: "Unknown hash", + message: "Unknown Map Hash", }); } const json = await data.json(); - BLMapStarCache.set(mapHash, json.difficulty.stars); - return res.json({ + RedisUtils.setValue(`${KEY}${mapHash}`, json.difficulty.stars); + res.setHeader("Cache-Status", "miss"); + return res.status(200).json({ status: "OK", - message: "Cache miss for " + mapHash, stars: json.difficulty.stars, }); } diff --git a/pages/api/beatsaver/art/[hash].js b/pages/api/beatsaver/art/[hash].js index 20d79c2..c81a9cc 100644 --- a/pages/api/beatsaver/art/[hash].js +++ b/pages/api/beatsaver/art/[hash].js @@ -1,26 +1,45 @@ -import fs from "fs"; import fetch from "node-fetch"; -import path from "path"; import sharp from "sharp"; -import cacheDir from "../../../../src/caches/SongArtCacheDir"; +import RedisUtils from "../../../../src/utils/redisUtils"; +const KEY = "BS_MAP_ART_"; + +/** + * + * @param {Request} req + * @param {Response} res + * @returns + */ export default async function handler(req, res) { const mapHash = req.query.hash.replace("custom_level_", "").toLowerCase(); - const ext = req.query.ext; + const ext = req.query.ext || "jpg"; - const imagePath = cacheDir + path.sep + mapHash + "." + ext; - const exists = fs.existsSync(imagePath); - if (!exists) { - const data = await fetch(`https://eu.cdn.beatsaver.com/${mapHash}.${ext}`); - let buffer = await data.buffer(); // Change to arrayBuffer at some point to make it shush - buffer = await sharp(buffer).resize(150, 150).toBuffer(); - fs.writeFileSync(imagePath, buffer); - res.setHeader("Content-Type", "image/" + ext); - res.send(buffer); - console.log('Song Art Cache - Added song "' + mapHash + '"'); - return; + const exists = await RedisUtils.exists(`${KEY}${mapHash}`); + if (exists) { + const data = await RedisUtils.getValue(`${KEY}${mapHash}`); + const buffer = Buffer.from(data, "base64"); + res.writeHead(200, { + "Content-Type": "image/" + ext, + "Content-Length": buffer.length, + "Cache-Status": "hit", + }); + return res.end(buffer); } - const buffer = fs.readFileSync(imagePath); + + const data = await fetch(`https://eu.cdn.beatsaver.com/${mapHash}.${ext}`); + if (data.status === 404) { + return res.status(404).json({ + status: 404, + message: "Unknown Map Hash", + }); + } + + let buffer = await data.buffer(); // Change to arrayBuffer at some point to make it shush + buffer = await sharp(buffer).resize(150, 150).toBuffer(); + const bytes = buffer.toString("base64"); + + await RedisUtils.setValue(`${KEY}${mapHash}`, bytes); + res.setHeader("Cache-Status", "miss"); res.setHeader("Content-Type", "image/" + ext); - res.send(buffer); + res.status(200).send(buffer); } diff --git a/pages/api/beatsaver/map.js b/pages/api/beatsaver/map.js index 22a573a..1a9b20c 100644 --- a/pages/api/beatsaver/map.js +++ b/pages/api/beatsaver/map.js @@ -6,7 +6,7 @@ export default async function handler(req, res) { const mapData = await Utils.getMapData(mapHash.replace("custom_level_", "")); if (mapData === undefined) { // Check if a map hash was provided - return res.json({ error: true, message: "Unknown map" }); + return res.status(200).json({ error: true, message: "Unknown map" }); } const data = { // The maps data from the provided map hash @@ -15,5 +15,5 @@ export default async function handler(req, res) { mapData.versions[0].coverURL.split("/")[3].split(".")[1] }`, }; - res.json({ error: false, data: data }); + res.status(200).json({ error: false, data: data }); } diff --git a/pages/api/steamavatar.js b/pages/api/steamavatar.js index 44867a0..fb493eb 100644 --- a/pages/api/steamavatar.js +++ b/pages/api/steamavatar.js @@ -1,35 +1,55 @@ -import fs from "fs"; import fetch from "node-fetch"; -import path from "path"; import sharp from "sharp"; -import cacheDir from "../../src/caches/SteamAvatarCacheDir"; -import Utils from "../../src/utils/utils"; +import { isValidSteamId } from "../../src/helpers/validateSteamId"; +import RedisUtils from "../../src/utils/redisUtils"; +const KEY = "STEAM_AVATAR_"; +const ext = "jpg"; + +/** + * + * @param {Request} req + * @param {Response} res + * @returns + */ export default async function handler(req, res) { const steamId = req.query.steamid; - const isValid = await Utils.isValidSteamId(steamId); - if (isValid == true) { - return res.json({ - status: "OK", - message: `Invalid steam id`, + const isValid = await isValidSteamId(steamId); + if (isValid == false) { + return res.status(404).json({ + status: 404, + message: "Unknown Steam Avatar", }); } - const imagePath = cacheDir + path.sep + steamId + ".jpg"; - const exists = fs.existsSync(imagePath); - if (!exists) { - const data = await fetch( - `https://cdn.scoresaber.com/avatars/${steamId}.jpg` - ); - let buffer = await data.buffer(); - buffer = await sharp(buffer).resize(150, 150).toBuffer(); - fs.writeFileSync(imagePath, buffer); - res.setHeader("Content-Type", "image/jpg"); - res.send(buffer); - console.log('Steam Avatar Cache - Added avatar "' + steamId + '"'); - return; + const exists = await RedisUtils.exists(`${KEY}${steamId}`); + if (exists) { + const data = await RedisUtils.getValue(`${KEY}${steamId}`); + const buffer = Buffer.from(data, "base64"); + res.writeHead(200, { + "Content-Type": "image/" + ext, + "Content-Length": buffer.length, + "Cache-Status": "hit", + }); + return res.end(buffer); } - const buffer = fs.readFileSync(imagePath); - res.setHeader("Content-Type", "image/jpg"); - res.send(buffer); + + const data = await fetch( + `https://cdn.scoresaber.com/avatars/${steamId}.${ext}` + ); + if (data.status === 404) { + return res.status(404).json({ + status: 404, + message: "Unknown Steam Avatar", + }); + } + + let buffer = await data.buffer(); // Change to arrayBuffer at some point to make it shush + buffer = await sharp(buffer).resize(150, 150).toBuffer(); + const bytes = buffer.toString("base64"); + + await RedisUtils.setValue(`${KEY}${steamId}`, bytes); + res.setHeader("Cache-Status", "miss"); + res.setHeader("Content-Type", "image/" + ext); + res.status(200).send(buffer); } diff --git a/pages/api/validateid.js b/pages/api/validateid.js index 962b5ec..a92ce5e 100644 --- a/pages/api/validateid.js +++ b/pages/api/validateid.js @@ -1,17 +1,17 @@ -import Utils from "../../src/utils/utils"; +import { isValidSteamId } from "../../src/helpers/validateSteamId"; export default async function handler(req, res) { const steamId = req.query.steamid; if (!steamId) { - return res.json({ + return res.status(404).json({ status: 404, - message: "Steam id not provided: Provide in the query.", + message: "Steam ID not provided", }); } - const isValid = await Utils.isValidSteamId(steamId); - return res.json({ + const isValid = await isValidSteamId(steamId); + return res.status(200).json({ status: "OK", - message: !isValid ? `Valid` : "Invalid", + message: isValid ? `Valid` : "Invalid", }); } diff --git a/src/caches/BLMapStarCache.js b/src/caches/BLMapStarCache.js deleted file mode 100644 index 2934fa0..0000000 --- a/src/caches/BLMapStarCache.js +++ /dev/null @@ -1,3 +0,0 @@ -const CACHE = new Map(); - -module.exports = CACHE; diff --git a/src/caches/SongArtCacheDir.js b/src/caches/SongArtCacheDir.js deleted file mode 100644 index c0858a6..0000000 --- a/src/caches/SongArtCacheDir.js +++ /dev/null @@ -1,10 +0,0 @@ -import fs from "fs"; -import path from "path"; - -const cacheDir = process.cwd() + path.sep + "cache"; -if (!fs.existsSync(cacheDir)) { - fs.mkdirSync(cacheDir); - console.log("Created art cache directory"); -} - -module.exports = cacheDir; diff --git a/src/caches/SteamAvatarCacheDir.js b/src/caches/SteamAvatarCacheDir.js deleted file mode 100644 index 098637b..0000000 --- a/src/caches/SteamAvatarCacheDir.js +++ /dev/null @@ -1,10 +0,0 @@ -import fs from "fs"; -import path from "path"; - -const cacheDir = process.cwd() + path.sep + "avatar_cache"; -if (!fs.existsSync(cacheDir)) { - fs.mkdirSync(cacheDir); - console.log("Created avatar cache directory"); -} - -module.exports = cacheDir; diff --git a/src/caches/SteamIdCache.js b/src/caches/SteamIdCache.js deleted file mode 100644 index 2934fa0..0000000 --- a/src/caches/SteamIdCache.js +++ /dev/null @@ -1,3 +0,0 @@ -const CACHE = new Map(); - -module.exports = CACHE; diff --git a/src/helpers/validateSteamId.js b/src/helpers/validateSteamId.js new file mode 100644 index 0000000..d05a811 --- /dev/null +++ b/src/helpers/validateSteamId.js @@ -0,0 +1,40 @@ +import { default as LeaderboardType } from "../consts/LeaderboardType"; +import RedisUtils from "../utils/redisUtils"; +import Utils from "../utils/utils"; + +const KEY = "VALID_STEAM_ID_"; +const TO_CHECK = [ + LeaderboardType.ScoreSaber.ApiUrl.PlayerData, + LeaderboardType.BeatLeader.ApiUrl.PlayerData, +]; + +async function isValidSteamId(steamId) { + if (!steamId) { + return false; + } + if (steamId.length !== 17) { + return false; + } + + const exists = await RedisUtils.exists(`${KEY}${steamId}`); + if (exists) { + const data = await RedisUtils.getValue(`${KEY}${steamId}`); + return Boolean(data); + } + + let valid = false; + for (const url of TO_CHECK) { + const isValid = await Utils.checkLeaderboard(url, steamId); + if (isValid) { + valid = true; + break; + } + } + + await RedisUtils.setValue(`${KEY}${steamId}`, valid); + return valid; +} + +module.exports = { + isValidSteamId, +}; diff --git a/src/utils/redisUtils.js b/src/utils/redisUtils.js new file mode 100644 index 0000000..150f7e1 --- /dev/null +++ b/src/utils/redisUtils.js @@ -0,0 +1,47 @@ +import Redis from "ioredis"; + +const client = new Redis({ + port: process.env.REDIS_PORT, + host: process.env.REDIS_HOST, + password: process.env.REDIS_PASSWORD, + db: process.env.REDIS_DATABASE, +}); +client.connect().catch(() => {}); + +async function setValue(key, value, expireAt = 86400) { + await client.set(key, value); +} + +async function getValue(key) { + const yes = new Promise((reject, resolve) => { + client.get(key, (err, result) => { + if (err !== null) { + resolve(err); + } else { + reject(result); + } + }); + }); + const maybe = await yes; + return maybe; +} + +async function exists(key) { + const yes = new Promise((reject, resolve) => { + client.exists(key, (err, result) => { + if (err !== null) { + resolve(err); + } else { + reject(result); + } + }); + }); + const maybe = await yes; + return maybe == 1 ? true : false; +} + +module.exports = { + getValue, + setValue, + exists, +}; diff --git a/src/utils/utils.js b/src/utils/utils.js index 4925145..fd2f615 100644 --- a/src/utils/utils.js +++ b/src/utils/utils.js @@ -1,14 +1,8 @@ -import SteamIdCache from "../../src/caches/SteamIdCache"; import { default as LeaderboardType, default as WebsiteTypes, } from "../consts/LeaderboardType"; -const TO_CHECK = [ - LeaderboardType.ScoreSaber.ApiUrl.PlayerData, - LeaderboardType.BeatLeader.ApiUrl.PlayerData, -]; - export default class Utils { constructor() {} @@ -26,35 +20,6 @@ export default class Utils { window.open(url, "_blank"); } - static async isValidSteamId(steamId) { - if (!steamId) { - return false; - } - if (steamId.length !== 17) { - return false; - } - - if (SteamIdCache.has(steamId)) { - return SteamIdCache.get(steamId); - } - - let invalid = false; - for (const url of TO_CHECK) { - const isValid = await Utils.checkLeaderboard(url, steamId); - - if (isValid) { - break; - } - if (!isValid) { - invalid = true; - break; - } - } - - SteamIdCache.set(steamId, invalid); - return invalid; - } - static async checkLeaderboard(url, steamId) { const data = await fetch(url.replace("%s", steamId), { headers: { @@ -74,4 +39,8 @@ export default class Utils { } return undefined; } + + static base64ToArrayBuffer(base64) { + return Uint8Array.from(atob(base64), (c) => c.charCodeAt(0)); + } } diff --git a/yarn.lock b/yarn.lock index 58e2dc6..8de897c 100644 --- a/yarn.lock +++ b/yarn.lock @@ -139,6 +139,11 @@ dependencies: "@babel/runtime" "^7.6.2" +"@ioredis/commands@^1.1.1": + version "1.2.0" + resolved "https://registry.yarnpkg.com/@ioredis/commands/-/commands-1.2.0.tgz#6d61b3097470af1fdbbe622795b8921d42018e11" + integrity sha512-Sx1pU8EM64o2BrqNpEO1CNLtKQwyhuXuqyfH7oGKCk+1a33d2r5saW8zNwm3j6BTExtjrv2BxTgzzkMwts6vGg== + "@next/env@12.3.1": version "12.3.1" resolved "https://registry.npmjs.org/@next/env/-/env-12.3.1.tgz" @@ -1024,6 +1029,11 @@ clsx@^1.1.1: version "1.2.1" resolved "https://registry.npmjs.org/clsx/-/clsx-1.2.1.tgz" +cluster-key-slot@^1.1.0: + version "1.1.1" + resolved "https://registry.yarnpkg.com/cluster-key-slot/-/cluster-key-slot-1.1.1.tgz#10ccb9ded0729464b6d2e7d714b100a2d1259d43" + integrity sha512-rwHwUfXL40Chm1r08yrhU3qpUvdVlgkKNeyeGPOxnW8/SyVDvgRaed/Uz54AqWNaTCAThlj6QAs3TZcKI0xDEw== + color-convert@^2.0.1: version "2.0.1" resolved "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz" @@ -1127,6 +1137,11 @@ define-properties@^1.1.4: has-property-descriptors "^1.0.0" object-keys "^1.1.1" +denque@^2.0.1: + version "2.1.0" + resolved "https://registry.yarnpkg.com/denque/-/denque-2.1.0.tgz#e93e1a6569fb5e66f16a3c2a2964617d349d6ab1" + integrity sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw== + detect-libc@^2.0.0, detect-libc@^2.0.1: version "2.0.1" resolved "https://registry.npmjs.org/detect-libc/-/detect-libc-2.0.1.tgz" @@ -1705,6 +1720,21 @@ intl-messageformat@^10.1.0: "@formatjs/icu-messageformat-parser" "2.1.7" tslib "2.4.0" +ioredis@^5.2.3: + version "5.2.3" + resolved "https://registry.yarnpkg.com/ioredis/-/ioredis-5.2.3.tgz#d5b37eb13e643241660d6cee4eeb41a026cda8c0" + integrity sha512-gQNcMF23/NpvjCaa1b5YycUyQJ9rBNH2xP94LWinNpodMWVUPP5Ai/xXANn/SM7gfIvI62B5CCvZxhg5pOgyMw== + dependencies: + "@ioredis/commands" "^1.1.1" + cluster-key-slot "^1.1.0" + debug "^4.3.4" + denque "^2.0.1" + lodash.defaults "^4.2.0" + lodash.isarguments "^3.1.0" + redis-errors "^1.2.0" + redis-parser "^3.0.0" + standard-as-callback "^2.1.0" + is-arrayish@^0.3.1: version "0.3.2" resolved "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.3.2.tgz" @@ -1877,6 +1907,16 @@ locate-path@^6.0.0: dependencies: p-locate "^5.0.0" +lodash.defaults@^4.2.0: + version "4.2.0" + resolved "https://registry.yarnpkg.com/lodash.defaults/-/lodash.defaults-4.2.0.tgz#d09178716ffea4dde9e5fb7b37f6f0802274580c" + integrity sha512-qjxPLHd3r5DnsdGacqOMU6pb/avJzdh9tFX2ymgoZE27BmjXrNy/y4LoaiTeAb+O3gL8AfpJGtqfX/ae2leYYQ== + +lodash.isarguments@^3.1.0: + version "3.1.0" + resolved "https://registry.yarnpkg.com/lodash.isarguments/-/lodash.isarguments-3.1.0.tgz#2f573d85c6a24289ff00663b491c1d338ff3458a" + integrity sha512-chi4NHZlZqZD18a0imDHnZPrDeBbTtVN7GXMwuGdRH9qotxAjYs3aVLKc7zNOG9eddR5Ksd8rvFEBc9SsggPpg== + lodash.merge@^4.6.2: version "4.6.2" resolved "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.2.tgz" @@ -2258,6 +2298,18 @@ readable-stream@~1.0.17, readable-stream@~1.0.27-1: isarray "0.0.1" string_decoder "~0.10.x" +redis-errors@^1.0.0, redis-errors@^1.2.0: + version "1.2.0" + resolved "https://registry.yarnpkg.com/redis-errors/-/redis-errors-1.2.0.tgz#eb62d2adb15e4eaf4610c04afe1529384250abad" + integrity sha512-1qny3OExCf0UvUV/5wpYKf2YwPcOqXzkwKKSmKHiE6ZMQs5heeE/c8eXK+PNllPvmjgAbfnsbpkGZWy8cBpn9w== + +redis-parser@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/redis-parser/-/redis-parser-3.0.0.tgz#b66d828cdcafe6b4b8a428a7def4c6bcac31c8b4" + integrity sha512-DJnGAeenTdpMEH6uAJRK/uiyEIH9WVsUmoLwzudwGJUwZPp80PDBWPHXSAGNPwNvIXAbe7MSUB1zQFugFml66A== + dependencies: + redis-errors "^1.0.0" + regenerator-runtime@^0.13.4: version "0.13.9" resolved "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.13.9.tgz" @@ -2406,6 +2458,11 @@ source-map-js@^1.0.2: version "1.0.2" resolved "https://registry.npmjs.org/source-map-js/-/source-map-js-1.0.2.tgz" +standard-as-callback@^2.1.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/standard-as-callback/-/standard-as-callback-2.1.0.tgz#8953fc05359868a77b5b9739a665c5977bb7df45" + integrity sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A== + string.prototype.matchall@^4.0.7: version "4.0.7" resolved "https://registry.yarnpkg.com/string.prototype.matchall/-/string.prototype.matchall-4.0.7.tgz#8e6ecb0d8a1fb1fda470d81acecb2dba057a481d"