Moved caches to redis

.env.local-example

@@ -5,7 +5,7 @@ SITE_COLOR=#0EBFE9
 SITE_URL=https://bs-overlay.fascinated.cc
 HTTP_PROXY=https://proxy.fascinated.cc
 
-REDIS_IP=127.0.0.1
+REDIS_PORT=6379
-REDIS_USERNAME=yes
+REDIS_HOST=127.0.0.1
 REDIS_PASSWORD=yes
 REDIS_DATABASE=0

package.json

@@ -11,6 +11,7 @@
     "@emotion/cache": "^11.10.3",
     "@emotion/server": "^11.10.0",
     "@nextui-org/react": "^1.0.0-beta.10",
+    "ioredis": "^5.2.3",
     "next": "12",
     "next-seo": "^5.6.0",
     "next-themes": "^0.2.1",

pages/api/beatleader/stars.js

@@ -1,17 +1,28 @@
 import fetch from "node-fetch";
-import BLMapStarCache from "../../../src/caches/BLMapStarCache";
 import WebsiteTypes from "../../../src/consts/LeaderboardType";
+import RedisUtils from "../../../src/utils/redisUtils";
 
+const KEY = "BL_MAP_STAR_";
+
+/**
+ *
+ * @param {Request} req
+ * @param {Response} res
+ * @returns
+ */
 export default async function handler(req, res) {
 	const mapHash = req.query.hash.replace("custom_level_", "").toLowerCase();
 	const difficulty = req.query.difficulty;
 	const characteristic = req.query.characteristic;
 
-	if (BLMapStarCache.has(mapHash)) {
+	const exists = await RedisUtils.exists(`${KEY}${mapHash}`);
-		return res.json({
+	if (exists) {
+		const data = await RedisUtils.getValue(`${KEY}${mapHash}`);
+		res.setHeader("Cache-Status", "hit");
+
+		return res.status(200).json({
 			status: "OK",
-			message: "Cache hit for " + mapHash,
+			stars: Number.parseFloat(data),
-			stars: BLMapStarCache.get(mapHash),
 		});
 	}
 
@@ -26,16 +37,16 @@ export default async function handler(req, res) {
 		}
 	);
 	if (data.status === 404) {
-		return res.json({
+		return res.status(404).json({
 			status: 404,
-			message: "Unknown hash",
+			message: "Unknown Map Hash",
 		});
 	}
 	const json = await data.json();
-	BLMapStarCache.set(mapHash, json.difficulty.stars);
+	RedisUtils.setValue(`${KEY}${mapHash}`, json.difficulty.stars);
-	return res.json({
+	res.setHeader("Cache-Status", "miss");
+	return res.status(200).json({
 		status: "OK",
-		message: "Cache miss for " + mapHash,
 		stars: json.difficulty.stars,
 	});
 }

pages/api/beatsaver/art/[hash].js

@@ -1,26 +1,45 @@
-import fs from "fs";
 import fetch from "node-fetch";
-import path from "path";
 import sharp from "sharp";
-import cacheDir from "../../../../src/caches/SongArtCacheDir";
+import RedisUtils from "../../../../src/utils/redisUtils";
 
+const KEY = "BS_MAP_ART_";
+
+/**
+ *
+ * @param {Request} req
+ * @param {Response} res
+ * @returns
+ */
 export default async function handler(req, res) {
 	const mapHash = req.query.hash.replace("custom_level_", "").toLowerCase();
-	const ext = req.query.ext;
+	const ext = req.query.ext || "jpg";
+
+	const exists = await RedisUtils.exists(`${KEY}${mapHash}`);
+	if (exists) {
+		const data = await RedisUtils.getValue(`${KEY}${mapHash}`);
+		const buffer = Buffer.from(data, "base64");
+		res.writeHead(200, {
+			"Content-Type": "image/" + ext,
+			"Content-Length": buffer.length,
+			"Cache-Status": "hit",
+		});
+		return res.end(buffer);
+	}
 
-	const imagePath = cacheDir + path.sep + mapHash + "." + ext;
-	const exists = fs.existsSync(imagePath);
-	if (!exists) {
 	const data = await fetch(`https://eu.cdn.beatsaver.com/${mapHash}.${ext}`);
+	if (data.status === 404) {
+		return res.status(404).json({
+			status: 404,
+			message: "Unknown Map Hash",
+		});
+	}
+
 	let buffer = await data.buffer(); // Change to arrayBuffer at some point to make it shush
 	buffer = await sharp(buffer).resize(150, 150).toBuffer();
-		fs.writeFileSync(imagePath, buffer);
+	const bytes = buffer.toString("base64");
+
+	await RedisUtils.setValue(`${KEY}${mapHash}`, bytes);
+	res.setHeader("Cache-Status", "miss");
 	res.setHeader("Content-Type", "image/" + ext);
-		res.send(buffer);
+	res.status(200).send(buffer);
-		console.log('Song Art Cache - Added song "' + mapHash + '"');
-		return;
-	}
-	const buffer = fs.readFileSync(imagePath);
-	res.setHeader("Content-Type", "image/" + ext);
-	res.send(buffer);
 }

pages/api/beatsaver/map.js

@@ -6,7 +6,7 @@ export default async function handler(req, res) {
 	const mapData = await Utils.getMapData(mapHash.replace("custom_level_", ""));
 	if (mapData === undefined) {
 		// Check if a map hash was provided
-		return res.json({ error: true, message: "Unknown map" });
+		return res.status(200).json({ error: true, message: "Unknown map" });
 	}
 	const data = {
 		// The maps data from the provided map hash
@@ -15,5 +15,5 @@ export default async function handler(req, res) {
 			mapData.versions[0].coverURL.split("/")[3].split(".")[1]
 		}`,
 	};
-	res.json({ error: false, data: data });
+	res.status(200).json({ error: false, data: data });
 }

pages/api/steamavatar.js

@@ -1,35 +1,55 @@
-import fs from "fs";
 import fetch from "node-fetch";
-import path from "path";
 import sharp from "sharp";
-import cacheDir from "../../src/caches/SteamAvatarCacheDir";
+import { isValidSteamId } from "../../src/helpers/validateSteamId";
-import Utils from "../../src/utils/utils";
+import RedisUtils from "../../src/utils/redisUtils";
 
+const KEY = "STEAM_AVATAR_";
+const ext = "jpg";
+
+/**
+ *
+ * @param {Request} req
+ * @param {Response} res
+ * @returns
+ */
 export default async function handler(req, res) {
 	const steamId = req.query.steamid;
-	const isValid = await Utils.isValidSteamId(steamId);
+	const isValid = await isValidSteamId(steamId);
-	if (isValid == true) {
+	if (isValid == false) {
-		return res.json({
+		return res.status(404).json({
-			status: "OK",
+			status: 404,
-			message: `Invalid steam id`,
+			message: "Unknown Steam Avatar",
 		});
 	}
 
-	const imagePath = cacheDir + path.sep + steamId + ".jpg";
+	const exists = await RedisUtils.exists(`${KEY}${steamId}`);
-	const exists = fs.existsSync(imagePath);
+	if (exists) {
-	if (!exists) {
+		const data = await RedisUtils.getValue(`${KEY}${steamId}`);
+		const buffer = Buffer.from(data, "base64");
+		res.writeHead(200, {
+			"Content-Type": "image/" + ext,
+			"Content-Length": buffer.length,
+			"Cache-Status": "hit",
+		});
+		return res.end(buffer);
+	}
+
 	const data = await fetch(
-			`https://cdn.scoresaber.com/avatars/${steamId}.jpg`
+		`https://cdn.scoresaber.com/avatars/${steamId}.${ext}`
 	);
-		let buffer = await data.buffer();
+	if (data.status === 404) {
+		return res.status(404).json({
+			status: 404,
+			message: "Unknown Steam Avatar",
+		});
+	}
+
+	let buffer = await data.buffer(); // Change to arrayBuffer at some point to make it shush
 	buffer = await sharp(buffer).resize(150, 150).toBuffer();
-		fs.writeFileSync(imagePath, buffer);
+	const bytes = buffer.toString("base64");
-		res.setHeader("Content-Type", "image/jpg");
+
-		res.send(buffer);
+	await RedisUtils.setValue(`${KEY}${steamId}`, bytes);
-		console.log('Steam Avatar Cache - Added avatar "' + steamId + '"');
+	res.setHeader("Cache-Status", "miss");
-		return;
+	res.setHeader("Content-Type", "image/" + ext);
-	}
+	res.status(200).send(buffer);
-	const buffer = fs.readFileSync(imagePath);
-	res.setHeader("Content-Type", "image/jpg");
-	res.send(buffer);
 }

pages/api/validateid.js

@@ -1,17 +1,17 @@
-import Utils from "../../src/utils/utils";
+import { isValidSteamId } from "../../src/helpers/validateSteamId";
 
 export default async function handler(req, res) {
 	const steamId = req.query.steamid;
 	if (!steamId) {
-		return res.json({
+		return res.status(404).json({
 			status: 404,
-			message: "Steam id not provided: Provide in the query.",
+			message: "Steam ID not provided",
 		});
 	}
 
-	const isValid = await Utils.isValidSteamId(steamId);
+	const isValid = await isValidSteamId(steamId);
-	return res.json({
+	return res.status(200).json({
 		status: "OK",
-		message: !isValid ? `Valid` : "Invalid",
+		message: isValid ? `Valid` : "Invalid",
 	});
 }

src/caches/BLMapStarCache.js

@@ -1,3 +0,0 @@
-const CACHE = new Map();
-
-module.exports = CACHE;

src/caches/SongArtCacheDir.js

@@ -1,10 +0,0 @@
-import fs from "fs";
-import path from "path";
-
-const cacheDir = process.cwd() + path.sep + "cache";
-if (!fs.existsSync(cacheDir)) {
-	fs.mkdirSync(cacheDir);
-	console.log("Created art cache directory");
-}
-
-module.exports = cacheDir;

src/caches/SteamAvatarCacheDir.js

@@ -1,10 +0,0 @@
-import fs from "fs";
-import path from "path";
-
-const cacheDir = process.cwd() + path.sep + "avatar_cache";
-if (!fs.existsSync(cacheDir)) {
-	fs.mkdirSync(cacheDir);
-	console.log("Created avatar cache directory");
-}
-
-module.exports = cacheDir;

src/caches/SteamIdCache.js

@@ -1,3 +0,0 @@
-const CACHE = new Map();
-
-module.exports = CACHE;

src/helpers/validateSteamId.js

@@ -0,0 +1,40 @@
+import { default as LeaderboardType } from "../consts/LeaderboardType";
+import RedisUtils from "../utils/redisUtils";
+import Utils from "../utils/utils";
+
+const KEY = "VALID_STEAM_ID_";
+const TO_CHECK = [
+	LeaderboardType.ScoreSaber.ApiUrl.PlayerData,
+	LeaderboardType.BeatLeader.ApiUrl.PlayerData,
+];
+
+async function isValidSteamId(steamId) {
+	if (!steamId) {
+		return false;
+	}
+	if (steamId.length !== 17) {
+		return false;
+	}
+
+	const exists = await RedisUtils.exists(`${KEY}${steamId}`);
+	if (exists) {
+		const data = await RedisUtils.getValue(`${KEY}${steamId}`);
+		return Boolean(data);
+	}
+
+	let valid = false;
+	for (const url of TO_CHECK) {
+		const isValid = await Utils.checkLeaderboard(url, steamId);
+		if (isValid) {
+			valid = true;
+			break;
+		}
+	}
+
+	await RedisUtils.setValue(`${KEY}${steamId}`, valid);
+	return valid;
+}
+
+module.exports = {
+	isValidSteamId,
+};

src/utils/redisUtils.js

@@ -0,0 +1,47 @@
+import Redis from "ioredis";
+
+const client = new Redis({
+	port: process.env.REDIS_PORT,
+	host: process.env.REDIS_HOST,
+	password: process.env.REDIS_PASSWORD,
+	db: process.env.REDIS_DATABASE,
+});
+client.connect().catch(() => {});
+
+async function setValue(key, value, expireAt = 86400) {
+	await client.set(key, value);
+}
+
+async function getValue(key) {
+	const yes = new Promise((reject, resolve) => {
+		client.get(key, (err, result) => {
+			if (err !== null) {
+				resolve(err);
+			} else {
+				reject(result);
+			}
+		});
+	});
+	const maybe = await yes;
+	return maybe;
+}
+
+async function exists(key) {
+	const yes = new Promise((reject, resolve) => {
+		client.exists(key, (err, result) => {
+			if (err !== null) {
+				resolve(err);
+			} else {
+				reject(result);
+			}
+		});
+	});
+	const maybe = await yes;
+	return maybe == 1 ? true : false;
+}
+
+module.exports = {
+	getValue,
+	setValue,
+	exists,
+};

src/utils/utils.js

@@ -1,14 +1,8 @@
-import SteamIdCache from "../../src/caches/SteamIdCache";
 import {
 	default as LeaderboardType,
 	default as WebsiteTypes,
 } from "../consts/LeaderboardType";
 
-const TO_CHECK = [
-	LeaderboardType.ScoreSaber.ApiUrl.PlayerData,
-	LeaderboardType.BeatLeader.ApiUrl.PlayerData,
-];
-
 export default class Utils {
 	constructor() {}
 
@@ -26,35 +20,6 @@ export default class Utils {
 		window.open(url, "_blank");
 	}
 
-	static async isValidSteamId(steamId) {
-		if (!steamId) {
-			return false;
-		}
-		if (steamId.length !== 17) {
-			return false;
-		}
-
-		if (SteamIdCache.has(steamId)) {
-			return SteamIdCache.get(steamId);
-		}
-
-		let invalid = false;
-		for (const url of TO_CHECK) {
-			const isValid = await Utils.checkLeaderboard(url, steamId);
-
-			if (isValid) {
-				break;
-			}
-			if (!isValid) {
-				invalid = true;
-				break;
-			}
-		}
-
-		SteamIdCache.set(steamId, invalid);
-		return invalid;
-	}
-
 	static async checkLeaderboard(url, steamId) {
 		const data = await fetch(url.replace("%s", steamId), {
 			headers: {
@@ -74,4 +39,8 @@ export default class Utils {
 		}
 		return undefined;
 	}
+
+	static base64ToArrayBuffer(base64) {
+		return Uint8Array.from(atob(base64), (c) => c.charCodeAt(0));
+	}
 }

yarn.lock

@@ -139,6 +139,11 @@
   dependencies:
     "@babel/runtime" "^7.6.2"
 
+"@ioredis/commands@^1.1.1":
+  version "1.2.0"
+  resolved "https://registry.yarnpkg.com/@ioredis/commands/-/commands-1.2.0.tgz#6d61b3097470af1fdbbe622795b8921d42018e11"
+  integrity sha512-Sx1pU8EM64o2BrqNpEO1CNLtKQwyhuXuqyfH7oGKCk+1a33d2r5saW8zNwm3j6BTExtjrv2BxTgzzkMwts6vGg==
+
 "@next/env@12.3.1":
   version "12.3.1"
   resolved "https://registry.npmjs.org/@next/env/-/env-12.3.1.tgz"
@@ -1024,6 +1029,11 @@ clsx@^1.1.1:
   version "1.2.1"
   resolved "https://registry.npmjs.org/clsx/-/clsx-1.2.1.tgz"
 
+cluster-key-slot@^1.1.0:
+  version "1.1.1"
+  resolved "https://registry.yarnpkg.com/cluster-key-slot/-/cluster-key-slot-1.1.1.tgz#10ccb9ded0729464b6d2e7d714b100a2d1259d43"
+  integrity sha512-rwHwUfXL40Chm1r08yrhU3qpUvdVlgkKNeyeGPOxnW8/SyVDvgRaed/Uz54AqWNaTCAThlj6QAs3TZcKI0xDEw==
+
 color-convert@^2.0.1:
   version "2.0.1"
   resolved "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz"
@@ -1127,6 +1137,11 @@ define-properties@^1.1.4:
     has-property-descriptors "^1.0.0"
     object-keys "^1.1.1"
 
+denque@^2.0.1:
+  version "2.1.0"
+  resolved "https://registry.yarnpkg.com/denque/-/denque-2.1.0.tgz#e93e1a6569fb5e66f16a3c2a2964617d349d6ab1"
+  integrity sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw==
+
 detect-libc@^2.0.0, detect-libc@^2.0.1:
   version "2.0.1"
   resolved "https://registry.npmjs.org/detect-libc/-/detect-libc-2.0.1.tgz"
@@ -1705,6 +1720,21 @@ intl-messageformat@^10.1.0:
     "@formatjs/icu-messageformat-parser" "2.1.7"
     tslib "2.4.0"
 
+ioredis@^5.2.3:
+  version "5.2.3"
+  resolved "https://registry.yarnpkg.com/ioredis/-/ioredis-5.2.3.tgz#d5b37eb13e643241660d6cee4eeb41a026cda8c0"
+  integrity sha512-gQNcMF23/NpvjCaa1b5YycUyQJ9rBNH2xP94LWinNpodMWVUPP5Ai/xXANn/SM7gfIvI62B5CCvZxhg5pOgyMw==
+  dependencies:
+    "@ioredis/commands" "^1.1.1"
+    cluster-key-slot "^1.1.0"
+    debug "^4.3.4"
+    denque "^2.0.1"
+    lodash.defaults "^4.2.0"
+    lodash.isarguments "^3.1.0"
+    redis-errors "^1.2.0"
+    redis-parser "^3.0.0"
+    standard-as-callback "^2.1.0"
+
 is-arrayish@^0.3.1:
   version "0.3.2"
   resolved "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.3.2.tgz"
@@ -1877,6 +1907,16 @@ locate-path@^6.0.0:
   dependencies:
     p-locate "^5.0.0"
 
+lodash.defaults@^4.2.0:
+  version "4.2.0"
+  resolved "https://registry.yarnpkg.com/lodash.defaults/-/lodash.defaults-4.2.0.tgz#d09178716ffea4dde9e5fb7b37f6f0802274580c"
+  integrity sha512-qjxPLHd3r5DnsdGacqOMU6pb/avJzdh9tFX2ymgoZE27BmjXrNy/y4LoaiTeAb+O3gL8AfpJGtqfX/ae2leYYQ==
+
+lodash.isarguments@^3.1.0:
+  version "3.1.0"
+  resolved "https://registry.yarnpkg.com/lodash.isarguments/-/lodash.isarguments-3.1.0.tgz#2f573d85c6a24289ff00663b491c1d338ff3458a"
+  integrity sha512-chi4NHZlZqZD18a0imDHnZPrDeBbTtVN7GXMwuGdRH9qotxAjYs3aVLKc7zNOG9eddR5Ksd8rvFEBc9SsggPpg==
+
 lodash.merge@^4.6.2:
   version "4.6.2"
   resolved "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.2.tgz"
@@ -2258,6 +2298,18 @@ readable-stream@~1.0.17, readable-stream@~1.0.27-1:
     isarray "0.0.1"
     string_decoder "~0.10.x"
 
+redis-errors@^1.0.0, redis-errors@^1.2.0:
+  version "1.2.0"
+  resolved "https://registry.yarnpkg.com/redis-errors/-/redis-errors-1.2.0.tgz#eb62d2adb15e4eaf4610c04afe1529384250abad"
+  integrity sha512-1qny3OExCf0UvUV/5wpYKf2YwPcOqXzkwKKSmKHiE6ZMQs5heeE/c8eXK+PNllPvmjgAbfnsbpkGZWy8cBpn9w==
+
+redis-parser@^3.0.0:
+  version "3.0.0"
+  resolved "https://registry.yarnpkg.com/redis-parser/-/redis-parser-3.0.0.tgz#b66d828cdcafe6b4b8a428a7def4c6bcac31c8b4"
+  integrity sha512-DJnGAeenTdpMEH6uAJRK/uiyEIH9WVsUmoLwzudwGJUwZPp80PDBWPHXSAGNPwNvIXAbe7MSUB1zQFugFml66A==
+  dependencies:
+    redis-errors "^1.0.0"
+
 regenerator-runtime@^0.13.4:
   version "0.13.9"
   resolved "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.13.9.tgz"
@@ -2406,6 +2458,11 @@ source-map-js@^1.0.2:
   version "1.0.2"
   resolved "https://registry.npmjs.org/source-map-js/-/source-map-js-1.0.2.tgz"
 
+standard-as-callback@^2.1.0:
+  version "2.1.0"
+  resolved "https://registry.yarnpkg.com/standard-as-callback/-/standard-as-callback-2.1.0.tgz#8953fc05359868a77b5b9739a665c5977bb7df45"
+  integrity sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A==
+
 string.prototype.matchall@^4.0.7:
   version "4.0.7"
   resolved "https://registry.yarnpkg.com/string.prototype.matchall/-/string.prototype.matchall-4.0.7.tgz#8e6ecb0d8a1fb1fda470d81acecb2dba057a481d"