Fascinated/beatsaber-overlay
Fascinated/beatsaber-overlay
Archived
1
1
Fork 0
Code Issues 1 Pull Requests 2 Actions Packages Wiki Activity

Moved caches to redis

Browse Source
This commit is contained in:
Liam 2022-10-20 18:00:27 +01:00
parent 3cb96a539a
commit 756138f8a0
15 changed files with 261 additions and 123 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
.env.local-example
View File

@ -5,7 +5,7 @@ SITE_COLOR=#0EBFE9
SITE_URL=https://bs-overlay.fascinated.cc
HTTP_PROXY=https://proxy.fascinated.cc
REDIS_IP=127.0.0.1
REDIS_USERNAME=yes
REDIS_PORT=6379
REDIS_HOST=127.0.0.1
REDIS_PASSWORD=yes
REDIS_DATABASE=0

1
package.json
View File

@ -11,6 +11,7 @@
"@emotion/cache": "^11.10.3",
"@emotion/server": "^11.10.0",
"@nextui-org/react": "^1.0.0-beta.10",
"ioredis": "^5.2.3",
"next": "12",
"next-seo": "^5.6.0",
"next-themes": "^0.2.1",

31
pages/api/beatleader/stars.js
View File

@ -1,17 +1,28 @@
import fetch from "node-fetch";
import BLMapStarCache from "../../../src/caches/BLMapStarCache";
import WebsiteTypes from "../../../src/consts/LeaderboardType";
import RedisUtils from "../../../src/utils/redisUtils";
const KEY = "BL_MAP_STAR_";
/**
*
* @param {Request} req
* @param {Response} res
* @returns
*/
export default async function handler(req, res) {
const mapHash = req.query.hash.replace("custom_level_", "").toLowerCase();
const difficulty = req.query.difficulty;
const characteristic = req.query.characteristic;
if (BLMapStarCache.has(mapHash)) {
return res.json({
const exists = await RedisUtils.exists(`${KEY}${mapHash}`);
if (exists) {
const data = await RedisUtils.getValue(`${KEY}${mapHash}`);
res.setHeader("Cache-Status", "hit");
return res.status(200).json({
status: "OK",
message: "Cache hit for " + mapHash,
stars: BLMapStarCache.get(mapHash),
stars: Number.parseFloat(data),
});
}
@ -26,16 +37,16 @@ export default async function handler(req, res) {
}
);
if (data.status === 404) {
return res.json({
return res.status(404).json({
status: 404,
message: "Unknown hash",
message: "Unknown Map Hash",
});
}
const json = await data.json();
BLMapStarCache.set(mapHash, json.difficulty.stars);
return res.json({
RedisUtils.setValue(`${KEY}${mapHash}`, json.difficulty.stars);
res.setHeader("Cache-Status", "miss");
return res.status(200).json({
status: "OK",
message: "Cache miss for " + mapHash,
stars: json.difficulty.stars,
});
}

53
pages/api/beatsaver/art/[hash].js
View File

@ -1,26 +1,45 @@
import fs from "fs";
import fetch from "node-fetch";
import path from "path";
import sharp from "sharp";
import cacheDir from "../../../../src/caches/SongArtCacheDir";
import RedisUtils from "../../../../src/utils/redisUtils";
const KEY = "BS_MAP_ART_";
/**
*
* @param {Request} req
* @param {Response} res
* @returns
*/
export default async function handler(req, res) {
const mapHash = req.query.hash.replace("custom_level_", "").toLowerCase();
const ext = req.query.ext;
const ext = req.query.ext || "jpg";
const imagePath = cacheDir + path.sep + mapHash + "." + ext;
const exists = fs.existsSync(imagePath);
if (!exists) {
const data = await fetch(`https://eu.cdn.beatsaver.com/${mapHash}.${ext}`);
let buffer = await data.buffer(); // Change to arrayBuffer at some point to make it shush
buffer = await sharp(buffer).resize(150, 150).toBuffer();
fs.writeFileSync(imagePath, buffer);
res.setHeader("Content-Type", "image/" + ext);
res.send(buffer);
console.log('Song Art Cache - Added song "' + mapHash + '"');
return;
const exists = await RedisUtils.exists(`${KEY}${mapHash}`);
if (exists) {
const data = await RedisUtils.getValue(`${KEY}${mapHash}`);
const buffer = Buffer.from(data, "base64");
res.writeHead(200, {
"Content-Type": "image/" + ext,
"Content-Length": buffer.length,
"Cache-Status": "hit",
});
return res.end(buffer);
}
const buffer = fs.readFileSync(imagePath);
const data = await fetch(`https://eu.cdn.beatsaver.com/${mapHash}.${ext}`);
if (data.status === 404) {
return res.status(404).json({
status: 404,
message: "Unknown Map Hash",
});
}
let buffer = await data.buffer(); // Change to arrayBuffer at some point to make it shush
buffer = await sharp(buffer).resize(150, 150).toBuffer();
const bytes = buffer.toString("base64");
await RedisUtils.setValue(`${KEY}${mapHash}`, bytes);
res.setHeader("Cache-Status", "miss");
res.setHeader("Content-Type", "image/" + ext);
res.send(buffer);
res.status(200).send(buffer);
}

4
pages/api/beatsaver/map.js
View File

@ -6,7 +6,7 @@ export default async function handler(req, res) {
const mapData = await Utils.getMapData(mapHash.replace("custom_level_", ""));
if (mapData === undefined) {
// Check if a map hash was provided
return res.json({ error: true, message: "Unknown map" });
return res.status(200).json({ error: true, message: "Unknown map" });
}
const data = {
// The maps data from the provided map hash
@ -15,5 +15,5 @@ export default async function handler(req, res) {
mapData.versions[0].coverURL.split("/")[3].split(".")[1]
}`,
};
res.json({ error: false, data: data });
res.status(200).json({ error: false, data: data });
}

70
pages/api/steamavatar.js
View File

@ -1,35 +1,55 @@
import fs from "fs";
import fetch from "node-fetch";
import path from "path";
import sharp from "sharp";
import cacheDir from "../../src/caches/SteamAvatarCacheDir";
import Utils from "../../src/utils/utils";
import { isValidSteamId } from "../../src/helpers/validateSteamId";
import RedisUtils from "../../src/utils/redisUtils";
const KEY = "STEAM_AVATAR_";
const ext = "jpg";
/**
*
* @param {Request} req
* @param {Response} res
* @returns
*/
export default async function handler(req, res) {
const steamId = req.query.steamid;
const isValid = await Utils.isValidSteamId(steamId);
if (isValid == true) {
return res.json({
status: "OK",
message: `Invalid steam id`,
const isValid = await isValidSteamId(steamId);
if (isValid == false) {
return res.status(404).json({
status: 404,
message: "Unknown Steam Avatar",
});
}
const imagePath = cacheDir + path.sep + steamId + ".jpg";
const exists = fs.existsSync(imagePath);
if (!exists) {
const data = await fetch(
`https://cdn.scoresaber.com/avatars/${steamId}.jpg`
);
let buffer = await data.buffer();
buffer = await sharp(buffer).resize(150, 150).toBuffer();
fs.writeFileSync(imagePath, buffer);
res.setHeader("Content-Type", "image/jpg");
res.send(buffer);
console.log('Steam Avatar Cache - Added avatar "' + steamId + '"');
return;
const exists = await RedisUtils.exists(`${KEY}${steamId}`);
if (exists) {
const data = await RedisUtils.getValue(`${KEY}${steamId}`);
const buffer = Buffer.from(data, "base64");
res.writeHead(200, {
"Content-Type": "image/" + ext,
"Content-Length": buffer.length,
"Cache-Status": "hit",
});
return res.end(buffer);
}
const buffer = fs.readFileSync(imagePath);
res.setHeader("Content-Type", "image/jpg");
res.send(buffer);
const data = await fetch(
`https://cdn.scoresaber.com/avatars/${steamId}.${ext}`
);
if (data.status === 404) {
return res.status(404).json({
status: 404,
message: "Unknown Steam Avatar",
});
}
let buffer = await data.buffer(); // Change to arrayBuffer at some point to make it shush
buffer = await sharp(buffer).resize(150, 150).toBuffer();
const bytes = buffer.toString("base64");
await RedisUtils.setValue(`${KEY}${steamId}`, bytes);
res.setHeader("Cache-Status", "miss");
res.setHeader("Content-Type", "image/" + ext);
res.status(200).send(buffer);
}

12
pages/api/validateid.js
View File

@ -1,17 +1,17 @@
import Utils from "../../src/utils/utils";
import { isValidSteamId } from "../../src/helpers/validateSteamId";
export default async function handler(req, res) {
const steamId = req.query.steamid;
if (!steamId) {
return res.json({
return res.status(404).json({
status: 404,
message: "Steam id not provided: Provide in the query.",
message: "Steam ID not provided",
});
}
const isValid = await Utils.isValidSteamId(steamId);
return res.json({
const isValid = await isValidSteamId(steamId);
return res.status(200).json({
status: "OK",
message: !isValid ? `Valid` : "Invalid",
message: isValid ? `Valid` : "Invalid",
});
}

3
src/caches/BLMapStarCache.js
View File

@ -1,3 +0,0 @@
const CACHE = new Map();
module.exports = CACHE;

10
src/caches/SongArtCacheDir.js
View File

@ -1,10 +0,0 @@
import fs from "fs";
import path from "path";
const cacheDir = process.cwd() + path.sep + "cache";
if (!fs.existsSync(cacheDir)) {
fs.mkdirSync(cacheDir);
console.log("Created art cache directory");
}
module.exports = cacheDir;

10
src/caches/SteamAvatarCacheDir.js
View File

@ -1,10 +0,0 @@
import fs from "fs";
import path from "path";
const cacheDir = process.cwd() + path.sep + "avatar_cache";
if (!fs.existsSync(cacheDir)) {
fs.mkdirSync(cacheDir);
console.log("Created avatar cache directory");
}
module.exports = cacheDir;

3
src/caches/SteamIdCache.js
View File

@ -1,3 +0,0 @@
const CACHE = new Map();
module.exports = CACHE;

40
src/helpers/validateSteamId.js Normal file
View File

@ -0,0 +1,40 @@
import { default as LeaderboardType } from "../consts/LeaderboardType";
import RedisUtils from "../utils/redisUtils";
import Utils from "../utils/utils";
const KEY = "VALID_STEAM_ID_";
const TO_CHECK = [
LeaderboardType.ScoreSaber.ApiUrl.PlayerData,
LeaderboardType.BeatLeader.ApiUrl.PlayerData,
];
async function isValidSteamId(steamId) {
if (!steamId) {
return false;
}
if (steamId.length !== 17) {
return false;
}
const exists = await RedisUtils.exists(`${KEY}${steamId}`);
if (exists) {
const data = await RedisUtils.getValue(`${KEY}${steamId}`);
return Boolean(data);
}
let valid = false;
for (const url of TO_CHECK) {
const isValid = await Utils.checkLeaderboard(url, steamId);
if (isValid) {
valid = true;
break;
}
}
await RedisUtils.setValue(`${KEY}${steamId}`, valid);
return valid;
}
module.exports = {
isValidSteamId,
};

47
src/utils/redisUtils.js Normal file
View File

@ -0,0 +1,47 @@
import Redis from "ioredis";
const client = new Redis({
port: process.env.REDIS_PORT,
host: process.env.REDIS_HOST,
password: process.env.REDIS_PASSWORD,
db: process.env.REDIS_DATABASE,
});
client.connect().catch(() => {});
async function setValue(key, value, expireAt = 86400) {
await client.set(key, value);
}
async function getValue(key) {
const yes = new Promise((reject, resolve) => {
client.get(key, (err, result) => {
if (err !== null) {
resolve(err);
} else {
reject(result);
}
});
});
const maybe = await yes;
return maybe;
}
async function exists(key) {
const yes = new Promise((reject, resolve) => {
client.exists(key, (err, result) => {
if (err !== null) {
resolve(err);
} else {
reject(result);
}
});
});
const maybe = await yes;
return maybe == 1 ? true : false;
}
module.exports = {
getValue,
setValue,
exists,
};

39
src/utils/utils.js
View File

@ -1,14 +1,8 @@
import SteamIdCache from "../../src/caches/SteamIdCache";
import {
default as LeaderboardType,
default as WebsiteTypes,
} from "../consts/LeaderboardType";
const TO_CHECK = [
LeaderboardType.ScoreSaber.ApiUrl.PlayerData,
LeaderboardType.BeatLeader.ApiUrl.PlayerData,
];
export default class Utils {
constructor() {}
@ -26,35 +20,6 @@ export default class Utils {
window.open(url, "_blank");
}
static async isValidSteamId(steamId) {
if (!steamId) {
return false;
}
if (steamId.length !== 17) {
return false;
}
if (SteamIdCache.has(steamId)) {
return SteamIdCache.get(steamId);
}
let invalid = false;
for (const url of TO_CHECK) {
const isValid = await Utils.checkLeaderboard(url, steamId);
if (isValid) {
break;
}
if (!isValid) {
invalid = true;
break;
}
}
SteamIdCache.set(steamId, invalid);
return invalid;
}
static async checkLeaderboard(url, steamId) {
const data = await fetch(url.replace("%s", steamId), {
headers: {
@ -74,4 +39,8 @@ export default class Utils {
}
return undefined;
}
static base64ToArrayBuffer(base64) {
return Uint8Array.from(atob(base64), (c) => c.charCodeAt(0));
}
}

57
yarn.lock
View File

@ -139,6 +139,11 @@
dependencies:
"@babel/runtime" "^7.6.2"
"@ioredis/commands@^1.1.1":
version "1.2.0"
resolved "https://registry.yarnpkg.com/@ioredis/commands/-/commands-1.2.0.tgz#6d61b3097470af1fdbbe622795b8921d42018e11"
integrity sha512-Sx1pU8EM64o2BrqNpEO1CNLtKQwyhuXuqyfH7oGKCk+1a33d2r5saW8zNwm3j6BTExtjrv2BxTgzzkMwts6vGg==
"@next/env@12.3.1":
version "12.3.1"
resolved "https://registry.npmjs.org/@next/env/-/env-12.3.1.tgz"
@ -1024,6 +1029,11 @@ clsx@^1.1.1:
version "1.2.1"
resolved "https://registry.npmjs.org/clsx/-/clsx-1.2.1.tgz"
cluster-key-slot@^1.1.0:
version "1.1.1"
resolved "https://registry.yarnpkg.com/cluster-key-slot/-/cluster-key-slot-1.1.1.tgz#10ccb9ded0729464b6d2e7d714b100a2d1259d43"
integrity sha512-rwHwUfXL40Chm1r08yrhU3qpUvdVlgkKNeyeGPOxnW8/SyVDvgRaed/Uz54AqWNaTCAThlj6QAs3TZcKI0xDEw==
color-convert@^2.0.1:
version "2.0.1"
resolved "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz"
@ -1127,6 +1137,11 @@ define-properties@^1.1.4:
has-property-descriptors "^1.0.0"
object-keys "^1.1.1"
denque@^2.0.1:
version "2.1.0"
resolved "https://registry.yarnpkg.com/denque/-/denque-2.1.0.tgz#e93e1a6569fb5e66f16a3c2a2964617d349d6ab1"
integrity sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw==
detect-libc@^2.0.0, detect-libc@^2.0.1:
version "2.0.1"
resolved "https://registry.npmjs.org/detect-libc/-/detect-libc-2.0.1.tgz"
@ -1705,6 +1720,21 @@ intl-messageformat@^10.1.0:
"@formatjs/icu-messageformat-parser" "2.1.7"
tslib "2.4.0"
ioredis@^5.2.3:
version "5.2.3"
resolved "https://registry.yarnpkg.com/ioredis/-/ioredis-5.2.3.tgz#d5b37eb13e643241660d6cee4eeb41a026cda8c0"
integrity sha512-gQNcMF23/NpvjCaa1b5YycUyQJ9rBNH2xP94LWinNpodMWVUPP5Ai/xXANn/SM7gfIvI62B5CCvZxhg5pOgyMw==
dependencies:
"@ioredis/commands" "^1.1.1"
cluster-key-slot "^1.1.0"
debug "^4.3.4"
denque "^2.0.1"
lodash.defaults "^4.2.0"
lodash.isarguments "^3.1.0"
redis-errors "^1.2.0"
redis-parser "^3.0.0"
standard-as-callback "^2.1.0"
is-arrayish@^0.3.1:
version "0.3.2"
resolved "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.3.2.tgz"
@ -1877,6 +1907,16 @@ locate-path@^6.0.0:
dependencies:
p-locate "^5.0.0"
lodash.defaults@^4.2.0:
version "4.2.0"
resolved "https://registry.yarnpkg.com/lodash.defaults/-/lodash.defaults-4.2.0.tgz#d09178716ffea4dde9e5fb7b37f6f0802274580c"
integrity sha512-qjxPLHd3r5DnsdGacqOMU6pb/avJzdh9tFX2ymgoZE27BmjXrNy/y4LoaiTeAb+O3gL8AfpJGtqfX/ae2leYYQ==
lodash.isarguments@^3.1.0:
version "3.1.0"
resolved "https://registry.yarnpkg.com/lodash.isarguments/-/lodash.isarguments-3.1.0.tgz#2f573d85c6a24289ff00663b491c1d338ff3458a"
integrity sha512-chi4NHZlZqZD18a0imDHnZPrDeBbTtVN7GXMwuGdRH9qotxAjYs3aVLKc7zNOG9eddR5Ksd8rvFEBc9SsggPpg==
lodash.merge@^4.6.2:
version "4.6.2"
resolved "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.2.tgz"
@ -2258,6 +2298,18 @@ readable-stream@~1.0.17, readable-stream@~1.0.27-1:
isarray "0.0.1"
string_decoder "~0.10.x"
redis-errors@^1.0.0, redis-errors@^1.2.0:
version "1.2.0"
resolved "https://registry.yarnpkg.com/redis-errors/-/redis-errors-1.2.0.tgz#eb62d2adb15e4eaf4610c04afe1529384250abad"
integrity sha512-1qny3OExCf0UvUV/5wpYKf2YwPcOqXzkwKKSmKHiE6ZMQs5heeE/c8eXK+PNllPvmjgAbfnsbpkGZWy8cBpn9w==
redis-parser@^3.0.0:
version "3.0.0"
resolved "https://registry.yarnpkg.com/redis-parser/-/redis-parser-3.0.0.tgz#b66d828cdcafe6b4b8a428a7def4c6bcac31c8b4"
integrity sha512-DJnGAeenTdpMEH6uAJRK/uiyEIH9WVsUmoLwzudwGJUwZPp80PDBWPHXSAGNPwNvIXAbe7MSUB1zQFugFml66A==
dependencies:
redis-errors "^1.0.0"
regenerator-runtime@^0.13.4:
version "0.13.9"
resolved "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.13.9.tgz"
@ -2406,6 +2458,11 @@ source-map-js@^1.0.2:
version "1.0.2"
resolved "https://registry.npmjs.org/source-map-js/-/source-map-js-1.0.2.tgz"
standard-as-callback@^2.1.0:
version "2.1.0"
resolved "https://registry.yarnpkg.com/standard-as-callback/-/standard-as-callback-2.1.0.tgz#8953fc05359868a77b5b9739a665c5977bb7df45"
integrity sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A==
string.prototype.matchall@^4.0.7:
version "4.0.7"
resolved "https://registry.yarnpkg.com/string.prototype.matchall/-/string.prototype.matchall-4.0.7.tgz#8e6ecb0d8a1fb1fda470d81acecb2dba057a481d"