From a421243973cd76191063b6a393f52d540805bc22 Mon Sep 17 00:00:00 2001 From: Liam Date: Fri, 25 Oct 2024 17:37:56 +0100 Subject: [PATCH] ensure scores are always up-to-date for players --- projects/backend/src/index.ts | 55 +++------ .../backend/src/service/player.service.ts | 109 +++++++++++++++++- projects/backend/src/service/score.service.ts | 105 ++++++++++++----- projects/common/src/model/player.ts | 6 + .../common/src/service/impl/scoresaber.ts | 7 +- 5 files changed, 209 insertions(+), 73 deletions(-) diff --git a/projects/backend/src/index.ts b/projects/backend/src/index.ts index 37d3e54..381661e 100644 --- a/projects/backend/src/index.ts +++ b/projects/backend/src/index.ts @@ -11,12 +11,10 @@ import mongoose from "mongoose"; import PlayerController from "./controller/player.controller"; import { PlayerService } from "./service/player.service"; import { cron } from "@elysiajs/cron"; -import { scoresaberService } from "@ssr/common/service/impl/scoresaber"; -import { delay, isProduction } from "@ssr/common/utils/utils"; +import { isProduction } from "@ssr/common/utils/utils"; import ImageController from "./controller/image.controller"; import { ScoreService } from "./service/score.service"; import { Config } from "@ssr/common/config"; -import { PlayerDocument, PlayerModel } from "@ssr/common/model/player"; import ScoresController from "./controller/scores.controller"; import LeaderboardController from "./controller/leaderboard.controller"; import { getAppVersion } from "./common/app.util"; @@ -38,7 +36,9 @@ await mongoose.connect(Config.mongoUri!); // Connect to MongoDB // Connect to websockets connectScoresaberWebsocket({ onScore: async score => { - await ScoreService.trackScoreSaberScore(score); + await ScoreService.trackScoreSaberScore(score.score, score.leaderboard); + await ScoreService.updatePlayerScoresSet(score); + await ScoreService.notifyNumberOne(score); }, onDisconnect: async error => { @@ -67,41 +67,18 @@ app.use( pattern: "1 0 * * *", // Every day at 00:01 timezone: "Europe/London", // UTC time run: async () => { - const pages = 20; // top 1000 players - const cooldown = 60_000 / 250; // 250 requests per minute - - let toTrack: PlayerDocument[] = await PlayerModel.find({}); - const toRemoveIds: string[] = []; - - // loop through pages to fetch the top players - console.log(`Fetching ${pages} pages of players from ScoreSaber...`); - for (let i = 0; i < pages; i++) { - const pageNumber = i + 1; - console.log(`Fetching page ${pageNumber}...`); - const page = await scoresaberService.lookupPlayers(pageNumber); - if (page === undefined) { - console.log(`Failed to fetch players on page ${pageNumber}, skipping page...`); - await delay(cooldown); - continue; - } - for (const player of page.players) { - const foundPlayer = await PlayerService.getPlayer(player.id, true, player); - await PlayerService.trackScoreSaberPlayer(foundPlayer, player); - toRemoveIds.push(foundPlayer.id); - } - await delay(cooldown); - } - console.log(`Finished tracking player statistics for ${pages} pages, found ${toRemoveIds.length} players.`); - - // remove all players that have been tracked - toTrack = toTrack.filter(player => !toRemoveIds.includes(player.id)); - - console.log(`Tracking ${toTrack.length} player statistics...`); - for (const player of toTrack) { - await PlayerService.trackScoreSaberPlayer(player); - await delay(cooldown); - } - console.log("Finished tracking player statistics."); + await PlayerService.updatePlayerStatistics(); + }, + }) +); +app.use( + cron({ + name: "scores-background-refresh", + pattern: "0 4 * * *", // Every day at 04:00 + timezone: "Europe/London", // UTC time + protect: true, + run: async () => { + await PlayerService.refreshPlayerScores(); }, }) ); diff --git a/projects/backend/src/service/player.service.ts b/projects/backend/src/service/player.service.ts index 3d2bc1c..1465d01 100644 --- a/projects/backend/src/service/player.service.ts +++ b/projects/backend/src/service/player.service.ts @@ -5,10 +5,13 @@ import { scoresaberService } from "@ssr/common/service/impl/scoresaber"; import ScoreSaberPlayerToken from "@ssr/common/types/token/scoresaber/score-saber-player-token"; import { InternalServerError } from "../error/internal-server-error"; import { formatPp } from "@ssr/common/utils/number-utils"; -import { getPageFromRank, isProduction } from "@ssr/common/utils/utils"; +import { delay, getPageFromRank, isProduction } from "@ssr/common/utils/utils"; import { DiscordChannels, logToChannel } from "../bot/bot"; import { EmbedBuilder } from "discord.js"; import { AroundPlayer } from "@ssr/common/types/around-player"; +import { ScoreSort } from "@ssr/common/score/score-sort"; +import { getScoreSaberLeaderboardFromToken } from "@ssr/common/token-creators"; +import { ScoreService } from "./score.service"; export class PlayerService { /** @@ -243,4 +246,108 @@ export class PlayerService { return players.slice(start, end); } + + /** + * Ensures all player scores are up-to-date. + */ + public static async refreshPlayerScores() { + const cooldown = 60_000 / 250; // 250 requests per minute + console.log(`Refreshing player score data...`); + + const players = await PlayerModel.find({}); + console.log(`Found ${players.length} players to refresh.`); + + for (const player of players) { + console.log(`Refreshing scores for ${player.id}...`); + let page = 1; + let hasMorePages = true; + + while (hasMorePages) { + const scoresPage = await scoresaberService.lookupPlayerScores({ + playerId: player.id, + page: page, + limit: 100, + sort: ScoreSort.recent, + }); + + if (!scoresPage) { + console.warn(`Failed to fetch scores for ${player.id} on page ${page}.`); + break; + } + + let missingScores = 0; + for (const score of scoresPage.playerScores) { + const leaderboard = getScoreSaberLeaderboardFromToken(score.leaderboard); + const scoreSaberScore = await ScoreService.getScoreSaberScore( + player.id, + leaderboard.id + "", + leaderboard.difficulty.difficulty, + leaderboard.difficulty.characteristic, + score.score.baseScore + ); + + if (scoreSaberScore == null) { + missingScores++; + } + await ScoreService.trackScoreSaberScore(score.score, score.leaderboard, player.id); + } + + // Stop paginating if no scores are missing OR if player has seededScores marked true + if ((missingScores === 0 && player.seededScores) || page >= Math.ceil(scoresPage.metadata.total / 100)) { + hasMorePages = false; + } + + page++; + await delay(cooldown); // Cooldown between page requests + } + + // Mark player as seeded + player.seededScores = true; + await player.save(); + + console.log(`Finished refreshing scores for ${player.id}, total pages refreshed: ${page - 1}.`); + await delay(cooldown); // Cooldown between players + } + } + + /** + * Updates the player statistics for all players. + */ + public static async updatePlayerStatistics() { + const pages = 20; // top 1000 players + const cooldown = 60_000 / 250; // 250 requests per minute + + let toTrack: PlayerDocument[] = await PlayerModel.find({}); + const toRemoveIds: string[] = []; + + // loop through pages to fetch the top players + console.log(`Fetching ${pages} pages of players from ScoreSaber...`); + for (let i = 0; i < pages; i++) { + const pageNumber = i + 1; + console.log(`Fetching page ${pageNumber}...`); + const page = await scoresaberService.lookupPlayers(pageNumber); + if (page === undefined) { + console.log(`Failed to fetch players on page ${pageNumber}, skipping page...`); + await delay(cooldown); + continue; + } + for (const player of page.players) { + const foundPlayer = await PlayerService.getPlayer(player.id, true, player); + await PlayerService.trackScoreSaberPlayer(foundPlayer, player); + toRemoveIds.push(foundPlayer.id); + } + await delay(cooldown); + } + console.log(`Finished tracking player statistics for ${pages} pages, found ${toRemoveIds.length} players.`); + + // remove all players that have been tracked + toTrack = toTrack.filter(player => !toRemoveIds.includes(player.id)); + + console.log(`Tracking ${toTrack.length} player statistics...`); + for (const player of toTrack) { + await PlayerService.trackScoreSaberPlayer(player); + await delay(cooldown); + } + console.log("Finished tracking player statistics."); + } } diff --git a/projects/backend/src/service/score.service.ts b/projects/backend/src/service/score.service.ts index 2815bdf..2f22bae 100644 --- a/projects/backend/src/service/score.service.ts +++ b/projects/backend/src/service/score.service.ts @@ -29,6 +29,10 @@ import { ScoreType } from "@ssr/common/model/score/score"; import { getScoreSaberLeaderboardFromToken, getScoreSaberScoreFromToken } from "@ssr/common/token-creators"; import { ScoreSaberScoreModel } from "@ssr/common/model/score/impl/scoresaber-score"; import ScoreSaberLeaderboard from "@ssr/common/leaderboard/impl/scoresaber-leaderboard"; +import ScoreSaberScoreToken from "@ssr/common/types/token/scoresaber/score-saber-score-token"; +import ScoreSaberLeaderboardToken from "@ssr/common/types/token/scoresaber/score-saber-leaderboard-token"; +import { MapDifficulty } from "@ssr/common/score/map-difficulty"; +import { MapCharacteristic } from "@ssr/common/types/map-characteristic"; const playerScoresCache = new SSRCache({ ttl: 1000 * 60, // 1 minute @@ -118,15 +122,17 @@ export class ScoreService { } /** - * Tracks ScoreSaber score. + * Updates the players set scores count for today. * - * @param score the score to track - * @param leaderboard the leaderboard to track + * @param score the score */ - public static async trackScoreSaberScore({ score, leaderboard: leaderboardToken }: ScoreSaberPlayerScoreToken) { + public static async updatePlayerScoresSet({ + score: scoreToken, + leaderboard: leaderboardToken, + }: ScoreSaberPlayerScoreToken) { + const playerId = scoreToken.leaderboardPlayerInfo.id; + const leaderboard = getScoreSaberLeaderboardFromToken(leaderboardToken); - const playerId = score.leaderboardPlayerInfo.id; - const playerName = score.leaderboardPlayerInfo.name; const player: PlayerDocument | null = await PlayerModel.findById(playerId); // Player is not tracked, so ignore the score. if (player == undefined) { @@ -147,37 +153,49 @@ export class ScoreService { history.scores = scores; player.setStatisticHistory(today, history); - player.sortStatisticHistory(); - - // Save the changes - player.markModified("statisticHistory"); await player.save(); + } - const scoreToken = getScoreSaberScoreFromToken(score, leaderboard, playerId); - // eslint-disable-next-line @typescript-eslint/ban-ts-comment - // @ts-expect-error - delete scoreToken.playerInfo; + /** + * Tracks ScoreSaber score. + * + * @param scoreToken the score to track + * @param leaderboardToken the leaderboard for the score + * @param playerId the id of the player + */ + public static async trackScoreSaberScore( + scoreToken: ScoreSaberScoreToken, + leaderboardToken: ScoreSaberLeaderboardToken, + playerId?: string + ) { + playerId = playerId || scoreToken.leaderboardPlayerInfo.id; - // Check if the score already exists - if ( - await ScoreSaberScoreModel.exists({ - playerId: playerId, - leaderboardId: leaderboard.id, - score: scoreToken.score, - difficulty: leaderboard.difficulty.difficulty, - characteristic: leaderboard.difficulty.characteristic, - }) - ) { - console.log( - `Score already exists for "${playerName}"(${playerId}), scoreId=${scoreToken.scoreId}, score=${scoreToken.score}` - ); + const leaderboard = getScoreSaberLeaderboardFromToken(leaderboardToken); + const score = getScoreSaberScoreFromToken(scoreToken, leaderboard, playerId); + const player: PlayerDocument | null = await PlayerModel.findById(playerId); + // Player is not tracked, so ignore the score. + if (player == undefined) { return; } - await ScoreSaberScoreModel.create(scoreToken); - console.log( - `Tracked score and updated scores set statistic for "${playerName}"(${playerId}), scores today: ${scores.rankedScores} ranked, ${scores.unrankedScores} unranked` - ); + // The score has already been tracked, so ignore it. + if ( + (await this.getScoreSaberScore( + playerId, + leaderboard.id + "", + leaderboard.difficulty.difficulty, + leaderboard.difficulty.characteristic, + score.score + )) !== null + ) { + return; + } + + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-expect-error + delete score.playerInfo; + + await ScoreSaberScoreModel.create(score); } /** @@ -286,6 +304,31 @@ export class ScoreService { return additionalData.toObject(); } + /** + * Gets a ScoreSaber score. + * + * @param playerId the player who set the score + * @param leaderboardId the leaderboard id the score was set on + * @param difficulty the difficulty played + * @param characteristic the characteristic played + * @param score the score of the score set + */ + public static async getScoreSaberScore( + playerId: string, + leaderboardId: string, + difficulty: MapDifficulty, + characteristic: MapCharacteristic, + score: number + ) { + return ScoreSaberScoreModel.findOne({ + playerId: playerId, + leaderboardId: leaderboardId, + difficulty: difficulty, + characteristic: characteristic, + score: score, + }); + } + /** * Gets scores for a player. * diff --git a/projects/common/src/model/player.ts b/projects/common/src/model/player.ts index e981081..0cc24c9 100644 --- a/projects/common/src/model/player.ts +++ b/projects/common/src/model/player.ts @@ -20,6 +20,12 @@ export class Player { @prop() private statisticHistory?: Record; + /** + * Whether the player has their scores seeded. + */ + @prop() + public seededScores?: boolean; + /** * The date the player was last tracked. */ diff --git a/projects/common/src/service/impl/scoresaber.ts b/projects/common/src/service/impl/scoresaber.ts index 2d618a1..b845049 100644 --- a/projects/common/src/service/impl/scoresaber.ts +++ b/projects/common/src/service/impl/scoresaber.ts @@ -167,18 +167,21 @@ class ScoreSaberService extends Service { * * @param playerId the ID of the player to look up * @param sort the sort to use + * @param limit the amount of sores to fetch * @param page the page to get scores for - * @param search + * @param search the query to search for * @returns the scores of the player, or undefined */ public async lookupPlayerScores({ playerId, sort, + limit = 8, page, search, }: { playerId: string; sort: ScoreSort; + limit?: number; page: number; search?: string; useProxy?: boolean; @@ -189,7 +192,7 @@ class ScoreSaberService extends Service { ); const response = await this.fetch( LOOKUP_PLAYER_SCORES_ENDPOINT.replace(":id", playerId) - .replace(":limit", 8 + "") + .replace(":limit", limit + "") .replace(":sort", sort) .replace(":page", page + "") + (search ? `&search=${search}` : "") );