ensure scores are always up-to-date for players

projects/backend/src/index.ts

@@ -11,12 +11,10 @@ import mongoose from "mongoose";
 import PlayerController from "./controller/player.controller";
 import { PlayerService } from "./service/player.service";
 import { cron } from "@elysiajs/cron";
-import { scoresaberService } from "@ssr/common/service/impl/scoresaber";
-import { delay, isProduction } from "@ssr/common/utils/utils";
+import { isProduction } from "@ssr/common/utils/utils";
 import ImageController from "./controller/image.controller";
 import { ScoreService } from "./service/score.service";
 import { Config } from "@ssr/common/config";
-import { PlayerDocument, PlayerModel } from "@ssr/common/model/player";
 import ScoresController from "./controller/scores.controller";
 import LeaderboardController from "./controller/leaderboard.controller";
 import { getAppVersion } from "./common/app.util";
@@ -38,7 +36,9 @@ await mongoose.connect(Config.mongoUri!); // Connect to MongoDB
 // Connect to websockets
 connectScoresaberWebsocket({
   onScore: async score => {
-    await ScoreService.trackScoreSaberScore(score);
+    await ScoreService.trackScoreSaberScore(score.score, score.leaderboard);
+    await ScoreService.updatePlayerScoresSet(score);
+
     await ScoreService.notifyNumberOne(score);
   },
   onDisconnect: async error => {
@@ -67,41 +67,18 @@ app.use(
     pattern: "1 0 * * *", // Every day at 00:01
     timezone: "Europe/London", // UTC time
     run: async () => {
-      const pages = 20; // top 1000 players
-      const cooldown = 60_000 / 250; // 250 requests per minute
-
-      let toTrack: PlayerDocument[] = await PlayerModel.find({});
-      const toRemoveIds: string[] = [];
-
-      // loop through pages to fetch the top players
-      console.log(`Fetching ${pages} pages of players from ScoreSaber...`);
-      for (let i = 0; i < pages; i++) {
-        const pageNumber = i + 1;
-        console.log(`Fetching page ${pageNumber}...`);
-        const page = await scoresaberService.lookupPlayers(pageNumber);
-        if (page === undefined) {
-          console.log(`Failed to fetch players on page ${pageNumber}, skipping page...`);
-          await delay(cooldown);
-          continue;
-        }
-        for (const player of page.players) {
-          const foundPlayer = await PlayerService.getPlayer(player.id, true, player);
-          await PlayerService.trackScoreSaberPlayer(foundPlayer, player);
-          toRemoveIds.push(foundPlayer.id);
-        }
-        await delay(cooldown);
-      }
-      console.log(`Finished tracking player statistics for ${pages} pages, found ${toRemoveIds.length} players.`);
-
-      // remove all players that have been tracked
-      toTrack = toTrack.filter(player => !toRemoveIds.includes(player.id));
-
-      console.log(`Tracking ${toTrack.length} player statistics...`);
-      for (const player of toTrack) {
-        await PlayerService.trackScoreSaberPlayer(player);
-        await delay(cooldown);
-      }
-      console.log("Finished tracking player statistics.");
+      await PlayerService.updatePlayerStatistics();
+    },
+  })
+);
+app.use(
+  cron({
+    name: "scores-background-refresh",
+    pattern: "0 4 * * *", // Every day at 04:00
+    timezone: "Europe/London", // UTC time
+    protect: true,
+    run: async () => {
+      await PlayerService.refreshPlayerScores();
     },
   })
 );

projects/backend/src/service/player.service.ts

@@ -5,10 +5,13 @@ import { scoresaberService } from "@ssr/common/service/impl/scoresaber";
 import ScoreSaberPlayerToken from "@ssr/common/types/token/scoresaber/score-saber-player-token";
 import { InternalServerError } from "../error/internal-server-error";
 import { formatPp } from "@ssr/common/utils/number-utils";
-import { getPageFromRank, isProduction } from "@ssr/common/utils/utils";
+import { delay, getPageFromRank, isProduction } from "@ssr/common/utils/utils";
 import { DiscordChannels, logToChannel } from "../bot/bot";
 import { EmbedBuilder } from "discord.js";
 import { AroundPlayer } from "@ssr/common/types/around-player";
+import { ScoreSort } from "@ssr/common/score/score-sort";
+import { getScoreSaberLeaderboardFromToken } from "@ssr/common/token-creators";
+import { ScoreService } from "./score.service";
 
 export class PlayerService {
   /**
@@ -243,4 +246,108 @@ export class PlayerService {
 
     return players.slice(start, end);
   }
+
+  /**
+   * Ensures all player scores are up-to-date.
+   */
+  public static async refreshPlayerScores() {
+    const cooldown = 60_000 / 250; // 250 requests per minute
+    console.log(`Refreshing player score data...`);
+
+    const players = await PlayerModel.find({});
+    console.log(`Found ${players.length} players to refresh.`);
+
+    for (const player of players) {
+      console.log(`Refreshing scores for ${player.id}...`);
+      let page = 1;
+      let hasMorePages = true;
+
+      while (hasMorePages) {
+        const scoresPage = await scoresaberService.lookupPlayerScores({
+          playerId: player.id,
+          page: page,
+          limit: 100,
+          sort: ScoreSort.recent,
+        });
+
+        if (!scoresPage) {
+          console.warn(`Failed to fetch scores for ${player.id} on page ${page}.`);
+          break;
+        }
+
+        let missingScores = 0;
+        for (const score of scoresPage.playerScores) {
+          const leaderboard = getScoreSaberLeaderboardFromToken(score.leaderboard);
+          const scoreSaberScore = await ScoreService.getScoreSaberScore(
+            player.id,
+            leaderboard.id + "",
+            leaderboard.difficulty.difficulty,
+            leaderboard.difficulty.characteristic,
+            score.score.baseScore
+          );
+
+          if (scoreSaberScore == null) {
+            missingScores++;
+          }
+          await ScoreService.trackScoreSaberScore(score.score, score.leaderboard, player.id);
+        }
+
+        // Stop paginating if no scores are missing OR if player has seededScores marked true
+        if ((missingScores === 0 && player.seededScores) || page >= Math.ceil(scoresPage.metadata.total / 100)) {
+          hasMorePages = false;
+        }
+
+        page++;
+        await delay(cooldown); // Cooldown between page requests
+      }
+
+      // Mark player as seeded
+      player.seededScores = true;
+      await player.save();
+
+      console.log(`Finished refreshing scores for ${player.id}, total pages refreshed: ${page - 1}.`);
+      await delay(cooldown); // Cooldown between players
+    }
+  }
+
+  /**
+   * Updates the player statistics for all players.
+   */
+  public static async updatePlayerStatistics() {
+    const pages = 20; // top 1000 players
+    const cooldown = 60_000 / 250; // 250 requests per minute
+
+    let toTrack: PlayerDocument[] = await PlayerModel.find({});
+    const toRemoveIds: string[] = [];
+
+    // loop through pages to fetch the top players
+    console.log(`Fetching ${pages} pages of players from ScoreSaber...`);
+    for (let i = 0; i < pages; i++) {
+      const pageNumber = i + 1;
+      console.log(`Fetching page ${pageNumber}...`);
+      const page = await scoresaberService.lookupPlayers(pageNumber);
+      if (page === undefined) {
+        console.log(`Failed to fetch players on page ${pageNumber}, skipping page...`);
+        await delay(cooldown);
+        continue;
+      }
+      for (const player of page.players) {
+        const foundPlayer = await PlayerService.getPlayer(player.id, true, player);
+        await PlayerService.trackScoreSaberPlayer(foundPlayer, player);
+        toRemoveIds.push(foundPlayer.id);
+      }
+      await delay(cooldown);
+    }
+    console.log(`Finished tracking player statistics for ${pages} pages, found ${toRemoveIds.length} players.`);
+
+    // remove all players that have been tracked
+    toTrack = toTrack.filter(player => !toRemoveIds.includes(player.id));
+
+    console.log(`Tracking ${toTrack.length} player statistics...`);
+    for (const player of toTrack) {
+      await PlayerService.trackScoreSaberPlayer(player);
+      await delay(cooldown);
+    }
+    console.log("Finished tracking player statistics.");
+  }
 }

projects/backend/src/service/score.service.ts

@@ -29,6 +29,10 @@ import { ScoreType } from "@ssr/common/model/score/score";
 import { getScoreSaberLeaderboardFromToken, getScoreSaberScoreFromToken } from "@ssr/common/token-creators";
 import { ScoreSaberScoreModel } from "@ssr/common/model/score/impl/scoresaber-score";
 import ScoreSaberLeaderboard from "@ssr/common/leaderboard/impl/scoresaber-leaderboard";
+import ScoreSaberScoreToken from "@ssr/common/types/token/scoresaber/score-saber-score-token";
+import ScoreSaberLeaderboardToken from "@ssr/common/types/token/scoresaber/score-saber-leaderboard-token";
+import { MapDifficulty } from "@ssr/common/score/map-difficulty";
+import { MapCharacteristic } from "@ssr/common/types/map-characteristic";
 
 const playerScoresCache = new SSRCache({
   ttl: 1000 * 60, // 1 minute
@@ -118,15 +122,17 @@ export class ScoreService {
   }
 
   /**
-   * Tracks ScoreSaber score.
+   * Updates the players set scores count for today.
    *
-   * @param score the score to track
-   * @param leaderboard the leaderboard to track
+   * @param score the score
    */
-  public static async trackScoreSaberScore({ score, leaderboard: leaderboardToken }: ScoreSaberPlayerScoreToken) {
+  public static async updatePlayerScoresSet({
+    score: scoreToken,
+    leaderboard: leaderboardToken,
+  }: ScoreSaberPlayerScoreToken) {
+    const playerId = scoreToken.leaderboardPlayerInfo.id;
+
     const leaderboard = getScoreSaberLeaderboardFromToken(leaderboardToken);
-    const playerId = score.leaderboardPlayerInfo.id;
-    const playerName = score.leaderboardPlayerInfo.name;
     const player: PlayerDocument | null = await PlayerModel.findById(playerId);
     // Player is not tracked, so ignore the score.
     if (player == undefined) {
@@ -147,37 +153,49 @@ export class ScoreService {
 
     history.scores = scores;
     player.setStatisticHistory(today, history);
-    player.sortStatisticHistory();
-
-    // Save the changes
-    player.markModified("statisticHistory");
     await player.save();
+  }
 
-    const scoreToken = getScoreSaberScoreFromToken(score, leaderboard, playerId);
-    // eslint-disable-next-line @typescript-eslint/ban-ts-comment
-    // @ts-expect-error
-    delete scoreToken.playerInfo;
-
-    // Check if the score already exists
-    if (
-      await ScoreSaberScoreModel.exists({
-        playerId: playerId,
-        leaderboardId: leaderboard.id,
-        score: scoreToken.score,
-        difficulty: leaderboard.difficulty.difficulty,
-        characteristic: leaderboard.difficulty.characteristic,
-      })
+  /**
+   * Tracks ScoreSaber score.
+   *
+   * @param scoreToken the score to track
+   * @param leaderboardToken the leaderboard for the score
+   * @param playerId the id of the player
+   */
+  public static async trackScoreSaberScore(
+    scoreToken: ScoreSaberScoreToken,
+    leaderboardToken: ScoreSaberLeaderboardToken,
+    playerId?: string
   ) {
-      console.log(
-        `Score already exists for "${playerName}"(${playerId}), scoreId=${scoreToken.scoreId}, score=${scoreToken.score}`
-      );
+    playerId = playerId || scoreToken.leaderboardPlayerInfo.id;
+
+    const leaderboard = getScoreSaberLeaderboardFromToken(leaderboardToken);
+    const score = getScoreSaberScoreFromToken(scoreToken, leaderboard, playerId);
+    const player: PlayerDocument | null = await PlayerModel.findById(playerId);
+    // Player is not tracked, so ignore the score.
+    if (player == undefined) {
       return;
     }
 
-    await ScoreSaberScoreModel.create(scoreToken);
-    console.log(
-      `Tracked score and updated scores set statistic for "${playerName}"(${playerId}), scores today: ${scores.rankedScores} ranked, ${scores.unrankedScores} unranked`
-    );
+    // The score has already been tracked, so ignore it.
+    if (
+      (await this.getScoreSaberScore(
+        playerId,
+        leaderboard.id + "",
+        leaderboard.difficulty.difficulty,
+        leaderboard.difficulty.characteristic,
+        score.score
+      )) !== null
+    ) {
+      return;
+    }
+
+    // eslint-disable-next-line @typescript-eslint/ban-ts-comment
+    // @ts-expect-error
+    delete score.playerInfo;
+
+    await ScoreSaberScoreModel.create(score);
   }
 
   /**
@@ -286,6 +304,31 @@ export class ScoreService {
     return additionalData.toObject();
   }
 
+  /**
+   * Gets a ScoreSaber score.
+   *
+   * @param playerId the player who set the score
+   * @param leaderboardId the leaderboard id the score was set on
+   * @param difficulty the difficulty played
+   * @param characteristic the characteristic played
+   * @param score the score of the score set
+   */
+  public static async getScoreSaberScore(
+    playerId: string,
+    leaderboardId: string,
+    difficulty: MapDifficulty,
+    characteristic: MapCharacteristic,
+    score: number
+  ) {
+    return ScoreSaberScoreModel.findOne({
+      playerId: playerId,
+      leaderboardId: leaderboardId,
+      difficulty: difficulty,
+      characteristic: characteristic,
+      score: score,
+    });
+  }
+
   /**
    * Gets scores for a player.
    *

projects/common/src/model/player.ts

@@ -20,6 +20,12 @@ export class Player {
   @prop()
   private statisticHistory?: Record<string, PlayerHistory>;
 
+  /**
+   * Whether the player has their scores seeded.
+   */
+  @prop()
+  public seededScores?: boolean;
+
   /**
    * The date the player was last tracked.
    */

projects/common/src/service/impl/scoresaber.ts

@@ -167,18 +167,21 @@ class ScoreSaberService extends Service {
    *
    * @param playerId the ID of the player to look up
    * @param sort the sort to use
+   * @param limit the amount of sores to fetch
    * @param page the page to get scores for
-   * @param search
+   * @param search the query to search for
    * @returns the scores of the player, or undefined
    */
   public async lookupPlayerScores({
     playerId,
     sort,
+    limit = 8,
     page,
     search,
   }: {
     playerId: string;
     sort: ScoreSort;
+    limit?: number;
     page: number;
     search?: string;
     useProxy?: boolean;
@@ -189,7 +192,7 @@ class ScoreSaberService extends Service {
     );
     const response = await this.fetch<ScoreSaberPlayerScoresPageToken>(
       LOOKUP_PLAYER_SCORES_ENDPOINT.replace(":id", playerId)
-        .replace(":limit", 8 + "")
+        .replace(":limit", limit + "")
         .replace(":sort", sort)
         .replace(":page", page + "") + (search ? `&search=${search}` : "")
     );