Delete old unnecessary ping records, store player count records in separate table

config.json

@@ -9,6 +9,7 @@
 	},
 	"logFailedPings": true,
 	"logToDatabase": false,
+	"deleteOldPings": false,
 	"graphDuration": 86400000,
 	"serverGraphDuration": 180000
 }

lib/app.js

@@ -22,6 +22,10 @@ class App {
     // Setup database instance
     this.database.ensureIndexes(() => {
       this.database.loadGraphPoints(config.graphDuration, () => {
+        if (config.deleteOldPings) {
+          this.database.initOldPingsDeleter()
+        }
+
         this.database.loadRecords(callback)
       })
     })

lib/database.js

@@ -52,6 +52,7 @@ class Database {
 
     this._sql.serialize(() => {
       this._sql.run('CREATE TABLE IF NOT EXISTS pings (timestamp BIGINT NOT NULL, ip TINYTEXT, playerCount MEDIUMINT)', handleError)
+      this._sql.run('CREATE TABLE IF NOT EXISTS players_record (timestamp BIGINT, ip TINYTEXT, playerCount MEDIUMINT)', handleError)
       this._sql.run('CREATE INDEX IF NOT EXISTS ip_index ON pings (ip, playerCount)', handleError)
       this._sql.run('CREATE INDEX IF NOT EXISTS timestamp_index on PINGS (timestamp)', [], err => {
         handleError(err)
@@ -130,6 +131,34 @@ class Database {
             playerCount,
             timestamp: TimeTracker.toSeconds(timestamp)
           }
+        } else {
+          this.getRecordLegacy(serverRegistration.data.ip, (hasRecordLegacy, playerCountLegacy, timestampLegacy) => {
+            // New values that will be inserted to table
+            let newTimestamp = null
+            let newPlayerCount = null
+
+            // If legacy record found, use it for insertion
+            if (hasRecordLegacy) {
+              newTimestamp = timestampLegacy
+              newPlayerCount = playerCountLegacy
+            }
+
+            // Set record to recordData
+            serverRegistration.recordData = {
+              playerCount: newPlayerCount,
+              timestamp: TimeTracker.toSeconds(newTimestamp)
+            }
+
+            // Insert server entry to records table
+            const statement = this._sql.prepare('INSERT INTO players_record (timestamp, ip, playerCount) VALUES (?, ?, ?)')
+            statement.run(newTimestamp, serverRegistration.data.ip, newPlayerCount, err => {
+              if (err) {
+                logger.error(`Cannot insert initial player count record of ${serverRegistration.data.ip}`)
+                throw err
+              }
+            })
+            statement.finalize()
+          })
         }
 
         // Check if completedTasks hit the finish value
@@ -155,7 +184,7 @@ class Database {
   }
 
   getRecord (ip, callback) {
-    this._sql.all('SELECT MAX(playerCount), timestamp FROM pings WHERE ip = ?', [
+    this._sql.all('SELECT playerCount, timestamp FROM players_record WHERE ip = ?', [
       ip
     ], (err, data) => {
       if (err) {
@@ -163,6 +192,32 @@ class Database {
         throw err
       }
 
+      // Record not found
+      if (data[0] === undefined) {
+        // eslint-disable-next-line node/no-callback-literal
+        callback(false)
+        return
+      }
+
+      const playerCount = data[0].playerCount
+      const timestamp = data[0].timestamp
+
+      // Allow null player counts and timestamps, the frontend will safely handle them
+      // eslint-disable-next-line node/no-callback-literal
+      callback(true, playerCount, timestamp)
+    })
+  }
+
+  // Retrieves record from pings table, used for converting to separate table
+  getRecordLegacy (ip, callback) {
+    this._sql.all('SELECT MAX(playerCount), timestamp FROM pings WHERE ip = ?', [
+      ip
+    ], (err, data) => {
+      if (err) {
+        logger.log('error', `Cannot get legacy ping record for ${ip}`)
+        throw err
+      }
+
       // For empty results, data will be length 1 with [null, null]
       const playerCount = data[0]['MAX(playerCount)']
       const timestamp = data[0].timestamp
@@ -200,6 +255,40 @@ class Database {
     })
     statement.finalize()
   }
+
+  updatePlayerCountRecord (ip, playerCount, timestamp) {
+    const statement = this._sql.prepare('UPDATE players_record SET timestamp = ?, playerCount = ? WHERE ip = ?')
+    statement.run(timestamp, playerCount, ip, err => {
+      if (err) {
+        logger.error(`Cannot update player count record of ${ip} at ${timestamp}`)
+        throw err
+      }
+    })
+    statement.finalize()
+  }
+
+  initOldPingsDeleter () {
+    // Delete old records every hour
+    setInterval(() => this.deleteOldPingRecords(), 3600000)
+  }
+
+  deleteOldPingRecords () {
+    // The oldest timestamp that will be kept
+    const oldestTimestamp = TimeTracker.getEpochMillis() - config.graphDuration
+
+    const deleteStart = TimeTracker.getEpochMillis()
+    const statement = this._sql.prepare('DELETE FROM pings WHERE timestamp < ?;')
+    statement.run(oldestTimestamp, err => {
+      if (err) {
+        logger.error('Cannot delete old ping records')
+        throw err
+      } else {
+        const deleteTook = TimeTracker.getEpochMillis() - deleteStart
+        logger.info(`Old ping records deleted in ${deleteTook}ms`)
+      }
+    })
+    statement.finalize()
+  }
 }
 
 module.exports = Database

lib/servers.js

@@ -63,6 +63,9 @@ class ServerRegistration {
 
         // Append an updated recordData
         update.recordData = this.recordData
+
+        // Update record in database
+        this._app.database.updatePlayerCountRecord(this.data.ip, resp.players.online, timestamp)
       }
 
       if (this.updateFavicon(resp.favicon)) {