Merge pull request #316 from mjezek09/feat/delete-unnecessary-pings

Option for old unnecessary pings deletion, separate table for player count records

config.json

@@ -7,6 +7,10 @@
 		"pingAll": 3000,
 		"connectTimeout": 2500
 	},
+	"oldPingsCleanup": {
+		"enabled": false,
+		"interval": 3600000
+	},
 	"logFailedPings": true,
 	"logToDatabase": false,
 	"graphDuration": 86400000,

lib/app.js

@@ -22,7 +22,13 @@ class App {
     // Setup database instance
     this.database.ensureIndexes(() => {
       this.database.loadGraphPoints(config.graphDuration, () => {
-        this.database.loadRecords(callback)
+        this.database.loadRecords(() => {
+          if (config.oldPingsCleanup && config.oldPingsCleanup.enabled) {
+            this.database.initOldPingsDelete(callback)
+          } else {
+            callback()
+          }
+        })
       })
     })
   }

lib/database.js

@@ -52,6 +52,7 @@ class Database {
 
     this._sql.serialize(() => {
       this._sql.run('CREATE TABLE IF NOT EXISTS pings (timestamp BIGINT NOT NULL, ip TINYTEXT, playerCount MEDIUMINT)', handleError)
+      this._sql.run('CREATE TABLE IF NOT EXISTS players_record (timestamp BIGINT, ip TINYTEXT NOT NULL PRIMARY KEY, playerCount MEDIUMINT)', handleError)
       this._sql.run('CREATE INDEX IF NOT EXISTS ip_index ON pings (ip, playerCount)', handleError)
       this._sql.run('CREATE INDEX IF NOT EXISTS timestamp_index on PINGS (timestamp)', [], err => {
         handleError(err)
@@ -130,6 +131,34 @@ class Database {
             playerCount,
             timestamp: TimeTracker.toSeconds(timestamp)
           }
+        } else {
+          this.getRecordLegacy(serverRegistration.data.ip, (hasRecordLegacy, playerCountLegacy, timestampLegacy) => {
+            // New values that will be inserted to table
+            let newTimestamp = null
+            let newPlayerCount = null
+
+            // If legacy record found, use it for insertion
+            if (hasRecordLegacy) {
+              newTimestamp = timestampLegacy
+              newPlayerCount = playerCountLegacy
+            }
+
+            // Set record to recordData
+            serverRegistration.recordData = {
+              playerCount: newPlayerCount,
+              timestamp: TimeTracker.toSeconds(newTimestamp)
+            }
+
+            // Insert server entry to records table
+            const statement = this._sql.prepare('INSERT INTO players_record (timestamp, ip, playerCount) VALUES (?, ?, ?)')
+            statement.run(newTimestamp, serverRegistration.data.ip, newPlayerCount, err => {
+              if (err) {
+                logger.error(`Cannot insert initial player count record of ${serverRegistration.data.ip}`)
+                throw err
+              }
+            })
+            statement.finalize()
+          })
         }
 
         // Check if completedTasks hit the finish value
@@ -155,7 +184,7 @@ class Database {
   }
 
   getRecord (ip, callback) {
-    this._sql.all('SELECT MAX(playerCount), timestamp FROM pings WHERE ip = ?', [
+    this._sql.all('SELECT playerCount, timestamp FROM players_record WHERE ip = ?', [
       ip
     ], (err, data) => {
       if (err) {
@@ -163,6 +192,32 @@ class Database {
         throw err
       }
 
+      // Record not found
+      if (data[0] === undefined) {
+        // eslint-disable-next-line node/no-callback-literal
+        callback(false)
+        return
+      }
+
+      const playerCount = data[0].playerCount
+      const timestamp = data[0].timestamp
+
+      // Allow null player counts and timestamps, the frontend will safely handle them
+      // eslint-disable-next-line node/no-callback-literal
+      callback(true, playerCount, timestamp)
+    })
+  }
+
+  // Retrieves record from pings table, used for converting to separate table
+  getRecordLegacy (ip, callback) {
+    this._sql.all('SELECT MAX(playerCount), timestamp FROM pings WHERE ip = ?', [
+      ip
+    ], (err, data) => {
+      if (err) {
+        logger.log('error', `Cannot get legacy ping record for ${ip}`)
+        throw err
+      }
+
       // For empty results, data will be length 1 with [null, null]
       const playerCount = data[0]['MAX(playerCount)']
       const timestamp = data[0].timestamp
@@ -200,6 +255,53 @@ class Database {
     })
     statement.finalize()
   }
+
+  updatePlayerCountRecord (ip, playerCount, timestamp) {
+    const statement = this._sql.prepare('UPDATE players_record SET timestamp = ?, playerCount = ? WHERE ip = ?')
+    statement.run(timestamp, playerCount, ip, err => {
+      if (err) {
+        logger.error(`Cannot update player count record of ${ip} at ${timestamp}`)
+        throw err
+      }
+    })
+    statement.finalize()
+  }
+
+  initOldPingsDelete (callback) {
+    // Delete old pings on startup
+    logger.info('Deleting old pings..')
+    this.deleteOldPings(() => {
+      const oldPingsCleanupInterval = config.oldPingsCleanup.interval || 3600000
+      if (oldPingsCleanupInterval > 0) {
+        // Delete old pings periodically
+        setInterval(() => this.deleteOldPings(), oldPingsCleanupInterval)
+      }
+
+      callback()
+    })
+  }
+
+  deleteOldPings (callback) {
+    // The oldest timestamp that will be kept
+    const oldestTimestamp = TimeTracker.getEpochMillis() - config.graphDuration
+
+    const deleteStart = TimeTracker.getEpochMillis()
+    const statement = this._sql.prepare('DELETE FROM pings WHERE timestamp < ?;')
+    statement.run(oldestTimestamp, err => {
+      if (err) {
+        logger.error('Cannot delete old pings')
+        throw err
+      } else {
+        const deleteTook = TimeTracker.getEpochMillis() - deleteStart
+        logger.info(`Old pings deleted in ${deleteTook}ms`)
+
+        if (callback) {
+          callback()
+        }
+      }
+    })
+    statement.finalize()
+  }
 }
 
 module.exports = Database

lib/servers.js

@@ -63,6 +63,9 @@ class ServerRegistration {
 
         // Append an updated recordData
         update.recordData = this.recordData
+
+        // Update record in database
+        this._app.database.updatePlayerCountRecord(this.data.ip, resp.players.online, timestamp)
       }
 
       if (this.updateFavicon(resp.favicon)) {