Merge pull request #316 from mjezek09/feat/delete-unnecessary-pings

Option for old unnecessary pings deletion, separate table for player count records
This commit is contained in:
Nick Krecklow 2022-11-11 07:53:10 -08:00 committed by GitHub
commit 47d48aa948
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 117 additions and 2 deletions

@ -7,6 +7,10 @@
"pingAll": 3000, "pingAll": 3000,
"connectTimeout": 2500 "connectTimeout": 2500
}, },
"oldPingsCleanup": {
"enabled": false,
"interval": 3600000
},
"logFailedPings": true, "logFailedPings": true,
"logToDatabase": false, "logToDatabase": false,
"graphDuration": 86400000, "graphDuration": 86400000,

@ -22,7 +22,13 @@ class App {
// Setup database instance // Setup database instance
this.database.ensureIndexes(() => { this.database.ensureIndexes(() => {
this.database.loadGraphPoints(config.graphDuration, () => { this.database.loadGraphPoints(config.graphDuration, () => {
this.database.loadRecords(callback) this.database.loadRecords(() => {
if (config.oldPingsCleanup && config.oldPingsCleanup.enabled) {
this.database.initOldPingsDelete(callback)
} else {
callback()
}
})
}) })
}) })
} }

@ -52,6 +52,7 @@ class Database {
this._sql.serialize(() => { this._sql.serialize(() => {
this._sql.run('CREATE TABLE IF NOT EXISTS pings (timestamp BIGINT NOT NULL, ip TINYTEXT, playerCount MEDIUMINT)', handleError) this._sql.run('CREATE TABLE IF NOT EXISTS pings (timestamp BIGINT NOT NULL, ip TINYTEXT, playerCount MEDIUMINT)', handleError)
this._sql.run('CREATE TABLE IF NOT EXISTS players_record (timestamp BIGINT, ip TINYTEXT NOT NULL PRIMARY KEY, playerCount MEDIUMINT)', handleError)
this._sql.run('CREATE INDEX IF NOT EXISTS ip_index ON pings (ip, playerCount)', handleError) this._sql.run('CREATE INDEX IF NOT EXISTS ip_index ON pings (ip, playerCount)', handleError)
this._sql.run('CREATE INDEX IF NOT EXISTS timestamp_index on PINGS (timestamp)', [], err => { this._sql.run('CREATE INDEX IF NOT EXISTS timestamp_index on PINGS (timestamp)', [], err => {
handleError(err) handleError(err)
@ -130,6 +131,34 @@ class Database {
playerCount, playerCount,
timestamp: TimeTracker.toSeconds(timestamp) timestamp: TimeTracker.toSeconds(timestamp)
} }
} else {
this.getRecordLegacy(serverRegistration.data.ip, (hasRecordLegacy, playerCountLegacy, timestampLegacy) => {
// New values that will be inserted to table
let newTimestamp = null
let newPlayerCount = null
// If legacy record found, use it for insertion
if (hasRecordLegacy) {
newTimestamp = timestampLegacy
newPlayerCount = playerCountLegacy
}
// Set record to recordData
serverRegistration.recordData = {
playerCount: newPlayerCount,
timestamp: TimeTracker.toSeconds(newTimestamp)
}
// Insert server entry to records table
const statement = this._sql.prepare('INSERT INTO players_record (timestamp, ip, playerCount) VALUES (?, ?, ?)')
statement.run(newTimestamp, serverRegistration.data.ip, newPlayerCount, err => {
if (err) {
logger.error(`Cannot insert initial player count record of ${serverRegistration.data.ip}`)
throw err
}
})
statement.finalize()
})
} }
// Check if completedTasks hit the finish value // Check if completedTasks hit the finish value
@ -155,7 +184,7 @@ class Database {
} }
getRecord (ip, callback) { getRecord (ip, callback) {
this._sql.all('SELECT MAX(playerCount), timestamp FROM pings WHERE ip = ?', [ this._sql.all('SELECT playerCount, timestamp FROM players_record WHERE ip = ?', [
ip ip
], (err, data) => { ], (err, data) => {
if (err) { if (err) {
@ -163,6 +192,32 @@ class Database {
throw err throw err
} }
// Record not found
if (data[0] === undefined) {
// eslint-disable-next-line node/no-callback-literal
callback(false)
return
}
const playerCount = data[0].playerCount
const timestamp = data[0].timestamp
// Allow null player counts and timestamps, the frontend will safely handle them
// eslint-disable-next-line node/no-callback-literal
callback(true, playerCount, timestamp)
})
}
// Retrieves record from pings table, used for converting to separate table
getRecordLegacy (ip, callback) {
this._sql.all('SELECT MAX(playerCount), timestamp FROM pings WHERE ip = ?', [
ip
], (err, data) => {
if (err) {
logger.log('error', `Cannot get legacy ping record for ${ip}`)
throw err
}
// For empty results, data will be length 1 with [null, null] // For empty results, data will be length 1 with [null, null]
const playerCount = data[0]['MAX(playerCount)'] const playerCount = data[0]['MAX(playerCount)']
const timestamp = data[0].timestamp const timestamp = data[0].timestamp
@ -200,6 +255,53 @@ class Database {
}) })
statement.finalize() statement.finalize()
} }
updatePlayerCountRecord (ip, playerCount, timestamp) {
const statement = this._sql.prepare('UPDATE players_record SET timestamp = ?, playerCount = ? WHERE ip = ?')
statement.run(timestamp, playerCount, ip, err => {
if (err) {
logger.error(`Cannot update player count record of ${ip} at ${timestamp}`)
throw err
}
})
statement.finalize()
}
initOldPingsDelete (callback) {
// Delete old pings on startup
logger.info('Deleting old pings..')
this.deleteOldPings(() => {
const oldPingsCleanupInterval = config.oldPingsCleanup.interval || 3600000
if (oldPingsCleanupInterval > 0) {
// Delete old pings periodically
setInterval(() => this.deleteOldPings(), oldPingsCleanupInterval)
}
callback()
})
}
deleteOldPings (callback) {
// The oldest timestamp that will be kept
const oldestTimestamp = TimeTracker.getEpochMillis() - config.graphDuration
const deleteStart = TimeTracker.getEpochMillis()
const statement = this._sql.prepare('DELETE FROM pings WHERE timestamp < ?;')
statement.run(oldestTimestamp, err => {
if (err) {
logger.error('Cannot delete old pings')
throw err
} else {
const deleteTook = TimeTracker.getEpochMillis() - deleteStart
logger.info(`Old pings deleted in ${deleteTook}ms`)
if (callback) {
callback()
}
}
})
statement.finalize()
}
} }
module.exports = Database module.exports = Database

@ -63,6 +63,9 @@ class ServerRegistration {
// Append an updated recordData // Append an updated recordData
update.recordData = this.recordData update.recordData = this.recordData
// Update record in database
this._app.database.updatePlayerCountRecord(this.data.ip, resp.players.online, timestamp)
} }
if (this.updateFavicon(resp.favicon)) { if (this.updateFavicon(resp.favicon)) {