From a3c88dc0c58b086e93b2da0cff3bb69d1c25f0cc Mon Sep 17 00:00:00 2001 From: Nick Krecklow Date: Thu, 7 May 2020 23:46:59 -0500 Subject: [PATCH 01/30] add ability to skip unfurlSrv calls to avoid ping cost --- config.json | 3 +++ lib/ping.js | 5 +++++ main.js | 4 ++++ 3 files changed, 12 insertions(+) diff --git a/config.json b/config.json index 3fd9359..6a4f90a 100644 --- a/config.json +++ b/config.json @@ -9,6 +9,9 @@ "pingAll": 3000, "connectTimeout": 2500 }, + "performance": { + "skipUnfurlSrv": false + }, "logToDatabase": false, "graphDuration": 86400000 } diff --git a/lib/ping.js b/lib/ping.js index 65ea8af..2f8216b 100644 --- a/lib/ping.js +++ b/lib/ping.js @@ -54,6 +54,11 @@ function ping (host, port, type, timeout, callback, version) { } function unfurlSrv (hostname, port, callback) { + if (config.performance && config.performance.skipUnfurlSrv) { + callback(hostname, port) + return + } + dns.resolveSrv('_minecraft._tcp.' + hostname, (_, records) => { if (!records || records.length < 1) { callback(hostname, port) diff --git a/main.js b/main.js index 44d05d4..e972b92 100644 --- a/main.js +++ b/main.js @@ -25,6 +25,10 @@ servers.forEach((server, serverId) => { app.serverRegistrations.push(new ServerRegistration(serverId, server)) }) +if (config.performance && config.performance.skipUnfurlSrv) { + logger.log('warn', '"performance.skipUnfurlSrv" is enabled. Any configured hosts using SRV records may not properly resolve.') +} + if (!config.logToDatabase) { logger.log('warn', 'Database logging is not enabled. You can enable it by setting "logToDatabase" to true in config.json. This requires sqlite3 to be installed.') From 3ddb2c9a08c972fa5c16de56ebb008e4619614c1 Mon Sep 17 00:00:00 2001 From: Nick Krecklow Date: Fri, 8 May 2020 00:36:15 -0500 Subject: [PATCH 02/30] work on bulking updateServer payloads and single timestamps --- assets/js/app.js | 2 +- assets/js/servers.js | 8 ++--- assets/js/socket.js | 37 ++++++++++++----------- lib/ping.js | 70 +++++++++++++++++++++++++++----------------- lib/servers.js | 26 +++++++++++++--- 5 files changed, 90 insertions(+), 53 deletions(-) diff --git a/assets/js/app.js b/assets/js/app.js index 0bd5d05..a3b3ab6 100644 --- a/assets/js/app.js +++ b/assets/js/app.js @@ -121,7 +121,7 @@ export class App { // Handle the last known state (if any) as an incoming update // This triggers the main update pipeline and enables centralized update handling - serverRegistration.updateServerStatus(latestPing, true, this.publicConfig.minecraftVersions) + serverRegistration.updateServerStatus(latestPing, latestPing.timestamp, true, this.publicConfig.minecraftVersions) // Allow the ServerRegistration to bind any DOM events with app instance context serverRegistration.initEventListeners() diff --git a/assets/js/servers.js b/assets/js/servers.js index 8c530e0..f0717da 100644 --- a/assets/js/servers.js +++ b/assets/js/servers.js @@ -113,14 +113,14 @@ export class ServerRegistration { this._plotInstance = $.plot('#chart_' + this.serverId, [this._graphData], SERVER_GRAPH_OPTIONS) } - handlePing (payload, pushToGraph) { + handlePing (payload, timestamp, pushToGraph) { if (payload.result) { this.playerCount = payload.result.players.online if (pushToGraph) { // Only update graph for successful pings // This intentionally pauses the server graph when pings begin to fail - this._graphData.push([payload.timestamp, this.playerCount]) + this._graphData.push([timestamp, this.playerCount]) // Trim graphData to within the max length by shifting out the leading elements if (this._graphData.length > SERVER_GRAPH_DATA_MAX_LENGTH) { @@ -169,10 +169,10 @@ export class ServerRegistration { this.lastPeakData = data } - updateServerStatus (ping, isInitialUpdate, minecraftVersions) { + updateServerStatus (ping, timestamp, isInitialUpdate, minecraftVersions) { // Only pushToGraph when initialUpdate === false // Otherwise the ping value is pushed into the graphData when already present - this.handlePing(ping, !isInitialUpdate) + this.handlePing(ping, timestamp, !isInitialUpdate) if (ping.versions) { const versionsElement = document.getElementById('version_' + this.serverId) diff --git a/assets/js/socket.js b/assets/js/socket.js index 256e9a0..9a30864 100644 --- a/assets/js/socket.js +++ b/assets/js/socket.js @@ -79,27 +79,30 @@ export class SocketManager { break - case 'updateServer': { - // The backend may send "update" events prior to receiving all "add" events - // A server has only been added once it's ServerRegistration is defined - // Checking undefined protects from this race condition - const serverRegistration = this._app.serverRegistry.getServerRegistration(payload.serverId) + case 'updateServers': { + for (let serverId = 0; serverId < payload.updates.length; serverId++) { + // The backend may send "update" events prior to receiving all "add" events + // A server has only been added once it's ServerRegistration is defined + // Checking undefined protects from this race condition + const serverRegistration = this._app.serverRegistry.getServerRegistration(serverId) + const serverUpdate = payload.updates[serverId] - if (serverRegistration) { - serverRegistration.updateServerStatus(payload, false, this._app.publicConfig.minecraftVersions) - } + if (serverRegistration) { + serverRegistration.updateServerStatus(serverUpdate, payload.timestamp, false, this._app.publicConfig.minecraftVersions) + } - // Use update payloads to conditionally append data to graph - // Skip any incoming updates if the graph is disabled - if (payload.updateHistoryGraph && this._app.graphDisplayManager.isVisible) { - // Update may not be successful, safely append 0 points - const playerCount = payload.result ? payload.result.players.online : 0 + // Use update payloads to conditionally append data to graph + // Skip any incoming updates if the graph is disabled + if (serverUpdate.updateHistoryGraph && this._app.graphDisplayManager.isVisible) { + // Update may not be successful, safely append 0 points + const playerCount = serverUpdate.result ? serverUpdate.result.players.online : 0 - this._app.graphDisplayManager.addGraphPoint(serverRegistration.serverId, payload.timestamp, playerCount) + this._app.graphDisplayManager.addGraphPoint(serverRegistration.serverId, payload.timestamp, playerCount) - // Only redraw the graph if not mutating hidden data - if (serverRegistration.isVisible) { - this._app.graphDisplayManager.requestRedraw() + // Only redraw the graph if not mutating hidden data + if (serverRegistration.isVisible) { + this._app.graphDisplayManager.requestRedraw() + } } } break diff --git a/lib/ping.js b/lib/ping.js index 2f8216b..31b0548 100644 --- a/lib/ping.js +++ b/lib/ping.js @@ -97,6 +97,40 @@ class PingController { } pingAll = () => { + const timestamp = new Date().getTime() + + this.startPingTasks(results => { + const updates = [] + + for (const serverRegistration of this._app.serverRegistrations) { + const result = results[serverRegistration.serverId] + + // Log to database if enabled + if (config.logToDatabase) { + const playerCount = result.resp ? result.resp.players.online : 0 + this._app.database.insertPing(serverRegistration.data.ip, timestamp, playerCount) + } + + // Generate a combined update payload + // This includes any modified fields and flags used by the frontend + // This will not be cached and can contain live metadata + const update = serverRegistration.handlePing(timestamp, result.resp, result.err, result.version) + updates[serverRegistration.serverId] = update + } + + // Send object since updates uses serverIds as keys + // Send a single timestamp entry since it is shared + this._app.server.broadcast(MessageOf('updateServers', { + timestamp, + updates + })) + }) + } + + startPingTasks = (callback) => { + const results = [] + let remainingTasks = this._app.serverRegistrations.length + for (const serverRegistration of this._app.serverRegistrations) { const version = serverRegistration.getNextProtocolVersion() @@ -105,36 +139,18 @@ class PingController { logger.log('error', 'Failed to ping %s: %s', serverRegistration.data.ip, err.message) } - this.handlePing(serverRegistration, resp, err, version) + results[serverRegistration.serverId] = { + resp, + err, + version + } + + if (--remainingTasks === 0) { + callback(results) + } }, version.protocolId) } } - - handlePing (serverRegistration, resp, err, version) { - const timestamp = new Date().getTime() - - serverRegistration.addPing(timestamp, resp) - - let updateHistoryGraph = false - - if (config.logToDatabase) { - const playerCount = resp ? resp.players.online : 0 - - // Log to database - this._app.database.insertPing(serverRegistration.data.ip, timestamp, playerCount) - - if (serverRegistration.addGraphPoint(resp !== undefined, playerCount, timestamp)) { - updateHistoryGraph = true - } - } - - // Generate a combined update payload - // This includes any modified fields and flags used by the frontend - // This will not be cached and can contain live metadata - const updateMessage = serverRegistration.getUpdate(timestamp, resp, err, version, updateHistoryGraph) - - this._app.server.broadcast(MessageOf('updateServer', updateMessage)) - } } module.exports = PingController diff --git a/lib/servers.js b/lib/servers.js index 7668866..5d18729 100644 --- a/lib/servers.js +++ b/lib/servers.js @@ -14,12 +14,30 @@ class ServerRegistration { this._pingHistory = [] } - getUpdate (timestamp, resp, err, version, updateHistoryGraph) { - const update = { - serverId: this.serverId, - timestamp: timestamp + handlePing (timestamp, resp, err, version) { + // Store into in-memory ping data + this.addPing(timestamp, resp) + + // Only notify the frontend to append to the historical graph + // if both the graphing behavior is enabled and the backend agrees + // that the ping is eligible for addition + let updateHistoryGraph = false + + if (config.logToDatabase) { + const playerCount = resp ? resp.players.online : 0 + + if (this.addGraphPoint(resp !== undefined, playerCount, timestamp)) { + updateHistoryGraph = true + } } + // Delegate out update payload generation + return this.getUpdate(timestamp, resp, err, version, updateHistoryGraph) + } + + getUpdate (timestamp, resp, err, version, updateHistoryGraph) { + const update = {} + if (resp) { if (resp.version && this.updateProtocolVersionCompat(resp.version, version.protocolId, version.protocolIndex)) { // Append an updated version listing From c2494af82db2cd55e672bb16d1c6d6e3d8ff3844 Mon Sep 17 00:00:00 2001 From: Nick Krecklow Date: Fri, 8 May 2020 01:22:07 -0500 Subject: [PATCH 03/30] use a single, shared timestamps array between all pings --- assets/js/app.js | 6 +++--- assets/js/servers.js | 35 +++++++++++++++++------------------ assets/js/socket.js | 6 +++++- lib/app.js | 3 +++ lib/ping.js | 2 +- lib/servers.js | 10 +++------- lib/time.js | 23 +++++++++++++++++++++++ 7 files changed, 55 insertions(+), 30 deletions(-) create mode 100644 lib/time.js diff --git a/assets/js/app.js b/assets/js/app.js index a3b3ab6..8458157 100644 --- a/assets/js/app.js +++ b/assets/js/app.js @@ -102,7 +102,7 @@ export class App { .reduce((sum, current) => sum + current, 0) } - addServer = (pings) => { + addServer = (pings, timestampPoints) => { // Even if the backend has never pinged the server, the frontend is promised a placeholder object. // result = undefined // error = defined with "Waiting" description @@ -114,14 +114,14 @@ export class App { // Push the historical data into the graph // This will trim and format the data so it is ready for the graph to render once init - serverRegistration.addGraphPoints(pings) + serverRegistration.addGraphPoints(pings, timestampPoints) // Create the plot instance internally with the restructured and cleaned data serverRegistration.buildPlotInstance() // Handle the last known state (if any) as an incoming update // This triggers the main update pipeline and enables centralized update handling - serverRegistration.updateServerStatus(latestPing, latestPing.timestamp, true, this.publicConfig.minecraftVersions) + serverRegistration.updateServerStatus(latestPing, true, this.publicConfig.minecraftVersions) // Allow the ServerRegistration to bind any DOM events with app instance context serverRegistration.initEventListeners() diff --git a/assets/js/servers.js b/assets/js/servers.js index f0717da..40cc517 100644 --- a/assets/js/servers.js +++ b/assets/js/servers.js @@ -94,7 +94,7 @@ export class ServerRegistration { this._failedSequentialPings = 0 } - addGraphPoints (points) { + addGraphPoints (points, timestampPoints) { // Test if the first point contains error.placeholder === true // This is sent by the backend when the server hasn't been pinged yet // These points will be disregarded to prevent the graph starting at 0 player count @@ -106,30 +106,33 @@ export class ServerRegistration { points.slice(points.length - SERVER_GRAPH_DATA_MAX_LENGTH, points.length) } - this._graphData = points.map(point => point.result ? [point.timestamp, point.result.players.online] : [point.timestamp, 0]) + for (let i = 0; i < points.length; i++) { + const point = points[i] + const timestamp = timestampPoints[i] + + this._graphData.push([timestamp, point.result ? point.result.players.online : 0]) + } } buildPlotInstance () { this._plotInstance = $.plot('#chart_' + this.serverId, [this._graphData], SERVER_GRAPH_OPTIONS) } - handlePing (payload, timestamp, pushToGraph) { + handlePing (payload, timestamp) { if (payload.result) { this.playerCount = payload.result.players.online - if (pushToGraph) { - // Only update graph for successful pings - // This intentionally pauses the server graph when pings begin to fail - this._graphData.push([timestamp, this.playerCount]) + // Only update graph for successful pings + // This intentionally pauses the server graph when pings begin to fail + this._graphData.push([timestamp, this.playerCount]) - // Trim graphData to within the max length by shifting out the leading elements - if (this._graphData.length > SERVER_GRAPH_DATA_MAX_LENGTH) { - this._graphData.shift() - } - - this.redraw() + // Trim graphData to within the max length by shifting out the leading elements + if (this._graphData.length > SERVER_GRAPH_DATA_MAX_LENGTH) { + this._graphData.shift() } + this.redraw() + // Reset failed ping counter to ensure the next connection error // doesn't instantly retrigger a layout change this._failedSequentialPings = 0 @@ -169,11 +172,7 @@ export class ServerRegistration { this.lastPeakData = data } - updateServerStatus (ping, timestamp, isInitialUpdate, minecraftVersions) { - // Only pushToGraph when initialUpdate === false - // Otherwise the ping value is pushed into the graphData when already present - this.handlePing(ping, timestamp, !isInitialUpdate) - + updateServerStatus (ping, isInitialUpdate, minecraftVersions) { if (ping.versions) { const versionsElement = document.getElementById('version_' + this.serverId) diff --git a/assets/js/socket.js b/assets/js/socket.js index 9a30864..5abd849 100644 --- a/assets/js/socket.js +++ b/assets/js/socket.js @@ -67,7 +67,9 @@ export class SocketManager { } } - payload.servers.forEach(this._app.addServer) + payload.servers.forEach(server => { + this._app.addServer(server, payload.timestampPoints) + }) if (payload.mojangServices) { this._app.mojangUpdater.updateStatus(payload.mojangServices) @@ -88,6 +90,8 @@ export class SocketManager { const serverUpdate = payload.updates[serverId] if (serverRegistration) { + serverRegistration.handlePing(serverUpdate, payload.timestamp) + serverRegistration.updateServerStatus(serverUpdate, payload.timestamp, false, this._app.publicConfig.minecraftVersions) } diff --git a/lib/app.js b/lib/app.js index 820b42d..28a302f 100644 --- a/lib/app.js +++ b/lib/app.js @@ -2,6 +2,7 @@ const Database = require('./database') const MojangUpdater = require('./mojang') const PingController = require('./ping') const Server = require('./server') +const TimeTracker = require('./time') const MessageOf = require('./message') const config = require('../config') @@ -14,6 +15,7 @@ class App { this.mojangUpdater = new MojangUpdater(this) this.pingController = new PingController(this) this.server = new Server(this.handleClientConnection) + this.timeTracker = new TimeTracker() } loadDatabase (callback) { @@ -73,6 +75,7 @@ class App { } })(), mojangServices: this.mojangUpdater.getLastUpdate(), + timestampPoints: this.timeTracker.getPoints(), servers: this.serverRegistrations.map(serverRegistration => serverRegistration.getPingHistory()) } diff --git a/lib/ping.js b/lib/ping.js index 31b0548..a9dda14 100644 --- a/lib/ping.js +++ b/lib/ping.js @@ -97,7 +97,7 @@ class PingController { } pingAll = () => { - const timestamp = new Date().getTime() + const timestamp = this._app.timeTracker.newTimestamp() this.startPingTasks(results => { const updates = [] diff --git a/lib/servers.js b/lib/servers.js index 5d18729..64feaa2 100644 --- a/lib/servers.js +++ b/lib/servers.js @@ -16,7 +16,7 @@ class ServerRegistration { handlePing (timestamp, resp, err, version) { // Store into in-memory ping data - this.addPing(timestamp, resp) + this.addPing(resp) // Only notify the frontend to append to the historical graph // if both the graphing behavior is enabled and the backend agrees @@ -89,10 +89,8 @@ class ServerRegistration { return update } - addPing (timestamp, resp) { - const ping = { - timestamp: timestamp - } + addPing (resp) { + const ping = {} if (resp) { // Append a result object @@ -127,7 +125,6 @@ class ServerRegistration { const payload = { serverId: this.serverId, - timestamp: lastPing.timestamp, versions: this.versions, recordData: this.recordData, favicon: this.lastFavicon @@ -157,7 +154,6 @@ class ServerRegistration { return [{ serverId: this.serverId, - timestamp: new Date().getTime(), error: { message: 'Waiting...', placeholder: true diff --git a/lib/time.js b/lib/time.js new file mode 100644 index 0000000..c820ae6 --- /dev/null +++ b/lib/time.js @@ -0,0 +1,23 @@ +class TimeTracker { + constructor () { + this._points = [] + } + + newTimestamp () { + const timestamp = new Date().getTime() + + this._points.push(timestamp) + + if (this._points.length > 72) { + this._points.shift() + } + + return timestamp + } + + getPoints () { + return this._points + } +} + +module.exports = TimeTracker From 7993ac00773c59dad12b392cec7fdf71b6838888 Mon Sep 17 00:00:00 2001 From: Nick Krecklow Date: Fri, 8 May 2020 01:25:38 -0500 Subject: [PATCH 04/30] do not append serverId in getPingHistory entries --- assets/js/app.js | 4 ++-- assets/js/socket.js | 4 ++-- config.json | 4 ++-- lib/servers.js | 2 -- 4 files changed, 6 insertions(+), 8 deletions(-) diff --git a/assets/js/app.js b/assets/js/app.js index 8458157..5804a20 100644 --- a/assets/js/app.js +++ b/assets/js/app.js @@ -102,13 +102,13 @@ export class App { .reduce((sum, current) => sum + current, 0) } - addServer = (pings, timestampPoints) => { + addServer = (serverId, pings, timestampPoints) => { // Even if the backend has never pinged the server, the frontend is promised a placeholder object. // result = undefined // error = defined with "Waiting" description // info = safely defined with configured data const latestPing = pings[pings.length - 1] - const serverRegistration = this.serverRegistry.createServerRegistration(latestPing.serverId) + const serverRegistration = this.serverRegistry.createServerRegistration(serverId) serverRegistration.initServerStatus(latestPing) diff --git a/assets/js/socket.js b/assets/js/socket.js index 5abd849..be79817 100644 --- a/assets/js/socket.js +++ b/assets/js/socket.js @@ -67,8 +67,8 @@ export class SocketManager { } } - payload.servers.forEach(server => { - this._app.addServer(server, payload.timestampPoints) + payload.servers.forEach((pings, serverId) => { + this._app.addServer(serverId, pings, payload.timestampPoints) }) if (payload.mojangServices) { diff --git a/config.json b/config.json index 6a4f90a..33ac9bf 100644 --- a/config.json +++ b/config.json @@ -10,8 +10,8 @@ "connectTimeout": 2500 }, "performance": { - "skipUnfurlSrv": false + "skipUnfurlSrv": true }, - "logToDatabase": false, + "logToDatabase": true, "graphDuration": 86400000 } diff --git a/lib/servers.js b/lib/servers.js index 64feaa2..d0ffd90 100644 --- a/lib/servers.js +++ b/lib/servers.js @@ -124,7 +124,6 @@ class ServerRegistration { const lastPing = this._pingHistory[this._pingHistory.length - 1] const payload = { - serverId: this.serverId, versions: this.versions, recordData: this.recordData, favicon: this.lastFavicon @@ -153,7 +152,6 @@ class ServerRegistration { } return [{ - serverId: this.serverId, error: { message: 'Waiting...', placeholder: true From aee7b565b232b8788d2fbd86b050fff60f375841 Mon Sep 17 00:00:00 2001 From: Nick Krecklow Date: Fri, 8 May 2020 01:27:21 -0500 Subject: [PATCH 05/30] unify 72 magic number behind named constant --- lib/servers.js | 4 +++- lib/time.js | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/lib/servers.js b/lib/servers.js index d0ffd90..f5c2f87 100644 --- a/lib/servers.js +++ b/lib/servers.js @@ -1,6 +1,8 @@ const config = require('../config') const minecraftVersions = require('../minecraft_versions') +const SERVER_GRAPH_DATA_MAX_LENGTH = 72 + class ServerRegistration { serverId lastFavicon @@ -105,7 +107,7 @@ class ServerRegistration { this._pingHistory.push(ping) // Trim pingHistory to avoid memory leaks - if (this._pingHistory.length > 72) { + if (this._pingHistory.length > SERVER_GRAPH_DATA_MAX_LENGTH) { this._pingHistory.shift() } } diff --git a/lib/time.js b/lib/time.js index c820ae6..a7ac7ec 100644 --- a/lib/time.js +++ b/lib/time.js @@ -1,3 +1,5 @@ +const SERVER_GRAPH_DATA_MAX_LENGTH = require('./servers').SERVER_GRAPH_DATA_MAX_LENGTH + class TimeTracker { constructor () { this._points = [] @@ -8,7 +10,7 @@ class TimeTracker { this._points.push(timestamp) - if (this._points.length > 72) { + if (this._points.length > SERVER_GRAPH_DATA_MAX_LENGTH) { this._points.shift() } From f467fa19381d56a01d9c1d4751498d84b6c661f3 Mon Sep 17 00:00:00 2001 From: Nick Krecklow Date: Fri, 8 May 2020 01:54:04 -0500 Subject: [PATCH 06/30] add serverGraphDuration config option --- assets/js/graph.js | 11 ++++++----- assets/js/servers.js | 14 ++------------ assets/js/sort.js | 2 +- config.json | 3 ++- lib/app.js | 8 +++++--- lib/database.js | 33 ++++++++++++--------------------- lib/ping.js | 8 ++++++++ lib/servers.js | 14 +++++++------- lib/time.js | 7 +++---- main.js | 7 ++++++- 10 files changed, 52 insertions(+), 55 deletions(-) diff --git a/assets/js/graph.js b/assets/js/graph.js index 8c72c97..058b10d 100644 --- a/assets/js/graph.js +++ b/assets/js/graph.js @@ -57,14 +57,15 @@ export class GraphDisplayManager { return } - // Trim any outdated entries by filtering the array into a new array - const startTimestamp = new Date().getTime() - const newGraphData = this._graphData[serverId].filter(point => startTimestamp - point[0] <= this._app.publicConfig.graphDuration) + const graphData = this._graphData[serverId] // Push the new data from the method call request - newGraphData.push([timestamp, playerCount]) + graphData.push([timestamp, playerCount]) - this._graphData[serverId] = newGraphData + // Trim any outdated entries by filtering the array into a new array + if (graphData.length > this._app.publicConfig.graphMaxLength) { + graphData.shift() + } } loadLocalStorage () { diff --git a/assets/js/servers.js b/assets/js/servers.js index 40cc517..1aee659 100644 --- a/assets/js/servers.js +++ b/assets/js/servers.js @@ -76,8 +76,6 @@ export class ServerRegistry { } } -const SERVER_GRAPH_DATA_MAX_LENGTH = 72 - export class ServerRegistration { playerCount = 0 isVisible = true @@ -100,12 +98,6 @@ export class ServerRegistration { // These points will be disregarded to prevent the graph starting at 0 player count points = points.filter(point => !point.error || !point.error.placeholder) - // The backend should never return more data elements than the max - // but trim the data result regardless for safety and performance purposes - if (points.length > SERVER_GRAPH_DATA_MAX_LENGTH) { - points.slice(points.length - SERVER_GRAPH_DATA_MAX_LENGTH, points.length) - } - for (let i = 0; i < points.length; i++) { const point = points[i] const timestamp = timestampPoints[i] @@ -127,7 +119,7 @@ export class ServerRegistration { this._graphData.push([timestamp, this.playerCount]) // Trim graphData to within the max length by shifting out the leading elements - if (this._graphData.length > SERVER_GRAPH_DATA_MAX_LENGTH) { + if (this._graphData.length > this._app.publicConfig.serverGraphMaxLength) { this._graphData.shift() } @@ -230,8 +222,6 @@ export class ServerRegistration { } initServerStatus (latestPing) { - const peakHourDuration = Math.floor(this._app.publicConfig.graphDuration / (60 * 60 * 1000)) + 'h Peak: ' - const serverElement = document.createElement('div') serverElement.id = 'container_' + this.serverId @@ -243,7 +233,7 @@ export class ServerRegistration { '

' + this.data.name + '

' + '' + 'Players: ' + - '' + peakHourDuration + '-' + + '' + this._app.publicConfig.graphDurationLabel + ' Peak: -' + 'Record: -' + '' + '' + diff --git a/assets/js/sort.js b/assets/js/sort.js index 74f7b89..ad6a25f 100644 --- a/assets/js/sort.js +++ b/assets/js/sort.js @@ -8,7 +8,7 @@ const SORT_OPTIONS = [ }, { getName: (app) => { - return Math.floor(app.publicConfig.graphDuration / (60 * 60 * 1000)) + 'h Peak' + return app.publicConfig.graphDurationLabel + ' Peak' }, sortFunc: (a, b) => { if (!a.lastPeakData && !b.lastPeakData) { diff --git a/config.json b/config.json index 33ac9bf..ce470b7 100644 --- a/config.json +++ b/config.json @@ -13,5 +13,6 @@ "skipUnfurlSrv": true }, "logToDatabase": true, - "graphDuration": 86400000 + "graphDuration": 86400000, + "serverGraphDuration": 180000 } diff --git a/lib/app.js b/lib/app.js index 28a302f..a94e238 100644 --- a/lib/app.js +++ b/lib/app.js @@ -15,7 +15,7 @@ class App { this.mojangUpdater = new MojangUpdater(this) this.pingController = new PingController(this) this.server = new Server(this.handleClientConnection) - this.timeTracker = new TimeTracker() + this.timeTracker = new TimeTracker(this) } loadDatabase (callback) { @@ -24,7 +24,7 @@ class App { // Setup database instance this.database.ensureIndexes() - this.database.loadGraphPoints(config.graphDuration, () => { + this.database.loadGraphPoints(this.pingController.getMaxGraphDataLength(), () => { this.database.loadRecords(callback) }) } @@ -68,7 +68,9 @@ class App { // Send configuration data for rendering the page return { - graphDuration: config.graphDuration, + graphDurationLabel: config.graphDurationLabel || (Math.floor(config.graphDuration / (60 * 60 * 1000)) + 'h'), + graphMaxLength: this.pingController.getMaxGraphDataLength(), + serverGraphMaxLength: this.pingController.getMaxServerGraphDataLength(), servers: this.serverRegistrations.map(serverRegistration => serverRegistration.data), minecraftVersions: minecraftVersionNames, isGraphVisible: config.logToDatabase diff --git a/lib/database.js b/lib/database.js index 821896a..aef9af9 100644 --- a/lib/database.js +++ b/lib/database.js @@ -14,27 +14,19 @@ class Database { }) } - loadGraphPoints (graphDuration, callback) { - // Query recent pings - const endTime = new Date().getTime() - const startTime = endTime - graphDuration - - this.getRecentPings(startTime, endTime, pingData => { + loadGraphPoints (length, callback) { + this.getRecentPings(length, pingData => { const graphPointsByIp = [] for (const row of pingData) { - // Avoid loading outdated records - // This shouldn't happen and is mostly a sanity measure - if (startTime - row.timestamp <= graphDuration) { - // Load into temporary array - // This will be culled prior to being pushed to the serverRegistration - let graphPoints = graphPointsByIp[row.ip] - if (!graphPoints) { - graphPoints = graphPointsByIp[row.ip] = [] - } - - graphPoints.push([row.timestamp, row.playerCount]) + // Load into temporary array + // This will be culled prior to being pushed to the serverRegistration + let graphPoints = graphPointsByIp[row.ip] + if (!graphPoints) { + graphPoints = graphPointsByIp[row.ip] = [] } + + graphPoints.push([row.timestamp, row.playerCount]) } Object.keys(graphPointsByIp).forEach(ip => { @@ -80,10 +72,9 @@ class Database { }) } - getRecentPings (startTime, endTime, callback) { - this._sql.all('SELECT * FROM pings WHERE timestamp >= ? AND timestamp <= ?', [ - startTime, - endTime + getRecentPings (length, callback) { + this._sql.all('SELECT * FROM pings WHERE 1 LIMIT ?', [ + length ], (_, data) => callback(data)) } diff --git a/lib/ping.js b/lib/ping.js index a9dda14..b19a37d 100644 --- a/lib/ping.js +++ b/lib/ping.js @@ -151,6 +151,14 @@ class PingController { }, version.protocolId) } } + + getMaxServerGraphDataLength () { + return Math.ceil(config.serverGraphDuration / config.rates.pingAll) + } + + getMaxGraphDataLength () { + return Math.ceil(config.graphDuration / config.rates.pingAll) + } } module.exports = PingController diff --git a/lib/servers.js b/lib/servers.js index f5c2f87..b5d51e7 100644 --- a/lib/servers.js +++ b/lib/servers.js @@ -1,8 +1,6 @@ const config = require('../config') const minecraftVersions = require('../minecraft_versions') -const SERVER_GRAPH_DATA_MAX_LENGTH = 72 - class ServerRegistration { serverId lastFavicon @@ -10,7 +8,8 @@ class ServerRegistration { recordData graphData = [] - constructor (serverId, data) { + constructor (app, serverId, data) { + this._app = app this.serverId = serverId this.data = data this._pingHistory = [] @@ -107,7 +106,7 @@ class ServerRegistration { this._pingHistory.push(ping) // Trim pingHistory to avoid memory leaks - if (this._pingHistory.length > SERVER_GRAPH_DATA_MAX_LENGTH) { + if (this._pingHistory.length > this._app.pingController.getMaxServerGraphDataLength()) { this._pingHistory.shift() } } @@ -203,9 +202,10 @@ class ServerRegistration { this.graphData.push([timestamp, playerCount]) this._lastGraphDataPush = timestamp - // Trim old graphPoints according to graphDuration - const filterTimestamp = new Date().getTime() - config.graphDuration - this.graphData = this.graphData.filter(point => point[0] >= filterTimestamp) + // Trim old graphPoints according to #getMaxGraphDataLength + if (this.graphData.length > this._app.pingController.getMaxGraphDataLength()) { + this.graphData.shift() + } return true } diff --git a/lib/time.js b/lib/time.js index a7ac7ec..1269b46 100644 --- a/lib/time.js +++ b/lib/time.js @@ -1,7 +1,6 @@ -const SERVER_GRAPH_DATA_MAX_LENGTH = require('./servers').SERVER_GRAPH_DATA_MAX_LENGTH - class TimeTracker { - constructor () { + constructor (app) { + this._app = app this._points = [] } @@ -10,7 +9,7 @@ class TimeTracker { this._points.push(timestamp) - if (this._points.length > SERVER_GRAPH_DATA_MAX_LENGTH) { + if (this._points.length > this._app.pingController.getMaxServerGraphDataLength()) { this._points.shift() } diff --git a/main.js b/main.js index e972b92..094e496 100644 --- a/main.js +++ b/main.js @@ -22,9 +22,14 @@ servers.forEach((server, serverId) => { } // Init a ServerRegistration instance of each entry in servers.json - app.serverRegistrations.push(new ServerRegistration(serverId, server)) + app.serverRegistrations.push(new ServerRegistration(app, serverId, server)) }) +if (!config.serverGraphDuration) { + logger.log('warn', '"serverGraphDuration" is not defined in config.json - defaulting to 3 minutes!') + config.serverGraphDuration = 3 * 60 * 10000 +} + if (config.performance && config.performance.skipUnfurlSrv) { logger.log('warn', '"performance.skipUnfurlSrv" is enabled. Any configured hosts using SRV records may not properly resolve.') } From 5f2c62c23afcadd427b8f15e4b7f42589aea7e7b Mon Sep 17 00:00:00 2001 From: Nick Krecklow Date: Fri, 8 May 2020 02:01:17 -0500 Subject: [PATCH 07/30] beta 5.4.0 --- config.json | 4 ++-- docs/CHANGELOG.md | 6 ++++++ package-lock.json | 2 +- package.json | 2 +- 4 files changed, 10 insertions(+), 4 deletions(-) diff --git a/config.json b/config.json index ce470b7..ccd1794 100644 --- a/config.json +++ b/config.json @@ -10,9 +10,9 @@ "connectTimeout": 2500 }, "performance": { - "skipUnfurlSrv": true + "skipUnfurlSrv": false }, - "logToDatabase": true, + "logToDatabase": false, "graphDuration": 86400000, "serverGraphDuration": 180000 } diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 52cb4a7..d9b7c99 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -1,3 +1,9 @@ +**5.4.0** *(May 8 2020)* +- Adds "graphDurationLabel" to `config.json` which allows you to manually modify the "24h Peak" label to a custom time duration. +- Adds "serverGraphDuration" to `config.json` which allows you to specify the max time duration for the individual server player count graphs. +- Adds "performance.skipUnfurlSrv" to `config.json` which allows you to skip SRV unfurling when pinging. For those who aren't pinging servers that use SRV records, this should help speed up ping times. +- Ping timestamps are now shared between all server pings. This means less data transfer when loading or updating the page, less memory usage by the backend and frontend, and less hectic updates on the frontend. + **5.3.1** *(May 5 2020)* - Fixes Mojang service status indicators not updating after initial page load. diff --git a/package-lock.json b/package-lock.json index 11cfe34..67e40be 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "minetrack", - "version": "5.3.1", + "version": "5.4.0", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/package.json b/package.json index 4560a96..9ce478a 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "minetrack", - "version": "5.3.1", + "version": "5.4.0", "description": "A Minecraft server tracker that lets you focus on the basics.", "main": "main.js", "dependencies": { From 2f1c9c1dce12bf308ffa12c0e7631718c7e84071 Mon Sep 17 00:00:00 2001 From: Nick Krecklow Date: Fri, 8 May 2020 02:06:39 -0500 Subject: [PATCH 08/30] use config.graphDuration when querying database, not LIMIT --- lib/app.js | 2 +- lib/database.js | 15 ++++++++++----- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/lib/app.js b/lib/app.js index a94e238..a5dc9d5 100644 --- a/lib/app.js +++ b/lib/app.js @@ -24,7 +24,7 @@ class App { // Setup database instance this.database.ensureIndexes() - this.database.loadGraphPoints(this.pingController.getMaxGraphDataLength(), () => { + this.database.loadGraphPoints(config.graphDuration, () => { this.database.loadRecords(callback) }) } diff --git a/lib/database.js b/lib/database.js index aef9af9..f94979c 100644 --- a/lib/database.js +++ b/lib/database.js @@ -14,8 +14,12 @@ class Database { }) } - loadGraphPoints (length, callback) { - this.getRecentPings(length, pingData => { + loadGraphPoints (graphDuration, callback) { + // Query recent pings + const endTime = new Date().getTime() + const startTime = endTime - graphDuration + + this.getRecentPings(startTime, endTime, length, pingData => { const graphPointsByIp = [] for (const row of pingData) { @@ -72,9 +76,10 @@ class Database { }) } - getRecentPings (length, callback) { - this._sql.all('SELECT * FROM pings WHERE 1 LIMIT ?', [ - length + getRecentPings (startTime, endTime, callback) { + this._sql.all('SELECT * FROM pings WHERE timestamp >= ? AND timestamp <= ?', [ + startTime, + endTime ], (_, data) => callback(data)) } From a3624d38ff3bb98f27a6cdb72b7f5774a95e302d Mon Sep 17 00:00:00 2001 From: Nick Krecklow Date: Fri, 8 May 2020 02:07:59 -0500 Subject: [PATCH 09/30] remove legacy length param --- lib/database.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/database.js b/lib/database.js index f94979c..bbfc336 100644 --- a/lib/database.js +++ b/lib/database.js @@ -19,7 +19,7 @@ class Database { const endTime = new Date().getTime() const startTime = endTime - graphDuration - this.getRecentPings(startTime, endTime, length, pingData => { + this.getRecentPings(startTime, endTime, pingData => { const graphPointsByIp = [] for (const row of pingData) { From e45f8b66397849c229daf36428812f85dc5cf8bf Mon Sep 17 00:00:00 2001 From: Nick Krecklow Date: Fri, 8 May 2020 02:39:30 -0500 Subject: [PATCH 10/30] simplify addServer/pingHistory & placeholder generation prior to optimizations --- assets/js/app.js | 17 ++++++++++------- assets/js/servers.js | 10 ++-------- assets/js/socket.js | 6 +++--- config.json | 4 ++-- lib/servers.js | 31 ++++++++++++------------------- 5 files changed, 29 insertions(+), 39 deletions(-) diff --git a/assets/js/app.js b/assets/js/app.js index 5804a20..a5b3283 100644 --- a/assets/js/app.js +++ b/assets/js/app.js @@ -102,26 +102,29 @@ export class App { .reduce((sum, current) => sum + current, 0) } - addServer = (serverId, pings, timestampPoints) => { + addServer = (serverId, payload, timestampPoints) => { // Even if the backend has never pinged the server, the frontend is promised a placeholder object. // result = undefined // error = defined with "Waiting" description // info = safely defined with configured data - const latestPing = pings[pings.length - 1] const serverRegistration = this.serverRegistry.createServerRegistration(serverId) - serverRegistration.initServerStatus(latestPing) + serverRegistration.initServerStatus(payload) - // Push the historical data into the graph - // This will trim and format the data so it is ready for the graph to render once init - serverRegistration.addGraphPoints(pings, timestampPoints) + // pingHistory is only defined when the backend has previous ping data + // undefined pingHistory means this is a placeholder ping generated by the backend + if (typeof payload.pingHistory !== 'undefined') { + // Push the historical data into the graph + // This will trim and format the data so it is ready for the graph to render once init + serverRegistration.addGraphPoints(payload.pingHistory, timestampPoints) + } // Create the plot instance internally with the restructured and cleaned data serverRegistration.buildPlotInstance() // Handle the last known state (if any) as an incoming update // This triggers the main update pipeline and enables centralized update handling - serverRegistration.updateServerStatus(latestPing, true, this.publicConfig.minecraftVersions) + serverRegistration.updateServerStatus(payload, this.publicConfig.minecraftVersions) // Allow the ServerRegistration to bind any DOM events with app instance context serverRegistration.initEventListeners() diff --git a/assets/js/servers.js b/assets/js/servers.js index 1aee659..cc518b8 100644 --- a/assets/js/servers.js +++ b/assets/js/servers.js @@ -93,11 +93,6 @@ export class ServerRegistration { } addGraphPoints (points, timestampPoints) { - // Test if the first point contains error.placeholder === true - // This is sent by the backend when the server hasn't been pinged yet - // These points will be disregarded to prevent the graph starting at 0 player count - points = points.filter(point => !point.error || !point.error.placeholder) - for (let i = 0; i < points.length; i++) { const point = points[i] const timestamp = timestampPoints[i] @@ -164,7 +159,7 @@ export class ServerRegistration { this.lastPeakData = data } - updateServerStatus (ping, isInitialUpdate, minecraftVersions) { + updateServerStatus (ping, minecraftVersions) { if (ping.versions) { const versionsElement = document.getElementById('version_' + this.serverId) @@ -214,8 +209,7 @@ export class ServerRegistration { document.getElementById('player-count-value_' + this.serverId).innerText = formatNumber(ping.result.players.online) // An updated favicon has been sent, update the src - // Ignore calls from 'add' events since they will have explicitly manually handled the favicon update - if (!isInitialUpdate && ping.favicon) { + if (ping.favicon) { document.getElementById('favicon_' + this.serverId).setAttribute('src', ping.favicon) } } diff --git a/assets/js/socket.js b/assets/js/socket.js index be79817..2263d64 100644 --- a/assets/js/socket.js +++ b/assets/js/socket.js @@ -67,8 +67,8 @@ export class SocketManager { } } - payload.servers.forEach((pings, serverId) => { - this._app.addServer(serverId, pings, payload.timestampPoints) + payload.servers.forEach((serverPayload, serverId) => { + this._app.addServer(serverId, serverPayload, payload.timestampPoints) }) if (payload.mojangServices) { @@ -92,7 +92,7 @@ export class SocketManager { if (serverRegistration) { serverRegistration.handlePing(serverUpdate, payload.timestamp) - serverRegistration.updateServerStatus(serverUpdate, payload.timestamp, false, this._app.publicConfig.minecraftVersions) + serverRegistration.updateServerStatus(serverUpdate, this._app.publicConfig.minecraftVersions) } // Use update payloads to conditionally append data to graph diff --git a/config.json b/config.json index ccd1794..ce470b7 100644 --- a/config.json +++ b/config.json @@ -10,9 +10,9 @@ "connectTimeout": 2500 }, "performance": { - "skipUnfurlSrv": false + "skipUnfurlSrv": true }, - "logToDatabase": false, + "logToDatabase": true, "graphDuration": 86400000, "serverGraphDuration": 180000 } diff --git a/lib/servers.js b/lib/servers.js index b5d51e7..164d532 100644 --- a/lib/servers.js +++ b/lib/servers.js @@ -113,17 +113,6 @@ class ServerRegistration { getPingHistory () { if (this._pingHistory.length > 0) { - const pingHistory = [] - - for (let i = 0; i < this._pingHistory.length - 1; i++) { - pingHistory[i] = this._pingHistory[i] - } - - // Insert the latest update manually into the array - // This is a mutated copy of the last update to contain live metadata - // The metadata is used by the frontend for rendering - const lastPing = this._pingHistory[this._pingHistory.length - 1] - const payload = { versions: this.versions, recordData: this.recordData, @@ -138,6 +127,11 @@ class ServerRegistration { payload.graphPeakData = graphPeakData } + // Insert the latest update manually into the array + // This is a mutated copy of the last update to contain live metadata + // The metadata is used by the frontend for rendering + const lastPing = this._pingHistory[this._pingHistory.length - 1] + // Conditionally append to avoid defining fields with undefined values if (lastPing.result) { payload.result = lastPing.result @@ -145,21 +139,20 @@ class ServerRegistration { payload.error = lastPing.error } - // Insert the reconstructed update as the last entry - // pingHistory is already sorted during its copy from _pingHistory - pingHistory.push(payload) + // Send a copy of pingHistory + // Omit the last value since it is contained within payload + payload.pingHistory = this._pingHistory.slice(0, this._pingHistory.length - 1) - return pingHistory + return payload } - return [{ + return { error: { - message: 'Waiting...', - placeholder: true + message: 'Pinging...' }, recordData: this.recordData, graphPeakData: this.getGraphPeak() - }] + } } loadGraphPoints (points) { From 4375798f6cfbbe151a1c685b054b627178103946 Mon Sep 17 00:00:00 2001 From: Nick Krecklow Date: Fri, 8 May 2020 02:40:29 -0500 Subject: [PATCH 11/30] update docs/CHANGELOG.md --- docs/CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index d9b7c99..688b03f 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -3,6 +3,7 @@ - Adds "serverGraphDuration" to `config.json` which allows you to specify the max time duration for the individual server player count graphs. - Adds "performance.skipUnfurlSrv" to `config.json` which allows you to skip SRV unfurling when pinging. For those who aren't pinging servers that use SRV records, this should help speed up ping times. - Ping timestamps are now shared between all server pings. This means less data transfer when loading or updating the page, less memory usage by the backend and frontend, and less hectic updates on the frontend. +- Fixes a bug where favicons may not be updated if the page is loaded prior to their initialization. **5.3.1** *(May 5 2020)* - Fixes Mojang service status indicators not updating after initial page load. From 37a88677cfe6ddc1b0902d8addca3c7f385cdf2d Mon Sep 17 00:00:00 2001 From: Nick Krecklow Date: Fri, 8 May 2020 02:42:46 -0500 Subject: [PATCH 12/30] always update favicon if supplied --- assets/js/servers.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/assets/js/servers.js b/assets/js/servers.js index cc518b8..71cb92b 100644 --- a/assets/js/servers.js +++ b/assets/js/servers.js @@ -207,11 +207,11 @@ export class ServerRegistration { errorElement.style.display = 'none' document.getElementById('player-count-value_' + this.serverId).innerText = formatNumber(ping.result.players.online) + } - // An updated favicon has been sent, update the src - if (ping.favicon) { - document.getElementById('favicon_' + this.serverId).setAttribute('src', ping.favicon) - } + // An updated favicon has been sent, update the src + if (ping.favicon) { + document.getElementById('favicon_' + this.serverId).setAttribute('src', ping.favicon) } } From 3491c73b893cf062294bf4686a38b8d482883cd8 Mon Sep 17 00:00:00 2001 From: Nick Krecklow Date: Fri, 8 May 2020 02:52:48 -0500 Subject: [PATCH 13/30] store pingHistory as player count directly --- assets/js/app.js | 8 +++---- assets/js/servers.js | 3 +-- assets/js/socket.js | 2 ++ lib/servers.js | 54 +++++++++++++++----------------------------- 4 files changed, 25 insertions(+), 42 deletions(-) diff --git a/assets/js/app.js b/assets/js/app.js index a5b3283..32d4992 100644 --- a/assets/js/app.js +++ b/assets/js/app.js @@ -111,12 +111,12 @@ export class App { serverRegistration.initServerStatus(payload) - // pingHistory is only defined when the backend has previous ping data - // undefined pingHistory means this is a placeholder ping generated by the backend - if (typeof payload.pingHistory !== 'undefined') { + // playerCountHistory is only defined when the backend has previous ping data + // undefined playerCountHistory means this is a placeholder ping generated by the backend + if (typeof payload.playerCountHistory !== 'undefined') { // Push the historical data into the graph // This will trim and format the data so it is ready for the graph to render once init - serverRegistration.addGraphPoints(payload.pingHistory, timestampPoints) + serverRegistration.addGraphPoints(payload.playerCountHistory, timestampPoints) } // Create the plot instance internally with the restructured and cleaned data diff --git a/assets/js/servers.js b/assets/js/servers.js index 71cb92b..1c8cc08 100644 --- a/assets/js/servers.js +++ b/assets/js/servers.js @@ -96,8 +96,7 @@ export class ServerRegistration { for (let i = 0; i < points.length; i++) { const point = points[i] const timestamp = timestampPoints[i] - - this._graphData.push([timestamp, point.result ? point.result.players.online : 0]) + this._graphData.push([timestamp, point]) } } diff --git a/assets/js/socket.js b/assets/js/socket.js index 2263d64..72770ed 100644 --- a/assets/js/socket.js +++ b/assets/js/socket.js @@ -67,6 +67,8 @@ export class SocketManager { } } + console.log(payload.servers) + payload.servers.forEach((serverPayload, serverId) => { this._app.addServer(serverId, serverPayload, payload.timestampPoints) }) diff --git a/lib/servers.js b/lib/servers.js index 164d532..7da838b 100644 --- a/lib/servers.js +++ b/lib/servers.js @@ -16,8 +16,15 @@ class ServerRegistration { } handlePing (timestamp, resp, err, version) { + const playerCount = resp ? resp.players.online : 0 + // Store into in-memory ping data - this.addPing(resp) + this._pingHistory.push(playerCount) + + // Trim pingHistory to avoid memory leaks + if (this._pingHistory.length > this._app.pingController.getMaxServerGraphDataLength()) { + this._pingHistory.shift() + } // Only notify the frontend to append to the historical graph // if both the graphing behavior is enabled and the backend agrees @@ -25,8 +32,6 @@ class ServerRegistration { let updateHistoryGraph = false if (config.logToDatabase) { - const playerCount = resp ? resp.players.online : 0 - if (this.addGraphPoint(resp !== undefined, playerCount, timestamp)) { updateHistoryGraph = true } @@ -90,27 +95,6 @@ class ServerRegistration { return update } - addPing (resp) { - const ping = {} - - if (resp) { - // Append a result object - // This filters out unwanted data from resp - ping.result = { - players: { - online: resp.players.online - } - } - } - - this._pingHistory.push(ping) - - // Trim pingHistory to avoid memory leaks - if (this._pingHistory.length > this._app.pingController.getMaxServerGraphDataLength()) { - this._pingHistory.shift() - } - } - getPingHistory () { if (this._pingHistory.length > 0) { const payload = { @@ -127,21 +111,19 @@ class ServerRegistration { payload.graphPeakData = graphPeakData } - // Insert the latest update manually into the array - // This is a mutated copy of the last update to contain live metadata - // The metadata is used by the frontend for rendering - const lastPing = this._pingHistory[this._pingHistory.length - 1] - - // Conditionally append to avoid defining fields with undefined values - if (lastPing.result) { - payload.result = lastPing.result - } else if (lastPing.error) { - payload.error = lastPing.error + // Assume the ping was a success and define result + // pingHistory does not keep error references, so its impossible to detect if this is an error + // It is also pointless to store that data since it will be short lived + payload.result = { + players: { + online: this._pingHistory[this._pingHistory.length - 1] + } } // Send a copy of pingHistory - // Omit the last value since it is contained within payload - payload.pingHistory = this._pingHistory.slice(0, this._pingHistory.length - 1) + // Include the last value even though it is contained within payload + // The frontend will only push to its graphData from playerCountHistory + payload.playerCountHistory = this._pingHistory return payload } From 024e605a413cb571eb77cdeafd2c0fe6a130940e Mon Sep 17 00:00:00 2001 From: Nick Krecklow Date: Fri, 8 May 2020 02:56:39 -0500 Subject: [PATCH 14/30] make getMaxGraphDataLength/getMaxServerGraphDataLength methods static --- lib/app.js | 4 ++-- lib/ping.js | 8 -------- lib/servers.js | 9 +++++---- lib/time.js | 12 +++++++++++- main.js | 2 +- 5 files changed, 19 insertions(+), 16 deletions(-) diff --git a/lib/app.js b/lib/app.js index a5dc9d5..7a330fc 100644 --- a/lib/app.js +++ b/lib/app.js @@ -69,8 +69,8 @@ class App { // Send configuration data for rendering the page return { graphDurationLabel: config.graphDurationLabel || (Math.floor(config.graphDuration / (60 * 60 * 1000)) + 'h'), - graphMaxLength: this.pingController.getMaxGraphDataLength(), - serverGraphMaxLength: this.pingController.getMaxServerGraphDataLength(), + graphMaxLength: TimeTracker.getMaxGraphDataLength(), + serverGraphMaxLength: TimeTracker.getMaxServerGraphDataLength(), servers: this.serverRegistrations.map(serverRegistration => serverRegistration.data), minecraftVersions: minecraftVersionNames, isGraphVisible: config.logToDatabase diff --git a/lib/ping.js b/lib/ping.js index b19a37d..a9dda14 100644 --- a/lib/ping.js +++ b/lib/ping.js @@ -151,14 +151,6 @@ class PingController { }, version.protocolId) } } - - getMaxServerGraphDataLength () { - return Math.ceil(config.serverGraphDuration / config.rates.pingAll) - } - - getMaxGraphDataLength () { - return Math.ceil(config.graphDuration / config.rates.pingAll) - } } module.exports = PingController diff --git a/lib/servers.js b/lib/servers.js index 7da838b..5db95dd 100644 --- a/lib/servers.js +++ b/lib/servers.js @@ -1,3 +1,5 @@ +const TimeTracker = require('./time') + const config = require('../config') const minecraftVersions = require('../minecraft_versions') @@ -8,8 +10,7 @@ class ServerRegistration { recordData graphData = [] - constructor (app, serverId, data) { - this._app = app + constructor (serverId, data) { this.serverId = serverId this.data = data this._pingHistory = [] @@ -22,7 +23,7 @@ class ServerRegistration { this._pingHistory.push(playerCount) // Trim pingHistory to avoid memory leaks - if (this._pingHistory.length > this._app.pingController.getMaxServerGraphDataLength()) { + if (this._pingHistory.length > TimeTracker.getMaxServerGraphDataLength()) { this._pingHistory.shift() } @@ -178,7 +179,7 @@ class ServerRegistration { this._lastGraphDataPush = timestamp // Trim old graphPoints according to #getMaxGraphDataLength - if (this.graphData.length > this._app.pingController.getMaxGraphDataLength()) { + if (this.graphData.length > TimeTracker.getMaxGraphDataLength()) { this.graphData.shift() } diff --git a/lib/time.js b/lib/time.js index 1269b46..966272a 100644 --- a/lib/time.js +++ b/lib/time.js @@ -1,3 +1,5 @@ +const config = require('../config.json') + class TimeTracker { constructor (app) { this._app = app @@ -9,7 +11,7 @@ class TimeTracker { this._points.push(timestamp) - if (this._points.length > this._app.pingController.getMaxServerGraphDataLength()) { + if (this._points.length > TimeTracker.getMaxServerGraphDataLength()) { this._points.shift() } @@ -19,6 +21,14 @@ class TimeTracker { getPoints () { return this._points } + + static getMaxServerGraphDataLength () { + return Math.ceil(config.serverGraphDuration / config.rates.pingAll) + } + + static getMaxGraphDataLength () { + return Math.ceil(config.graphDuration / config.rates.pingAll) + } } module.exports = TimeTracker diff --git a/main.js b/main.js index 094e496..5f4c56e 100644 --- a/main.js +++ b/main.js @@ -22,7 +22,7 @@ servers.forEach((server, serverId) => { } // Init a ServerRegistration instance of each entry in servers.json - app.serverRegistrations.push(new ServerRegistration(app, serverId, server)) + app.serverRegistrations.push(new ServerRegistration(serverId, server)) }) if (!config.serverGraphDuration) { From 63a01ace95efc0b85938e9e8dc33082118e20467 Mon Sep 17 00:00:00 2001 From: Nick Krecklow Date: Fri, 8 May 2020 02:59:51 -0500 Subject: [PATCH 15/30] remove console.log debug call --- assets/js/socket.js | 2 -- 1 file changed, 2 deletions(-) diff --git a/assets/js/socket.js b/assets/js/socket.js index 72770ed..2263d64 100644 --- a/assets/js/socket.js +++ b/assets/js/socket.js @@ -67,8 +67,6 @@ export class SocketManager { } } - console.log(payload.servers) - payload.servers.forEach((serverPayload, serverId) => { this._app.addServer(serverId, serverPayload, payload.timestampPoints) }) From 4104c4144a779c3276b7215bcd82e5211da318fe Mon Sep 17 00:00:00 2001 From: Nick Krecklow Date: Fri, 8 May 2020 03:27:01 -0500 Subject: [PATCH 16/30] remove timeout tasks since updates occur in bulk now --- assets/js/app.js | 2 -- assets/js/graph.js | 26 -------------------------- assets/js/socket.js | 13 ++++++++++++- 3 files changed, 12 insertions(+), 29 deletions(-) diff --git a/assets/js/app.js b/assets/js/app.js index 32d4992..1ce52fe 100644 --- a/assets/js/app.js +++ b/assets/js/app.js @@ -62,8 +62,6 @@ export class App { initTasks () { this._taskIds.push(setInterval(this.sortController.sortServers, 5000)) - this._taskIds.push(setInterval(this.updateGlobalStats, 1000)) - this._taskIds.push(setInterval(this.percentageBar.redraw, 1000)) } handleDisconnect () { diff --git a/assets/js/graph.js b/assets/js/graph.js index 058b10d..9c88a11 100644 --- a/assets/js/graph.js +++ b/assets/js/graph.js @@ -161,18 +161,6 @@ export class GraphDisplayManager { document.getElementById('settings-toggle').style.display = 'inline-block' } - // requestRedraw allows usages to request a redraw that may be performed, or cancelled, sometime later - // This allows multiple rapid, but individual updates, to clump into a single redraw instead - requestRedraw () { - if (this._redrawRequestTimeout) { - clearTimeout(this._redrawRequestTimeout) - } - - // Schedule new delayed redraw call - // This can be cancelled by #requestRedraw, #redraw and #reset - this._redrawRequestTimeout = setTimeout(this.redraw, 1000) - } - redraw = () => { // Use drawing as a hint to update settings // This may cause unnessecary localStorage updates, but its a rare and harmless outcome @@ -183,14 +171,6 @@ export class GraphDisplayManager { this._plotInstance.setData(this.getVisibleGraphData()) this._plotInstance.setupGrid() this._plotInstance.draw() - - // undefine value so #clearTimeout is not called - // This is safe even if #redraw is manually called since it removes the pending work - if (this._redrawRequestTimeout) { - clearTimeout(this._redrawRequestTimeout) - } - - this._redrawRequestTimeout = undefined } requestResize () { @@ -348,12 +328,6 @@ export class GraphDisplayManager { this._resizeRequestTimeout = undefined } - if (this._redrawRequestTimeout) { - clearTimeout(this._redrawRequestTimeout) - - this._redrawRequestTimeout = undefined - } - // Reset modified DOM structures document.getElementById('big-graph-checkboxes').innerHTML = '' document.getElementById('big-graph-controls').style.display = 'none' diff --git a/assets/js/socket.js b/assets/js/socket.js index 2263d64..2bb3600 100644 --- a/assets/js/socket.js +++ b/assets/js/socket.js @@ -82,6 +82,8 @@ export class SocketManager { break case 'updateServers': { + let requestGraphRedraw = false + for (let serverId = 0; serverId < payload.updates.length; serverId++) { // The backend may send "update" events prior to receiving all "add" events // A server has only been added once it's ServerRegistration is defined @@ -105,10 +107,19 @@ export class SocketManager { // Only redraw the graph if not mutating hidden data if (serverRegistration.isVisible) { - this._app.graphDisplayManager.requestRedraw() + requestGraphRedraw = true } } } + + // Run redraw tasks after handling bulk updates + if (requestGraphRedraw) { + this._app.graphDisplayManager.redraw() + } + + this._app.percentageBar.redraw() + this._app.updateGlobalStats() + break } From 96acf3614b64b21c0690d09ddbfbfe85e9d6ce0c Mon Sep 17 00:00:00 2001 From: Nick Krecklow Date: Fri, 8 May 2020 03:35:22 -0500 Subject: [PATCH 17/30] explicitly set serverRegistration.playerCount when firing #addServer --- assets/js/app.js | 5 +++++ lib/ping.js | 2 ++ 2 files changed, 7 insertions(+) diff --git a/assets/js/app.js b/assets/js/app.js index 1ce52fe..d5446ce 100644 --- a/assets/js/app.js +++ b/assets/js/app.js @@ -115,6 +115,11 @@ export class App { // Push the historical data into the graph // This will trim and format the data so it is ready for the graph to render once init serverRegistration.addGraphPoints(payload.playerCountHistory, timestampPoints) + + // Set initial playerCount to the payload's value + // This will always exist since it is explicitly generated by the backend + // This is used for any post-add rendering of things like the percentageBar + serverRegistration.playerCount = payload.result.players.online } // Create the plot instance internally with the restructured and cleaned data diff --git a/lib/ping.js b/lib/ping.js index a9dda14..e9ae8ba 100644 --- a/lib/ping.js +++ b/lib/ping.js @@ -108,6 +108,7 @@ class PingController { // Log to database if enabled if (config.logToDatabase) { const playerCount = result.resp ? result.resp.players.online : 0 + this._app.database.insertPing(serverRegistration.data.ip, timestamp, playerCount) } @@ -115,6 +116,7 @@ class PingController { // This includes any modified fields and flags used by the frontend // This will not be cached and can contain live metadata const update = serverRegistration.handlePing(timestamp, result.resp, result.err, result.version) + updates[serverRegistration.serverId] = update } From 66da5f6497fa1801ebde269999e72f4dc631e5ca Mon Sep 17 00:00:00 2001 From: Nick Krecklow Date: Fri, 8 May 2020 03:41:19 -0500 Subject: [PATCH 18/30] remove useless sql.serialize call --- lib/database.js | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/lib/database.js b/lib/database.js index bbfc336..60f795a 100644 --- a/lib/database.js +++ b/lib/database.js @@ -91,9 +91,7 @@ class Database { insertPing (ip, timestamp, playerCount) { const statement = this._sql.prepare('INSERT INTO pings (timestamp, ip, playerCount) VALUES (?, ?, ?)') - this._sql.serialize(() => { - statement.run(timestamp, ip, playerCount) - }) + statement.run(timestamp, ip, playerCount) statement.finalize() } } From 4dfecce966f95e5176c9bb1d7ad2135e88afa513 Mon Sep 17 00:00:00 2001 From: Nick Krecklow Date: Fri, 8 May 2020 04:47:10 -0500 Subject: [PATCH 19/30] serve favicons over hashed paths for improved caching --- assets/js/servers.js | 8 +++++++- lib/app.js | 2 +- lib/server.js | 45 +++++++++++++++++++++++++++++++++++--------- lib/servers.js | 36 ++++++++++++++++++++++++++++------- 4 files changed, 73 insertions(+), 18 deletions(-) diff --git a/assets/js/servers.js b/assets/js/servers.js index 1c8cc08..3b4bd62 100644 --- a/assets/js/servers.js +++ b/assets/js/servers.js @@ -210,7 +210,13 @@ export class ServerRegistration { // An updated favicon has been sent, update the src if (ping.favicon) { - document.getElementById('favicon_' + this.serverId).setAttribute('src', ping.favicon) + const faviconElement = document.getElementById('favicon_' + this.serverId) + + // Since favicons may be URLs, only update the attribute when it has changed + // Otherwise the browser may send multiple requests to the same URL + if (faviconElement.getAttribute('src') !== ping.favicon) { + faviconElement.setAttribute('src', ping.favicon) + } } } diff --git a/lib/app.js b/lib/app.js index 7a330fc..22127ad 100644 --- a/lib/app.js +++ b/lib/app.js @@ -14,7 +14,7 @@ class App { constructor () { this.mojangUpdater = new MojangUpdater(this) this.pingController = new PingController(this) - this.server = new Server(this.handleClientConnection) + this.server = new Server(this) this.timeTracker = new TimeTracker(this) } diff --git a/lib/server.js b/lib/server.js index 9a6b371..49cf340 100644 --- a/lib/server.js +++ b/lib/server.js @@ -11,9 +11,11 @@ function getRemoteAddr (req) { } class Server { - constructor (clientSocketHandler) { + constructor (app) { + this._app = app + this.createHttpServer() - this.createWebSocketServer(clientSocketHandler) + this.createWebSocketServer() } createHttpServer () { @@ -23,15 +25,40 @@ class Server { this._http = http.createServer((req, res) => { logger.log('info', '%s requested: %s', getRemoteAddr(req), req.url) - // Attempt to handle req using distServeStatic, otherwise fail over to faviconServeStatic - // If faviconServeStatic fails, pass to finalHttpHandler to terminate - distServeStatic(req, res, () => { - faviconsServeStatic(req, res, finalHttpHandler(req, res)) - }) + if (req.url.startsWith('/hashedfavicon?')) { + this.handleFaviconRequest(req, res) + } else { + // Attempt to handle req using distServeStatic, otherwise fail over to faviconServeStatic + // If faviconServeStatic fails, pass to finalHttpHandler to terminate + distServeStatic(req, res, () => { + faviconsServeStatic(req, res, finalHttpHandler(req, res)) + }) + } }) } - createWebSocketServer (proxyClientSocketHandler) { + handleFaviconRequest = (req, res) => { + const hash = req.url.split('?')[1] + + for (const serverRegistration of this._app.serverRegistrations) { + if (serverRegistration.faviconHash && serverRegistration.faviconHash === hash) { + const buf = Buffer.from(serverRegistration.lastFavicon.split(',')[1], 'base64') + + res.writeHead(200, { + 'Content-Type': 'image/png', + 'Content-Length': buf.length + }).end(buf) + + return + } + } + + // Terminate request, no match + res.writeHead(404) + res.end() + } + + createWebSocketServer () { this._wss = new WebSocket.Server({ server: this._http }) @@ -45,7 +72,7 @@ class Server { }) // Pass client off to proxy handler - proxyClientSocketHandler(client) + this._app.handleClientConnection(client) }) } diff --git a/lib/servers.js b/lib/servers.js index 5db95dd..61777e2 100644 --- a/lib/servers.js +++ b/lib/servers.js @@ -1,3 +1,5 @@ +const crypto = require('crypto') + const TimeTracker = require('./time') const config = require('../config') @@ -61,11 +63,8 @@ class ServerRegistration { update.recordData = this.recordData } - // Compare against this.data.favicon to support favicon overrides - const newFavicon = resp.favicon || this.data.favicon - if (this.updateFavicon(newFavicon)) { - // Append an updated favicon - update.favicon = newFavicon + if (this.updateFavicon(resp.favicon)) { + update.favicon = this.getFaviconUrl() } // Append a result object @@ -101,7 +100,7 @@ class ServerRegistration { const payload = { versions: this.versions, recordData: this.recordData, - favicon: this.lastFavicon + favicon: this.getFaviconUrl() } // Only append graphPeakData if defined @@ -134,7 +133,8 @@ class ServerRegistration { message: 'Pinging...' }, recordData: this.recordData, - graphPeakData: this.getGraphPeak() + graphPeakData: this.getGraphPeak(), + favicon: this.data.favicon } } @@ -216,13 +216,35 @@ class ServerRegistration { } updateFavicon (favicon) { + // If data.favicon is defined, then a favicon override is present + // Disregard the incoming favicon, regardless if it is different + if (this.data.favicon) { + return false + } + if (favicon && favicon !== this.lastFavicon) { this.lastFavicon = favicon + + // Generate an updated hash + // This is used by #getFaviconUrl + if (!this._faviconHasher) { + this._faviconHasher = crypto.createHash('md5') + } + this.faviconHash = this._faviconHasher.update(favicon).digest('hex').toString() + return true } return false } + getFaviconUrl () { + if (this.faviconHash) { + return '/hashedfavicon?' + this.faviconHash + } else if (this.data.favicon) { + return this.data.favicon + } + } + updateProtocolVersionCompat (incomingId, outgoingId, protocolIndex) { // If the result version matches the attempted version, the version is supported const isSuccess = incomingId === outgoingId From 02071a43c31d5f6059e8c9b67afb92009f893cbf Mon Sep 17 00:00:00 2001 From: Nick Krecklow Date: Fri, 8 May 2020 04:51:42 -0500 Subject: [PATCH 20/30] send 7 day Cache-Control header --- lib/server.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/server.js b/lib/server.js index 49cf340..372c238 100644 --- a/lib/server.js +++ b/lib/server.js @@ -46,7 +46,8 @@ class Server { res.writeHead(200, { 'Content-Type': 'image/png', - 'Content-Length': buf.length + 'Content-Length': buf.length, + 'Cache-Control': 'maxage=604800' // Cache hashed favicon for 7 days }).end(buf) return From 169ec4bfde0b27868d146e587293c6031a10cf29 Mon Sep 17 00:00:00 2001 From: Nick Krecklow Date: Fri, 8 May 2020 04:54:28 -0500 Subject: [PATCH 21/30] update docs/CHANGELOG.md --- docs/CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 688b03f..2e90949 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -1,4 +1,5 @@ **5.4.0** *(May 8 2020)* +- Favicons are now served over the http server (using a unique hash). This allows the favicons to be safely cached for long durations and still support dynamic updates. - Adds "graphDurationLabel" to `config.json` which allows you to manually modify the "24h Peak" label to a custom time duration. - Adds "serverGraphDuration" to `config.json` which allows you to specify the max time duration for the individual server player count graphs. - Adds "performance.skipUnfurlSrv" to `config.json` which allows you to skip SRV unfurling when pinging. For those who aren't pinging servers that use SRV records, this should help speed up ping times. From 82e8db0128ce8140a3a234758b836e76f1e55d88 Mon Sep 17 00:00:00 2001 From: Nick Krecklow Date: Fri, 8 May 2020 04:55:43 -0500 Subject: [PATCH 22/30] maxage->max-age in Cache-Control header --- lib/server.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/server.js b/lib/server.js index 372c238..6c74c56 100644 --- a/lib/server.js +++ b/lib/server.js @@ -47,7 +47,7 @@ class Server { res.writeHead(200, { 'Content-Type': 'image/png', 'Content-Length': buf.length, - 'Cache-Control': 'maxage=604800' // Cache hashed favicon for 7 days + 'Cache-Control': 'max-age=604800' // Cache hashed favicon for 7 days }).end(buf) return From 15814cf86bcca4f9ce64ca094d4cb5e464cceb5a Mon Sep 17 00:00:00 2001 From: Nick Krecklow Date: Fri, 8 May 2020 05:05:46 -0500 Subject: [PATCH 23/30] serve hashedfavicon path as png --- lib/server.js | 8 ++++++-- lib/servers.js | 5 ++++- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/lib/server.js b/lib/server.js index 6c74c56..063b484 100644 --- a/lib/server.js +++ b/lib/server.js @@ -6,6 +6,9 @@ const serveStatic = require('serve-static') const logger = require('./logger') +const HASHED_FAVICON_URL = '/hashedfavicon_' +const HASHED_FAVICON_EXTENSION = '.png' + function getRemoteAddr (req) { return req.headers['cf-connecting-ip'] || req.headers['x-forwarded-for'] || req.connection.remoteAddress } @@ -25,7 +28,7 @@ class Server { this._http = http.createServer((req, res) => { logger.log('info', '%s requested: %s', getRemoteAddr(req), req.url) - if (req.url.startsWith('/hashedfavicon?')) { + if (req.url.startsWith(HASHED_FAVICON_URL) && req.url.endsWith(HASHED_FAVICON_EXTENSION)) { this.handleFaviconRequest(req, res) } else { // Attempt to handle req using distServeStatic, otherwise fail over to faviconServeStatic @@ -38,7 +41,8 @@ class Server { } handleFaviconRequest = (req, res) => { - const hash = req.url.split('?')[1] + let hash = req.url.substring(HASHED_FAVICON_URL.length) + hash = hash.substring(0, hash.length - HASHED_FAVICON_EXTENSION.length) for (const serverRegistration of this._app.serverRegistrations) { if (serverRegistration.faviconHash && serverRegistration.faviconHash === hash) { diff --git a/lib/servers.js b/lib/servers.js index 61777e2..11b8d3d 100644 --- a/lib/servers.js +++ b/lib/servers.js @@ -5,6 +5,9 @@ const TimeTracker = require('./time') const config = require('../config') const minecraftVersions = require('../minecraft_versions') +const HASHED_FAVICON_URL = '/hashedfavicon_' +const HASHED_FAVICON_EXTENSION = '.png' + class ServerRegistration { serverId lastFavicon @@ -239,7 +242,7 @@ class ServerRegistration { getFaviconUrl () { if (this.faviconHash) { - return '/hashedfavicon?' + this.faviconHash + return HASHED_FAVICON_URL + this.faviconHash + HASHED_FAVICON_EXTENSION } else if (this.data.favicon) { return this.data.favicon } From 04d94a9461f71b24ecd6e9b290ffeb3828ebfed5 Mon Sep 17 00:00:00 2001 From: Nick Krecklow Date: Fri, 8 May 2020 16:00:36 -0500 Subject: [PATCH 24/30] use regex + static method to centralize hashedfavicon URL behavior --- lib/server.js | 43 ++++++++++++++++++++++++------------------- lib/servers.js | 8 ++++---- 2 files changed, 28 insertions(+), 23 deletions(-) diff --git a/lib/server.js b/lib/server.js index 063b484..1dfbbad 100644 --- a/lib/server.js +++ b/lib/server.js @@ -1,4 +1,5 @@ const http = require('http') +const format = require('util').format const WebSocket = require('ws') const finalHttpHandler = require('finalhandler') @@ -6,14 +7,18 @@ const serveStatic = require('serve-static') const logger = require('./logger') -const HASHED_FAVICON_URL = '/hashedfavicon_' -const HASHED_FAVICON_EXTENSION = '.png' +const HASHED_FAVICON_URL_REGEX = /hashedfavicon_([a-z0-9]{32}).png/g function getRemoteAddr (req) { return req.headers['cf-connecting-ip'] || req.headers['x-forwarded-for'] || req.connection.remoteAddress } class Server { + static getHashedFaviconUrl (hash) { + // Format must be compatible with HASHED_FAVICON_URL_REGEX + return format('/hashedfavicon_%s.png', hash) + } + constructor (app) { this._app = app @@ -28,24 +33,26 @@ class Server { this._http = http.createServer((req, res) => { logger.log('info', '%s requested: %s', getRemoteAddr(req), req.url) - if (req.url.startsWith(HASHED_FAVICON_URL) && req.url.endsWith(HASHED_FAVICON_EXTENSION)) { - this.handleFaviconRequest(req, res) - } else { - // Attempt to handle req using distServeStatic, otherwise fail over to faviconServeStatic - // If faviconServeStatic fails, pass to finalHttpHandler to terminate - distServeStatic(req, res, () => { - faviconsServeStatic(req, res, finalHttpHandler(req, res)) - }) + // Test the URL against a regex for hashed favicon URLs + // Require only 1 match ([0]) and test its first captured group ([1]) + // Any invalid value or hit miss will pass into static handlers below + const faviconHash = [...req.url.matchAll(HASHED_FAVICON_URL_REGEX)] + + if (faviconHash.length === 1 && this.handleFaviconRequest(res, faviconHash[0][1])) { + return } + + // Attempt to handle req using distServeStatic, otherwise fail over to faviconServeStatic + // If faviconServeStatic fails, pass to finalHttpHandler to terminate + distServeStatic(req, res, () => { + faviconsServeStatic(req, res, finalHttpHandler(req, res)) + }) }) } - handleFaviconRequest = (req, res) => { - let hash = req.url.substring(HASHED_FAVICON_URL.length) - hash = hash.substring(0, hash.length - HASHED_FAVICON_EXTENSION.length) - + handleFaviconRequest = (res, faviconHash) => { for (const serverRegistration of this._app.serverRegistrations) { - if (serverRegistration.faviconHash && serverRegistration.faviconHash === hash) { + if (serverRegistration.faviconHash && serverRegistration.faviconHash === faviconHash) { const buf = Buffer.from(serverRegistration.lastFavicon.split(',')[1], 'base64') res.writeHead(200, { @@ -54,13 +61,11 @@ class Server { 'Cache-Control': 'max-age=604800' // Cache hashed favicon for 7 days }).end(buf) - return + return true } } - // Terminate request, no match - res.writeHead(404) - res.end() + return false } createWebSocketServer () { diff --git a/lib/servers.js b/lib/servers.js index 11b8d3d..064903b 100644 --- a/lib/servers.js +++ b/lib/servers.js @@ -1,13 +1,11 @@ const crypto = require('crypto') const TimeTracker = require('./time') +const Server = require('./server') const config = require('../config') const minecraftVersions = require('../minecraft_versions') -const HASHED_FAVICON_URL = '/hashedfavicon_' -const HASHED_FAVICON_EXTENSION = '.png' - class ServerRegistration { serverId lastFavicon @@ -233,16 +231,18 @@ class ServerRegistration { if (!this._faviconHasher) { this._faviconHasher = crypto.createHash('md5') } + this.faviconHash = this._faviconHasher.update(favicon).digest('hex').toString() return true } + return false } getFaviconUrl () { if (this.faviconHash) { - return HASHED_FAVICON_URL + this.faviconHash + HASHED_FAVICON_EXTENSION + return Server.getHashedFaviconUrl(this.faviconHash) } else if (this.data.favicon) { return this.data.favicon } From 320dc3c2fb96bfc81695e0862b6ab6c341cd1d05 Mon Sep 17 00:00:00 2001 From: Nick Krecklow Date: Fri, 8 May 2020 16:13:24 -0500 Subject: [PATCH 25/30] send playerCount in payload directly instead of nesting into legacy data structure --- assets/js/app.js | 2 +- assets/js/servers.js | 8 ++++---- assets/js/socket.js | 2 +- lib/servers.js | 10 ++-------- 4 files changed, 8 insertions(+), 14 deletions(-) diff --git a/assets/js/app.js b/assets/js/app.js index d5446ce..7c2afe0 100644 --- a/assets/js/app.js +++ b/assets/js/app.js @@ -119,7 +119,7 @@ export class App { // Set initial playerCount to the payload's value // This will always exist since it is explicitly generated by the backend // This is used for any post-add rendering of things like the percentageBar - serverRegistration.playerCount = payload.result.players.online + serverRegistration.playerCount = payload.playerCount } // Create the plot instance internally with the restructured and cleaned data diff --git a/assets/js/servers.js b/assets/js/servers.js index 3b4bd62..bffe350 100644 --- a/assets/js/servers.js +++ b/assets/js/servers.js @@ -105,8 +105,8 @@ export class ServerRegistration { } handlePing (payload, timestamp) { - if (payload.result) { - this.playerCount = payload.result.players.online + if (typeof payload.playerCount !== 'undefined') { + this.playerCount = payload.playerCount // Only update graph for successful pings // This intentionally pauses the server graph when pings begin to fail @@ -200,12 +200,12 @@ export class ServerRegistration { errorElement.style.display = 'block' errorElement.innerText = ping.error.message - } else if (ping.result) { + } else if (typeof ping.playerCount !== 'undefined') { // Ensure the player-count element is visible and hide the error element playerCountLabelElement.style.display = 'block' errorElement.style.display = 'none' - document.getElementById('player-count-value_' + this.serverId).innerText = formatNumber(ping.result.players.online) + document.getElementById('player-count-value_' + this.serverId).innerText = formatNumber(ping.playerCount) } // An updated favicon has been sent, update the src diff --git a/assets/js/socket.js b/assets/js/socket.js index 2bb3600..4fb764a 100644 --- a/assets/js/socket.js +++ b/assets/js/socket.js @@ -101,7 +101,7 @@ export class SocketManager { // Skip any incoming updates if the graph is disabled if (serverUpdate.updateHistoryGraph && this._app.graphDisplayManager.isVisible) { // Update may not be successful, safely append 0 points - const playerCount = serverUpdate.result ? serverUpdate.result.players.online : 0 + const playerCount = serverUpdate.playerCount || 0 this._app.graphDisplayManager.addGraphPoint(serverRegistration.serverId, payload.timestamp, playerCount) diff --git a/lib/servers.js b/lib/servers.js index 064903b..3fd5c6f 100644 --- a/lib/servers.js +++ b/lib/servers.js @@ -70,9 +70,7 @@ class ServerRegistration { // Append a result object // This filters out unwanted data from resp - update.result = { - players: resp.players - } + update.playerCount = resp.players.online if (config.logToDatabase) { // Update calculated graph peak regardless if the graph is being updated @@ -115,11 +113,7 @@ class ServerRegistration { // Assume the ping was a success and define result // pingHistory does not keep error references, so its impossible to detect if this is an error // It is also pointless to store that data since it will be short lived - payload.result = { - players: { - online: this._pingHistory[this._pingHistory.length - 1] - } - } + payload.playerCount = this._pingHistory[this._pingHistory.length - 1] // Send a copy of pingHistory // Include the last value even though it is contained within payload From 9cb9e4a0622cf839f9ae4074a6beb174fb738973 Mon Sep 17 00:00:00 2001 From: Nick Krecklow Date: Fri, 8 May 2020 16:17:38 -0500 Subject: [PATCH 26/30] update docs/CHANGELOG.md --- docs/CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 2e90949..21bfa12 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -4,6 +4,7 @@ - Adds "serverGraphDuration" to `config.json` which allows you to specify the max time duration for the individual server player count graphs. - Adds "performance.skipUnfurlSrv" to `config.json` which allows you to skip SRV unfurling when pinging. For those who aren't pinging servers that use SRV records, this should help speed up ping times. - Ping timestamps are now shared between all server pings. This means less data transfer when loading or updating the page, less memory usage by the backend and frontend, and less hectic updates on the frontend. +- Optimized several protocol level schemas to remove legacy format waste. Less bandwidth! - Fixes a bug where favicons may not be updated if the page is loaded prior to their initialization. **5.3.1** *(May 5 2020)* From f6780e7a9b153f98a02277e77ef7be3e668c0f62 Mon Sep 17 00:00:00 2001 From: Nick Krecklow Date: Fri, 8 May 2020 16:22:24 -0500 Subject: [PATCH 27/30] prevent data leakage when sending server.data --- lib/app.js | 2 +- lib/servers.js | 10 ++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/lib/app.js b/lib/app.js index 22127ad..82c2b0a 100644 --- a/lib/app.js +++ b/lib/app.js @@ -71,7 +71,7 @@ class App { graphDurationLabel: config.graphDurationLabel || (Math.floor(config.graphDuration / (60 * 60 * 1000)) + 'h'), graphMaxLength: TimeTracker.getMaxGraphDataLength(), serverGraphMaxLength: TimeTracker.getMaxServerGraphDataLength(), - servers: this.serverRegistrations.map(serverRegistration => serverRegistration.data), + servers: this.serverRegistrations.map(serverRegistration => serverRegistration.getPublicData()), minecraftVersions: minecraftVersionNames, isGraphVisible: config.logToDatabase } diff --git a/lib/servers.js b/lib/servers.js index 3fd5c6f..9b1a372 100644 --- a/lib/servers.js +++ b/lib/servers.js @@ -303,6 +303,16 @@ class ServerRegistration { message: message } } + + getPublicData () { + // Return a custom object instead of data directly to avoid data leakage + return { + name: this.data.name, + ip: this.data.ip, + type: this.data.type, + color: this.data.color + } + } } module.exports = ServerRegistration From 11d3031b6cd4947baeb9fd3eb79dda1fdc9a2b3a Mon Sep 17 00:00:00 2001 From: Nick Krecklow Date: Fri, 8 May 2020 16:53:48 -0500 Subject: [PATCH 28/30] add config.performance.unfurlSrvCacheTtl option for caching resolveSrv calls --- config.json | 3 ++- docs/CHANGELOG.md | 5 +++-- lib/ping.js | 31 ++++++-------------------- lib/servers.js | 57 +++++++++++++++++++++++++++++++++++++++++++++++ main.js | 5 +++++ 5 files changed, 74 insertions(+), 27 deletions(-) diff --git a/config.json b/config.json index ce470b7..157ed92 100644 --- a/config.json +++ b/config.json @@ -10,7 +10,8 @@ "connectTimeout": 2500 }, "performance": { - "skipUnfurlSrv": true + "skipUnfurlSrv": false, + "unfurlSrvCacheTtl": 120000 }, "logToDatabase": true, "graphDuration": 86400000, diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 21bfa12..713b868 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -1,8 +1,9 @@ **5.4.0** *(May 8 2020)* - Favicons are now served over the http server (using a unique hash). This allows the favicons to be safely cached for long durations and still support dynamic updates. - Adds "graphDurationLabel" to `config.json` which allows you to manually modify the "24h Peak" label to a custom time duration. -- Adds "serverGraphDuration" to `config.json` which allows you to specify the max time duration for the individual server player count graphs. -- Adds "performance.skipUnfurlSrv" to `config.json` which allows you to skip SRV unfurling when pinging. For those who aren't pinging servers that use SRV records, this should help speed up ping times. +- Adds "serverGraphDuration" (default 3 minutes) to `config.json` which allows you to specify the max time duration for the individual server player count graphs. +- Adds "performance.skipUnfurlSrv" (default false) to `config.json` which allows you to skip SRV unfurling when pinging. For those who aren't pinging servers that use SRV records, this should help speed up ping times. +- Adds "performance.skipUnfurlSrv" (default 120 seconds) to `config.json` which allows you specify how long a SRV unfurl should be cached for. This prevents repeated, potentially slow lookups. Set to 0 to disable caching. - Ping timestamps are now shared between all server pings. This means less data transfer when loading or updating the page, less memory usage by the backend and frontend, and less hectic updates on the frontend. - Optimized several protocol level schemas to remove legacy format waste. Less bandwidth! - Fixes a bug where favicons may not be updated if the page is loaded prior to their initialization. diff --git a/lib/ping.js b/lib/ping.js index e9ae8ba..6434e3a 100644 --- a/lib/ping.js +++ b/lib/ping.js @@ -1,5 +1,3 @@ -const dns = require('dns') - const minecraftJavaPing = require('mc-ping-updated') const minecraftBedrockPing = require('mcpe-ping-fixed') @@ -8,10 +6,10 @@ const MessageOf = require('./message') const config = require('../config') -function ping (host, port, type, timeout, callback, version) { - switch (type) { +function ping (serverRegistration, timeout, callback, version) { + switch (serverRegistration.data.type) { case 'PC': - unfurlSrv(host, port, (host, port) => { + serverRegistration.unfurlSrv((host, port) => { minecraftJavaPing(host, port || 25565, (err, res) => { if (err) { callback(err) @@ -35,13 +33,13 @@ function ping (host, port, type, timeout, callback, version) { break case 'PE': - minecraftBedrockPing(host, port || 19132, (err, res) => { + minecraftBedrockPing(serverRegistration.data.ip, serverRegistration.data.port || 19132, (err, res) => { if (err) { callback(err) } else { callback(null, { players: { - online: capPlayerCount(host, parseInt(res.currentPlayers)) + online: capPlayerCount(serverRegistration.data.ip, parseInt(res.currentPlayers)) } }) } @@ -49,25 +47,10 @@ function ping (host, port, type, timeout, callback, version) { break default: - throw new Error('Unsupported type: ' + type) + throw new Error('Unsupported type: ' + serverRegistration.data.type) } } -function unfurlSrv (hostname, port, callback) { - if (config.performance && config.performance.skipUnfurlSrv) { - callback(hostname, port) - return - } - - dns.resolveSrv('_minecraft._tcp.' + hostname, (_, records) => { - if (!records || records.length < 1) { - callback(hostname, port) - } else { - callback(records[0].name, records[0].port) - } - }) -} - // player count can be up to 1^32-1, which is a massive scale and destroys browser performance when rendering graphs // Artificially cap and warn to prevent propogating garbage function capPlayerCount (host, playerCount) { @@ -136,7 +119,7 @@ class PingController { for (const serverRegistration of this._app.serverRegistrations) { const version = serverRegistration.getNextProtocolVersion() - ping(serverRegistration.data.ip, serverRegistration.data.port, serverRegistration.data.type, config.rates.connectTimeout, (err, resp) => { + ping(serverRegistration, config.rates.connectTimeout, (err, resp) => { if (err) { logger.log('error', 'Failed to ping %s: %s', serverRegistration.data.ip, err.message) } diff --git a/lib/servers.js b/lib/servers.js index 9b1a372..52b93d5 100644 --- a/lib/servers.js +++ b/lib/servers.js @@ -1,4 +1,5 @@ const crypto = require('crypto') +const dns = require('dns') const TimeTracker = require('./time') const Server = require('./server') @@ -313,6 +314,62 @@ class ServerRegistration { color: this.data.color } } + + unfurlSrv (callback) { + // Skip unfurling SRV, instantly return pre-configured data + if (config.performance && config.performance.skipUnfurlSrv) { + callback(this.data.ip, this.data.port) + return + } + + const timestamp = new Date().getTime() + + // If a cached copy exists and is within its TTL, instantly return + if (this._lastUnfurlSrv && timestamp - this._lastUnfurlSrv.timestamp <= config.performance.unfurlSrvCacheTtl) { + callback(this._lastUnfurlSrv.ip, this._lastUnfurlSrv.port) + return + } + + // Group callbacks into an array + // Once resolved, fire callbacks sequentially + // This avoids callbacks possibly executing out of order + if (!this._unfurlSrvCallbackQueue) { + this._unfurlSrvCallbackQueue = [] + } + + this._unfurlSrvCallbackQueue.push(callback) + + // Prevent multiple #resolveSrv calls per ServerRegistration + if (!this._isUnfurlingSrv) { + this._isUnfurlingSrv = true + + dns.resolveSrv('_minecraft._tcp' + this.data.ip, (_, records) => { + this._lastUnfurlSrv = { + timestamp + } + + if (records && records.length > 0) { + this._lastUnfurlSrv.ip = records[0].name + this._lastUnfurlSrv.port = records[0].port + } else { + // Provide fallback to pre-configured data + this._lastUnfurlSrv.ip = this.data.ip + this._lastUnfurlSrv.port = this.data.port + } + + // Fire the waiting callbacks in queue + // Release blocking flag to allow new #resolveSrv calls + this._isUnfurlingSrv = false + + for (const callback of this._unfurlSrvCallbackQueue) { + callback(this._lastUnfurlSrv.ip, this._lastUnfurlSrv.port) + } + + // Reset the callback queue + this._unfurlSrvCallbackQueue = [] + }) + } + } } module.exports = ServerRegistration diff --git a/main.js b/main.js index 5f4c56e..e0518f7 100644 --- a/main.js +++ b/main.js @@ -34,6 +34,11 @@ if (config.performance && config.performance.skipUnfurlSrv) { logger.log('warn', '"performance.skipUnfurlSrv" is enabled. Any configured hosts using SRV records may not properly resolve.') } +if (!config.performance || typeof config.performance.unfurlSrvCacheTtl === 'undefined') { + logger.log('warn', '"performance.unfurlSrvCacheTtl" is not defined in config.json - defaulting to 120 seconds!') + config.performance.unfurlSrvCacheTtl = 2 * 60 * 1000 +} + if (!config.logToDatabase) { logger.log('warn', 'Database logging is not enabled. You can enable it by setting "logToDatabase" to true in config.json. This requires sqlite3 to be installed.') From 4c20843744ba56bc9a6717dd2227eb852297f4b3 Mon Sep 17 00:00:00 2001 From: Nick Krecklow Date: Fri, 8 May 2020 17:22:25 -0500 Subject: [PATCH 29/30] set Cache-Control header for hashed favicons to public --- lib/server.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/server.js b/lib/server.js index 1dfbbad..969ad68 100644 --- a/lib/server.js +++ b/lib/server.js @@ -58,7 +58,7 @@ class Server { res.writeHead(200, { 'Content-Type': 'image/png', 'Content-Length': buf.length, - 'Cache-Control': 'max-age=604800' // Cache hashed favicon for 7 days + 'Cache-Control': 'public, max-age=604800' // Cache hashed favicon for 7 days }).end(buf) return true From 6ed4b7e151656aa5c7db5221f329a825d7c1b044 Mon Sep 17 00:00:00 2001 From: Nick Krecklow Date: Sat, 9 May 2020 18:48:24 -0500 Subject: [PATCH 30/30] update 5.4.0 release date --- docs/CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 713b868..5ac082b 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -1,4 +1,4 @@ -**5.4.0** *(May 8 2020)* +**5.4.0** *(May 9 2020)* - Favicons are now served over the http server (using a unique hash). This allows the favicons to be safely cached for long durations and still support dynamic updates. - Adds "graphDurationLabel" to `config.json` which allows you to manually modify the "24h Peak" label to a custom time duration. - Adds "serverGraphDuration" (default 3 minutes) to `config.json` which allows you to specify the max time duration for the individual server player count graphs.