Merge pull request #164 from Cryptkeeper/bulk-pings

5.4.0 release preview

assets/js/app.js

@@ -62,8 +62,6 @@ export class App {
 
   initTasks () {
     this._taskIds.push(setInterval(this.sortController.sortServers, 5000))
-    this._taskIds.push(setInterval(this.updateGlobalStats, 1000))
-    this._taskIds.push(setInterval(this.percentageBar.redraw, 1000))
   }
 
   handleDisconnect () {
@@ -102,26 +100,34 @@ export class App {
       .reduce((sum, current) => sum + current, 0)
   }
 
-  addServer = (pings) => {
+  addServer = (serverId, payload, timestampPoints) => {
     // Even if the backend has never pinged the server, the frontend is promised a placeholder object.
     // result = undefined
     // error = defined with "Waiting" description
     // info = safely defined with configured data
-    const latestPing = pings[pings.length - 1]
+    const serverRegistration = this.serverRegistry.createServerRegistration(serverId)
-    const serverRegistration = this.serverRegistry.createServerRegistration(latestPing.serverId)
 
-    serverRegistration.initServerStatus(latestPing)
+    serverRegistration.initServerStatus(payload)
 
-    // Push the historical data into the graph
+    // playerCountHistory is only defined when the backend has previous ping data
-    // This will trim and format the data so it is ready for the graph to render once init
+    // undefined playerCountHistory means this is a placeholder ping generated by the backend
-    serverRegistration.addGraphPoints(pings)
+    if (typeof payload.playerCountHistory !== 'undefined') {
+      // Push the historical data into the graph
+      // This will trim and format the data so it is ready for the graph to render once init
+      serverRegistration.addGraphPoints(payload.playerCountHistory, timestampPoints)
+
+      // Set initial playerCount to the payload's value
+      // This will always exist since it is explicitly generated by the backend
+      // This is used for any post-add rendering of things like the percentageBar
+      serverRegistration.playerCount = payload.playerCount
+    }
 
     // Create the plot instance internally with the restructured and cleaned data
     serverRegistration.buildPlotInstance()
 
     // Handle the last known state (if any) as an incoming update
     // This triggers the main update pipeline and enables centralized update handling
-    serverRegistration.updateServerStatus(latestPing, true, this.publicConfig.minecraftVersions)
+    serverRegistration.updateServerStatus(payload, this.publicConfig.minecraftVersions)
 
     // Allow the ServerRegistration to bind any DOM events with app instance context
     serverRegistration.initEventListeners()

assets/js/graph.js

@@ -57,14 +57,15 @@ export class GraphDisplayManager {
       return
     }
 
-    // Trim any outdated entries by filtering the array into a new array
+    const graphData = this._graphData[serverId]
-    const startTimestamp = new Date().getTime()
-    const newGraphData = this._graphData[serverId].filter(point => startTimestamp - point[0] <= this._app.publicConfig.graphDuration)
 
     // Push the new data from the method call request
-    newGraphData.push([timestamp, playerCount])
+    graphData.push([timestamp, playerCount])
 
-    this._graphData[serverId] = newGraphData
+    // Trim any outdated entries by filtering the array into a new array
+    if (graphData.length > this._app.publicConfig.graphMaxLength) {
+      graphData.shift()
+    }
   }
 
   loadLocalStorage () {
@@ -160,18 +161,6 @@ export class GraphDisplayManager {
     document.getElementById('settings-toggle').style.display = 'inline-block'
   }
 
-  // requestRedraw allows usages to request a redraw that may be performed, or cancelled, sometime later
-  // This allows multiple rapid, but individual updates, to clump into a single redraw instead
-  requestRedraw () {
-    if (this._redrawRequestTimeout) {
-      clearTimeout(this._redrawRequestTimeout)
-    }
-
-    // Schedule new delayed redraw call
-    // This can be cancelled by #requestRedraw, #redraw and #reset
-    this._redrawRequestTimeout = setTimeout(this.redraw, 1000)
-  }
-
   redraw = () => {
     // Use drawing as a hint to update settings
     // This may cause unnessecary localStorage updates, but its a rare and harmless outcome
@@ -182,14 +171,6 @@ export class GraphDisplayManager {
     this._plotInstance.setData(this.getVisibleGraphData())
     this._plotInstance.setupGrid()
     this._plotInstance.draw()
-
-    // undefine value so #clearTimeout is not called
-    // This is safe even if #redraw is manually called since it removes the pending work
-    if (this._redrawRequestTimeout) {
-      clearTimeout(this._redrawRequestTimeout)
-    }
-
-    this._redrawRequestTimeout = undefined
   }
 
   requestResize () {
@@ -347,12 +328,6 @@ export class GraphDisplayManager {
       this._resizeRequestTimeout = undefined
     }
 
-    if (this._redrawRequestTimeout) {
-      clearTimeout(this._redrawRequestTimeout)
-
-      this._redrawRequestTimeout = undefined
-    }
-
     // Reset modified DOM structures
     document.getElementById('big-graph-checkboxes').innerHTML = ''
     document.getElementById('big-graph-controls').style.display = 'none'

assets/js/servers.js

@@ -76,8 +76,6 @@ export class ServerRegistry {
   }
 }
 
-const SERVER_GRAPH_DATA_MAX_LENGTH = 72
-
 export class ServerRegistration {
   playerCount = 0
   isVisible = true
@@ -94,42 +92,33 @@ export class ServerRegistration {
     this._failedSequentialPings = 0
   }
 
-  addGraphPoints (points) {
+  addGraphPoints (points, timestampPoints) {
-    // Test if the first point contains error.placeholder === true
+    for (let i = 0; i < points.length; i++) {
-    // This is sent by the backend when the server hasn't been pinged yet
+      const point = points[i]
-    // These points will be disregarded to prevent the graph starting at 0 player count
+      const timestamp = timestampPoints[i]
-    points = points.filter(point => !point.error || !point.error.placeholder)
+      this._graphData.push([timestamp, point])
-
-    // The backend should never return more data elements than the max
-    // but trim the data result regardless for safety and performance purposes
-    if (points.length > SERVER_GRAPH_DATA_MAX_LENGTH) {
-      points.slice(points.length - SERVER_GRAPH_DATA_MAX_LENGTH, points.length)
     }
-
-    this._graphData = points.map(point => point.result ? [point.timestamp, point.result.players.online] : [point.timestamp, 0])
   }
 
   buildPlotInstance () {
     this._plotInstance = $.plot('#chart_' + this.serverId, [this._graphData], SERVER_GRAPH_OPTIONS)
   }
 
-  handlePing (payload, pushToGraph) {
+  handlePing (payload, timestamp) {
-    if (payload.result) {
+    if (typeof payload.playerCount !== 'undefined') {
-      this.playerCount = payload.result.players.online
+      this.playerCount = payload.playerCount
 
-      if (pushToGraph) {
+      // Only update graph for successful pings
-        // Only update graph for successful pings
+      // This intentionally pauses the server graph when pings begin to fail
-        // This intentionally pauses the server graph when pings begin to fail
+      this._graphData.push([timestamp, this.playerCount])
-        this._graphData.push([payload.timestamp, this.playerCount])
 
-        // Trim graphData to within the max length by shifting out the leading elements
+      // Trim graphData to within the max length by shifting out the leading elements
-        if (this._graphData.length > SERVER_GRAPH_DATA_MAX_LENGTH) {
+      if (this._graphData.length > this._app.publicConfig.serverGraphMaxLength) {
-          this._graphData.shift()
+        this._graphData.shift()
-        }
-
-        this.redraw()
       }
 
+      this.redraw()
+
       // Reset failed ping counter to ensure the next connection error
       // doesn't instantly retrigger a layout change
       this._failedSequentialPings = 0
@@ -169,11 +158,7 @@ export class ServerRegistration {
     this.lastPeakData = data
   }
 
-  updateServerStatus (ping, isInitialUpdate, minecraftVersions) {
+  updateServerStatus (ping, minecraftVersions) {
-    // Only pushToGraph when initialUpdate === false
-    // Otherwise the ping value is pushed into the graphData when already present
-    this.handlePing(ping, !isInitialUpdate)
-
     if (ping.versions) {
       const versionsElement = document.getElementById('version_' + this.serverId)
 
@@ -215,24 +200,27 @@ export class ServerRegistration {
       errorElement.style.display = 'block'
 
       errorElement.innerText = ping.error.message
-    } else if (ping.result) {
+    } else if (typeof ping.playerCount !== 'undefined') {
       // Ensure the player-count element is visible and hide the error element
       playerCountLabelElement.style.display = 'block'
       errorElement.style.display = 'none'
 
-      document.getElementById('player-count-value_' + this.serverId).innerText = formatNumber(ping.result.players.online)
+      document.getElementById('player-count-value_' + this.serverId).innerText = formatNumber(ping.playerCount)
+    }
 
-      // An updated favicon has been sent, update the src
+    // An updated favicon has been sent, update the src
-      // Ignore calls from 'add' events since they will have explicitly manually handled the favicon update
+    if (ping.favicon) {
-      if (!isInitialUpdate && ping.favicon) {
+      const faviconElement = document.getElementById('favicon_' + this.serverId)
-        document.getElementById('favicon_' + this.serverId).setAttribute('src', ping.favicon)
+
+      // Since favicons may be URLs, only update the attribute when it has changed
+      // Otherwise the browser may send multiple requests to the same URL
+      if (faviconElement.getAttribute('src') !== ping.favicon) {
+        faviconElement.setAttribute('src', ping.favicon)
       }
     }
   }
 
   initServerStatus (latestPing) {
-    const peakHourDuration = Math.floor(this._app.publicConfig.graphDuration / (60 * 60 * 1000)) + 'h Peak: '
-
     const serverElement = document.createElement('div')
 
     serverElement.id = 'container_' + this.serverId
@@ -244,7 +232,7 @@ export class ServerRegistration {
         '<h3 class="server-name"><span class="' + this._app.favoritesManager.getIconClass(this.isFavorite) + '" id="favorite-toggle_' + this.serverId + '"></span> ' + this.data.name + '</h3>' +
         '<span class="server-error" id="error_' + this.serverId + '"></span>' +
         '<span class="server-label" id="player-count_' + this.serverId + '">Players: <span class="server-value" id="player-count-value_' + this.serverId + '"></span></span>' +
-        '<span class="server-label" id="peak_' + this.serverId + '">' + peakHourDuration + '<span class="server-value" id="peak-value_' + this.serverId + '">-</span></span>' +
+        '<span class="server-label" id="peak_' + this.serverId + '">' + this._app.publicConfig.graphDurationLabel + ' Peak: <span class="server-value" id="peak-value_' + this.serverId + '">-</span></span>' +
         '<span class="server-label" id="record_' + this.serverId + '">Record: <span class="server-value" id="record-value_' + this.serverId + '">-</span></span>' +
         '<span class="server-label" id="version_' + this.serverId + '"></span>' +
       '</div>' +

assets/js/socket.js

@@ -67,7 +67,9 @@ export class SocketManager {
             }
           }
 
-          payload.servers.forEach(this._app.addServer)
+          payload.servers.forEach((serverPayload, serverId) => {
+            this._app.addServer(serverId, serverPayload, payload.timestampPoints)
+          })
 
           if (payload.mojangServices) {
             this._app.mojangUpdater.updateStatus(payload.mojangServices)
@@ -79,29 +81,45 @@ export class SocketManager {
 
           break
 
-        case 'updateServer': {
+        case 'updateServers': {
-          // The backend may send "update" events prior to receiving all "add" events
+          let requestGraphRedraw = false
-          // A server has only been added once it's ServerRegistration is defined
-          // Checking undefined protects from this race condition
-          const serverRegistration = this._app.serverRegistry.getServerRegistration(payload.serverId)
 
-          if (serverRegistration) {
+          for (let serverId = 0; serverId < payload.updates.length; serverId++) {
-            serverRegistration.updateServerStatus(payload, false, this._app.publicConfig.minecraftVersions)
+            // The backend may send "update" events prior to receiving all "add" events
-          }
+            // A server has only been added once it's ServerRegistration is defined
+            // Checking undefined protects from this race condition
+            const serverRegistration = this._app.serverRegistry.getServerRegistration(serverId)
+            const serverUpdate = payload.updates[serverId]
 
-          // Use update payloads to conditionally append data to graph
+            if (serverRegistration) {
-          // Skip any incoming updates if the graph is disabled
+              serverRegistration.handlePing(serverUpdate, payload.timestamp)
-          if (payload.updateHistoryGraph && this._app.graphDisplayManager.isVisible) {
-            // Update may not be successful, safely append 0 points
-            const playerCount = payload.result ? payload.result.players.online : 0
 
-            this._app.graphDisplayManager.addGraphPoint(serverRegistration.serverId, payload.timestamp, playerCount)
+              serverRegistration.updateServerStatus(serverUpdate, this._app.publicConfig.minecraftVersions)
+            }
 
-            // Only redraw the graph if not mutating hidden data
+            // Use update payloads to conditionally append data to graph
-            if (serverRegistration.isVisible) {
+            // Skip any incoming updates if the graph is disabled
-              this._app.graphDisplayManager.requestRedraw()
+            if (serverUpdate.updateHistoryGraph && this._app.graphDisplayManager.isVisible) {
+              // Update may not be successful, safely append 0 points
+              const playerCount = serverUpdate.playerCount || 0
+
+              this._app.graphDisplayManager.addGraphPoint(serverRegistration.serverId, payload.timestamp, playerCount)
+
+              // Only redraw the graph if not mutating hidden data
+              if (serverRegistration.isVisible) {
+                requestGraphRedraw = true
+              }
             }
           }
+
+          // Run redraw tasks after handling bulk updates
+          if (requestGraphRedraw) {
+            this._app.graphDisplayManager.redraw()
+          }
+
+          this._app.percentageBar.redraw()
+          this._app.updateGlobalStats()
+
           break
         }
 

assets/js/sort.js

@@ -8,7 +8,7 @@ const SORT_OPTIONS = [
   },
   {
     getName: (app) => {
-      return Math.floor(app.publicConfig.graphDuration / (60 * 60 * 1000)) + 'h Peak'
+      return app.publicConfig.graphDurationLabel + ' Peak'
     },
     sortFunc: (a, b) => {
       if (!a.lastPeakData && !b.lastPeakData) {

config.json

@@ -9,6 +9,11 @@
 		"pingAll": 3000,
 		"connectTimeout": 2500
 	},
-	"logToDatabase": false,
+	"performance": {
-	"graphDuration": 86400000
+		"skipUnfurlSrv": false,
+		"unfurlSrvCacheTtl": 120000
+	},
+	"logToDatabase": true,
+	"graphDuration": 86400000,
+	"serverGraphDuration": 180000
 }

docs/CHANGELOG.md

@@ -1,3 +1,13 @@
+**5.4.0** *(May 9 2020)*
+- Favicons are now served over the http server (using a unique hash). This allows the favicons to be safely cached for long durations and still support dynamic updates.
+- Adds "graphDurationLabel" to `config.json` which allows you to manually modify the "24h Peak" label to a custom time duration.
+- Adds "serverGraphDuration" (default 3 minutes) to `config.json` which allows you to specify the max time duration for the individual server player count graphs.
+- Adds "performance.skipUnfurlSrv" (default false) to `config.json` which allows you to skip SRV unfurling when pinging. For those who aren't pinging servers that use SRV records, this should help speed up ping times.
+- Adds "performance.skipUnfurlSrv" (default 120 seconds) to `config.json` which allows you specify how long a SRV unfurl should be cached for. This prevents repeated, potentially slow lookups. Set to 0 to disable caching.
+- Ping timestamps are now shared between all server pings. This means less data transfer when loading or updating the page, less memory usage by the backend and frontend, and less hectic updates on the frontend.
+- Optimized several protocol level schemas to remove legacy format waste. Less bandwidth!
+- Fixes a bug where favicons may not be updated if the page is loaded prior to their initialization.
+
 **5.3.1** *(May 5 2020)*
 - Fixes Mojang service status indicators not updating after initial page load.
 

lib/app.js

@@ -2,6 +2,7 @@ const Database = require('./database')
 const MojangUpdater = require('./mojang')
 const PingController = require('./ping')
 const Server = require('./server')
+const TimeTracker = require('./time')
 const MessageOf = require('./message')
 
 const config = require('../config')
@@ -13,7 +14,8 @@ class App {
   constructor () {
     this.mojangUpdater = new MojangUpdater(this)
     this.pingController = new PingController(this)
-    this.server = new Server(this.handleClientConnection)
+    this.server = new Server(this)
+    this.timeTracker = new TimeTracker(this)
   }
 
   loadDatabase (callback) {
@@ -66,13 +68,16 @@ class App {
 
         // Send configuration data for rendering the page
         return {
-          graphDuration: config.graphDuration,
+          graphDurationLabel: config.graphDurationLabel || (Math.floor(config.graphDuration / (60 * 60 * 1000)) + 'h'),
-          servers: this.serverRegistrations.map(serverRegistration => serverRegistration.data),
+          graphMaxLength: TimeTracker.getMaxGraphDataLength(),
+          serverGraphMaxLength: TimeTracker.getMaxServerGraphDataLength(),
+          servers: this.serverRegistrations.map(serverRegistration => serverRegistration.getPublicData()),
           minecraftVersions: minecraftVersionNames,
           isGraphVisible: config.logToDatabase
         }
       })(),
       mojangServices: this.mojangUpdater.getLastUpdate(),
+      timestampPoints: this.timeTracker.getPoints(),
       servers: this.serverRegistrations.map(serverRegistration => serverRegistration.getPingHistory())
     }
 

lib/database.js

@@ -23,18 +23,14 @@ class Database {
       const graphPointsByIp = []
 
       for (const row of pingData) {
-        // Avoid loading outdated records
+        // Load into temporary array
-        // This shouldn't happen and is mostly a sanity measure
+        // This will be culled prior to being pushed to the serverRegistration
-        if (startTime - row.timestamp <= graphDuration) {
+        let graphPoints = graphPointsByIp[row.ip]
-          // Load into temporary array
+        if (!graphPoints) {
-          // This will be culled prior to being pushed to the serverRegistration
+          graphPoints = graphPointsByIp[row.ip] = []
-          let graphPoints = graphPointsByIp[row.ip]
-          if (!graphPoints) {
-            graphPoints = graphPointsByIp[row.ip] = []
-          }
-
-          graphPoints.push([row.timestamp, row.playerCount])
         }
+
+        graphPoints.push([row.timestamp, row.playerCount])
       }
 
       Object.keys(graphPointsByIp).forEach(ip => {
@@ -95,9 +91,7 @@ class Database {
 
   insertPing (ip, timestamp, playerCount) {
     const statement = this._sql.prepare('INSERT INTO pings (timestamp, ip, playerCount) VALUES (?, ?, ?)')
-    this._sql.serialize(() => {
+    statement.run(timestamp, ip, playerCount)
-      statement.run(timestamp, ip, playerCount)
-    })
     statement.finalize()
   }
 }

lib/ping.js

@@ -1,5 +1,3 @@
-const dns = require('dns')
-
 const minecraftJavaPing = require('mc-ping-updated')
 const minecraftBedrockPing = require('mcpe-ping-fixed')
 
@@ -8,10 +6,10 @@ const MessageOf = require('./message')
 
 const config = require('../config')
 
-function ping (host, port, type, timeout, callback, version) {
+function ping (serverRegistration, timeout, callback, version) {
-  switch (type) {
+  switch (serverRegistration.data.type) {
     case 'PC':
-      unfurlSrv(host, port, (host, port) => {
+      serverRegistration.unfurlSrv((host, port) => {
         minecraftJavaPing(host, port || 25565, (err, res) => {
           if (err) {
             callback(err)
@@ -35,13 +33,13 @@ function ping (host, port, type, timeout, callback, version) {
       break
 
     case 'PE':
-      minecraftBedrockPing(host, port || 19132, (err, res) => {
+      minecraftBedrockPing(serverRegistration.data.ip, serverRegistration.data.port || 19132, (err, res) => {
         if (err) {
           callback(err)
         } else {
           callback(null, {
             players: {
-              online: capPlayerCount(host, parseInt(res.currentPlayers))
+              online: capPlayerCount(serverRegistration.data.ip, parseInt(res.currentPlayers))
             }
           })
         }
@@ -49,20 +47,10 @@ function ping (host, port, type, timeout, callback, version) {
       break
 
     default:
-      throw new Error('Unsupported type: ' + type)
+      throw new Error('Unsupported type: ' + serverRegistration.data.type)
   }
 }
 
-function unfurlSrv (hostname, port, callback) {
-  dns.resolveSrv('_minecraft._tcp.' + hostname, (_, records) => {
-    if (!records || records.length < 1) {
-      callback(hostname, port)
-    } else {
-      callback(records[0].name, records[0].port)
-    }
-  })
-}
-
 // player count can be up to 1^32-1, which is a massive scale and destroys browser performance when rendering graphs
 // Artificially cap and warn to prevent propogating garbage
 function capPlayerCount (host, playerCount) {
@@ -92,44 +80,62 @@ class PingController {
   }
 
   pingAll = () => {
+    const timestamp = this._app.timeTracker.newTimestamp()
+
+    this.startPingTasks(results => {
+      const updates = []
+
+      for (const serverRegistration of this._app.serverRegistrations) {
+        const result = results[serverRegistration.serverId]
+
+        // Log to database if enabled
+        if (config.logToDatabase) {
+          const playerCount = result.resp ? result.resp.players.online : 0
+
+          this._app.database.insertPing(serverRegistration.data.ip, timestamp, playerCount)
+        }
+
+        // Generate a combined update payload
+        // This includes any modified fields and flags used by the frontend
+        // This will not be cached and can contain live metadata
+        const update = serverRegistration.handlePing(timestamp, result.resp, result.err, result.version)
+
+        updates[serverRegistration.serverId] = update
+      }
+
+      // Send object since updates uses serverIds as keys
+      // Send a single timestamp entry since it is shared
+      this._app.server.broadcast(MessageOf('updateServers', {
+        timestamp,
+        updates
+      }))
+    })
+  }
+
+  startPingTasks = (callback) => {
+    const results = []
+    let remainingTasks = this._app.serverRegistrations.length
+
     for (const serverRegistration of this._app.serverRegistrations) {
       const version = serverRegistration.getNextProtocolVersion()
 
-      ping(serverRegistration.data.ip, serverRegistration.data.port, serverRegistration.data.type, config.rates.connectTimeout, (err, resp) => {
+      ping(serverRegistration, config.rates.connectTimeout, (err, resp) => {
         if (err) {
           logger.log('error', 'Failed to ping %s: %s', serverRegistration.data.ip, err.message)
         }
 
-        this.handlePing(serverRegistration, resp, err, version)
+        results[serverRegistration.serverId] = {
+          resp,
+          err,
+          version
+        }
+
+        if (--remainingTasks === 0) {
+          callback(results)
+        }
       }, version.protocolId)
     }
   }
-
-  handlePing (serverRegistration, resp, err, version) {
-    const timestamp = new Date().getTime()
-
-    serverRegistration.addPing(timestamp, resp)
-
-    let updateHistoryGraph = false
-
-    if (config.logToDatabase) {
-      const playerCount = resp ? resp.players.online : 0
-
-      // Log to database
-      this._app.database.insertPing(serverRegistration.data.ip, timestamp, playerCount)
-
-      if (serverRegistration.addGraphPoint(resp !== undefined, playerCount, timestamp)) {
-        updateHistoryGraph = true
-      }
-    }
-
-    // Generate a combined update payload
-    // This includes any modified fields and flags used by the frontend
-    // This will not be cached and can contain live metadata
-    const updateMessage = serverRegistration.getUpdate(timestamp, resp, err, version, updateHistoryGraph)
-
-    this._app.server.broadcast(MessageOf('updateServer', updateMessage))
-  }
 }
 
 module.exports = PingController

lib/server.js

@@ -1,4 +1,5 @@
 const http = require('http')
+const format = require('util').format
 
 const WebSocket = require('ws')
 const finalHttpHandler = require('finalhandler')
@@ -6,14 +7,23 @@ const serveStatic = require('serve-static')
 
 const logger = require('./logger')
 
+const HASHED_FAVICON_URL_REGEX = /hashedfavicon_([a-z0-9]{32}).png/g
+
 function getRemoteAddr (req) {
   return req.headers['cf-connecting-ip'] || req.headers['x-forwarded-for'] || req.connection.remoteAddress
 }
 
 class Server {
-  constructor (clientSocketHandler) {
+  static getHashedFaviconUrl (hash) {
+    // Format must be compatible with HASHED_FAVICON_URL_REGEX
+    return format('/hashedfavicon_%s.png', hash)
+  }
+
+  constructor (app) {
+    this._app = app
+
     this.createHttpServer()
-    this.createWebSocketServer(clientSocketHandler)
+    this.createWebSocketServer()
   }
 
   createHttpServer () {
@@ -23,6 +33,15 @@ class Server {
     this._http = http.createServer((req, res) => {
       logger.log('info', '%s requested: %s', getRemoteAddr(req), req.url)
 
+      // Test the URL against a regex for hashed favicon URLs
+      // Require only 1 match ([0]) and test its first captured group ([1])
+      // Any invalid value or hit miss will pass into static handlers below
+      const faviconHash = [...req.url.matchAll(HASHED_FAVICON_URL_REGEX)]
+
+      if (faviconHash.length === 1 && this.handleFaviconRequest(res, faviconHash[0][1])) {
+        return
+      }
+
       // Attempt to handle req using distServeStatic, otherwise fail over to faviconServeStatic
       // If faviconServeStatic fails, pass to finalHttpHandler to terminate
       distServeStatic(req, res, () => {
@@ -31,7 +50,25 @@ class Server {
     })
   }
 
-  createWebSocketServer (proxyClientSocketHandler) {
+  handleFaviconRequest = (res, faviconHash) => {
+    for (const serverRegistration of this._app.serverRegistrations) {
+      if (serverRegistration.faviconHash && serverRegistration.faviconHash === faviconHash) {
+        const buf = Buffer.from(serverRegistration.lastFavicon.split(',')[1], 'base64')
+
+        res.writeHead(200, {
+          'Content-Type': 'image/png',
+          'Content-Length': buf.length,
+          'Cache-Control': 'public, max-age=604800' // Cache hashed favicon for 7 days
+        }).end(buf)
+
+        return true
+      }
+    }
+
+    return false
+  }
+
+  createWebSocketServer () {
     this._wss = new WebSocket.Server({
       server: this._http
     })
@@ -45,7 +82,7 @@ class Server {
       })
 
       // Pass client off to proxy handler
-      proxyClientSocketHandler(client)
+      this._app.handleClientConnection(client)
     })
   }
 

lib/servers.js

@@ -1,3 +1,9 @@
+const crypto = require('crypto')
+const dns = require('dns')
+
+const TimeTracker = require('./time')
+const Server = require('./server')
+
 const config = require('../config')
 const minecraftVersions = require('../minecraft_versions')
 
@@ -14,12 +20,35 @@ class ServerRegistration {
     this._pingHistory = []
   }
 
-  getUpdate (timestamp, resp, err, version, updateHistoryGraph) {
+  handlePing (timestamp, resp, err, version) {
-    const update = {
+    const playerCount = resp ? resp.players.online : 0
-      serverId: this.serverId,
+
-      timestamp: timestamp
+    // Store into in-memory ping data
+    this._pingHistory.push(playerCount)
+
+    // Trim pingHistory to avoid memory leaks
+    if (this._pingHistory.length > TimeTracker.getMaxServerGraphDataLength()) {
+      this._pingHistory.shift()
     }
 
+    // Only notify the frontend to append to the historical graph
+    // if both the graphing behavior is enabled and the backend agrees
+    // that the ping is eligible for addition
+    let updateHistoryGraph = false
+
+    if (config.logToDatabase) {
+      if (this.addGraphPoint(resp !== undefined, playerCount, timestamp)) {
+        updateHistoryGraph = true
+      }
+    }
+
+    // Delegate out update payload generation
+    return this.getUpdate(timestamp, resp, err, version, updateHistoryGraph)
+  }
+
+  getUpdate (timestamp, resp, err, version, updateHistoryGraph) {
+    const update = {}
+
     if (resp) {
       if (resp.version && this.updateProtocolVersionCompat(resp.version, version.protocolId, version.protocolIndex)) {
         // Append an updated version listing
@@ -36,18 +65,13 @@ class ServerRegistration {
         update.recordData = this.recordData
       }
 
-      // Compare against this.data.favicon to support favicon overrides
+      if (this.updateFavicon(resp.favicon)) {
-      const newFavicon = resp.favicon || this.data.favicon
+        update.favicon = this.getFaviconUrl()
-      if (this.updateFavicon(newFavicon)) {
-        // Append an updated favicon
-        update.favicon = newFavicon
       }
 
       // Append a result object
       // This filters out unwanted data from resp
-      update.result = {
+      update.playerCount = resp.players.online
-        players: resp.players
-      }
 
       if (config.logToDatabase) {
         // Update calculated graph peak regardless if the graph is being updated
@@ -71,48 +95,12 @@ class ServerRegistration {
     return update
   }
 
-  addPing (timestamp, resp) {
-    const ping = {
-      timestamp: timestamp
-    }
-
-    if (resp) {
-      // Append a result object
-      // This filters out unwanted data from resp
-      ping.result = {
-        players: {
-          online: resp.players.online
-        }
-      }
-    }
-
-    this._pingHistory.push(ping)
-
-    // Trim pingHistory to avoid memory leaks
-    if (this._pingHistory.length > 72) {
-      this._pingHistory.shift()
-    }
-  }
-
   getPingHistory () {
     if (this._pingHistory.length > 0) {
-      const pingHistory = []
-
-      for (let i = 0; i < this._pingHistory.length - 1; i++) {
-        pingHistory[i] = this._pingHistory[i]
-      }
-
-      // Insert the latest update manually into the array
-      // This is a mutated copy of the last update to contain live metadata
-      // The metadata is used by the frontend for rendering
-      const lastPing = this._pingHistory[this._pingHistory.length - 1]
-
       const payload = {
-        serverId: this.serverId,
-        timestamp: lastPing.timestamp,
         versions: this.versions,
         recordData: this.recordData,
-        favicon: this.lastFavicon
+        favicon: this.getFaviconUrl()
       }
 
       // Only append graphPeakData if defined
@@ -123,30 +111,27 @@ class ServerRegistration {
         payload.graphPeakData = graphPeakData
       }
 
-      // Conditionally append to avoid defining fields with undefined values
+      // Assume the ping was a success and define result
-      if (lastPing.result) {
+      // pingHistory does not keep error references, so its impossible to detect if this is an error
-        payload.result = lastPing.result
+      // It is also pointless to store that data since it will be short lived
-      } else if (lastPing.error) {
+      payload.playerCount = this._pingHistory[this._pingHistory.length - 1]
-        payload.error = lastPing.error
-      }
 
-      // Insert the reconstructed update as the last entry
+      // Send a copy of pingHistory
-      // pingHistory is already sorted during its copy from _pingHistory
+      // Include the last value even though it is contained within payload
-      pingHistory.push(payload)
+      // The frontend will only push to its graphData from playerCountHistory
+      payload.playerCountHistory = this._pingHistory
 
-      return pingHistory
+      return payload
     }
 
-    return [{
+    return {
-      serverId: this.serverId,
-      timestamp: new Date().getTime(),
       error: {
-        message: 'Waiting...',
+        message: 'Pinging...'
-        placeholder: true
       },
       recordData: this.recordData,
-      graphPeakData: this.getGraphPeak()
+      graphPeakData: this.getGraphPeak(),
-    }]
+      favicon: this.data.favicon
+    }
   }
 
   loadGraphPoints (points) {
@@ -189,9 +174,10 @@ class ServerRegistration {
     this.graphData.push([timestamp, playerCount])
     this._lastGraphDataPush = timestamp
 
-    // Trim old graphPoints according to graphDuration
+    // Trim old graphPoints according to #getMaxGraphDataLength
-    const filterTimestamp = new Date().getTime() - config.graphDuration
+    if (this.graphData.length > TimeTracker.getMaxGraphDataLength()) {
-    this.graphData = this.graphData.filter(point => point[0] >= filterTimestamp)
+      this.graphData.shift()
+    }
 
     return true
   }
@@ -226,13 +212,37 @@ class ServerRegistration {
   }
 
   updateFavicon (favicon) {
+    // If data.favicon is defined, then a favicon override is present
+    // Disregard the incoming favicon, regardless if it is different
+    if (this.data.favicon) {
+      return false
+    }
+
     if (favicon && favicon !== this.lastFavicon) {
       this.lastFavicon = favicon
+
+      // Generate an updated hash
+      // This is used by #getFaviconUrl
+      if (!this._faviconHasher) {
+        this._faviconHasher = crypto.createHash('md5')
+      }
+
+      this.faviconHash = this._faviconHasher.update(favicon).digest('hex').toString()
+
       return true
     }
+
     return false
   }
 
+  getFaviconUrl () {
+    if (this.faviconHash) {
+      return Server.getHashedFaviconUrl(this.faviconHash)
+    } else if (this.data.favicon) {
+      return this.data.favicon
+    }
+  }
+
   updateProtocolVersionCompat (incomingId, outgoingId, protocolIndex) {
     // If the result version matches the attempted version, the version is supported
     const isSuccess = incomingId === outgoingId
@@ -294,6 +304,72 @@ class ServerRegistration {
       message: message
     }
   }
+
+  getPublicData () {
+    // Return a custom object instead of data directly to avoid data leakage
+    return {
+      name: this.data.name,
+      ip: this.data.ip,
+      type: this.data.type,
+      color: this.data.color
+    }
+  }
+
+  unfurlSrv (callback) {
+    // Skip unfurling SRV, instantly return pre-configured data
+    if (config.performance && config.performance.skipUnfurlSrv) {
+      callback(this.data.ip, this.data.port)
+      return
+    }
+
+    const timestamp = new Date().getTime()
+
+    // If a cached copy exists and is within its TTL, instantly return
+    if (this._lastUnfurlSrv && timestamp - this._lastUnfurlSrv.timestamp <= config.performance.unfurlSrvCacheTtl) {
+      callback(this._lastUnfurlSrv.ip, this._lastUnfurlSrv.port)
+      return
+    }
+
+    // Group callbacks into an array
+    // Once resolved, fire callbacks sequentially
+    // This avoids callbacks possibly executing out of order
+    if (!this._unfurlSrvCallbackQueue) {
+      this._unfurlSrvCallbackQueue = []
+    }
+
+    this._unfurlSrvCallbackQueue.push(callback)
+
+    // Prevent multiple #resolveSrv calls per ServerRegistration
+    if (!this._isUnfurlingSrv) {
+      this._isUnfurlingSrv = true
+
+      dns.resolveSrv('_minecraft._tcp' + this.data.ip, (_, records) => {
+        this._lastUnfurlSrv = {
+          timestamp
+        }
+
+        if (records && records.length > 0) {
+          this._lastUnfurlSrv.ip = records[0].name
+          this._lastUnfurlSrv.port = records[0].port
+        } else {
+          // Provide fallback to pre-configured data
+          this._lastUnfurlSrv.ip = this.data.ip
+          this._lastUnfurlSrv.port = this.data.port
+        }
+
+        // Fire the waiting callbacks in queue
+        // Release blocking flag to allow new #resolveSrv calls
+        this._isUnfurlingSrv = false
+
+        for (const callback of this._unfurlSrvCallbackQueue) {
+          callback(this._lastUnfurlSrv.ip, this._lastUnfurlSrv.port)
+        }
+
+        // Reset the callback queue
+        this._unfurlSrvCallbackQueue = []
+      })
+    }
+  }
 }
 
 module.exports = ServerRegistration

lib/time.js

@@ -0,0 +1,34 @@
+const config = require('../config.json')
+
+class TimeTracker {
+  constructor (app) {
+    this._app = app
+    this._points = []
+  }
+
+  newTimestamp () {
+    const timestamp = new Date().getTime()
+
+    this._points.push(timestamp)
+
+    if (this._points.length > TimeTracker.getMaxServerGraphDataLength()) {
+      this._points.shift()
+    }
+
+    return timestamp
+  }
+
+  getPoints () {
+    return this._points
+  }
+
+  static getMaxServerGraphDataLength () {
+    return Math.ceil(config.serverGraphDuration / config.rates.pingAll)
+  }
+
+  static getMaxGraphDataLength () {
+    return Math.ceil(config.graphDuration / config.rates.pingAll)
+  }
+}
+
+module.exports = TimeTracker

main.js

@@ -25,6 +25,20 @@ servers.forEach((server, serverId) => {
   app.serverRegistrations.push(new ServerRegistration(serverId, server))
 })
 
+if (!config.serverGraphDuration) {
+  logger.log('warn', '"serverGraphDuration" is not defined in config.json - defaulting to 3 minutes!')
+  config.serverGraphDuration = 3 * 60 * 10000
+}
+
+if (config.performance && config.performance.skipUnfurlSrv) {
+  logger.log('warn', '"performance.skipUnfurlSrv" is enabled. Any configured hosts using SRV records may not properly resolve.')
+}
+
+if (!config.performance || typeof config.performance.unfurlSrvCacheTtl === 'undefined') {
+  logger.log('warn', '"performance.unfurlSrvCacheTtl" is not defined in config.json - defaulting to 120 seconds!')
+  config.performance.unfurlSrvCacheTtl = 2 * 60 * 1000
+}
+
 if (!config.logToDatabase) {
   logger.log('warn', 'Database logging is not enabled. You can enable it by setting "logToDatabase" to true in config.json. This requires sqlite3 to be installed.')
 

package-lock.json

@@ -1,6 +1,6 @@
 {
   "name": "minetrack",
-  "version": "5.3.1",
+  "version": "5.4.0",
   "lockfileVersion": 1,
   "requires": true,
   "dependencies": {

package.json

@@ -1,6 +1,6 @@
 {
   "name": "minetrack",
-  "version": "5.3.1",
+  "version": "5.4.0",
   "description": "A Minecraft server tracker that lets you focus on the basics.",
   "main": "main.js",
   "dependencies": {