Fascinated/mc-tracker
1
0
Fork 0
Code Issues 2 Pull Requests 6 Actions Activity

influx-only #2

Merged
Fascinated merged 20 commits from influx-only into master 2024-01-03 08:32:27 +00:00
Conversation 0 Commits 20 Files Changed 19 +1296 -699
19 changed files with 1296 additions and 699 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
.dockerignore Normal file
View File

@ -0,0 +1,24 @@
**/.classpath
**/.dockerignore
**/.env
**/.git
**/.gitignore
**/.project
**/.settings
**/.toolstarget
**/.vs
**/.vscode
**/*.*proj.user
**/*.dbmdl
**/*.jfm
**/charts
**/docker-compose*
**/compose*
**/Dockerfile*
**/node_modules
**/npm-debug.log
**/obj
**/secrets.dev.yaml
**/values.dev.yaml
LICENSE
README.md

62
.gitea/workflows/publish.yaml Normal file
View File

@ -0,0 +1,62 @@
name: Publish Docker Image
on:
push:
branches:
- "master"
- "influx-only"
jobs:
docker:
runs-on: ubuntu-22.04
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Restore Docker Cache
uses: actions/cache@v3
id: docker-cache
with:
path: /usr/bin/docker
key: ${{ runner.os }}-docker
- name: Install Docker (if not cached)
if: steps.docker-cache.outputs.cache-hit != 'true'
run: |
wget -q -O /tmp/docker.tgz https://download.docker.com/linux/static/stable/x86_64/docker-20.10.23.tgz \
&& tar --extract --file /tmp/docker.tgz --directory /usr/bin --strip-components 1 --no-same-owner docker/docker \
&& rm -rf /tmp/* &&
echo "Done"
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
- name: Login to Repo
uses: docker/login-action@v3
with:
username: ${{ secrets.REPO_USERNAME }}
password: ${{ secrets.REPO_TOKEN }}
- name: Restore Docker Build Cache
uses: actions/cache@v3
id: build-cache
with:
path: /tmp/.buildx-cache
key: ${{ runner.os }}-buildx
- name: Build and Push (Latest)
uses: docker/build-push-action@v5
with:
push: true
context: .
tags: fascinated/mc-tracker-backend:influx
cache-from: type=local,src=/tmp/.buildx-cache
cache-to: type=local,dest=/tmp/.buildx-cache
- name: Save Docker Build Cache
if: steps.build-cache.outputs.cache-hit != 'true'
run: |
mkdir -p /tmp/.buildx-cache
cp -r /tmp/.buildx-cache/. /tmp/.buildx-cache-new
rm -rf /tmp/.buildx-cache
mv /tmp/.buildx-cache-new /tmp/.buildx-cache

2
.gitignore vendored
View File

@ -228,3 +228,5 @@ data/db.sqlite
data/db.sqlite-shm
data/db.sqlite-wal
data/database-backups
data/config.json

11
.vscode/launch.json vendored Normal file
View File

@ -0,0 +1,11 @@
{
"configurations": [
{
"name": "Docker Node.js Launch",
"type": "docker",
"request": "launch",
"preLaunchTask": "docker-run: debug",
"platform": "node"
}
]
}

35
.vscode/tasks.json vendored Normal file
View File

@ -0,0 +1,35 @@
{
"version": "2.0.0",
"tasks": [
{
"type": "docker-build",
"label": "docker-build",
"platform": "node",
"dockerBuild": {
"dockerfile": "${workspaceFolder}/Dockerfile",
"context": "${workspaceFolder}",
"pull": true
}
},
{
"type": "docker-run",
"label": "docker-run: release",
"dependsOn": ["docker-build"],
"platform": "node"
},
{
"type": "docker-run",
"label": "docker-run: debug",
"dependsOn": ["docker-build"],
"dockerRun": {
"env": {
"DEBUG": "*",
"NODE_ENV": "development"
}
},
"node": {
"enableDebugging": true
}
}
]
}

11
Dockerfile Normal file
View File

@ -0,0 +1,11 @@
FROM fascinated/docker-images:node-pnpm-latest AS base
ENV NODE_ENV=production
WORKDIR /usr/src/app
COPY ["package.json", "pnpm-lock.yaml", "./"]
COPY . .
RUN pnpm install --production --silent
CMD pnpm run build && pnpm run start

7
data/config.json → data/config.json-example
View File

@ -1,6 +1,9 @@
{
"websocket": {
"port": 3000
"influx": {
"url": "http://localhost:8086",
"token": "my-token",
"org": "my-org",
"bucket": "my-bucket"
},
"scanner": {
"updateCron": "*/1 * * * *",

42
data/servers.json
View File

@ -1,74 +1,62 @@
[
{
"name": "WildPrison",
"ip": "wildprison.net",
"type": "PC",
"id": 0
"name": "WildNetwork",
"ip": "wildnetwork.net",
"type": "PC"
},
{
"name": "Hypixel",
"ip": "mc.hypixel.net",
"type": "PC",
"id": 1
"type": "PC"
},
{
"name": "CubeCraft",
"name": "Cubecraft",
"ip": "play.cubecraft.net",
"type": "PC",
"id": 2
"type": "PC"
},
{
"name": "Mineplex",
"ip": "mineplex.com",
"type": "PC",
"id": 3
"type": "PC"
},
{
"name": "2b2t",
"ip": "2b2t.org",
"type": "PC",
"id": 4
"type": "PC"
},
{
"name": "AkumaMC",
"ip": "akumamc.net",
"type": "PC",
"id": 5
"type": "PC"
},
{
"name": "Wynncraft",
"ip": "play.wynncraft.com",
"type": "PC",
"id": 6
"type": "PC"
},
{
"name": "Minehut",
"ip": "minehut.com",
"type": "PC",
"id": 7
"type": "PC"
},
{
"name": "Grand Theft Minecraft",
"ip": "gtm.network",
"type": "PC",
"id": 8
"type": "PC"
},
{
"name": "HiveMC",
"ip": "geo.hivebedrock.network",
"type": "PE",
"id": 9
"type": "PE"
},
{
"name": "Purple Prison",
"ip": "MCSL.PURPLE.WTF",
"type": "PC",
"id": 10
"type": "PC"
},
{
"name": "MinecraftOnline",
"ip": "minecraftonline.com",
"type": "PC",
"id": 11
"type": "PC"
}
]

22
docker-compose.yml Normal file
View File

@ -0,0 +1,22 @@
version: "3"
services:
tracker:
restart: always
image: fascinated/mc-tracker-backend:influx
volumes:
- ./data:/usr/src/app/data
influxdb:
image: influxdb:latest
container_name: influxdb
networks:
- default
security_opt:
- no-new-privileges:true
restart: always
ports:
- "8086:8086"
volumes:
- ./influx/config:/etc/influxdb2
- ./influx/db:/var/lib/influxdb2

13
package.json
View File

@ -4,26 +4,25 @@
"description": "",
"main": "index.js",
"scripts": {
"dev": "nodemon --exec ts-node src/index.ts"
"dev": "nodemon --exec ts-node src/index.ts",
"build": "tsup src/index.ts --format cjs",
"start": "node dist/index.js"
},
"keywords": [],
"author": "",
"license": "ISC",
"dependencies": {
"@types/better-sqlite3": "^7.6.8",
"@influxdata/influxdb-client": "^1.33.2",
"@types/mcping-js": "^1.5.4",
"@types/node-cron": "^3.0.11",
"better-sqlite3": "^9.2.2",
"dns": "^0.2.2",
"mcpe-ping-fixed": "^0.0.3",
"mcping-js": "^1.5.0",
"node-cron": "^3.0.3",
"socket.io": "^4.7.2",
"ts-node": "^10.9.2",
"tsup": "^8.0.1",
"typescript": "^5.3.3",
"winston": "^3.11.0"
},
"devDependencies": {
"winston": "^3.11.0",
"@types/node": "^20.10.6"
}
}

1346
pnpm-lock.yaml generated
View File

File diff suppressed because it is too large Load Diff

158
src/database/database.ts
View File

@ -1,158 +0,0 @@
import SQLiteDatabase from "better-sqlite3";
import cron from "node-cron";
import Server from "../server/server";
import { logger } from "../utils/logger";
import { getFormattedDate } from "../utils/timeUtils";
import Config from "../../data/config.json";
import { Ping } from "../types/ping";
import { createDirectory } from "../utils/fsUtils";
const DATA_DIR = "data";
const BACKUP_DIR = `${DATA_DIR}/database-backups`;
const PINGS_TABLE = "pings";
const RECORD_TABLE = "record";
/**
* SQL Queries
*/
const CREATE_PINGS_TABLE = `
CREATE TABLE IF NOT EXISTS pings (
id INTEGER NOT NULL,
timestamp BIGINT NOT NULL,
ip TINYTEXT NOT NULL,
playerCount MEDIUMINT NOT NULL
);
`;
const CREATE_RECORD_TABLE = `
CREATE TABLE IF NOT EXISTS record (
id INTEGER PRIMARY KEY,
timestamp BIGINT NOT NULL,
ip TINYTEXT NOT NULL,
playerCount MEDIUMINT NOT NULL
);
`;
const CREATE_PINGS_INDEX = `CREATE INDEX IF NOT EXISTS ip_index ON pings (id, ip, playerCount)`;
const CREATE_TIMESTAMP_INDEX = `CREATE INDEX IF NOT EXISTS timestamp_index on PINGS (id, timestamp)`;
const INSERT_PING = `
INSERT INTO ${PINGS_TABLE} (id, timestamp, ip, playerCount)
VALUES (?, ?, ?, ?)
`;
const INSERT_RECORD = `
INSERT INTO ${RECORD_TABLE} (id, timestamp, ip, playerCount)
VALUES (?, ?, ?, ?)
ON CONFLICT(id) DO UPDATE SET
timestamp = excluded.timestamp,
playerCount = excluded.playerCount,
ip = excluded.ip
`;
const SELECT_PINGS = `
SELECT * FROM ${PINGS_TABLE} WHERE id = ? AND timestamp >= ? AND timestamp <= ?
`;
const SELECT_RECORD = `
SELECT playerCount, timestamp FROM ${RECORD_TABLE} WHERE {} = ?
`;
const SELECT_RECORD_BY_ID = SELECT_RECORD.replace("{}", "id");
const SELECT_RECORD_BY_IP = SELECT_RECORD.replace("{}", "ip");
export default class Database {
private db: SQLiteDatabase.Database;
constructor() {
this.db = new SQLiteDatabase(`${DATA_DIR}/db.sqlite`);
this.db.pragma("journal_mode = WAL"); // Enable WAL mode for better performance
logger.info("Ensuring tables exist");
this.db.exec(CREATE_PINGS_TABLE); // Ensure the pings table exists
this.db.exec(CREATE_RECORD_TABLE); // Ensure the record table exists
logger.info("Ensuring indexes exist");
this.db.exec(CREATE_PINGS_INDEX); // Ensure the pings index exists
this.db.exec(CREATE_TIMESTAMP_INDEX); // Ensure the timestamp index exists
cron.schedule(Config.backup.cron, () => {
this.createBackup();
});
}
/**
* Gets the pings for a server.
*
* @param id the server ID
* @param startTime the start time
* @param endTime the end time
* @returns the pings for the server
*/
public getPings(id: number, startTime: number, endTime: number): Ping[] | [] {
return this.db.prepare(SELECT_PINGS).all(id, startTime, endTime) as
| Ping[]
| [];
}
/**
* Gets the record player count for a server.
*
* @param value the server ID or IP
* @returns the record for the server
*/
public getRecord(value: any): Ping | undefined {
if (typeof value === "number") {
return this.db.prepare(SELECT_RECORD_BY_ID).get(value) as
| Ping
| undefined;
}
return this.db.prepare(SELECT_RECORD_BY_IP).get(value) as Ping | undefined;
}
/**
* Creates a full backup of the database.
*/
public async createBackup() {
logger.info("Creating database backup");
createDirectory(BACKUP_DIR);
await this.db.backup(`${BACKUP_DIR}/${getFormattedDate()}.sqlite`);
logger.info("Finished creating database backup");
}
/**
* Inserts a ping into the database.
*
* @param timestamp the timestamp of the ping
* @param ip the IP address of the server
* @param playerCount the number of players online
*/
public insertPing(server: Server, response: Ping) {
const { timestamp, playerCount } = response;
const id = server.getID();
const ip = server.getIP();
const statement = this.db.prepare(INSERT_PING);
statement.run(id, timestamp, ip, playerCount); // Insert the ping into the database
}
/**
* Inserts a record into the database.
*
* @param server the server to insert
* @param response the response to insert
* @returns true if the a new record was set, false otherwise
*/
public insertRecord(server: Server, response: Ping): boolean {
const { timestamp, playerCount } = response;
const id = server.getID();
const ip = server.getIP();
const oldRecord = this.getRecord(id);
if (oldRecord && oldRecord.playerCount >= playerCount) {
return false; // Don't update the record if the player count is lower
}
const statement = this.db.prepare(INSERT_RECORD);
statement.run(id, timestamp, ip, playerCount); // Insert the record into the database
return true;
}
}

29
src/index.ts
View File

@ -1,14 +1,6 @@
import Database from "./database/database";
import Influx from "./influx/influx";
import Scanner from "./scanner/scanner";
import ServerManager from "./server/serverManager";
import WebsocketServer from "./websocket/websocket";
import Config from "../data/config.json";
/**
* The database instance.
*/
export const database = new Database();
/**
* The server manager instance.
@ -16,9 +8,9 @@ export const database = new Database();
export const serverManager = new ServerManager();
/**
* The websocket server instance.
* The influx database instance.
*/
export const websocketServer = new WebsocketServer(Config.websocket.port);
export const influx = new Influx();
(async () => {
await serverManager.init();
@ -26,18 +18,3 @@ export const websocketServer = new WebsocketServer(Config.websocket.port);
// The scanner is responsible for scanning all servers
new Scanner();
})();
// The websocket server is responsible for
// sending data to the client in real time
// serverManager.getServers().forEach((server) => {
// const record = database.getRecord(server.getID());
// if (!record) {
// return;
// }
// console.log(
// `Record for "${server.getName()}": ${record.playerCount} (${formatTimestamp(
// record.timestamp
// )})`
// );
// });

34
src/influx/influx.ts Normal file
View File

@ -0,0 +1,34 @@
import { InfluxDB, Point, WriteApi } from "@influxdata/influxdb-client";
import Config from "../../data/config.json";
import { logger } from "../utils/logger";
export default class Influx {
private influx: InfluxDB;
private writeApi: WriteApi;
constructor() {
logger.info("Loading influx database");
this.influx = new InfluxDB({
url: Config.influx.url,
token: Config.influx.token,
});
logger.info("InfluxDB initialized");
this.writeApi = this.influx.getWriteApi(
Config.influx.org,
Config.influx.bucket,
"ms"
);
}
/**
* Write a point to the database.
*
* @param point the point to write
*/
public async writePoint(point: Point): Promise<void> {
this.writeApi.writePoint(point);
}
}

41
src/scanner/scanner.ts
View File

@ -1,14 +1,11 @@
import cron from "node-cron";
import { database, serverManager, websocketServer } from "..";
import Server, { ServerStatus } from "../server/server";
import { serverManager } from "..";
import Config from "../../data/config.json";
import { logger } from "../utils/logger";
export default class Scanner {
constructor() {
logger.info("Loading scanner database");
cron.schedule(Config.scanner.updateCron, () => {
this.scanServers();
});
@ -22,43 +19,9 @@ export default class Scanner {
// ping all servers in parallel
await Promise.all(
serverManager.getServers().map((server) => this.scanServer(server))
serverManager.getServers().map((server) => server.pingServer())
);
logger.info("Finished scanning servers");
}
/**
* Scans a server and inserts the ping into the database.
*
* @param server the server to scan
* @returns a promise that resolves when the server has been scanned
*/
async scanServer(server: Server): Promise<void> {
//logger.info(`Scanning server ${server.getIP()} - ${server.getType()}`);
let response;
let online = false;
try {
response = await server.pingServer();
if (response == undefined) {
return; // Server is offline
}
online = true;
} catch (err) {
logger.info(`Failed to ping ${server.getIP()}`, err);
websocketServer.sendServerError(server, ServerStatus.OFFLINE);
return;
}
if (!online || !response) {
return; // Server is offline
}
database.insertPing(server, response);
const isNewRecord = database.insertRecord(server, response);
// todo: send all server pings at once
websocketServer.sendNewPing(server, response, isNewRecord);
}
}

66
src/server/server.ts
View File

@ -2,8 +2,11 @@ import javaPing from "mcping-js";
import { ResolvedServer, resolveDns } from "../utils/dnsResolver";
const bedrockPing = require("mcpe-ping-fixed"); // Doesn't have typescript definitions
import { Point } from "@influxdata/influxdb-client";
import { influx } from "..";
import Config from "../../data/config.json";
import { Ping } from "../types/ping";
import { logger } from "../utils/logger";
/**
* The type of server.
@ -17,7 +20,6 @@ export enum ServerStatus {
}
type ServerOptions = {
id: number;
name: string;
ip: string;
port?: number;
@ -30,11 +32,6 @@ type DnsInfo = {
};
export default class Server {
/**
* The ID of the server.
*/
private id: number;
/**
* The name of the server.
*/
@ -68,8 +65,7 @@ export default class Server {
hasResolved: false,
};
constructor({ id, name, ip, port, type }: ServerOptions) {
this.id = id;
constructor({ name, ip, port, type }: ServerOptions) {
this.name = name;
this.ip = ip;
this.port = port;
@ -79,23 +75,42 @@ export default class Server {
/**
* Pings a server and gets the response.
*
* @param server the server to ping
* @param insertPing whether to insert the ping into the database
* @returns the ping response or undefined if the server is offline
*/
public pingServer(): Promise<Ping | undefined> {
switch (this.getType()) {
case "PC": {
return this.pingPCServer();
public async pingServer(): Promise<Ping | undefined> {
try {
let response;
switch (this.getType()) {
case "PC": {
response = await this.pingPCServer();
break;
}
case "PE": {
response = await this.pingPEServer();
break;
}
}
case "PE": {
return this.pingPEServer();
if (!response) {
return Promise.resolve(undefined);
}
default: {
throw new Error(
`Unknown server type ${this.getType()} for ${this.getName()}`
try {
influx.writePoint(
new Point("playerCount")
.tag("name", this.getName())
.intField("playerCount", response.playerCount)
.timestamp(response.timestamp)
);
} catch (err) {
logger.warn(`Failed to write ping to influxdb`, err);
}
return Promise.resolve(response);
} catch (err) {
logger.warn(`Failed to ping ${this.getIP()}`, err);
return Promise.resolve(undefined);
}
}
@ -133,14 +148,13 @@ export default class Server {
const serverPing = new javaPing.MinecraftServer(ip, port);
return new Promise((resolve, reject) => {
serverPing.ping(Config.scanner.timeout, 700, (err, res) => {
serverPing.ping(Config.scanner.timeout, 765, (err, res) => {
if (err || res == undefined) {
return reject(err);
}
this.favicon = res.favicon; // Set the favicon
resolve({
id: this.getID(),
timestamp: Date.now(),
ip: ip,
playerCount: res.players.online,
@ -166,7 +180,6 @@ export default class Server {
}
resolve({
id: this.getID(),
timestamp: Date.now(),
ip: this.getIP(),
playerCount: res.currentPlayers,
@ -176,15 +189,6 @@ export default class Server {
});
}
/**
* Returns the ID of the server.
*
* @returns the ID
*/
public getID(): number {
return this.id;
}
/**
* Returns the name of the server.
*

7
src/server/serverManager.ts
View File

@ -1,6 +1,7 @@
import Server, { ServerType } from "./server";
import Servers from "../../data/servers.json";
import { logger } from "../utils/logger";
export default class ServerManager {
private servers: Server[] = [];
@ -11,18 +12,16 @@ export default class ServerManager {
* Loads the servers from the config file.
*/
async init() {
logger.info("Loading servers");
for (const configServer of Servers) {
const server = new Server({
id: configServer.id,
ip: configServer.ip,
name: configServer.name,
type: configServer.type as ServerType,
});
try {
await server.pingServer();
} catch (err) {}
this.servers.push(server);
}
logger.info(`Loaded ${this.servers.length} servers`);
}
/**

1
src/types/ping.ts
View File

@ -1,5 +1,4 @@
export type Ping = {
id: number;
timestamp: number;
ip: string;
playerCount: number;

82
src/websocket/websocket.ts
View File

@ -1,82 +0,0 @@
import { Socket, Server as SocketServer } from "socket.io";
import { serverManager } from "..";
import Server, { ServerStatus } from "../server/server";
import { Ping } from "../types/ping";
import { logger } from "../utils/logger";
export default class WebsocketServer {
private server: SocketServer;
constructor(port: number) {
logger.info(`Starting websocket server on port ${port}`);
this.server = new SocketServer(port);
this.server.on("connection", (socket) => {
logger.debug("ws: Client connected");
this.sendServerList(socket);
});
}
/**
* Sends the server list to the given socket.
*
* @param socket the socket to send the server list to
*/
public sendServerList(socket: Socket): void {
logger.debug(`ws: Sending server list to ${socket.id}`);
const servers = [];
for (const server of serverManager.getServers()) {
servers.push({
id: server.getID(),
name: server.getName(),
ip: server.getIP(),
port: server.getPort(),
favicon: server.getFavicon(),
});
}
socket.emit("serverList", servers);
}
/**
* Sends the latest ping data for the given server to all clients.
*
* @param server the server to send the ping for
* @param pingResponse the ping data to send
* @param isNewRecord whether a new record has been set
*/
public sendNewPing(
server: Server,
pingResponse: Ping,
isNewRecord: boolean
): void {
logger.debug(`ws: Sending new ping for ${server.getName()}`);
this.server.emit("newPing", {
server: server.getID(),
timestamp: pingResponse.timestamp,
playerCount: pingResponse.playerCount,
});
if (isNewRecord) {
logger.debug(`ws: Sending new record for ${server.getName()}`);
this.server.emit("newRecord", {
server: server.getID(),
timestamp: pingResponse.timestamp,
playerCount: pingResponse.playerCount,
});
}
}
/**
* Sends the server status for the given server to all clients.
*
* @param server the server to send the status for
* @param status the status to send
*/
public sendServerError(server: Server, status: ServerStatus): void {
logger.debug(`ws: Sending server status for ${server.getName()}`);
this.server.emit("serverStatus", {
server: server.getID(),
status: status,
});
}
}