metrics!!!!!!!!!!!

apps/proxy/package.json

@@ -18,6 +18,8 @@
     "ts-node": "^10.9.1",
     "typescript": "^5.2.2",
     "utils": "workspace:*",
-    "node-cache": "^5.1.2"
+    "node-cache": "^5.1.2",
+    "@influxdata/influxdb-client": "^1.33.2",
+    "mongoose": "^7.6.3"
   }
 }

apps/proxy/src/db/metrics.ts

@@ -0,0 +1,11 @@
+import mongoose, { Model } from "mongoose";
+const { Schema } = mongoose;
+
+const metricsSchema = new Schema({
+  _id: String,
+  totalRequests: Number,
+});
+
+export const MetricsSchema =
+  (mongoose.models.Metrics as Model<typeof metricsSchema>) ||
+  mongoose.model("Metrics", metricsSchema);

apps/proxy/src/db/mongo.ts

@@ -0,0 +1,18 @@
+import mongoose from "mongoose";
+
+/**
+ * Creates a connection to Mongo
+ *
+ * @param uri the URI of the mongo instance
+ * @returns the mongoose connection
+ */
+export async function connectMongo(uri: string) {
+  // Check if mongoose is already connected
+  if (mongoose.connection.readyState) {
+    return;
+  }
+  return mongoose.connect(uri, {
+    autoCreate: true,
+    family: 4,
+  });
+}

apps/proxy/src/index.ts

@@ -1,9 +1,18 @@
+import { QueryApi, WriteApi } from "@influxdata/influxdb-client";
 import dotenv from "dotenv";
 import { RouteManager, RouteMessages, createServer } from "server";
-import { checkEnvironmentVariables } from "utils";
+import { checkEnvironmentVariables, createInfluxClient } from "utils";
+import { connectMongo } from "./db/mongo";
 import NodeManager from "./node/nodeManager";
 import ProxyRoute from "./routes/proxy";
-import { initSecrets } from "./secrets";
+import {
+  INFLUXDB_BUCKET,
+  INFLUXDB_ORG,
+  INFLUXDB_TOKEN,
+  INFLUXDB_URL,
+  MONGO_URI,
+  initSecrets,
+} from "./secrets";
 
 dotenv.config(); // load .env file
 
@@ -13,19 +22,41 @@ if (!envVarsValid) {
   process.exit(1);
 }
 
+export let InfluxWriteApi: WriteApi;
+export let InfluxQueryApi: QueryApi;
+
 /**
  * The node manager for all of the loaded nodes
  */
 export const nodeManager = new NodeManager();
 
-const server = createServer({
-  port: process.env.API_PORT || 3000,
-  onLoaded: async () => {
+(async () => {
   await initSecrets(process.env.INFISICAL_TOKEN!); // Load the infisical secrets
 
+  // Init MongoDB
+  await connectMongo(MONGO_URI);
+
   const routeManager = new RouteManager();
   routeManager.addRoute(new ProxyRoute());
 
+  // Init InfluxDB
+  const influxClient = createInfluxClient(
+    INFLUXDB_TOKEN,
+    INFLUXDB_URL,
+    INFLUXDB_ORG,
+    INFLUXDB_BUCKET
+  );
+  if (!influxClient) {
+    console.log("Influx client failed to initialize, exiting...");
+    process.exit(1);
+  }
+  InfluxWriteApi = influxClient.influxWriteClient;
+  InfluxQueryApi = influxClient.influxQueryClient;
+
+  const server = createServer({
+    port: process.env.API_PORT || 3000,
+  });
+
   server.all("*", (req, res) => {
     // Handle all paths to the proxy route
     const routes = routeManager.getRoutes();
@@ -35,5 +66,4 @@ const server = createServer({
     }
     routes[0].handle(req, res);
   });
-  },
-});
+})();

apps/proxy/src/routes/proxy.ts

@@ -1,10 +1,18 @@
+import { Point } from "@influxdata/influxdb-client";
 import { Request, Response } from "express";
 import Cache from "node-cache";
 import { Route, RouteMessages } from "server";
-import { nodeManager } from "..";
+import { InfluxWriteApi, nodeManager } from "..";
+import { MetricsSchema } from "../db/metrics";
 
 const IGNORED_PATHS = ["/favicon.ico"];
 
+/**
+ * The total requests that have been made
+ * TODO: move this to a metrics file
+ */
+let totalRequests: number | undefined;
+
 const cache = new Cache({
   stdTTL: 300, // 5 minutes
 });
@@ -36,6 +44,56 @@ function log(
   );
 }
 
+type InfluxLog = {
+  nodeId: string;
+  url: string;
+  status: number;
+  time?: number;
+  cached?: boolean;
+};
+/**
+ * Creates an InfluxDB point and writes it to the database
+ *
+ * @param nodeId the node ID that handled the request
+ * @param url the URL of the request
+ * @param status the status code of the request
+ * @param time the time it took to handle the request (or true if it was cached)
+ */
+async function logRequestToDatabase({
+  nodeId,
+  url,
+  status,
+  time,
+  cached,
+}: InfluxLog) {
+  totalRequests = totalRequests ? totalRequests + 1 : 1;
+  MetricsSchema.updateOne(
+    { _id: "proxy" },
+    { $set: { totalRequests: totalRequests } },
+    { upsert: true }
+  ).exec();
+
+  const point = new Point("proxy");
+  point.tag("type", "request");
+  point.tag("node", nodeId);
+  point.stringField("url", url);
+  point.intField("status", status);
+  if (cached) {
+    point.tag("cached", "true");
+  } else {
+    point.intField("time", time as number);
+  }
+  try {
+    InfluxWriteApi.writePoint(point);
+    InfluxWriteApi.writePoint(
+      new Point("proxy").intField("totalRequests", totalRequests)
+    );
+  } catch (ex) {
+    console.log("Failed to write to influx");
+    console.log(ex);
+  }
+}
+
 export default class ProxyRoute extends Route {
   constructor() {
     super({ path: "/" });
@@ -63,11 +121,17 @@ export default class ProxyRoute extends Route {
     try {
       const cachedRequest = cache.get<CachedRequest>(url);
       if (cachedRequest) {
-        log(cachedRequest.nodeId, url, cachedRequest.status, true);
         res
           .status(cachedRequest.status)
           .set(cachedRequest.headers)
           .json(cachedRequest.data);
+        log(cachedRequest.nodeId, url, cachedRequest.status, true);
+        logRequestToDatabase({
+          nodeId: cachedRequest.nodeId,
+          url,
+          status: cachedRequest.status,
+          cached: true,
+        });
         return;
       }
 
@@ -91,6 +155,12 @@ export default class ProxyRoute extends Route {
         headers: response.headers,
         data: data,
       } as CachedRequest);
+      logRequestToDatabase({
+        nodeId,
+        url,
+        status: response.status,
+        time: Date.now() - before,
+      });
     } catch (ex: any) {
       res.status(500).json(RouteMessages.internalServerError(ex.message || ex));
     }

apps/proxy/src/secrets.ts

@@ -2,6 +2,15 @@ import { createInfisicalClient } from "utils";
 
 export let PROXY_SECRET: string;
 
+// InfluxDB
+export let INFLUXDB_URL: string;
+export let INFLUXDB_ORG: string;
+export let INFLUXDB_BUCKET: string;
+export let INFLUXDB_TOKEN: string;
+
+// MongoDB
+export let MONGO_URI: string;
+
 /**
  * Initialize the secrets from Infisical
  */
@@ -9,12 +18,54 @@ export async function initSecrets(token: string) {
   console.log("Initializing secrets...");
 
   const infisicalClient = createInfisicalClient(token);
+
   const proxySecret = (await infisicalClient.getSecret("PROXY_SECRET"))
     .secretValue;
 
+  // InfluxDB
+  const influxDBUrl = (await infisicalClient.getSecret("INFLUXDB_URL"))
+    .secretValue;
+  const influxDBOrg = (await infisicalClient.getSecret("INFLUXDB_ORG"))
+    .secretValue;
+  const influxDBBucket = (await infisicalClient.getSecret("INFLUXDB_BUCKET"))
+    .secretValue;
+  const influxDBToken = (await infisicalClient.getSecret("INFLUXDB_TOKEN"))
+    .secretValue;
+
+  // Mongo
+  const mongoUri = (await infisicalClient.getSecret("MONGO_URI")).secretValue;
+
   if (!proxySecret) {
     throw new Error("PROXY_SECRET not set in Infisical");
   }
 
+  // InfluxDB
+  if (!influxDBUrl) {
+    throw new Error("INFLUXDB_URL not set in Infisical");
+  }
+  if (!influxDBOrg) {
+    throw new Error("INFLUXDB_ORG not set in Infisical");
+  }
+  if (!influxDBBucket) {
+    throw new Error("INFLUXDB_BUCKET not set in Infisical");
+  }
+  if (!influxDBToken) {
+    throw new Error("INFLUXDB_TOKEN not set in Infisical");
+  }
+
+  // Mongo
+  if (!mongoUri) {
+    throw new Error("MONGO_URI not set in Infisical");
+  }
+
   PROXY_SECRET = proxySecret;
+
+  // InfluxDB
+  INFLUXDB_URL = influxDBUrl;
+  INFLUXDB_ORG = influxDBOrg;
+  INFLUXDB_BUCKET = influxDBBucket;
+  INFLUXDB_TOKEN = influxDBToken;
+
+  // Mongo
+  MONGO_URI = mongoUri;
 }

apps/proxy/tsconfig.json

@@ -1,3 +1,6 @@
 {
-  "extends": "tsconfig/server.json"
+  "extends": "tsconfig/server.json",
+  "compilerOptions": {
+    "esModuleInterop": true
+  }
 }

package.json

@@ -9,17 +9,19 @@
     "format": "prettier --write \"**/*.{ts,tsx,md}\""
   },
   "devDependencies": {
+    "@influxdata/influxdb-client": "^1.33.2",
     "dotenv": "^16.3.1",
     "eslint": "^8.48.0",
+    "infisical-node": "^1.5.0",
+    "node-cache": "^5.1.2",
     "nodemon": "^3.0.1",
     "prettier": "^3.0.3",
     "tsconfig": "workspace:*",
+    "tsup": "^7.2.0",
     "turbo": "latest"
   },
   "dependencies": {
     "@types/express": "^4.17.21",
-    "infisical-node": "^1.5.0",
-    "node-cache": "^5.1.2",
-    "tsup": "^7.2.0"
+    "mongoose": "7.6.3"
   }
 }

packages/utils/src/index.ts

@@ -1,2 +1,3 @@
 export * from "./envVariables";
-export * from "./secrets";
+export * from "./influx/influx";
+export * from "./secrets/secrets";

packages/utils/src/influx/influx.ts

@@ -0,0 +1,29 @@
+import { InfluxDB } from "@influxdata/influxdb-client";
+
+/**
+ * Creates an InfluxDB client
+ *
+ * @param token the influx token
+ * @param url the url of the influx instance
+ * @param org the org of the influx instance
+ * @param bucket the bucket of the influx instance
+ * @returns
+ */
+function createInfluxClient(
+  token: string,
+  url: string,
+  org: string,
+  bucket: string
+) {
+  const client = new InfluxDB({
+    url: url,
+    token: token,
+  });
+
+  return {
+    influxWriteClient: client.getWriteApi(org, bucket, "ms"),
+    influxQueryClient: client.getQueryApi(org),
+  };
+}
+
+export { createInfluxClient };