log to loki

packages/doorman-api/.env.example

@@ -13,4 +13,9 @@ DISCORD_BOT_TOKEN=
 PUSHGATEWAY_URL=https://metrics.chromart.cc
 STAGE=prod
 PUSHGATEWAY_USER=doorman
-PUSHGATEWAY_PW=doormanmetrics
+PUSHGATEWAY_PW=doormanmetrics
+
+# logs
+LOKI_URL=https://logs.chromart.cc
+LOKI_USER=doorman
+LOKI_PW=doormanlogs

packages/doorman-api/package.json

@@ -12,11 +12,13 @@
   },
   "dependencies": {
     "@aws-sdk/client-dynamodb": "^3.609.0",
+    "@twilio-labs/serverless-runtime-types": "^3.0.0",
     "@twilio/runtime-handler": "1.3.0",
     "discord.js": "^14.16.3",
     "prom-client": "^15.1.3",
     "twilio": "^3.56",
-    "@twilio-labs/serverless-runtime-types": "^3.0.0"
+    "winston": "^3.17.0",
+    "winston-loki": "^6.1.3"
   },
   "devDependencies": {
     "twilio-run": "^3.5.4",

packages/doorman-api/src/common/DoormanHandler.ts

@@ -5,6 +5,9 @@ import { shouldBlockRequest } from "../utils/blockUserAgent";
 import { RequestOptions } from "https";
 import { FastHttpPushgateway } from "../metrics/FastHttpPromGateway";
 
+import { createLogger, format, Logger, transports } from "winston";
+import LokiTransport from "winston-loki";
+
 export type BaseEvent = { request: { cookies: {}; headers: {}; }; }
 
 export type DoormanLambda<T extends DoormanLambdaContext, U extends BaseEvent> = (
@@ -12,12 +15,15 @@ export type DoormanLambda<T extends DoormanLambdaContext, U extends BaseEvent> =
   event: Parameters<ServerlessFunctionSignature<T, U>>[1],
   callback: Parameters<ServerlessFunctionSignature<T, U>>[2],
   metricsRegistry: Registry<PrometheusContentType>,
+  logger: Logger,
 ) => void;
 
 export enum CommonMetrics {
   RUNTIME = "FunctionRuntime",
   BLOCKED_REQUEST = "BlockedRequest",
   HTTP_CLIENT_ERROR = "HttpClientError",
+  LOKI_ERROR = "LokiError",
+  LOKI_LOG_AFTER_CLOSE = "LokiLogAfterClose",
 };
 
 export function getMetricFromRegistry<T>(metricsRegistry: Registry, metric: string): T {
@@ -34,6 +40,7 @@ export function withMetrics<T extends DoormanLambdaContext, U extends BaseEvent>
   handler: DoormanLambda<T, U>
 ): ServerlessFunctionSignature<T, U> {
   return async (context, event, callback) => {
+    console.log("[CommonHandler] created loki logger");
     console.log("[CommonHandler] clearing old metrics");
     register.clear();
     
@@ -63,6 +70,80 @@ export function withMetrics<T extends DoormanLambdaContext, U extends BaseEvent>
       labelNames: [ "ErrorCode" ],
     }));
 
+    metricsRegistry.registerMetric(new Counter({
+      name: CommonMetrics.LOKI_ERROR,
+      help: "Error connecting to loki for logs",
+    }));
+
+    metricsRegistry.registerMetric(new Counter({
+      name: CommonMetrics.LOKI_LOG_AFTER_CLOSE,
+      help: "Tried to write logs after log stream closed",
+    }));
+
+    // Create a Winston logger
+    const logger = createLogger({
+      level: 'info',
+      format: format.json(),
+      transports: [
+        new LokiTransport({
+          host: context.LOKI_URL,
+          labels: {
+            service: "doorman",
+            stage: context.STAGE,
+            handler: functionName,
+          },
+          basicAuth: `${context.LOKI_USER}:${context.LOKI_PW}`,
+          interval: 500,
+          gracefulShutdown: true,
+          clearOnError: true,
+          batching: true,
+          onConnectionError: (error) => {
+            console.log("in error block");
+            console.error(error);
+            getMetricFromRegistry<Counter>(metricsRegistry, CommonMetrics.LOKI_ERROR).inc(1);
+          },
+        }),
+        new transports.Console(), // Add other transports if needed
+      ],
+    });
+
+    const orignalLog = {
+      log: console.log,
+      error: console.error,
+      warn: console.warn,
+      info: console.info,
+    }
+
+    // Override the base console log with winston
+    console.log = function (...args) {
+      if (logger.writable) {
+        return logger.info.apply(logger, [...args]);
+      }
+      getMetricFromRegistry<Counter>(metricsRegistry, CommonMetrics.LOKI_LOG_AFTER_CLOSE).inc(1);
+      return orignalLog.log(...args);
+    };
+    console.error = function (...args) {
+      if (logger.writable) {
+        return logger.error.apply(logger, [...args]);
+      }
+      getMetricFromRegistry<Counter>(metricsRegistry, CommonMetrics.LOKI_LOG_AFTER_CLOSE).inc(1);
+      return orignalLog.error(...args);
+    };
+    console.warn = function (...args) {
+      if (logger.writable) {
+        return logger.warn.apply(logger, [...args]);
+      }
+      getMetricFromRegistry<Counter>(metricsRegistry, CommonMetrics.LOKI_LOG_AFTER_CLOSE).inc(1);
+      return orignalLog.warn(...args);
+    };
+    console.info = function (...args) {
+      if (logger.writable) {
+        return logger.info.apply(logger, [...args]);
+      }
+      getMetricFromRegistry<Counter>(metricsRegistry, CommonMetrics.LOKI_LOG_AFTER_CLOSE).inc(1);
+      return orignalLog.info(...args);
+    };
+
     const summaryTimer = getMetricFromRegistry<Summary>(metricsRegistry, CommonMetrics.RUNTIME).startTimer();
 
     const startTime = Date.now();
@@ -82,7 +163,7 @@ export function withMetrics<T extends DoormanLambdaContext, U extends BaseEvent>
         response.setStatusCode(200);
         callbackResult = [null, response];
       } else {
-        await handler(context, event, tempCallback, metricsRegistry);
+        await handler(context, event, tempCallback, metricsRegistry, logger);
       }
 
       if (!callbackResult) {
@@ -112,26 +193,29 @@ export function withMetrics<T extends DoormanLambdaContext, U extends BaseEvent>
     console.log(`[CommonHandler] there is ${remainingTime} ms left to send metrics`);
 
     let metricsTimeout = setTimeout(() => {
-      console.log("[CommonHandler] cutting it too close, abandoning metrics");
       callback(...result);
     }, remainingTime - 250);
 
     summaryTimer();
-
     console.log("[CommonHandler] attempting to push metrics...");
+
     try {
-      await pushGateway.push({
+      logger.end(async () => {
-        jobName: functionName,
+        await pushGateway.push({
-        groupings: {
+          jobName: functionName,
-          stage: context.STAGE,
+          groupings: {
-        },
+            stage: context.STAGE,
+          },
+        });
+        console.log("[CommonHandler] pushed metrics successfully");
+        clearTimeout(metricsTimeout);
+        callback(...result);
       });
-      console.log("[CommonHandler] pushed metrics successfully");
+      
     } catch (e: any) {
       console.log("[CommonHandler] failed to push metrics, quietly discarding them", e);
+      clearTimeout(metricsTimeout);
+      callback(...result);
     }
-    
-    clearTimeout(metricsTimeout);
-    callback(...result);
   };
 };

packages/doorman-api/src/common/DoormanHandlerContext.ts

@@ -10,4 +10,7 @@ export interface DoormanLambdaContext extends EnvironmentVariables {
   PUSHGATEWAY_USER: string;
   PUSHGATEWAY_PW: string;
   STAGE: string;
+  LOKI_URL: string;
+  LOKI_USER: string;
+  LOKI_PW: string;
 };

packages/doorman-client/.env.example

@@ -8,4 +8,9 @@ AUTH_TOKEN=
 PUSHGATEWAY_URL=https://metrics.chromart.cc
 STAGE=prod
 PUSHGATEWAY_USER=doorman
-PUSHGATEWAY_PW=doormanmetrics
+PUSHGATEWAY_PW=doormanmetrics
+
+# logs
+LOKI_URL=https://logs.chromart.cc
+LOKI_USER=doorman
+LOKI_PW=doormanlogs

packages/doorman-client/package.json

@@ -11,12 +11,14 @@
     "deploy": "twilio-run deploy --load-system-env --env .env.example --service-name buzzer --environment=prod --override-existing-project"
   },
   "dependencies": {
+    "@twilio-labs/serverless-runtime-types": "^3.0.0",
     "@twilio/runtime-handler": "1.3.0",
     "node-fetch": "^2.7.0",
     "prom-client": "^15.1.3",
     "prometheus-remote-write": "^0.5.0",
     "twilio": "^3.84.1",
-    "@twilio-labs/serverless-runtime-types": "^3.0.0"
+    "winston": "^3.17.0",
+    "winston-loki": "^6.1.3"
   },
   "devDependencies": {
     "@types/bun": "latest",