diff --git a/bun.lockb b/bun.lockb index ec7491b..79d6fb3 100755 Binary files a/bun.lockb and b/bun.lockb differ diff --git a/packages/doorman-api/.env.example b/packages/doorman-api/.env.example index 224f279..982e8ae 100644 --- a/packages/doorman-api/.env.example +++ b/packages/doorman-api/.env.example @@ -13,4 +13,9 @@ DISCORD_BOT_TOKEN= PUSHGATEWAY_URL=https://metrics.chromart.cc STAGE=prod PUSHGATEWAY_USER=doorman -PUSHGATEWAY_PW=doormanmetrics \ No newline at end of file +PUSHGATEWAY_PW=doormanmetrics + +# logs +LOKI_URL=https://logs.chromart.cc +LOKI_USER=doorman +LOKI_PW=doormanlogs diff --git a/packages/doorman-api/package.json b/packages/doorman-api/package.json index dc25e95..7707587 100644 --- a/packages/doorman-api/package.json +++ b/packages/doorman-api/package.json @@ -12,11 +12,13 @@ }, "dependencies": { "@aws-sdk/client-dynamodb": "^3.609.0", + "@twilio-labs/serverless-runtime-types": "^3.0.0", "@twilio/runtime-handler": "1.3.0", "discord.js": "^14.16.3", "prom-client": "^15.1.3", "twilio": "^3.56", - "@twilio-labs/serverless-runtime-types": "^3.0.0" + "winston": "^3.17.0", + "winston-loki": "^6.1.3" }, "devDependencies": { "twilio-run": "^3.5.4", diff --git a/packages/doorman-api/src/common/DoormanHandler.ts b/packages/doorman-api/src/common/DoormanHandler.ts index 7953ab6..dac3ca4 100644 --- a/packages/doorman-api/src/common/DoormanHandler.ts +++ b/packages/doorman-api/src/common/DoormanHandler.ts @@ -5,6 +5,9 @@ import { shouldBlockRequest } from "../utils/blockUserAgent"; import { RequestOptions } from "https"; import { FastHttpPushgateway } from "../metrics/FastHttpPromGateway"; +import { createLogger, format, Logger, transports } from "winston"; +import LokiTransport from "winston-loki"; + export type BaseEvent = { request: { cookies: {}; headers: {}; }; } export type DoormanLambda = ( @@ -12,12 +15,15 @@ export type DoormanLambda = event: Parameters>[1], callback: Parameters>[2], metricsRegistry: Registry, + logger: Logger, ) => void; export enum CommonMetrics { RUNTIME = "FunctionRuntime", BLOCKED_REQUEST = "BlockedRequest", HTTP_CLIENT_ERROR = "HttpClientError", + LOKI_ERROR = "LokiError", + LOKI_LOG_AFTER_CLOSE = "LokiLogAfterClose", }; export function getMetricFromRegistry(metricsRegistry: Registry, metric: string): T { @@ -34,6 +40,7 @@ export function withMetrics handler: DoormanLambda ): ServerlessFunctionSignature { return async (context, event, callback) => { + console.log("[CommonHandler] created loki logger"); console.log("[CommonHandler] clearing old metrics"); register.clear(); @@ -63,6 +70,80 @@ export function withMetrics labelNames: [ "ErrorCode" ], })); + metricsRegistry.registerMetric(new Counter({ + name: CommonMetrics.LOKI_ERROR, + help: "Error connecting to loki for logs", + })); + + metricsRegistry.registerMetric(new Counter({ + name: CommonMetrics.LOKI_LOG_AFTER_CLOSE, + help: "Tried to write logs after log stream closed", + })); + + // Create a Winston logger + const logger = createLogger({ + level: 'info', + format: format.json(), + transports: [ + new LokiTransport({ + host: context.LOKI_URL, + labels: { + service: "doorman", + stage: context.STAGE, + handler: functionName, + }, + basicAuth: `${context.LOKI_USER}:${context.LOKI_PW}`, + interval: 500, + gracefulShutdown: true, + clearOnError: true, + batching: true, + onConnectionError: (error) => { + console.log("in error block"); + console.error(error); + getMetricFromRegistry(metricsRegistry, CommonMetrics.LOKI_ERROR).inc(1); + }, + }), + new transports.Console(), // Add other transports if needed + ], + }); + + const orignalLog = { + log: console.log, + error: console.error, + warn: console.warn, + info: console.info, + } + + // Override the base console log with winston + console.log = function (...args) { + if (logger.writable) { + return logger.info.apply(logger, [...args]); + } + getMetricFromRegistry(metricsRegistry, CommonMetrics.LOKI_LOG_AFTER_CLOSE).inc(1); + return orignalLog.log(...args); + }; + console.error = function (...args) { + if (logger.writable) { + return logger.error.apply(logger, [...args]); + } + getMetricFromRegistry(metricsRegistry, CommonMetrics.LOKI_LOG_AFTER_CLOSE).inc(1); + return orignalLog.error(...args); + }; + console.warn = function (...args) { + if (logger.writable) { + return logger.warn.apply(logger, [...args]); + } + getMetricFromRegistry(metricsRegistry, CommonMetrics.LOKI_LOG_AFTER_CLOSE).inc(1); + return orignalLog.warn(...args); + }; + console.info = function (...args) { + if (logger.writable) { + return logger.info.apply(logger, [...args]); + } + getMetricFromRegistry(metricsRegistry, CommonMetrics.LOKI_LOG_AFTER_CLOSE).inc(1); + return orignalLog.info(...args); + }; + const summaryTimer = getMetricFromRegistry(metricsRegistry, CommonMetrics.RUNTIME).startTimer(); const startTime = Date.now(); @@ -82,7 +163,7 @@ export function withMetrics response.setStatusCode(200); callbackResult = [null, response]; } else { - await handler(context, event, tempCallback, metricsRegistry); + await handler(context, event, tempCallback, metricsRegistry, logger); } if (!callbackResult) { @@ -112,26 +193,29 @@ export function withMetrics console.log(`[CommonHandler] there is ${remainingTime} ms left to send metrics`); let metricsTimeout = setTimeout(() => { - console.log("[CommonHandler] cutting it too close, abandoning metrics"); callback(...result); }, remainingTime - 250); summaryTimer(); - console.log("[CommonHandler] attempting to push metrics..."); + try { - await pushGateway.push({ - jobName: functionName, - groupings: { - stage: context.STAGE, - }, + logger.end(async () => { + await pushGateway.push({ + jobName: functionName, + groupings: { + stage: context.STAGE, + }, + }); + console.log("[CommonHandler] pushed metrics successfully"); + clearTimeout(metricsTimeout); + callback(...result); }); - console.log("[CommonHandler] pushed metrics successfully"); + } catch (e: any) { console.log("[CommonHandler] failed to push metrics, quietly discarding them", e); + clearTimeout(metricsTimeout); + callback(...result); } - - clearTimeout(metricsTimeout); - callback(...result); }; }; diff --git a/packages/doorman-api/src/common/DoormanHandlerContext.ts b/packages/doorman-api/src/common/DoormanHandlerContext.ts index 259bfdb..2d0588b 100644 --- a/packages/doorman-api/src/common/DoormanHandlerContext.ts +++ b/packages/doorman-api/src/common/DoormanHandlerContext.ts @@ -10,4 +10,7 @@ export interface DoormanLambdaContext extends EnvironmentVariables { PUSHGATEWAY_USER: string; PUSHGATEWAY_PW: string; STAGE: string; + LOKI_URL: string; + LOKI_USER: string; + LOKI_PW: string; }; diff --git a/packages/doorman-client/.env.example b/packages/doorman-client/.env.example index 3d98815..10a3eae 100644 --- a/packages/doorman-client/.env.example +++ b/packages/doorman-client/.env.example @@ -8,4 +8,9 @@ AUTH_TOKEN= PUSHGATEWAY_URL=https://metrics.chromart.cc STAGE=prod PUSHGATEWAY_USER=doorman -PUSHGATEWAY_PW=doormanmetrics \ No newline at end of file +PUSHGATEWAY_PW=doormanmetrics + +# logs +LOKI_URL=https://logs.chromart.cc +LOKI_USER=doorman +LOKI_PW=doormanlogs diff --git a/packages/doorman-client/package.json b/packages/doorman-client/package.json index 850016d..6e4f891 100644 --- a/packages/doorman-client/package.json +++ b/packages/doorman-client/package.json @@ -11,12 +11,14 @@ "deploy": "twilio-run deploy --load-system-env --env .env.example --service-name buzzer --environment=prod --override-existing-project" }, "dependencies": { + "@twilio-labs/serverless-runtime-types": "^3.0.0", "@twilio/runtime-handler": "1.3.0", "node-fetch": "^2.7.0", "prom-client": "^15.1.3", "prometheus-remote-write": "^0.5.0", "twilio": "^3.84.1", - "@twilio-labs/serverless-runtime-types": "^3.0.0" + "winston": "^3.17.0", + "winston-loki": "^6.1.3" }, "devDependencies": { "@types/bun": "latest",