increase global timeout to 9s to prevent timeouts and graceful fallback sooner at 7.25s
This commit is contained in:
parent
f3863719b5
commit
512feb0268
@ -51,7 +51,7 @@ export function gracefullyEndLogger(logger: Logger): Promise<void> {
|
|||||||
logger.on('finish', () => {
|
logger.on('finish', () => {
|
||||||
fulfill();
|
fulfill();
|
||||||
});
|
});
|
||||||
|
|
||||||
logger.end();
|
logger.end();
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
@ -64,7 +64,7 @@ const FALLBACK_CALLBACK: CallbackResult = [null, REJECT_RESPONSE];
|
|||||||
|
|
||||||
const MINIMUM_MS_TO_SEND_RESPONSE: number = 250;
|
const MINIMUM_MS_TO_SEND_RESPONSE: number = 250;
|
||||||
const FUNCTION_MAXIMUM_DURATION_MS: number = 10_000;
|
const FUNCTION_MAXIMUM_DURATION_MS: number = 10_000;
|
||||||
const INNER_HANDLER_MAXIMUM_DURATION_MS: number = 8_500;
|
const INNER_HANDLER_MAXIMUM_DURATION_MS: number = 9_000;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* A decorator for twilio handlers. It provides a metrics registry and
|
* A decorator for twilio handlers. It provides a metrics registry and
|
||||||
@ -79,7 +79,7 @@ export function withMetrics<T extends DoormanLambdaContext, U extends BaseEvent>
|
|||||||
console.log("[CommonHandler] created loki logger");
|
console.log("[CommonHandler] created loki logger");
|
||||||
console.log("[CommonHandler] clearing old metrics");
|
console.log("[CommonHandler] clearing old metrics");
|
||||||
register.clear();
|
register.clear();
|
||||||
|
|
||||||
console.log("[CommonHandler] creating metrics registry");
|
console.log("[CommonHandler] creating metrics registry");
|
||||||
const metricsRegistry = new Registry();
|
const metricsRegistry = new Registry();
|
||||||
const requestOptions: RequestOptions = {
|
const requestOptions: RequestOptions = {
|
||||||
@ -103,7 +103,7 @@ export function withMetrics<T extends DoormanLambdaContext, U extends BaseEvent>
|
|||||||
metricsRegistry.registerMetric(new Counter({
|
metricsRegistry.registerMetric(new Counter({
|
||||||
name: CommonMetrics.HTTP_CLIENT_ERROR,
|
name: CommonMetrics.HTTP_CLIENT_ERROR,
|
||||||
help: "Client side HTTP error",
|
help: "Client side HTTP error",
|
||||||
labelNames: [ "ErrorCode" ],
|
labelNames: ["ErrorCode"],
|
||||||
}));
|
}));
|
||||||
|
|
||||||
metricsRegistry.registerMetric(new Counter({
|
metricsRegistry.registerMetric(new Counter({
|
||||||
@ -231,7 +231,7 @@ export function withMetrics<T extends DoormanLambdaContext, U extends BaseEvent>
|
|||||||
// wait for response for up to x ms, otherwise we call the failFastCallbackMethod
|
// wait for response for up to x ms, otherwise we call the failFastCallbackMethod
|
||||||
try {
|
try {
|
||||||
await pTimeout(
|
await pTimeout(
|
||||||
() => handler(context, event, tempCallback, metricsRegistry, logger, failFastCallbackProvider),
|
() => handler(context, event, tempCallback, metricsRegistry, logger, failFastCallbackProvider),
|
||||||
INNER_HANDLER_MAXIMUM_DURATION_MS
|
INNER_HANDLER_MAXIMUM_DURATION_MS
|
||||||
)();
|
)();
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
@ -254,7 +254,7 @@ export function withMetrics<T extends DoormanLambdaContext, U extends BaseEvent>
|
|||||||
setResponseJson(response, 400, {
|
setResponseJson(response, 400, {
|
||||||
msg: fromError(e).toString(),
|
msg: fromError(e).toString(),
|
||||||
});
|
});
|
||||||
|
|
||||||
callbackResult = [null, response];
|
callbackResult = [null, response];
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@ -19,10 +19,10 @@ import { getMetricFromRegistry, withMetrics } from '../../../doorman-api/src/com
|
|||||||
import { Counter, Summary } from 'prom-client';
|
import { Counter, Summary } from 'prom-client';
|
||||||
import { BuzzerActivatedMetrics, registerMetrics } from '../metrics/BuzzerActivatedMetrics';
|
import { BuzzerActivatedMetrics, registerMetrics } from '../metrics/BuzzerActivatedMetrics';
|
||||||
|
|
||||||
export const handler: ServerlessFunctionSignature<TwilioContext, BuzzerDialEvent> = withMetrics('buzzer-activated', async function(context, event, callback, metricsRegistry, logger, failFastCallback) {
|
export const handler: ServerlessFunctionSignature<TwilioContext, BuzzerDialEvent> = withMetrics('buzzer-activated', async function (context, event, callback, metricsRegistry, logger, failFastCallback) {
|
||||||
// metrics
|
// metrics
|
||||||
registerMetrics(metricsRegistry);
|
registerMetrics(metricsRegistry);
|
||||||
|
|
||||||
let invokeId = `[${randomUUID()}]`;
|
let invokeId = `[${randomUUID()}]`;
|
||||||
let configString = event.config;
|
let configString = event.config;
|
||||||
let config: InfoResponseClient | undefined;
|
let config: InfoResponseClient | undefined;
|
||||||
@ -34,7 +34,7 @@ export const handler: ServerlessFunctionSignature<TwilioContext, BuzzerDialEvent
|
|||||||
} else {
|
} else {
|
||||||
try {
|
try {
|
||||||
config = JSON.parse(configString);
|
config = JSON.parse(configString);
|
||||||
} catch(e) {
|
} catch (e) {
|
||||||
config = await getConfig(context, event.From);
|
config = await getConfig(context, event.From);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -52,7 +52,7 @@ export const handler: ServerlessFunctionSignature<TwilioContext, BuzzerDialEvent
|
|||||||
|
|
||||||
// let users know someone is currently buzzing, and allow unlock by discord user
|
// let users know someone is currently buzzing, and allow unlock by discord user
|
||||||
let msg = `🔔 Someone is dialing right now @ Door "${config.door}" [Click to unlock](${context.DOORMAN_URL}/api/door/auth?door=${config.door}&key=`;
|
let msg = `🔔 Someone is dialing right now @ Door "${config.door}" [Click to unlock](${context.DOORMAN_URL}/api/door/auth?door=${config.door}&key=`;
|
||||||
let msgs = config.discordUsers.map((u) =>
|
let msgs = config.discordUsers.map((u) =>
|
||||||
msg + u + ')'
|
msg + u + ')'
|
||||||
);
|
);
|
||||||
|
|
||||||
@ -69,7 +69,7 @@ export const handler: ServerlessFunctionSignature<TwilioContext, BuzzerDialEvent
|
|||||||
|
|
||||||
const recordPollLatency = getMetricFromRegistry<Summary>(metricsRegistry, BuzzerActivatedMetrics.POLL_LATENCY)
|
const recordPollLatency = getMetricFromRegistry<Summary>(metricsRegistry, BuzzerActivatedMetrics.POLL_LATENCY)
|
||||||
.startTimer({ door: config.door });
|
.startTimer({ door: config.door });
|
||||||
|
|
||||||
fetch(context.DOORMAN_URL + `/api/door/status?door=${config.door}`)
|
fetch(context.DOORMAN_URL + `/api/door/status?door=${config.door}`)
|
||||||
.then(res => res.json())
|
.then(res => res.json())
|
||||||
.then(async (rawBody) => {
|
.then(async (rawBody) => {
|
||||||
@ -87,7 +87,7 @@ export const handler: ServerlessFunctionSignature<TwilioContext, BuzzerDialEvent
|
|||||||
getMetricFromRegistry<Counter>(metricsRegistry, BuzzerActivatedMetrics.API_UNLOCK)
|
getMetricFromRegistry<Counter>(metricsRegistry, BuzzerActivatedMetrics.API_UNLOCK)
|
||||||
.inc({ door: config.door }, 1);
|
.inc({ door: config.door }, 1);
|
||||||
await notifyAllDiscord(context, config, `🔓 Doorman buzzed someone up @ Door "${config.door}"`, metricsRegistry, JSON.stringify(body.fingerprint));
|
await notifyAllDiscord(context, config, `🔓 Doorman buzzed someone up @ Door "${config.door}"`, metricsRegistry, JSON.stringify(body.fingerprint));
|
||||||
|
|
||||||
resolve(twiml);
|
resolve(twiml);
|
||||||
} else {
|
} else {
|
||||||
console.log(
|
console.log(
|
||||||
@ -112,8 +112,8 @@ export const handler: ServerlessFunctionSignature<TwilioContext, BuzzerDialEvent
|
|||||||
invokeId + " GracefulFallbackPromise: I was the fastest, so I will attempt to notify discord users before resolving with a call"
|
invokeId + " GracefulFallbackPromise: I was the fastest, so I will attempt to notify discord users before resolving with a call"
|
||||||
);
|
);
|
||||||
await notifyAllDiscord(
|
await notifyAllDiscord(
|
||||||
context,
|
context,
|
||||||
config,
|
config,
|
||||||
`📞 Somebody buzzed the door and it dialed through to fallback phone numbers @ Door "${config.door}"`,
|
`📞 Somebody buzzed the door and it dialed through to fallback phone numbers @ Door "${config.door}"`,
|
||||||
metricsRegistry,
|
metricsRegistry,
|
||||||
);
|
);
|
||||||
@ -123,7 +123,7 @@ export const handler: ServerlessFunctionSignature<TwilioContext, BuzzerDialEvent
|
|||||||
invokeId + " GracefulFallbackPromise: dropping out of the race, another response is already underway"
|
invokeId + " GracefulFallbackPromise: dropping out of the race, another response is already underway"
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
}, 7500));
|
}, 7_250));
|
||||||
});
|
});
|
||||||
|
|
||||||
// provide a method to the decorator to use as a fast fallback that shouldn't have any await
|
// provide a method to the decorator to use as a fast fallback that shouldn't have any await
|
||||||
@ -132,16 +132,16 @@ export const handler: ServerlessFunctionSignature<TwilioContext, BuzzerDialEvent
|
|||||||
responseLock = true;
|
responseLock = true;
|
||||||
|
|
||||||
getMetricFromRegistry<Counter>(metricsRegistry, BuzzerActivatedMetrics.RESULT_NOTIFICATION_FATE_UNKNOWN)
|
getMetricFromRegistry<Counter>(metricsRegistry, BuzzerActivatedMetrics.RESULT_NOTIFICATION_FATE_UNKNOWN)
|
||||||
.inc({ door: config.door }, 1);
|
.inc({ door: config.door }, 1);
|
||||||
|
|
||||||
getMetricFromRegistry<Counter>(metricsRegistry, BuzzerActivatedMetrics.DIAL_THROUGH)
|
|
||||||
.inc({ door: config.door }, 1);
|
|
||||||
|
|
||||||
const twiml = dialFallbackTwiml(config);
|
getMetricFromRegistry<Counter>(metricsRegistry, BuzzerActivatedMetrics.DIAL_THROUGH)
|
||||||
console.log(
|
.inc({ door: config.door }, 1);
|
||||||
invokeId + " UngracefulFallback: Cutting it too close to timeout! Skipping notifying users and responding fallback"
|
|
||||||
);
|
const twiml = dialFallbackTwiml(config);
|
||||||
return [null, twiml];
|
console.log(
|
||||||
|
invokeId + " UngracefulFallback: Cutting it too close to timeout! Skipping notifying users and responding fallback"
|
||||||
|
);
|
||||||
|
return [null, twiml];
|
||||||
});
|
});
|
||||||
|
|
||||||
const twiml = await Promise.race([unlockPromise, gracefulFallbackPromise]);
|
const twiml = await Promise.race([unlockPromise, gracefulFallbackPromise]);
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user