From 4d599a2118fa6af96111725745cc3a142d62be9e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gast=C3=B3n=20Fournier?= Date: Thu, 23 Oct 2025 12:21:12 +0200 Subject: [PATCH 1/5] feat: include readiness check option --- src/lib/create-config.ts | 6 +++ src/lib/openapi/spec/index.ts | 1 + src/lib/openapi/spec/ready-check-schema.ts | 22 ++++++++ src/lib/routes/health-check.test.ts | 4 +- src/lib/routes/index.ts | 5 ++ src/lib/routes/ready-check.test.ts | 44 +++++++++++++++ src/lib/routes/ready-check.ts | 62 ++++++++++++++++++++++ src/lib/types/option.ts | 3 ++ src/test/e2e/ready.e2e.test.ts | 27 ++++++++++ 9 files changed, 172 insertions(+), 2 deletions(-) create mode 100644 src/lib/openapi/spec/ready-check-schema.ts create mode 100644 src/lib/routes/ready-check.test.ts create mode 100644 src/lib/routes/ready-check.ts create mode 100644 src/test/e2e/ready.e2e.test.ts diff --git a/src/lib/create-config.ts b/src/lib/create-config.ts index 3512094ffd..a2bd9bf595 100644 --- a/src/lib/create-config.ts +++ b/src/lib/create-config.ts @@ -786,6 +786,11 @@ export function createConfig(options: IUnleashOptions): IUnleashConfig { options.prometheusImpactMetricsApi || process.env.PROMETHEUS_IMPACT_METRICS_API; + const checkDbOnReady = + typeof options.checkDbOnReady === 'boolean' + ? options.checkDbOnReady + : parseEnvVarBoolean(process.env.CHECK_DB_ON_READY, false); + return { db, session, @@ -830,5 +835,6 @@ export function createConfig(options: IUnleashOptions): IUnleashConfig { userInactivityThresholdInDays, buildDate: process.env.BUILD_DATE, unleashFrontendToken, + checkDbOnReady, }; } diff --git a/src/lib/openapi/spec/index.ts b/src/lib/openapi/spec/index.ts index b7d69c8ee2..6788e0a44c 100644 --- a/src/lib/openapi/spec/index.ts +++ b/src/lib/openapi/spec/index.ts @@ -169,6 +169,7 @@ export * from './public-signup-token-schema.js'; export * from './public-signup-token-update-schema.js'; export * from './public-signup-tokens-schema.js'; export * from './push-variants-schema.js'; +export * from './ready-check-schema.js'; export * from './record-ui-error-schema.js'; export * from './release-plan-milestone-schema.js'; export * from './release-plan-milestone-strategy-schema.js'; diff --git a/src/lib/openapi/spec/ready-check-schema.ts b/src/lib/openapi/spec/ready-check-schema.ts new file mode 100644 index 0000000000..bab0a0c454 --- /dev/null +++ b/src/lib/openapi/spec/ready-check-schema.ts @@ -0,0 +1,22 @@ +import type { FromSchema } from 'json-schema-to-ts'; + +export const readyCheckSchema = { + $id: '#/components/schemas/readyCheckSchema', + type: 'object', + description: + 'Used by service orchestrators to decide whether this Unleash instance should be considered ready to serve requests.', + additionalProperties: false, + required: ['health'], + properties: { + health: { + description: + 'The readiness state this Unleash instance is in. GOOD if the server is up and running. It never returns BAD; if the server is unhealthy you will get an unsuccessful http response.', + type: 'string', + enum: ['GOOD', 'BAD'], + example: 'GOOD', + }, + }, + components: {}, +} as const; + +export type ReadyCheckSchema = FromSchema; diff --git a/src/lib/routes/health-check.test.ts b/src/lib/routes/health-check.test.ts index 89ab26e8ec..b32176f398 100644 --- a/src/lib/routes/health-check.test.ts +++ b/src/lib/routes/health-check.test.ts @@ -28,11 +28,11 @@ afterEach(() => { getLogger.setMuteError(false); }); -test('should give 200 when ready', async () => { +test('should give 200 when healthy', async () => { await request.get('/health').expect(200); }); -test('should give health=GOOD when ready', async () => { +test('should give health=GOOD when healthy', async () => { expect.assertions(2); await request .get('/health') diff --git a/src/lib/routes/index.ts b/src/lib/routes/index.ts index 79e4d6d061..d59a8bae36 100644 --- a/src/lib/routes/index.ts +++ b/src/lib/routes/index.ts @@ -8,6 +8,7 @@ import Controller from './controller.js'; import { AdminApi } from './admin-api/index.js'; import ClientApi from './client-api/index.js'; +import { ReadyCheckController } from './ready-check.js'; import { HealthCheckController } from './health-check.js'; import FrontendAPIController from '../features/frontend-api/frontend-api-controller.js'; import EdgeController from './edge-api/index.js'; @@ -25,6 +26,10 @@ class IndexRouter extends Controller { ) { super(config); + this.use( + '/ready', + new ReadyCheckController(config, services, db).router, + ); this.use('/health', new HealthCheckController(config, services).router); this.use( '/invite', diff --git a/src/lib/routes/ready-check.test.ts b/src/lib/routes/ready-check.test.ts new file mode 100644 index 0000000000..005dcbe9db --- /dev/null +++ b/src/lib/routes/ready-check.test.ts @@ -0,0 +1,44 @@ +import supertest, { type Test } from 'supertest'; +import { createServices } from '../services/index.js'; +import { createTestConfig } from '../../test/config/test-config.js'; + +import createStores from '../../test/fixtures/store.js'; +import getLogger from '../../test/fixtures/no-logger.js'; +import getApp from '../app.js'; +import type TestAgent from 'supertest/lib/agent.d.ts'; + +async function getSetup() { + const stores = createStores(); + const config = createTestConfig(); + const services = createServices(stores, config); + const app = await getApp(config, stores, services); + + return { + request: supertest(app), + stores, + }; +} +let request: TestAgent; +beforeEach(async () => { + const setup = await getSetup(); + request = setup.request; +}); + +afterEach(() => { + getLogger.setMuteError(false); +}); + +test('should give 200 when ready', async () => { + await request.get('/ready').expect(200); +}); + +test('should give health=GOOD when ready', async () => { + expect.assertions(2); + await request + .get('/ready') + .expect(200) + .expect((res) => { + expect(res.status).toBe(200); + expect(res.body.health).toBe('GOOD'); + }); +}); diff --git a/src/lib/routes/ready-check.ts b/src/lib/routes/ready-check.ts new file mode 100644 index 0000000000..26221d7dca --- /dev/null +++ b/src/lib/routes/ready-check.ts @@ -0,0 +1,62 @@ +import type { Request, Response } from 'express'; +import type { IUnleashConfig } from '../types/option.js'; +import type { IUnleashServices } from '../services/index.js'; +import type { Db } from '../db/db.js'; +import type { Logger } from '../logger.js'; + +import Controller from './controller.js'; +import { NONE } from '../types/permissions.js'; +import { createResponseSchema } from '../openapi/util/create-response-schema.js'; +import type { ReadyCheckSchema } from '../openapi/spec/ready-check-schema.js'; + +export class ReadyCheckController extends Controller { + private logger: Logger; + + private db?: Db; + + constructor( + config: IUnleashConfig, + { openApiService }: Pick, + db?: Db, + ) { + super(config); + this.logger = config.getLogger('ready-check.js'); + this.db = db; + + this.route({ + method: 'get', + path: '', + handler: this.getReady, + permission: NONE, + middleware: [ + openApiService.validPath({ + tags: ['Operational'], + operationId: 'getReady', + summary: 'Get instance readiness status', + description: + 'This operation returns information about whether this Unleash instance is ready to serve requests or not. Typically used by your deployment orchestrator (e.g. Kubernetes, Docker Swarm, Mesos, et al.).', + responses: { + 200: createResponseSchema('readyCheckSchema'), + 500: createResponseSchema('readyCheckSchema'), + }, + }), + ], + }); + } + + async getReady(_: Request, res: Response): Promise { + if (this.config.checkDbOnReady && this.db) { + try { + await this.db.raw('select 1'); + res.status(200).json({ health: 'GOOD' }); + return; + } catch (err: any) { + this.logger.warn('Database readiness check failed', err); + res.status(500).json({ health: 'BAD' }); + return; + } + } + + res.status(200).json({ health: 'GOOD' }); + } +} diff --git a/src/lib/types/option.ts b/src/lib/types/option.ts index 9a5d7a8ed6..44de4e6240 100644 --- a/src/lib/types/option.ts +++ b/src/lib/types/option.ts @@ -175,6 +175,7 @@ export interface IUnleashOptions { resourceLimits?: Partial; userInactivityThresholdInDays?: number; unleashFrontendToken?: string; + checkDbOnReady?: boolean; } export interface IEmailOption { @@ -301,4 +302,6 @@ export interface IUnleashConfig { userInactivityThresholdInDays: number; buildDate?: string; unleashFrontendToken?: string; + /** If true, the readiness endpoint will attempt a simple Postgres query to verify DB availability */ + checkDbOnReady?: boolean; } diff --git a/src/test/e2e/ready.e2e.test.ts b/src/test/e2e/ready.e2e.test.ts new file mode 100644 index 0000000000..9e60c4a30a --- /dev/null +++ b/src/test/e2e/ready.e2e.test.ts @@ -0,0 +1,27 @@ +import { setupApp } from './helpers/test-helper.js'; +import dbInit, { type ITestDb } from './helpers/database-init.js'; +import getLogger from '../fixtures/no-logger.js'; +import type { IUnleashStores } from '../../lib/types/index.js'; + +let stores: IUnleashStores; +let db: ITestDb; + +beforeAll(async () => { + db = await dbInit('ready_api', getLogger); + stores = db.stores; +}); + +afterAll(async () => { + await db.destroy(); +}); + +test('returns readiness good', async () => { + expect.assertions(0); + const { request, destroy } = await setupApp(stores); + await request + .get('/ready') + .expect('Content-Type', /json/) + .expect(200) + .expect('{"health":"GOOD"}'); + await destroy(); +}); From a0ac6502fad34db2cfce222fc1ebf5d44e0ef445 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gast=C3=B3n=20Fournier?= Date: Thu, 23 Oct 2025 12:40:17 +0200 Subject: [PATCH 2/5] Fix test snapshot --- src/lib/__snapshots__/create-config.test.ts.snap | 1 + 1 file changed, 1 insertion(+) diff --git a/src/lib/__snapshots__/create-config.test.ts.snap b/src/lib/__snapshots__/create-config.test.ts.snap index fe6bece2cb..b407729b53 100644 --- a/src/lib/__snapshots__/create-config.test.ts.snap +++ b/src/lib/__snapshots__/create-config.test.ts.snap @@ -24,6 +24,7 @@ exports[`should create default config 1`] = ` "type": "open-source", }, "buildDate": undefined, + "checkDbOnReady": false, "clientFeatureCaching": { "enabled": true, "maxAge": 3600000, From 9cb27364a01c0cd82a7144f6e63ec7f7f33fdbdd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gast=C3=B3n=20Fournier?= Date: Thu, 23 Oct 2025 13:47:26 +0200 Subject: [PATCH 3/5] Use coalesce instead of if --- src/lib/create-config.ts | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/lib/create-config.ts b/src/lib/create-config.ts index a2bd9bf595..426e86787c 100644 --- a/src/lib/create-config.ts +++ b/src/lib/create-config.ts @@ -787,9 +787,8 @@ export function createConfig(options: IUnleashOptions): IUnleashConfig { process.env.PROMETHEUS_IMPACT_METRICS_API; const checkDbOnReady = - typeof options.checkDbOnReady === 'boolean' - ? options.checkDbOnReady - : parseEnvVarBoolean(process.env.CHECK_DB_ON_READY, false); + Boolean(options.checkDbOnReady) ?? + parseEnvVarBoolean(process.env.CHECK_DB_ON_READY, false); return { db, From e4c322d633663e568100decff3f1924b1b6cb191 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gast=C3=B3n=20Fournier?= Date: Thu, 23 Oct 2025 16:18:17 +0200 Subject: [PATCH 4/5] Checking for readiness tests --- src/lib/openapi/spec/ready-check-schema.ts | 4 +- src/lib/routes/ready-check.test.ts | 44 ------------- src/lib/routes/ready-check.ts | 45 ++++++++++++- src/test/e2e/ready.e2e.test.ts | 73 ++++++++++++++++------ 4 files changed, 98 insertions(+), 68 deletions(-) delete mode 100644 src/lib/routes/ready-check.test.ts diff --git a/src/lib/openapi/spec/ready-check-schema.ts b/src/lib/openapi/spec/ready-check-schema.ts index bab0a0c454..0e0e0fee5e 100644 --- a/src/lib/openapi/spec/ready-check-schema.ts +++ b/src/lib/openapi/spec/ready-check-schema.ts @@ -10,9 +10,9 @@ export const readyCheckSchema = { properties: { health: { description: - 'The readiness state this Unleash instance is in. GOOD if the server is up and running. It never returns BAD; if the server is unhealthy you will get an unsuccessful http response.', + 'The readiness state this Unleash instance is in. GOOD if the server is up and running. If the server is unhealthy you will get an unsuccessful http response.', type: 'string', - enum: ['GOOD', 'BAD'], + enum: ['GOOD'], example: 'GOOD', }, }, diff --git a/src/lib/routes/ready-check.test.ts b/src/lib/routes/ready-check.test.ts deleted file mode 100644 index 005dcbe9db..0000000000 --- a/src/lib/routes/ready-check.test.ts +++ /dev/null @@ -1,44 +0,0 @@ -import supertest, { type Test } from 'supertest'; -import { createServices } from '../services/index.js'; -import { createTestConfig } from '../../test/config/test-config.js'; - -import createStores from '../../test/fixtures/store.js'; -import getLogger from '../../test/fixtures/no-logger.js'; -import getApp from '../app.js'; -import type TestAgent from 'supertest/lib/agent.d.ts'; - -async function getSetup() { - const stores = createStores(); - const config = createTestConfig(); - const services = createServices(stores, config); - const app = await getApp(config, stores, services); - - return { - request: supertest(app), - stores, - }; -} -let request: TestAgent; -beforeEach(async () => { - const setup = await getSetup(); - request = setup.request; -}); - -afterEach(() => { - getLogger.setMuteError(false); -}); - -test('should give 200 when ready', async () => { - await request.get('/ready').expect(200); -}); - -test('should give health=GOOD when ready', async () => { - expect.assertions(2); - await request - .get('/ready') - .expect(200) - .expect((res) => { - expect(res.status).toBe(200); - expect(res.body.health).toBe('GOOD'); - }); -}); diff --git a/src/lib/routes/ready-check.ts b/src/lib/routes/ready-check.ts index 26221d7dca..860c020732 100644 --- a/src/lib/routes/ready-check.ts +++ b/src/lib/routes/ready-check.ts @@ -1,4 +1,5 @@ import type { Request, Response } from 'express'; +import type { PoolClient } from 'pg'; import type { IUnleashConfig } from '../types/option.js'; import type { IUnleashServices } from '../services/index.js'; import type { Db } from '../db/db.js'; @@ -8,6 +9,7 @@ import Controller from './controller.js'; import { NONE } from '../types/permissions.js'; import { createResponseSchema } from '../openapi/util/create-response-schema.js'; import type { ReadyCheckSchema } from '../openapi/spec/ready-check-schema.js'; +import { emptyResponse, parseEnvVarNumber } from '../server-impl.js'; export class ReadyCheckController extends Controller { private logger: Logger; @@ -37,7 +39,7 @@ export class ReadyCheckController extends Controller { 'This operation returns information about whether this Unleash instance is ready to serve requests or not. Typically used by your deployment orchestrator (e.g. Kubernetes, Docker Swarm, Mesos, et al.).', responses: { 200: createResponseSchema('readyCheckSchema'), - 500: createResponseSchema('readyCheckSchema'), + 503: emptyResponse, }, }), ], @@ -47,16 +49,53 @@ export class ReadyCheckController extends Controller { async getReady(_: Request, res: Response): Promise { if (this.config.checkDbOnReady && this.db) { try { - await this.db.raw('select 1'); + const timeoutMs = parseEnvVarNumber( + process.env.DATABASE_STATEMENT_TIMEOUT_MS, + 200, + ); + + await this.runReadinessQuery(timeoutMs); res.status(200).json({ health: 'GOOD' }); return; } catch (err: any) { this.logger.warn('Database readiness check failed', err); - res.status(500).json({ health: 'BAD' }); + res.status(503).end(); return; } } res.status(200).json({ health: 'GOOD' }); } + + private async runReadinessQuery(timeoutMs: number): Promise { + const client = getKnexClient(this.db!); + const pendingAcquire = client.pool.acquire(); + + const abortDelay = setTimeout(() => pendingAcquire.abort(), timeoutMs); + let connection: PoolClient | undefined; + + try { + connection = (await pendingAcquire.promise) as PoolClient; + } finally { + clearTimeout(abortDelay); + } + + try { + connection.query({ + text: 'SELECT 1', + }); + } finally { + if (connection) { + await client.releaseConnection(connection); + } + } + } +} + +function getKnexClient(db: Db): Db['client'] { + if (db.client?.pool && typeof db.client.releaseConnection === 'function') { + return db.client as Db['client']; + } + + throw new Error('Unsupported database handle for readiness check'); } diff --git a/src/test/e2e/ready.e2e.test.ts b/src/test/e2e/ready.e2e.test.ts index 9e60c4a30a..7ddb95fc4e 100644 --- a/src/test/e2e/ready.e2e.test.ts +++ b/src/test/e2e/ready.e2e.test.ts @@ -1,27 +1,62 @@ -import { setupApp } from './helpers/test-helper.js'; +import { setupAppWithCustomConfig } from './helpers/test-helper.js'; import dbInit, { type ITestDb } from './helpers/database-init.js'; -import getLogger from '../fixtures/no-logger.js'; -import type { IUnleashStores } from '../../lib/types/index.js'; -let stores: IUnleashStores; let db: ITestDb; -beforeAll(async () => { - db = await dbInit('ready_api', getLogger); - stores = db.stores; +describe('DB is up', () => { + beforeAll(async () => { + db = await dbInit(); + }); + + test('when checkDb is disabled, returns ready', async () => { + const { request } = await setupAppWithCustomConfig( + db.stores, + undefined, + db.rawDatabase, + ); + await request + .get('/ready') + .expect('Content-Type', /json/) + .expect(200) + .expect('{"health":"GOOD"}'); + }); + + test('when checkDb is enabled, returns ready', async () => { + const { request } = await setupAppWithCustomConfig( + db.stores, + { checkDbOnReady: true }, + db.rawDatabase, + ); + await request.get('/ready').expect(200).expect('{"health":"GOOD"}'); + }); }); -afterAll(async () => { - await db.destroy(); -}); +describe('DB is down', () => { + beforeAll(async () => { + db = await dbInit(); + }); -test('returns readiness good', async () => { - expect.assertions(0); - const { request, destroy } = await setupApp(stores); - await request - .get('/ready') - .expect('Content-Type', /json/) - .expect(200) - .expect('{"health":"GOOD"}'); - await destroy(); + test('when checkDb is disabled, returns readiness good', async () => { + const { request } = await setupAppWithCustomConfig( + db.stores, + undefined, + db.rawDatabase, + ); + await db.destroy(); + await request + .get('/ready') + .expect('Content-Type', /json/) + .expect(200) + .expect('{"health":"GOOD"}'); + }); + + test('when checkDb is enabled, fails readiness check', async () => { + const { request } = await setupAppWithCustomConfig( + db.stores, + { checkDbOnReady: true }, + db.rawDatabase, + ); + await db.destroy(); + await request.get('/ready').expect(503); + }); }); From 887d154cfe1ac40730d5eb03f9020c63bfe1f3ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gast=C3=B3n=20Fournier?= Date: Thu, 23 Oct 2025 16:49:15 +0200 Subject: [PATCH 5/5] Validate also that if query halts readiness fails --- src/lib/routes/ready-check.ts | 17 +++++++++++++-- src/lib/types/option.ts | 1 - src/test/e2e/ready.e2e.test.ts | 40 ++++++++++++++++++++++++++++++++++ 3 files changed, 55 insertions(+), 3 deletions(-) diff --git a/src/lib/routes/ready-check.ts b/src/lib/routes/ready-check.ts index 860c020732..381e90e1d4 100644 --- a/src/lib/routes/ready-check.ts +++ b/src/lib/routes/ready-check.ts @@ -81,9 +81,22 @@ export class ReadyCheckController extends Controller { } try { - connection.query({ - text: 'SELECT 1', + await connection.query('BEGIN'); + await connection.query({ + text: `SET LOCAL statement_timeout = ${timeoutMs}`, }); + await connection.query('SELECT 1'); + await connection.query('COMMIT'); + } catch (error) { + try { + await connection.query('ROLLBACK'); + } catch (rollbackError) { + this.logger.debug( + 'Failed to rollback readiness timeout transaction', + rollbackError, + ); + } + throw error; } finally { if (connection) { await client.releaseConnection(connection); diff --git a/src/lib/types/option.ts b/src/lib/types/option.ts index 44de4e6240..ef0df45051 100644 --- a/src/lib/types/option.ts +++ b/src/lib/types/option.ts @@ -302,6 +302,5 @@ export interface IUnleashConfig { userInactivityThresholdInDays: number; buildDate?: string; unleashFrontendToken?: string; - /** If true, the readiness endpoint will attempt a simple Postgres query to verify DB availability */ checkDbOnReady?: boolean; } diff --git a/src/test/e2e/ready.e2e.test.ts b/src/test/e2e/ready.e2e.test.ts index 7ddb95fc4e..f84ef6c388 100644 --- a/src/test/e2e/ready.e2e.test.ts +++ b/src/test/e2e/ready.e2e.test.ts @@ -29,6 +29,46 @@ describe('DB is up', () => { ); await request.get('/ready').expect(200).expect('{"health":"GOOD"}'); }); + + test('fails fast when readiness query hangs', async () => { + const { request } = await setupAppWithCustomConfig( + db.stores, + { checkDbOnReady: true }, + db.rawDatabase, + ); + + const pool = db.rawDatabase.client.pool; + const originalAcquire = pool.acquire.bind(pool); + + pool.acquire = () => { + const pending = originalAcquire(); + pending.promise = pending.promise.then((conn: any) => { + const originalQuery = conn.query; + conn.query = (queryConfig: any, ...args: any[]) => { + const isSelectOne = + queryConfig?.toUpperCase() === 'SELECT 1'; + + if (isSelectOne) { + return originalQuery.call( + conn, + 'SELECT pg_sleep(1)', + ...args, + ); + } + + return originalQuery.call(conn, queryConfig, ...args); + }; + return conn; + }); + return pending; + }; + + try { + await request.get('/ready').expect(503); + } finally { + pool.acquire = originalAcquire; + } + }); }); describe('DB is down', () => {