diff --git a/apps/cli/e2e/server/backend-tls.e2e-spec.ts b/apps/cli/e2e/server/backend-tls.e2e-spec.ts new file mode 100644 index 00000000..a787e138 --- /dev/null +++ b/apps/cli/e2e/server/backend-tls.e2e-spec.ts @@ -0,0 +1,209 @@ +import { readFileSync } from 'node:fs'; +import * as https from 'node:https'; +import { join } from 'node:path'; +import request from 'supertest'; + +import * as commandUtils from '../../src/command/utils'; +import { ADCServer } from '../../src/server'; +import { mockBackend } from '../support/utils'; + +const tlsAssetsDir = join(__dirname, '../assets/tls'); +const readCert = (fileName: string) => + readFileSync(join(tlsAssetsDir, fileName), 'utf-8'); + +describe('Server - Backend TLS', () => { + let server: ADCServer; + + beforeAll(() => { + server = new ADCServer({ + listen: new URL('http://127.0.1:3000'), + listenStatus: 3002, + }); + }); + + afterEach(() => { + vi.restoreAllMocks(); + }); + + it('rejects a request with tlsClientCert but no tlsClientKey', async () => { + const { status, body } = await request(server.TEST_ONLY_getExpress()) + .put('/sync') + .send({ + task: { + opts: { + backend: 'mock', + server: 'http://1.1.1.1:3000', + token: 'mock', + cacheKey: 'default', + tlsClientCert: readCert('client.cer'), + }, + config: {}, + }, + }); + + expect(status).toEqual(400); + expect( + (body.errors as Array<{ path: string[] }>).some((issue) => + issue.path.includes('tlsClientKey'), + ), + ).toBe(true); + }); + + it('rejects a caCert that does not look like PEM content', async () => { + const { status, body } = await request(server.TEST_ONLY_getExpress()) + .put('/sync') + .send({ + task: { + opts: { + backend: 'mock', + server: 'http://1.1.1.1:3000', + token: 'mock', + cacheKey: 'default', + caCert: 'not-a-pem', + }, + config: {}, + }, + }); + + expect(status).toEqual(400); + expect( + (body.errors as Array<{ path: string[] }>).some((issue) => + issue.path.includes('caCert'), + ), + ).toBe(true); + }); + + it('reuses the same pooled agent across requests with identical TLS material, but not across different material', async () => { + const loadBackendSpy = vi + .spyOn(commandUtils, 'loadBackend') + .mockImplementation(() => mockBackend()); + + const sendSync = (caCert?: string) => + request(server.TEST_ONLY_getExpress()) + .put('/sync') + .send({ + task: { + opts: { + backend: 'mock', + server: 'http://1.1.1.1:3000', + token: 'mock', + cacheKey: 'default', + ...(caCert ? { caCert } : {}), + }, + config: {}, + }, + }); + + const ca = readCert('ca.cer'); + await sendSync(ca); + await sendSync(ca); + await sendSync(); // no TLS material at all -> a different, insecure-default agent + + expect(loadBackendSpy).toHaveBeenCalledTimes(3); + const httpsAgents = loadBackendSpy.mock.calls.map( + ([, opts]) => (opts as { httpsAgent: unknown }).httpsAgent, + ); + expect(httpsAgents[0]).toBeDefined(); + expect(httpsAgents[2]).toBeDefined(); + expect(httpsAgents[0]).toBe(httpsAgents[1]); + expect(httpsAgents[0]).not.toBe(httpsAgents[2]); + }); + + it.each(['/sync', '/validate'] as const)( + 'does not forward raw TLS material to loadBackend for %s', + async (route) => { + const loadBackendSpy = vi + .spyOn(commandUtils, 'loadBackend') + .mockImplementation(() => mockBackend()); + + await request(server.TEST_ONLY_getExpress()) + .put(route) + .send({ + task: { + opts: { + backend: 'mock', + server: 'http://1.1.1.1:3000', + token: 'mock', + cacheKey: 'default', + caCert: readCert('ca.cer'), + tlsClientCert: readCert('client.cer'), + tlsClientKey: readCert('client.key'), + }, + config: {}, + }, + }); + + expect(loadBackendSpy).toHaveBeenCalledTimes(1); + const [, opts] = loadBackendSpy.mock.calls[0]; + expect(opts).not.toHaveProperty('caCert'); + expect(opts).not.toHaveProperty('tlsClientCert'); + expect(opts).not.toHaveProperty('tlsClientKey'); + expect(opts).not.toHaveProperty('tlsSkipVerify'); + // secure default: no tlsSkipVerify means the pooled agent must still verify + expect((opts as { httpsAgent: https.Agent }).httpsAgent.options.rejectUnauthorized).toBe( + true, + ); + }, + ); + + describe('real backend connection', () => { + let backendServer: https.Server; + let backendPort: number; + + beforeAll(async () => { + backendServer = https.createServer( + { cert: readCert('server.cer'), key: readCert('server.key') }, + (_, res) => res.end('{}'), + ); + await new Promise((resolve) => + backendServer.listen(0, '127.0.0.1', resolve), + ); + backendPort = (backendServer.address() as { port: number }).port; + }); + + afterAll(async () => { + await new Promise((resolve) => backendServer.close(() => resolve())); + }); + + it('fails with a certificate verification error when no caCert is provided', async () => { + const { status, body } = await request(server.TEST_ONLY_getExpress()) + .put('/sync') + .send({ + task: { + opts: { + backend: 'apisix', + server: `https://127.0.0.1:${backendPort}`, + token: 'mock', + cacheKey: 'default', + }, + config: {}, + }, + }); + + expect(status).toEqual(500); + expect(body.message).toMatch(/self-signed certificate|unable to verify/i); + }); + + it('does not fail on certificate verification once the signing caCert is provided', async () => { + const { body } = await request(server.TEST_ONLY_getExpress()) + .put('/sync') + .send({ + task: { + opts: { + backend: 'apisix', + server: `https://127.0.0.1:${backendPort}`, + token: 'mock', + cacheKey: 'default', + caCert: readCert('ca.cer'), + }, + config: {}, + }, + }); + + // the fake backend doesn't implement the real Admin API, so the request + // may still fail (with any status) for unrelated reasons; the point of + // this assertion is that it no longer fails on TLS certificate verification + expect(body.message).not.toMatch(/self-signed certificate|unable to verify/i); + }); + }); +}); diff --git a/apps/cli/eslint.config.ts b/apps/cli/eslint.config.ts index dbf8ff37..10ff6893 100644 --- a/apps/cli/eslint.config.ts +++ b/apps/cli/eslint.config.ts @@ -15,6 +15,10 @@ export default config([ '{projectRoot}/vitest.config.{js,ts,mjs,mts}', '{projectRoot}/e2e/**/*', ], + // false positive: this workspace also resolves an unrelated, + // transitive lru-cache@5.1.1 (via @babel/helper-compilation-targets), + // which confuses the rule's usage detection for our direct dependency + ignoredDependencies: ['lru-cache'], }, ], }, diff --git a/apps/cli/package.json b/apps/cli/package.json index 6af03256..58b1ee7d 100644 --- a/apps/cli/package.json +++ b/apps/cli/package.json @@ -32,6 +32,7 @@ "js-yaml": "catalog:", "listr2": "catalog:", "lodash-es": "catalog:", + "lru-cache": "catalog:", "parse-duration": "^2.1.5", "pluralize": "^8.0.0", "qs": "^6.14.1", diff --git a/apps/cli/src/server/agent-pool.spec.ts b/apps/cli/src/server/agent-pool.spec.ts new file mode 100644 index 00000000..e755097c --- /dev/null +++ b/apps/cli/src/server/agent-pool.spec.ts @@ -0,0 +1,186 @@ +import { HttpsAgent } from 'agentkeepalive'; +import * as https from 'node:https'; +import { join } from 'node:path'; +import { readFileSync } from 'node:fs'; + +import { + fingerprintTlsMaterial, + getHttpsAgent, + type TlsMaterial, +} from './agent-pool'; + +const tlsAssetsDir = join(__dirname, '../../e2e/assets/tls'); +const readAsset = (fileName: string) => + readFileSync(join(tlsAssetsDir, fileName), 'utf-8'); + +describe('agent-pool fingerprintTlsMaterial', () => { + it('produces the same fingerprint for identical TLS material', () => { + const tls: TlsMaterial = { caCert: 'ca-content', tlsSkipVerify: true }; + expect(fingerprintTlsMaterial(tls)).toEqual( + fingerprintTlsMaterial({ ...tls }), + ); + }); + + it('treats a missing tlsSkipVerify the same as an explicit false', () => { + expect(fingerprintTlsMaterial({ caCert: 'ca-content' })).toEqual( + fingerprintTlsMaterial({ caCert: 'ca-content', tlsSkipVerify: false }), + ); + }); + + it('produces different fingerprints when any field differs', () => { + const base = fingerprintTlsMaterial({ caCert: 'ca-content' }); + expect(fingerprintTlsMaterial({ caCert: 'other-content' })).not.toEqual( + base, + ); + expect(fingerprintTlsMaterial({ tlsSkipVerify: true })).not.toEqual(base); + expect( + fingerprintTlsMaterial({ + caCert: 'ca-content', + tlsClientCert: 'cert', + }), + ).not.toEqual(base); + expect( + fingerprintTlsMaterial({ + caCert: 'ca-content', + tlsClientKey: 'key', + }), + ).not.toEqual(base); + }); +}); + +describe('agent-pool getHttpsAgent pooling', () => { + it('reuses the same agent instance for identical TLS material', () => { + const agent1 = getHttpsAgent({ caCert: 'shared-ca' }); + const agent2 = getHttpsAgent({ caCert: 'shared-ca' }); + expect(agent1).toBe(agent2); + }); + + it('returns isolated agent instances for different TLS material', () => { + const agent1 = getHttpsAgent({ caCert: 'ca-a' }); + const agent2 = getHttpsAgent({ caCert: 'ca-b' }); + expect(agent1).not.toBe(agent2); + }); + + it('builds an agent with the requested TLS options', () => { + const agent = getHttpsAgent({ tlsSkipVerify: true, caCert: 'ca-c' }); + expect(agent).toBeInstanceOf(HttpsAgent); + expect(agent.options.rejectUnauthorized).toBe(false); + expect(agent.options.ca).toEqual('ca-c'); + }); + + it('builds an agent with the requested mTLS client cert and key', () => { + const agent = getHttpsAgent({ + tlsClientCert: 'client-cert', + tlsClientKey: 'client-key', + }); + expect(agent.options.cert).toEqual('client-cert'); + expect(agent.options.key).toEqual('client-key'); + }); + + it('defaults to rejectUnauthorized: true when no TLS material is given', () => { + const agent = getHttpsAgent(); + expect(agent.options.rejectUnauthorized).toBe(true); + }); +}); + +describe('agent-pool getHttpsAgent real TLS handshake', () => { + let server: https.Server; + let port: number; + + beforeAll(async () => { + server = https.createServer( + { + cert: readAsset('server.cer'), + key: readAsset('server.key'), + }, + (_, res) => res.end('ok'), + ); + await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)); + port = (server.address() as { port: number }).port; + }); + + afterAll(async () => { + await new Promise((resolve, reject) => + server.close((err) => (err ? reject(err) : resolve())), + ); + }); + + // server.cer's CN is "localhost" (no IP SAN), so pin SNI/hostname + // verification to "localhost" while still dialing the loopback IP directly + const request = (agent: https.Agent) => + new Promise((resolve, reject) => { + https + .get( + { hostname: '127.0.0.1', servername: 'localhost', port, path: '/', agent }, + (res) => { + res.resume(); + res.on('end', resolve); + }, + ) + .on('error', reject); + }); + + it('connects successfully when trusting the signing CA', async () => { + const agent = getHttpsAgent({ caCert: readAsset('ca.cer') }); + await expect(request(agent)).resolves.toBeUndefined(); + }); + + it('fails certificate verification without the CA', async () => { + const agent = getHttpsAgent(); + await expect(request(agent)).rejects.toThrow(/self.signed|unable to verify/i); + }); +}); + +describe('agent-pool LRU eviction', () => { + beforeEach(() => { + vi.resetModules(); + vi.stubEnv('ADC_INGRESS_TLS_AGENT_POOL_MAX', '2'); + }); + + afterEach(() => { + vi.unstubAllEnvs(); + }); + + it('evicts and destroys the least-recently-used agent, not merely the first-inserted one', async () => { + const pool = await import('./agent-pool'); + + const agentA = pool.getHttpsAgent({ caCert: 'a' }); + pool.releaseHttpsAgent(agentA); // a's modeled request completes + const agentB = pool.getHttpsAgent({ caCert: 'b' }); + pool.releaseHttpsAgent(agentB); // b's modeled request completes + // re-fetching "a" refreshes its recency, so "b" (not "a") becomes the + // least-recently-used entry despite "a" having been inserted first + pool.getHttpsAgent({ caCert: 'a' }); + pool.releaseHttpsAgent(agentA); + const destroySpyA = vi.spyOn(agentA, 'destroy'); + const destroySpyB = vi.spyOn(agentB, 'destroy'); + + // exceeding max size (2) evicts the least-recently-used entry (b) + const agentC = pool.getHttpsAgent({ caCert: 'c' }); + pool.releaseHttpsAgent(agentC); // c's modeled request completes + + expect(destroySpyB).toHaveBeenCalledTimes(1); + expect(destroySpyA).not.toHaveBeenCalled(); + }); + + it('defers destroying an evicted agent until its active request finishes', async () => { + const pool = await import('./agent-pool'); + + // "a" is kept checked out (simulating a request still in flight) until + // the final assertion below; every other checkout is released as soon as + // its modeled request completes + const agentA = pool.getHttpsAgent({ caCert: 'a' }); + const agentB = pool.getHttpsAgent({ caCert: 'b' }); + pool.releaseHttpsAgent(agentB); + const destroySpy = vi.spyOn(agentA, 'destroy'); + + // evicts "a" (LRU) while its request is still active; must not destroy yet + const agentC = pool.getHttpsAgent({ caCert: 'c' }); + pool.releaseHttpsAgent(agentC); + expect(destroySpy).not.toHaveBeenCalled(); + + // the in-flight request using "a" now completes + pool.releaseHttpsAgent(agentA); + expect(destroySpy).toHaveBeenCalledTimes(1); + }); +}); diff --git a/apps/cli/src/server/agent-pool.ts b/apps/cli/src/server/agent-pool.ts new file mode 100644 index 00000000..2d69d840 --- /dev/null +++ b/apps/cli/src/server/agent-pool.ts @@ -0,0 +1,104 @@ +import { HttpAgent, HttpOptions, HttpsAgent } from 'agentkeepalive'; +import { LRUCache } from 'lru-cache'; +import { createHash } from 'node:crypto'; + +const keepAlive: HttpOptions = { + keepAlive: true, + maxSockets: 256, // per host + maxFreeSockets: 16, // per host free + freeSocketTimeout: + parseInt(process.env.ADC_INGRESS_FREE_SOCKET_TIMEOUT ?? '') || 50000, // free socket keepalive for 50 seconds, and if the ADC_INGRESS_FREE_SOCKET_TIMEOUT environment variable is provided, it takes precedence. +}; + +// plain http:// backends have no TLS material to distinguish, so a single +// shared agent is enough +export const httpAgent = new HttpAgent(keepAlive); + +export interface TlsMaterial { + tlsSkipVerify?: boolean; + caCert?: string; + tlsClientCert?: string; + tlsClientKey?: string; +} + +const parseEnvInt = (value: string | undefined, defaultVal: number): number => { + const n = Number(value ?? defaultVal); + return Number.isFinite(n) && n >= 1 ? Math.floor(n) : defaultVal; +}; +const maxPoolSize = parseEnvInt(process.env.ADC_INGRESS_TLS_AGENT_POOL_MAX, 16); + +interface PooledAgent { + agent: HttpsAgent; + activeRequests: number; + evicted: boolean; +} + +// only safe to close an evicted agent's sockets once nothing is still using +// it for an in-flight request +const destroyIfIdle = (entry: PooledAgent) => { + if (entry.evicted && entry.activeRequests === 0) entry.agent.destroy(); +}; + +// key: sha256 fingerprint of the TLS material -> value: a pooled agent entry +const httpsAgentPool = new LRUCache({ + max: maxPoolSize, + dispose: (entry) => { + entry.evicted = true; + destroyIfIdle(entry); + }, +}); + +// reverse lookup so releaseHttpsAgent can find an agent's bookkeeping entry +// even after it has been evicted from httpsAgentPool +const entriesByAgent = new WeakMap(); + +// Fingerprint the TLS material into a fixed-size cache key instead of using +// the raw PEM strings as the Map key. `\0` separators avoid ambiguous +// concatenation collisions between fields. +export const fingerprintTlsMaterial = (tls: TlsMaterial = {}): string => + createHash('sha256') + .update(tls.tlsSkipVerify ? '1' : '0') + .update('\0') + .update(tls.caCert ?? '') + .update('\0') + .update(tls.tlsClientCert ?? '') + .update('\0') + .update(tls.tlsClientKey ?? '') + .digest('hex'); + +/** + * Returns a pooled HttpsAgent for the given TLS material. Requests with + * identical material share (and thus keep-alive-reuse) the same agent and + * connection pool; different material gets an isolated agent so certs/keys + * are never cross-contaminated between backends. + * + * Every call must be paired with a `releaseHttpsAgent` call once the request + * using the returned agent has finished, so an agent evicted from the pool + * while still in flight isn't destroyed out from under that request. + */ +export const getHttpsAgent = (tls: TlsMaterial = {}): HttpsAgent => { + const key = fingerprintTlsMaterial(tls); + let entry = httpsAgentPool.get(key); // also refreshes LRU recency + if (!entry) { + const agent = new HttpsAgent({ + ...keepAlive, + rejectUnauthorized: !tls.tlsSkipVerify, + ...(tls.caCert ? { ca: tls.caCert } : {}), + ...(tls.tlsClientCert ? { cert: tls.tlsClientCert } : {}), + ...(tls.tlsClientKey ? { key: tls.tlsClientKey } : {}), + }); + entry = { agent, activeRequests: 0, evicted: false }; + httpsAgentPool.set(key, entry); + entriesByAgent.set(agent, entry); + } + entry.activeRequests++; + return entry.agent; +}; + +/** Releases an agent obtained from `getHttpsAgent` once its request is done. */ +export const releaseHttpsAgent = (agent: HttpsAgent): void => { + const entry = entriesByAgent.get(agent); + if (!entry) return; + entry.activeRequests = Math.max(0, entry.activeRequests - 1); + destroyIfIdle(entry); +}; diff --git a/apps/cli/src/server/logger.spec.ts b/apps/cli/src/server/logger.spec.ts new file mode 100644 index 00000000..91d4cbe8 --- /dev/null +++ b/apps/cli/src/server/logger.spec.ts @@ -0,0 +1,36 @@ +import { redactRequestBody } from './logger'; + +describe('redactRequestBody', () => { + it('redacts tlsClientKey while preserving other fields', () => { + expect( + redactRequestBody({ + task: { + opts: { backend: 'apisix', tlsClientKey: 'SECRET', tlsClientCert: 'cert' }, + config: {}, + }, + }), + ).toEqual({ + task: { + opts: { backend: 'apisix', tlsClientKey: '***', tlsClientCert: 'cert' }, + config: {}, + }, + }); + }); + + it('returns the body unchanged when tlsClientKey is absent', () => { + const body = { task: { opts: { backend: 'apisix' }, config: {} } }; + expect(redactRequestBody(body)).toBe(body); + }); + + it.each([ + { task: { opts: 1 } }, + { task: { opts: 'not-an-object' } }, + { task: { opts: null } }, + { task: {} }, + {}, + undefined, + null, + ])('does not throw for malformed body %j', (body) => { + expect(() => redactRequestBody(body)).not.toThrow(); + }); +}); diff --git a/apps/cli/src/server/logger.ts b/apps/cli/src/server/logger.ts index ff518a93..549de53c 100644 --- a/apps/cli/src/server/logger.ts +++ b/apps/cli/src/server/logger.ts @@ -23,6 +23,17 @@ export const logger = winston.createLogger({ transports: [new winston.transports.Console({})], }); +// task.opts.tlsClientKey carries a raw mTLS private key; never let it reach +// the debug request-body log +export const redactRequestBody = (body: unknown) => { + const opts = (body as { task?: { opts?: { tlsClientKey?: unknown } } })?.task + ?.opts; + if (typeof opts !== 'object' || opts === null || !('tlsClientKey' in opts)) + return body; + const { task, ...rest } = body as { task: { opts: object } }; + return { ...rest, task: { ...task, opts: { ...task.opts, tlsClientKey: '***' } } }; +}; + export const loggerMiddleware: RequestHandler = (req, res, next) => { req.requestId = randomUUID(); @@ -36,7 +47,7 @@ export const loggerMiddleware: RequestHandler = (req, res, next) => { logger.log({ level: 'debug', message: '', - requestBody: req.body, + requestBody: redactRequestBody(req.body), requestId: req.requestId, }); diff --git a/apps/cli/src/server/schema.ts b/apps/cli/src/server/schema.ts index ce37d8d6..d4ea0a7a 100644 --- a/apps/cli/src/server/schema.ts +++ b/apps/cli/src/server/schema.ts @@ -1,18 +1,55 @@ import * as ADCSDK from '@api7/adc-sdk'; import { z } from 'zod'; +import type { TlsMaterial } from './agent-pool'; + +const isPemLike = (value?: string) => !value || value.trim().startsWith('-----BEGIN'); + +// tlsClientCert/tlsClientKey must be provided together, and any provided PEM field +// must at least look like PEM content (full certificate/key parsing happens at the +// TLS layer when the connection is actually established). +const tlsCertKeyPaired = (o: TlsMaterial) => !!o.tlsClientCert === !!o.tlsClientKey; +const caCertIsPemLike = (o: TlsMaterial) => isPemLike(o.caCert); +const tlsClientCertIsPemLike = (o: TlsMaterial) => isPemLike(o.tlsClientCert); +const tlsClientKeyIsPemLike = (o: TlsMaterial) => isPemLike(o.tlsClientKey); + +const tlsShape = { + tlsSkipVerify: z.boolean().optional(), + caCert: z.string().min(1).optional(), + tlsClientCert: z.string().min(1).optional(), + tlsClientKey: z.string().min(1).optional(), +}; + const SyncTask = z.strictObject({ - opts: z.looseObject({ - backend: z.string().min(1), - server: z.union([z.url().min(1), z.array(z.url().min(1))]), - token: z.string().min(1), - lint: z.boolean().optional().default(true), - includeResourceType: z.array(z.enum(ADCSDK.ResourceType)).optional(), - excludeResourceType: z.array(z.enum(ADCSDK.ResourceType)).optional(), - labelSelector: z.record(z.string(), z.string()).optional(), - cacheKey: z.string(), - bypassCache: z.boolean().optional().default(false), - }), + opts: z + .looseObject({ + backend: z.string().min(1), + server: z.union([z.url().min(1), z.array(z.url().min(1))]), + token: z.string().min(1), + lint: z.boolean().optional().default(true), + includeResourceType: z.array(z.enum(ADCSDK.ResourceType)).optional(), + excludeResourceType: z.array(z.enum(ADCSDK.ResourceType)).optional(), + labelSelector: z.record(z.string(), z.string()).optional(), + cacheKey: z.string(), + bypassCache: z.boolean().optional().default(false), + ...tlsShape, + }) + .refine(tlsCertKeyPaired, { + error: 'tlsClientCert and tlsClientKey must be provided together', + path: ['tlsClientKey'], + }) + .refine(caCertIsPemLike, { + error: 'caCert does not look like a PEM-encoded certificate', + path: ['caCert'], + }) + .refine(tlsClientCertIsPemLike, { + error: 'tlsClientCert does not look like a PEM-encoded certificate', + path: ['tlsClientCert'], + }) + .refine(tlsClientKeyIsPemLike, { + error: 'tlsClientKey does not look like a PEM-encoded key', + path: ['tlsClientKey'], + }), config: z.looseObject({}), }); @@ -22,16 +59,34 @@ export const SyncInput = z.strictObject({ export type SyncInputType = z.infer; const ValidateTask = z.strictObject({ - opts: z.looseObject({ - backend: z.string().min(1), - server: z.union([z.url().min(1), z.array(z.url().min(1))]), - token: z.string().min(1), - lint: z.boolean().optional().default(true), - includeResourceType: z.array(z.enum(ADCSDK.ResourceType)).optional(), - excludeResourceType: z.array(z.enum(ADCSDK.ResourceType)).optional(), - labelSelector: z.record(z.string(), z.string()).optional(), - cacheKey: z.string(), - }), + opts: z + .looseObject({ + backend: z.string().min(1), + server: z.union([z.url().min(1), z.array(z.url().min(1))]), + token: z.string().min(1), + lint: z.boolean().optional().default(true), + includeResourceType: z.array(z.enum(ADCSDK.ResourceType)).optional(), + excludeResourceType: z.array(z.enum(ADCSDK.ResourceType)).optional(), + labelSelector: z.record(z.string(), z.string()).optional(), + cacheKey: z.string(), + ...tlsShape, + }) + .refine(tlsCertKeyPaired, { + error: 'tlsClientCert and tlsClientKey must be provided together', + path: ['tlsClientKey'], + }) + .refine(caCertIsPemLike, { + error: 'caCert does not look like a PEM-encoded certificate', + path: ['caCert'], + }) + .refine(tlsClientCertIsPemLike, { + error: 'tlsClientCert does not look like a PEM-encoded certificate', + path: ['tlsClientCert'], + }) + .refine(tlsClientKeyIsPemLike, { + error: 'tlsClientKey does not look like a PEM-encoded key', + path: ['tlsClientKey'], + }), config: z.looseObject({}), }); diff --git a/apps/cli/src/server/sync.ts b/apps/cli/src/server/sync.ts index d9de553b..abd7d6f8 100644 --- a/apps/cli/src/server/sync.ts +++ b/apps/cli/src/server/sync.ts @@ -1,6 +1,6 @@ import { Differ } from '@api7/adc-differ'; import * as ADCSDK from '@api7/adc-sdk'; -import { HttpAgent, HttpOptions, HttpsAgent } from 'agentkeepalive'; +import type { HttpsAgent } from 'agentkeepalive'; import { AxiosResponse } from 'axios'; import type { RequestHandler } from 'express'; import { omit, toString } from 'lodash-es'; @@ -13,34 +13,19 @@ import { loadBackend, } from '../command/utils'; import { check } from '../linter'; +import { getHttpsAgent, httpAgent, releaseHttpsAgent } from './agent-pool'; import { logger } from './logger'; import { SyncInput, type SyncInputType } from './schema'; -// create connection pool -const keepAlive: HttpOptions = { - keepAlive: true, - maxSockets: 256, // per host - maxFreeSockets: 16, // per host free - freeSocketTimeout: - parseInt(process.env.ADC_INGRESS_FREE_SOCKET_TIMEOUT ?? '') || 50000, // free socket keepalive for 50 seconds, and if the ADC_INGRESS_FREE_SOCKET_TIMEOUT environment variable is provided, it takes precedence. -}; -const httpAgent = new HttpAgent(keepAlive); - -//TODO: dynamic rejectUnauthorized and support mTLS -const httpsAgent = new HttpsAgent({ - rejectUnauthorized: true, - ...keepAlive, -}); -const httpsInsecureAgent = new HttpsAgent({ - rejectUnauthorized: false, - ...keepAlive, -}); - export const syncHandler: RequestHandler< unknown, unknown, SyncInputType > = async (req, res) => { + // checked out from the TLS agent pool once the backend is initialized below, + // and released in `finally` so an agent evicted mid-request isn't destroyed + // while this request is still using it + let httpsAgent: HttpsAgent | undefined; try { const parsedInput = SyncInput.safeParse(req.body); if (!parsedInput.success) @@ -68,13 +53,16 @@ export const syncHandler: RequestHandler< fillLabels(local, task.opts.labelSelector); // load and filter remote configuration + const { tlsSkipVerify, caCert, tlsClientCert, tlsClientKey, ...restOpts } = + task.opts; + httpsAgent = getHttpsAgent({ tlsSkipVerify, caCert, tlsClientCert, tlsClientKey }); const backend = loadBackend(task.opts.backend, { - ...task.opts, + ...restOpts, server: (Array.isArray(task.opts.server) ? task.opts.server.join(',') : task.opts.server) as string, httpAgent, - httpsAgent: (task.opts as any).tlsSkipVerify ? httpsInsecureAgent : httpsAgent, + httpsAgent, }); backend.on('TASK_START', ({ name }) => @@ -158,6 +146,8 @@ export const syncHandler: RequestHandler< res.status(500).json({ message: toString(err), }); + } finally { + if (httpsAgent) releaseHttpsAgent(httpsAgent); } }; diff --git a/apps/cli/src/server/validate.ts b/apps/cli/src/server/validate.ts index e2a70940..56595a3a 100644 --- a/apps/cli/src/server/validate.ts +++ b/apps/cli/src/server/validate.ts @@ -1,40 +1,25 @@ import { Differ } from '@api7/adc-differ'; import * as ADCSDK from '@api7/adc-sdk'; -import { HttpAgent, HttpOptions, HttpsAgent } from 'agentkeepalive'; +import type { HttpsAgent } from 'agentkeepalive'; import type { RequestHandler } from 'express'; import { toString } from 'lodash-es'; import { lastValueFrom } from 'rxjs'; import { fillLabels, filterResourceType, loadBackend } from '../command/utils'; import { check } from '../linter'; +import { getHttpsAgent, httpAgent, releaseHttpsAgent } from './agent-pool'; import { logger } from './logger'; import { ValidateInput, type ValidateInputType } from './schema'; -// create connection pool -const keepAlive: HttpOptions = { - keepAlive: true, - maxSockets: 256, // per host - maxFreeSockets: 16, // per host free - freeSocketTimeout: - parseInt(process.env.ADC_INGRESS_FREE_SOCKET_TIMEOUT ?? '') || 50000, -}; -const httpAgent = new HttpAgent(keepAlive); - -//TODO: dynamic rejectUnauthorized and support mTLS -const httpsAgent = new HttpsAgent({ - rejectUnauthorized: true, - ...keepAlive, -}); -const httpsInsecureAgent = new HttpsAgent({ - rejectUnauthorized: false, - ...keepAlive, -}); - export const validateHandler: RequestHandler< unknown, unknown, ValidateInputType > = async (req, res) => { + // checked out from the TLS agent pool once the backend is initialized below, + // and released in `finally` so an agent evicted mid-request isn't destroyed + // while this request is still using it + let httpsAgent: HttpsAgent | undefined; try { const parsedInput = ValidateInput.safeParse(req.body); if (!parsedInput.success) @@ -67,13 +52,16 @@ export const validateHandler: RequestHandler< fillLabels(local, task.opts.labelSelector); // initialize backend + const { tlsSkipVerify, caCert, tlsClientCert, tlsClientKey, ...restOpts } = + task.opts; + httpsAgent = getHttpsAgent({ tlsSkipVerify, caCert, tlsClientCert, tlsClientKey }); const backend = loadBackend(task.opts.backend, { - ...task.opts, + ...restOpts, server: (Array.isArray(task.opts.server) ? task.opts.server.join(',') : task.opts.server) as string, httpAgent, - httpsAgent: (task.opts as any).tlsSkipVerify ? httpsInsecureAgent : httpsAgent, + httpsAgent, }); backend.on('AXIOS_DEBUG', ({ description, response }) => @@ -149,5 +137,7 @@ export const validateHandler: RequestHandler< message: toString(err), errors: [], }); + } finally { + if (httpsAgent) releaseHttpsAgent(httpsAgent); } }; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 9ea93900..80c6bb85 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -196,6 +196,9 @@ importers: lodash-es: specifier: '>=4.17.24' version: 4.18.1 + lru-cache: + specifier: 'catalog:' + version: 11.5.1 parse-duration: specifier: ^2.1.5 version: 2.1.6