diff --git a/packages/dashmate/src/commands/ssl/obtain.js b/packages/dashmate/src/commands/ssl/obtain.js index 8bd6f7e389c..d23a1b29b3b 100644 --- a/packages/dashmate/src/commands/ssl/obtain.js +++ b/packages/dashmate/src/commands/ssl/obtain.js @@ -1,5 +1,6 @@ import { Listr } from 'listr2'; import { Flags } from '@oclif/core'; +import ServiceIsNotRunningError from '../../docker/errors/ServiceIsNotRunningError.js'; import ConfigBaseCommand from '../../oclif/command/ConfigBaseCommand.js'; import MuteOneLineError from '../../oclif/errors/MuteOneLineError.js'; import Certificate from '../../ssl/zerossl/Certificate.js'; @@ -40,6 +41,7 @@ Certificate will be renewed if it is about to expire (see 'expiration-days' flag * @param {obtainLetsEncryptCertificateTask} obtainLetsEncryptCertificateTask * @param {ConfigFileJsonRepository} configFileRepository * @param {ConfigFile} configFile + * @param {DockerCompose} dockerCompose * @return {Promise} */ async runWithDependencies( @@ -100,15 +102,25 @@ Certificate will be renewed if it is about to expire (see 'expiration-days' flag // and nothing on disk reveals which certificate Envoy currently // holds, so an obtain that skipped the write is also how an operator // retries a reload that failed earlier. + // + // The gateway is signalled without asking first whether it is running. + // execCommand makes that check itself, and asking separately leaves a + // gap in which the answer can change - the certificate has already + // been obtained by then, so failing there would report the whole + // command as failed and send the operator back to a provider that may + // have nothing left to issue. title: 'Reload gateway', - skip: async () => { - if (!await dockerCompose.isServiceRunning(config, 'gateway')) { - return 'Gateway is not running'; - } + task: async (ctx, listrTask) => { + try { + await dockerCompose.execCommand(config, 'gateway', 'kill -SIGHUP 1'); + } catch (e) { + if (!(e instanceof ServiceIsNotRunningError)) { + throw e; + } - return false; + listrTask.skip('Gateway is not running'); + } }, - task: () => dockerCompose.execCommand(config, 'gateway', 'kill -SIGHUP 1'), }, ], { diff --git a/packages/dashmate/src/createDIContainer.js b/packages/dashmate/src/createDIContainer.js index 9b8b1c74e96..97a4ff67671 100644 --- a/packages/dashmate/src/createDIContainer.js +++ b/packages/dashmate/src/createDIContainer.js @@ -18,6 +18,7 @@ import createConfigFileFactory from './config/configFile/createConfigFileFactory import migrateConfigFileFactory from './config/configFile/migrateConfigFileFactory.js'; import DefaultConfigs from './config/DefaultConfigs.js'; import analyseConfigFactory from './doctor/analyse/analyseConfigFactory.js'; +import analyseGatewayCertificateFactory from './doctor/analyse/analyseGatewayCertificateFactory.js'; import analyseCoreFactory from './doctor/analyse/analyseCoreFactory.js'; import analysePlatformFactory from './doctor/analyse/analysePlatformFactory.js'; import analyseServiceContainersFactory from './doctor/analyse/analyseServiceContainersFactory.js'; @@ -365,6 +366,7 @@ export default async function createDIContainer(options = {}) { analyseSystemResources: asFunction(analyseSystemResourcesFactory).singleton(), analyseServiceContainers: asFunction(analyseServiceContainersFactory).singleton(), analyseConfig: asFunction(analyseConfigFactory).singleton(), + analyseGatewayCertificate: asFunction(analyseGatewayCertificateFactory).singleton(), analyseCore: asFunction(analyseCoreFactory).singleton(), analysePlatform: asFunction(analysePlatformFactory).singleton(), unarchiveSamples: asFunction(unarchiveSamplesFactory).singleton(), diff --git a/packages/dashmate/src/doctor/analyse/analyseConfigFactory.js b/packages/dashmate/src/doctor/analyse/analyseConfigFactory.js index 80007be0129..67e897bd736 100644 --- a/packages/dashmate/src/doctor/analyse/analyseConfigFactory.js +++ b/packages/dashmate/src/doctor/analyse/analyseConfigFactory.js @@ -5,6 +5,16 @@ import { ERRORS as ZEROSSL_ERRORS } from '../../ssl/zerossl/validateZeroSslCerti import { SEVERITY } from '../Prescription.js'; import Problem from '../Problem.js'; +/** + * Whether a ZeroSSL certificate can be renewed depends on the operator's plan, which dashmate + * cannot see, so both routes are offered rather than assuming which one applies. + */ +const LETSENCRYPT_ALTERNATIVE = chalk`Or switch to Let's Encrypt, which issues certificates for IP addresses free +of charge: + {bold.cyanBright dashmate config set platform.gateway.ssl.provider letsencrypt} + {bold.cyanBright dashmate config set platform.gateway.ssl.providerConfigs.letsencrypt.email EMAIL} + {bold.cyanBright dashmate ssl obtain}`; + export default function analyseConfigFactory() { /** * @typedef analyseConfig @@ -60,10 +70,7 @@ export default function analyseConfigFactory() { } break; default: { - const { - description, - solution, - } = { + const fileProblems = { // File provider error 'not-valid': { description: 'SSL certificate files are not valid', @@ -82,15 +89,17 @@ Private key file path: {bold.cyanBright ${ssl?.data?.privateFilePath}} Or use ZeroSSL https://docs.dash.org/en/stable/masternodes/dashmate.html#ssl-certificate`, }, - // ZeroSSL validation errors + }; + + const zeroSslProblems = { [ZEROSSL_ERRORS.API_KEY_IS_NOT_SET]: { description: 'ZeroSSL API key is not set.', solution: chalk`Please obtain your API key from {underline.cyanBright https://app.zerossl.com/developer} -And then update your configuration with {block.cyanBright dashmate config set platform.gateway.ssl.providerConfigs.zerossl.apiKey [KEY]}`, +And then update your configuration with {bold.cyanBright dashmate config set platform.gateway.ssl.providerConfigs.zerossl.apiKey [KEY]}`, }, [ZEROSSL_ERRORS.EXTERNAL_IP_IS_NOT_SET]: { description: 'External IP is not set.', - solution: chalk`Please update your configuration to include your external IP using {block.cyanBright dashmate config set externalIp [IP]}`, + solution: chalk`Please update your configuration to include your external IP using {bold.cyanBright dashmate config set externalIp [IP]}`, }, [ZEROSSL_ERRORS.CERTIFICATE_ID_IS_NOT_SET]: { description: 'ZeroSSL certificate is not configured', @@ -102,7 +111,7 @@ And then update your configuration with {block.cyanBright dashmate config set pl and revoke the previous certificate in the ZeroSSL dashboard`, }, [ZEROSSL_ERRORS.EXTERNAL_IP_MISMATCH]: { - description: chalk`ZeroSSL IP ${ssl?.data?.certificate.common_name} does not match external IP ${ssl?.data?.externalIp}.`, + description: chalk`ZeroSSL IP ${ssl?.data?.certificate?.common_name} does not match external IP ${ssl?.data?.externalIp}.`, solution: chalk`Please regenerate the certificate using {bold.cyanBright dashmate ssl obtain --force} and revoke the previous certificate in the ZeroSSL dashboard`, }, @@ -113,8 +122,11 @@ This makes auto-renewal impossible.`, and revoke the previous certificate in the ZeroSSL dashboard`, }, [ZEROSSL_ERRORS.CERTIFICATE_EXPIRES_SOON]: { - description: chalk`ZeroSSL certificate expires at ${ssl?.data?.certificate.expires}.`, - solution: chalk`Please run {bold.cyanBright dashmate ssl obtain} to get a new one`, + description: chalk`ZeroSSL certificate expires at ${ssl?.data?.certificate?.expires}.`, + solution: chalk`Please run {bold.cyanBright dashmate ssl obtain} to get a new one, which needs an +available certificate on your ZeroSSL plan. + +${LETSENCRYPT_ALTERNATIVE}`, }, [ZEROSSL_ERRORS.CERTIFICATE_IS_NOT_VALIDATED]: { description: chalk`ZeroSSL certificate is not approved.`, @@ -122,13 +134,26 @@ and revoke the previous certificate in the ZeroSSL dashboard`, }, [ZEROSSL_ERRORS.CERTIFICATE_IS_NOT_VALID]: { description: chalk`ZeroSSL certificate is not valid.`, - solution: chalk`Please run {bold.cyanBright dashmate ssl zerossl obtain} to get a new one.`, + solution: chalk`Please run {bold.cyanBright dashmate ssl obtain} to get a new one. + +${LETSENCRYPT_ALTERNATIVE}`, }, [ZEROSSL_ERRORS.ZERO_SSL_API_ERROR]: { - description: ssl?.data?.error?.message, - solution: chalk`Please contact ZeroSSL support if needed.`, + // ZeroSSL's own wording is the most accurate account of what went wrong - it + // names an exhausted certificate limit, an unpaid invoice or a rejected key + // directly. The fallback keeps the problem reported when it sends none, since + // an empty description would otherwise drop it silently. + description: ssl?.data?.error?.message + ? chalk`ZeroSSL rejected the request: ${ssl.data.error.message}` + : chalk`The ZeroSSL API could not be reached, so the certificate cannot be checked or renewed.`, + solution: chalk`If this is something you can resolve with ZeroSSL, such as an expired plan or a +rejected API key, fix it there and run {bold.cyanBright dashmate ssl obtain}. + +${LETSENCRYPT_ALTERNATIVE}`, }, - // Let's Encrypt validation errors + }; + + const letsEncryptProblems = { [LETSENCRYPT_ERRORS.EMAIL_IS_NOT_SET]: { description: 'Let\'s Encrypt email is not set.', solution: chalk`Please update your configuration with {bold.cyanBright dashmate config set platform.gateway.ssl.providerConfigs.letsencrypt.email [EMAIL]}`, @@ -153,10 +178,31 @@ and revoke the previous certificate in the ZeroSSL dashboard`, description: chalk`Let's Encrypt certificate expires at ${ssl?.data?.certificate?.expires}.`, solution: chalk`Please run {bold.cyanBright dashmate ssl obtain --provider=letsencrypt} to renew`, }, + [LETSENCRYPT_ERRORS.CERTIFICATE_NOT_INSTALLED]: { + description: chalk`A renewed Let's Encrypt certificate has not been installed for the gateway.`, + solution: chalk`The gateway keeps serving the previous certificate until it is reloaded, +and will stop accepting clients when that one expires. +Please restart Platform: {bold.cyanBright dashmate restart --platform}`, + }, [LETSENCRYPT_ERRORS.CERTIFICATE_NOT_VALID]: { description: chalk`Let's Encrypt certificate is not valid.`, solution: chalk`Please run {bold.cyanBright dashmate ssl obtain --provider=letsencrypt --force} to get a new one.`, }, + }; + + // Both providers report some errors under the same name, so only the + // configured provider's messages are considered. Otherwise one provider's + // message would describe a problem found by the other one. + const providerProblems = config.get('platform.gateway.ssl.provider') === 'letsencrypt' + ? letsEncryptProblems + : zeroSslProblems; + + const { + description, + solution, + } = { + ...fileProblems, + ...providerProblems, }[ssl.error] ?? {}; if (description) { diff --git a/packages/dashmate/src/doctor/analyse/analyseGatewayCertificateFactory.js b/packages/dashmate/src/doctor/analyse/analyseGatewayCertificateFactory.js new file mode 100644 index 00000000000..23d77a4e270 --- /dev/null +++ b/packages/dashmate/src/doctor/analyse/analyseGatewayCertificateFactory.js @@ -0,0 +1,143 @@ +import chalk from 'chalk'; +import { SEVERITY } from '../Prescription.js'; +import Problem from '../Problem.js'; + +/** + * The manual obtain command writes certificate files but does not signal the gateway, so an + * operator following the advice can succeed and see no change on the wire. Every message about + * a certificate the gateway has not picked up has to say this. + */ +const RESTART_HINT = chalk`Then restart Platform so the gateway picks it up: {bold.cyanBright dashmate restart --platform}`; + +export default function analyseGatewayCertificateFactory() { + /** + * Analyse the certificate the gateway actually serves. + * + * @typedef analyseGatewayCertificate + * @param {Samples} samples + * @return {Problem[]} + */ + function analyseGatewayCertificate(samples) { + const config = samples.getDashmateConfig(); + + if (!config?.get('platform.enable')) { + return []; + } + + const served = samples.getServiceInfo('gateway', 'servedCertificate'); + + if (!served) { + return []; + } + + const problems = []; + + // Certificate validity is judged against the moment the samples were taken, not the moment + // they are analysed. A report is often opened days after it was collected, and the node's + // certificate may be renewed every few days, so judging at analysis time would report every + // healthy node as expired. + const now = samples.date?.getTime() ?? Date.now(); + + if (served.state === 'unreachable') { + problems.push(new Problem( + `The gateway did not answer a TLS connection (${served.reason}). Clients may not be able to connect`, + chalk`Please check that the gateway is running and listening: {bold.cyanBright dashmate status platform}`, + SEVERITY.MEDIUM, + )); + + return problems; + } + + if (served.state !== 'served') { + return problems; + } + + const externalIp = config.get('externalIp'); + + // An identity mismatch is evaluated first and stops the comparisons below. It means the + // connection did not reach this node's gateway at all - another config or a proxy answering + // on the same port - and in that case the certificate it returned says nothing about this + // node, so reporting it as a wrong or stale certificate would be misleading. + if (served.identityVerified === false) { + problems.push(new Problem( + `The certificate served on port ${served.port} is not valid for ${externalIp}: ${served.identityError}`, + chalk`Either the certificate is issued for the wrong address, or something other than this +node's gateway is answering on that port. Check that no other node or proxy is using it, then +regenerate the certificate if needed: {bold.cyanBright dashmate ssl obtain --force} +${RESTART_HINT}`, + SEVERITY.HIGH, + )); + + return problems; + } + + const servedExpiresAt = new Date(served.certificate.validTo).getTime(); + const isServedExpired = servedExpiresAt <= now; + const onDiskDiffers = served.matchesOnDisk === false; + + if (isServedExpired && onDiskDiffers) { + problems.push(new Problem( + `The gateway is serving a certificate that expired on ${served.certificate.validTo}, ` + + 'while a newer one is already present on disk', + chalk`The certificate was renewed but never reached the gateway. +{bold.cyanBright dashmate restart --platform}`, + SEVERITY.HIGH, + )); + } else if (isServedExpired) { + problems.push(new Problem( + `The gateway is serving a certificate that expired on ${served.certificate.validTo}. ` + + 'Clients cannot connect to this node', + chalk`Renewal has not succeeded. Check the renewal logs: +{bold.cyanBright dashmate logs dashmate_helper} +Then obtain a new certificate: {bold.cyanBright dashmate ssl obtain} +${RESTART_HINT}`, + SEVERITY.HIGH, + )); + } else if (onDiskDiffers) { + // Still serving a valid certificate, but the renewed one has not been picked up, so this + // node goes dark when the served certificate expires. + problems.push(new Problem( + 'The gateway is serving an older certificate than the one on disk. ' + + `It will stop accepting clients on ${served.certificate.validTo}`, + chalk`The certificate was renewed but never reached the gateway. +{bold.cyanBright dashmate restart --platform}`, + SEVERITY.HIGH, + )); + } + + // Reported separately from expiry because the connection surfaces only its first + // verification failure: a certificate that is both expired and untrusted reports only the + // expiry, and the second fault would otherwise stay hidden until the first was fixed. + if (!served.chainVerified && !isServedExpired) { + problems.push(new Problem( + `The certificate served by the gateway is not trusted by standard clients (${served.chainError})`, + chalk`Clients verifying against public certificate authorities will reject this node. +If the certificate chain is incomplete, make sure the bundle contains the issuing +certificates as well as the server certificate. +${RESTART_HINT}`, + SEVERITY.HIGH, + )); + } + + // Both obtainable providers reach this node on port 80 to validate it. Being closed is + // only reported alongside a certificate problem: the port is bound just for the seconds a + // validation takes, so an external check finds it closed on healthy nodes too and on its + // own would be noise. + const validationHttpPort = samples.getServiceInfo('gateway', 'validationHttpPort'); + + if (problems.length > 0 && validationHttpPort && validationHttpPort !== 'OPEN') { + problems.push(new Problem( + 'Inbound port 80 is not reachable, which is how certificates are validated. ' + + 'This may be why renewal is failing', + chalk`Please make sure port 80 on ${externalIp} accepts incoming connections from the +internet. Both certificate providers connect back to it to validate this node's +address before issuing a certificate. If you are behind NAT, forward port 80 as well.`, + SEVERITY.MEDIUM, + )); + } + + return problems; + } + + return analyseGatewayCertificate; +} diff --git a/packages/dashmate/src/doctor/analyseSamplesFactory.js b/packages/dashmate/src/doctor/analyseSamplesFactory.js index f3aae62e8b7..3ce87feaed4 100644 --- a/packages/dashmate/src/doctor/analyseSamplesFactory.js +++ b/packages/dashmate/src/doctor/analyseSamplesFactory.js @@ -5,6 +5,7 @@ import Problem from './Problem.js'; * @param {analyseSystemResources} analyseSystemResources * @param {analyseServiceContainers} analyseServiceContainers * @param {analyseConfig} analyseConfig + * @param {analyseGatewayCertificate} analyseGatewayCertificate * @param {analyseCore} analyseCore * @param {analysePlatform} analysePlatform * @return {analyseSamples} @@ -13,6 +14,7 @@ export default function analyseSamplesFactory( analyseSystemResources, analyseServiceContainers, analyseConfig, + analyseGatewayCertificate, analyseCore, analysePlatform, ) { @@ -41,6 +43,8 @@ export default function analyseSamplesFactory( problems.push(...analyseConfig(samples)); + problems.push(...analyseGatewayCertificate(samples)); + problems.push(...analyseCore(samples)); problems.push(...analysePlatform(samples)); diff --git a/packages/dashmate/src/listr/tasks/doctor/collectSamplesTaskFactory.js b/packages/dashmate/src/listr/tasks/doctor/collectSamplesTaskFactory.js index 46d8f4fd7ad..632a57371a0 100644 --- a/packages/dashmate/src/listr/tasks/doctor/collectSamplesTaskFactory.js +++ b/packages/dashmate/src/listr/tasks/doctor/collectSamplesTaskFactory.js @@ -7,6 +7,8 @@ import obfuscateConfig from '../../../config/obfuscateConfig.js'; import { DASHMATE_VERSION } from '../../../constants.js'; import LegoCertificate from '../../../ssl/letsencrypt/LegoCertificate.js'; import Certificate from '../../../ssl/zerossl/Certificate.js'; +import probeServedCertificate, { STATE as PROBE_STATE } from '../../../ssl/probeServedCertificate.js'; +import readCertificateBundle from '../../../ssl/readCertificateBundle.js'; import providers from '../../../status/providers.js'; import hideString from '../../../util/hideString.js'; import obfuscateObjectRecursive from '../../../util/obfuscateObjectRecursive.js'; @@ -105,7 +107,10 @@ export default function collectSamplesTaskFactory( const { error, data, - } = validateZeroSslCertificate(config, Certificate.EXPIRATION_LIMIT_DAYS); + } = await validateZeroSslCertificate( + config, + Certificate.EXPIRATION_LIMIT_DAYS, + ); obfuscateObjectRecursive(data, (_field, value) => (typeof value === 'string' ? value.replaceAll( process.env.USER, @@ -187,6 +192,65 @@ export default function collectSamplesTaskFactory( } }, }, + { + // Every other certificate check reads a file or the provider's API, so a + // certificate that was renewed on disk but never reached the gateway looks + // healthy to all of them. This connects to the gateway and records what it + // actually serves. Doctor is run by an operator on the node, so the gateway's + // listener is reached at the address it is published on. + enabled: () => config.get('platform.enable') + && config.get('platform.gateway.ssl.provider') !== 'self-signed', + title: 'Gateway served certificate', + task: async () => { + const listenerHost = config.get('platform.gateway.listeners.dapiAndDrive.host'); + const port = config.get('platform.gateway.listeners.dapiAndDrive.port'); + + const result = await probeServedCertificate({ + host: listenerHost === '0.0.0.0' ? '127.0.0.1' : listenerHost, + port, + externalIp: config.get('externalIp'), + }); + + result.port = port; + + if (result.state === PROBE_STATE.SERVED) { + // Read beside the probe rather than at analysis time: renewal replaces the + // file and signals the gateway moments apart, and the rest of the sample + // collection takes long enough that the two would routinely be read from + // either side of a renewal and reported as a mismatch. + const onDisk = readCertificateBundle(path.join( + homeDir.joinPath(config.getName(), 'platform', 'gateway', 'ssl'), + 'bundle.crt', + )); + + result.onDisk = onDisk && { + fingerprint256: onDisk.fingerprint256, + validTo: onDisk.validTo.toUTCString(), + }; + + result.matchesOnDisk = onDisk + ? onDisk.fingerprint256 === result.certificate.fingerprint256 + : null; + } + + ctx.samples.setServiceInfo('gateway', 'servedCertificate', result); + }, + }, + { + // Both obtainable providers reach this node on port 80 to prove it controls + // its address before issuing: Let's Encrypt over ACME, ZeroSSL over its own + // verification server. A self-signed or operator-supplied certificate is + // never validated, so the port means nothing for those. + enabled: () => config.get('platform.enable') + && ['zerossl', 'letsencrypt'].includes(config.get('platform.gateway.ssl.provider')), + title: 'Certificate validation port', + task: async () => { + const response = await providers.mnowatch.checkPortStatus(80, config.get('externalIp')) + .catch((e) => e.toString()); + + ctx.samples.setServiceInfo('gateway', 'validationHttpPort', response); + }, + }, { title: 'Core P2P port', task: async () => { @@ -311,7 +375,7 @@ export default function collectSamplesTaskFactory( const url = `http://${config.get('platform.drive.tenderdash.rpc.host')}:${config.get('platform.drive.tenderdash.rpc.port')}/metrics`; - const result = fetchTextOrError(url); + const result = await fetchTextOrError(url); ctx.samples.setServiceInfo('drive_tenderdash', 'metrics', result); } @@ -322,7 +386,7 @@ export default function collectSamplesTaskFactory( const url = `http://${config.get('platform.drive.abci.metrics.host')}:${config.get('platform.drive.abci.metrics.port')}/metrics`; - const result = fetchTextOrError(url); + const result = await fetchTextOrError(url); ctx.samples.setServiceInfo('drive_abci', 'metrics', result); } @@ -333,7 +397,7 @@ export default function collectSamplesTaskFactory( const url = `http://${config.get('platform.gateway.metrics.host')}:${config.get('platform.gateway.metrics.port')}/metrics`; - const result = fetchTextOrError(url); + const result = await fetchTextOrError(url); ctx.samples.setServiceInfo('gateway', 'metrics', result); } diff --git a/packages/dashmate/src/ssl/letsencrypt/validateLetsEncryptCertificateFactory.js b/packages/dashmate/src/ssl/letsencrypt/validateLetsEncryptCertificateFactory.js index 7a9effe67fb..86970dec505 100644 --- a/packages/dashmate/src/ssl/letsencrypt/validateLetsEncryptCertificateFactory.js +++ b/packages/dashmate/src/ssl/letsencrypt/validateLetsEncryptCertificateFactory.js @@ -12,6 +12,7 @@ export const ERRORS = { CERTIFICATE_EXPIRES_SOON: 'CERTIFICATE_EXPIRES_SOON', CERTIFICATE_IP_MISMATCH: 'CERTIFICATE_IP_MISMATCH', CERTIFICATE_NOT_VALID: 'CERTIFICATE_NOT_VALID', + CERTIFICATE_NOT_INSTALLED: 'CERTIFICATE_NOT_INSTALLED', }; /** @@ -133,6 +134,16 @@ export default function validateLetsEncryptCertificateFactory(homeDir) { }; } + // The certificate is valid, but the gateway loads its own copy rather than the issued + // file. Until the two match the node keeps serving whatever was installed last, which + // stays invisible to every check that only looks at the issued certificate. + if (!data.isCertificatePairInstalled) { + return { + error: ERRORS.CERTIFICATE_NOT_INSTALLED, + data, + }; + } + // Certificate is valid return { data, diff --git a/packages/dashmate/src/ssl/probeServedCertificate.js b/packages/dashmate/src/ssl/probeServedCertificate.js new file mode 100644 index 00000000000..6678d681b36 --- /dev/null +++ b/packages/dashmate/src/ssl/probeServedCertificate.js @@ -0,0 +1,155 @@ +import tls from 'node:tls'; + +/** + * How long the whole probe may take, matching the timeout used for the port checks. + * + * This is an absolute budget covering the TCP connect and the TLS handshake together. The + * socket's own timeout option cannot serve as one: it is an inactivity timer that resets on + * every byte received and does not close the socket, so a peer trickling data keeps a probe + * alive indefinitely. + */ +export const PROBE_TIMEOUT_MS = 5000; + +export const STATE = { + SERVED: 'served', + UNREACHABLE: 'unreachable', + SKIPPED: 'skipped', +}; + +/** + * Flatten a TLS peer certificate into plain values. + * + * The peer certificate must never be stored as it is: a chain that verifies ends at a + * self-signed root whose issuerCertificate points back at itself, and that cycle makes both + * JSON serialisation and the sample obfuscation pass fail. Its raw and pubkey fields are also + * buffers that serialise into thousands of numbers. + * + * @param {Object} peerCertificate + * @return {Object} + */ +function flattenCertificate(peerCertificate) { + return { + fingerprint256: peerCertificate.fingerprint256, + validFrom: peerCertificate.valid_from, + validTo: peerCertificate.valid_to, + subject: peerCertificate.subject?.CN ?? null, + issuer: peerCertificate.issuer?.CN ?? null, + subjectAltName: peerCertificate.subjectaltname ?? null, + serialNumber: peerCertificate.serialNumber ?? null, + }; +} + +/** + * Connect to the gateway and report the certificate it actually serves. + * + * Every other certificate check reads a file or asks the provider's API. A certificate that was + * renewed on disk but never reached the gateway is indistinguishable from a healthy one to all + * of them, so this opens a real connection and looks at what the gateway presents. + * + * @param {Object} options + * @param {string} options.host - address the gateway listener is reachable on + * @param {number} options.port + * @param {string} options.externalIp - the address clients use, which the certificate must name + * @param {number} [options.timeout] + * @return {Promise} never rejects + */ +export default async function probeServedCertificate({ + host, + port, + externalIp, + timeout = PROBE_TIMEOUT_MS, +}) { + return new Promise((resolve) => { + let settled = false; + let deadline; + + const settle = (result) => { + if (settled) { + return; + } + + settled = true; + + clearTimeout(deadline); + + resolve(result); + }; + + let socket; + + const fail = (reason) => { + socket?.destroy(); + + settle({ + state: STATE.UNREACHABLE, + reason, + }); + }; + + deadline = setTimeout(() => fail('ETIMEDOUT'), timeout); + + try { + socket = tls.connect({ + host, + port, + // A certificate identifying a node by IP address cannot be requested by name: SNI must + // not carry an IP literal, and the gateway selects its filter chain without it. + servername: undefined, + // The handshake has to complete even when the certificate is expired or untrusted, + // otherwise the probe learns nothing in the cases it exists for. Verification still + // runs and its verdict is read from the socket below. Nothing here grants trust: the + // result is reported, never used to authorise a connection. + rejectUnauthorized: false, + // Node would otherwise check the certificate against the address being dialled, which + // is the local address the gateway happens to be reachable on rather than the one + // clients use, and a correct certificate would fail that on every healthy node. The + // check is done separately below, against the address that matters. + checkServerIdentity: () => undefined, + }); + } catch (e) { + fail(e.code ?? 'CONNECT_FAILED'); + + return; + } + + // Stays attached for the socket's lifetime. A connection can fail after the certificate has + // already been read, and settle() ignores anything that arrives once a result is decided. + socket.on('error', (e) => fail(e.code ?? 'CONNECT_FAILED')); + + socket.on('timeout', () => fail('ETIMEDOUT')); + + socket.on('secureConnect', () => { + const peerCertificate = socket.getPeerCertificate(true); + + // An absent peer certificate is reported as an empty object, which would otherwise be + // taken for a served certificate with no fields and compared against the one on disk. + if (!peerCertificate?.fingerprint256) { + fail('NO_PEER_CERTIFICATE'); + + return; + } + + const { authorized, authorizationError } = socket; + + // Identity is checked separately from the chain because the socket reports only one + // error: an expired certificate that also names the wrong address reports just the + // expiry, so a single verdict would hide the second fault until the first was fixed. + // Delegating to Node handles the common-name fallback and address normalisation that a + // hand-rolled comparison against the alternative names gets wrong. + const identityError = externalIp + ? tls.checkServerIdentity(externalIp, peerCertificate) + : undefined; + + socket.destroy(); + + settle({ + state: STATE.SERVED, + certificate: flattenCertificate(peerCertificate), + chainVerified: authorized, + chainError: authorized ? null : (authorizationError?.code ?? String(authorizationError)), + identityVerified: externalIp ? identityError === undefined : null, + identityError: identityError ? identityError.message : null, + }); + }); + }); +} diff --git a/packages/dashmate/src/ssl/readCertificateBundle.js b/packages/dashmate/src/ssl/readCertificateBundle.js new file mode 100644 index 00000000000..649cf7e9a94 --- /dev/null +++ b/packages/dashmate/src/ssl/readCertificateBundle.js @@ -0,0 +1,76 @@ +import crypto from 'node:crypto'; +import fs from 'node:fs'; + +/** + * Extract IP addresses from the OpenSSL rendering of a subject alternative name extension. + * + * The value is a single string such as "IP Address:1.2.3.4, DNS:example.com", so entries are + * split out rather than substring-matched: searching the raw string for "1.2.3.4" would also + * match inside "11.2.3.44" and inside a DNS entry. + * + * @param {string|undefined} subjectAltName + * @return {string[]} + */ +function parseIpAddresses(subjectAltName) { + if (!subjectAltName) { + return []; + } + + return subjectAltName + .split(',') + .map((entry) => entry.trim()) + .filter((entry) => entry.startsWith('IP Address:')) + .map((entry) => entry.slice('IP Address:'.length)); +} + +/** + * Read the server certificate from a PEM bundle. + * + * The server certificate is expected first, but an operator supplying their own bundle can + * order it the other way round, so the first block is only accepted when it is not a CA. + * Comparing a served certificate against an intermediate would report a permanent mismatch. + * + * The fingerprint is the same uppercase colon-separated SHA-256 that a TLS peer certificate + * reports, so the two can be compared directly. + * + * @param {string} filePath + * @return {{fingerprint256: string, validFrom: Date, validTo: Date, subject: string, + * issuer: string, ipAddresses: string[]}|null} null when the file is missing or unparseable + */ +export default function readCertificateBundle(filePath) { + let pem; + + try { + pem = fs.readFileSync(filePath, 'utf8'); + } catch { + return null; + } + + const blocks = pem.match(/-----BEGIN CERTIFICATE-----[\s\S]*?-----END CERTIFICATE-----/g) ?? []; + + for (const block of blocks) { + let certificate; + + try { + certificate = new crypto.X509Certificate(block); + } catch { + // Skip a block we cannot parse rather than failing the whole bundle + continue; + } + + if (certificate.ca) { + continue; + } + + return { + fingerprint256: certificate.fingerprint256, + validFrom: new Date(certificate.validFrom), + validTo: new Date(certificate.validTo), + subject: certificate.subject, + issuer: certificate.issuer, + ipAddresses: parseIpAddresses(certificate.subjectAltName), + }; + } + + return null; +} diff --git a/packages/dashmate/src/test/createCertificateForTest.js b/packages/dashmate/src/test/createCertificateForTest.js new file mode 100644 index 00000000000..ec78288605a --- /dev/null +++ b/packages/dashmate/src/test/createCertificateForTest.js @@ -0,0 +1,45 @@ +import forge from 'node-forge'; + +/** + * Create a self-signed certificate with a chosen validity window and IP address. + * + * Certificates are generated when a test runs rather than committed, so a fixture cannot + * expire and fail the suite on a date nobody chose. Built with node-forge rather than the + * openssl binary because the flags needed to place a certificate in the past arrived in + * OpenSSL 3.5, which is newer than the version on the CI image. + * + * @param {Object} [options] + * @param {string} [options.ip] - placed in the subject alternative name and common name + * @param {number} [options.days] - days from now the certificate expires, negative for expired + * @return {{cert: string, key: string}} PEM encoded + */ +export default function createCertificateForTest({ ip = '127.0.0.1', days = 30 } = {}) { + const keys = forge.pki.rsa.generateKeyPair(2048); + const certificate = forge.pki.createCertificate(); + + certificate.publicKey = keys.publicKey; + certificate.serialNumber = '01'; + + // Anchored to the expiry so an already-expired certificate still starts before it ends + certificate.validity.notAfter = new Date(Date.now() + days * 24 * 60 * 60 * 1000); + certificate.validity.notBefore = new Date( + certificate.validity.notAfter.getTime() - 30 * 24 * 60 * 60 * 1000, + ); + + const attributes = [{ name: 'commonName', value: ip }]; + + certificate.setSubject(attributes); + certificate.setIssuer(attributes); + certificate.setExtensions([ + { name: 'basicConstraints', cA: false }, + // Type 7 is an IP address. An evonode is identified by its address, not by a name. + { name: 'subjectAltName', altNames: [{ type: 7, ip }] }, + ]); + + certificate.sign(keys.privateKey, forge.md.sha256.create()); + + return { + cert: forge.pki.certificateToPem(certificate), + key: forge.pki.privateKeyToPem(keys.privateKey), + }; +} diff --git a/packages/dashmate/test/unit/commands/ssl/obtain.spec.js b/packages/dashmate/test/unit/commands/ssl/obtain.spec.js index 5e80a1967f4..c142f820496 100644 --- a/packages/dashmate/test/unit/commands/ssl/obtain.spec.js +++ b/packages/dashmate/test/unit/commands/ssl/obtain.spec.js @@ -1,5 +1,6 @@ import { Listr } from 'listr2'; import ObtainCommand from '../../../../src/commands/ssl/obtain.js'; +import ServiceIsNotRunningError from '../../../../src/docker/errors/ServiceIsNotRunningError.js'; describe('SSL obtain command', () => { /** @@ -92,13 +93,24 @@ describe('SSL obtain command', () => { .to.have.been.calledOnceWith(dependencies.config, 'gateway', 'kill -SIGHUP 1'); }); - it('should not reload a gateway that is not running', async function it() { + // The certificate has already been obtained by the time the gateway is signalled, so a + // gateway that is down must not turn the whole command into a failure - that would send the + // operator back to a provider that may have nothing left to issue. + it('should not fail when the gateway is not running', async function it() { const dependencies = obtainDependencies(this.sinon); - dependencies.dockerCompose.isServiceRunning.resolves(false); + dependencies.dockerCompose.execCommand + .rejects(new ServiceIsNotRunningError('testnet', 'gateway')); await runObtain(dependencies); - expect(dependencies.dockerCompose.execCommand).to.have.not.been.called(); + expect(dependencies.dockerCompose.execCommand).to.have.been.calledOnce(); + }); + + it('should still fail on a reload error that is not a stopped gateway', async function it() { + const dependencies = obtainDependencies(this.sinon); + dependencies.dockerCompose.execCommand.rejects(new Error('docker daemon is unreachable')); + + await expect(runObtain(dependencies)).to.be.rejected(); }); it('should checkpoint a newly created ZeroSSL certificate before a later failure', async function it() { diff --git a/packages/dashmate/test/unit/doctor/analyse/analyseConfigFactory.spec.js b/packages/dashmate/test/unit/doctor/analyse/analyseConfigFactory.spec.js new file mode 100644 index 00000000000..602fc78c3ae --- /dev/null +++ b/packages/dashmate/test/unit/doctor/analyse/analyseConfigFactory.spec.js @@ -0,0 +1,157 @@ +import getBaseConfigFactory from '../../../../configs/defaults/getBaseConfigFactory.js'; +import analyseConfigFactory from '../../../../src/doctor/analyse/analyseConfigFactory.js'; +import { SEVERITY } from '../../../../src/doctor/Prescription.js'; +import Samples from '../../../../src/doctor/Samples.js'; +import { ERRORS as LETSENCRYPT_ERRORS } from '../../../../src/ssl/letsencrypt/validateLetsEncryptCertificateFactory.js'; +import { ERRORS as ZEROSSL_ERRORS } from '../../../../src/ssl/zerossl/validateZeroSslCertificateFactory.js'; + +describe('analyseConfigFactory', () => { + let analyseConfig; + let config; + let samples; + + /** + * @param {Object} ssl + * @param {string} [provider=zerossl] + * @return {Problem[]} + */ + function analyseSslSample(ssl, provider = 'zerossl') { + config.set('platform.gateway.ssl.provider', provider); + + samples.setServiceInfo('gateway', 'ssl', ssl); + + return analyseConfig(samples); + } + + beforeEach(() => { + config = getBaseConfigFactory()(); + + config.set('platform.enable', true); + + samples = new Samples(); + samples.setDashmateConfig(config); + + // Ports are reported healthy so that only certificate problems are analysed + samples.setServiceInfo('core', 'p2pPort', 'OPEN'); + samples.setServiceInfo('gateway', 'httpPort', 'OPEN'); + samples.setServiceInfo('drive_tenderdash', 'p2pPort', 'OPEN'); + + analyseConfig = analyseConfigFactory(); + }); + + it('should report a problem for a Let\'s Encrypt certificate that expires soon', () => { + const problems = analyseSslSample({ + error: LETSENCRYPT_ERRORS.CERTIFICATE_EXPIRES_SOON, + data: { certificate: { expires: '2026-01-01' } }, + }, 'letsencrypt'); + + expect(problems).to.have.lengthOf(1); + expect(problems[0].getDescription()).to.include('Let\'s Encrypt certificate expires at'); + expect(problems[0].getSeverity()).to.equal(SEVERITY.HIGH); + }); + + it('should report a problem for a ZeroSSL certificate that expires soon', () => { + const problems = analyseSslSample({ + error: ZEROSSL_ERRORS.CERTIFICATE_EXPIRES_SOON, + data: { certificate: { expires: '2026-01-01' } }, + }); + + expect(problems).to.have.lengthOf(1); + expect(problems[0].getDescription()).to.include('ZeroSSL certificate expires at'); + expect(problems[0].getSeverity()).to.equal(SEVERITY.HIGH); + }); + + it('should report a problem when the ZeroSSL API key is not set', () => { + const problems = analyseSslSample({ + error: ZEROSSL_ERRORS.API_KEY_IS_NOT_SET, + data: {}, + }); + + expect(problems).to.have.lengthOf(1); + expect(problems[0].getDescription()).to.include('ZeroSSL API key is not set'); + expect(problems[0].getSolution()).to.include('dashmate config set platform.gateway.ssl.providerConfigs.zerossl.apiKey'); + }); + + it('should report a problem when the external IP is not set', () => { + const problems = analyseSslSample({ + error: ZEROSSL_ERRORS.EXTERNAL_IP_IS_NOT_SET, + data: {}, + }); + + expect(problems).to.have.lengthOf(1); + expect(problems[0].getDescription()).to.include('External IP is not set'); + expect(problems[0].getSolution()).to.include('dashmate config set externalIp'); + }); + + it('should report a problem when certificate files are not found', () => { + const problems = analyseSslSample({ + error: 'not-exist', + data: { + chainFilePath: '/home/dashmate/ssl/bundle.crt', + privateFilePath: '/home/dashmate/ssl/private.key', + }, + }, 'file'); + + expect(problems).to.have.lengthOf(1); + expect(problems[0].getDescription()).to.include('SSL certificate files are not found'); + }); + + describe('ZeroSSL remediation', () => { + it('should offer both renewing with ZeroSSL and switching to Let\'s Encrypt', () => { + // Whether renewing works depends on the operator's ZeroSSL plan, which dashmate cannot + // see, so both routes are offered rather than one being asserted to be the answer. + const [problem] = analyseSslSample({ + error: ZEROSSL_ERRORS.CERTIFICATE_EXPIRES_SOON, + data: { certificate: { expires: '2026-01-01' } }, + }); + + expect(problem.getSolution()).to.include('dashmate ssl obtain'); + expect(problem.getSolution()).to.include('platform.gateway.ssl.provider letsencrypt'); + }); + + it('should name what makes the alternative worth taking', () => { + // Both providers renew on their own, so that is not what separates them + const [problem] = analyseSslSample({ + error: ZEROSSL_ERRORS.CERTIFICATE_EXPIRES_SOON, + data: { certificate: { expires: '2026-01-01' } }, + }); + + expect(problem.getSolution()).to.include('free'); + }); + + it('should surface the reason ZeroSSL itself gave', () => { + const [problem] = analyseSslSample({ + error: ZEROSSL_ERRORS.ZERO_SSL_API_ERROR, + data: { error: { message: 'Limit of certificates on your ZeroSSL account was reached' } }, + }); + + expect(problem.getDescription()).to.include('Limit of certificates'); + expect(problem.getSolution()).to.include('platform.gateway.ssl.provider letsencrypt'); + }); + + it('should still report an API failure that carried no message', () => { + // The description doubles as the presence check, so an empty one dropped the problem + const problems = analyseSslSample({ + error: ZEROSSL_ERRORS.ZERO_SSL_API_ERROR, + data: {}, + }); + + expect(problems).to.have.lengthOf(1); + }); + + it('should not suggest a command that does not exist', () => { + const [problem] = analyseSslSample({ + error: ZEROSSL_ERRORS.CERTIFICATE_IS_NOT_VALID, + data: {}, + }); + + expect(problem.getSolution()).to.not.include('ssl zerossl obtain'); + }); + }); + + it('should not report a problem for a valid certificate', () => { + const problems = analyseSslSample({ data: {} }); + + expect(problems).to.be.empty(); + }); +}); diff --git a/packages/dashmate/test/unit/doctor/analyse/analyseGatewayCertificateFactory.spec.js b/packages/dashmate/test/unit/doctor/analyse/analyseGatewayCertificateFactory.spec.js new file mode 100644 index 00000000000..f7f7a022b47 --- /dev/null +++ b/packages/dashmate/test/unit/doctor/analyse/analyseGatewayCertificateFactory.spec.js @@ -0,0 +1,167 @@ +import getBaseConfigFactory from '../../../../configs/defaults/getBaseConfigFactory.js'; +import analyseGatewayCertificateFactory from '../../../../src/doctor/analyse/analyseGatewayCertificateFactory.js'; +import { SEVERITY } from '../../../../src/doctor/Prescription.js'; +import Samples from '../../../../src/doctor/Samples.js'; + +const EXTERNAL_IP = '198.51.100.7'; + +/** + * @param {number} days - relative to now, negative for an expired certificate + * @return {string} + */ +function validTo(days) { + return new Date(Date.now() + days * 24 * 60 * 60 * 1000).toUTCString(); +} + +describe('analyseGatewayCertificateFactory', () => { + let analyseGatewayCertificate; + let config; + let samples; + + /** + * @param {Object} servedCertificate + * @return {Problem[]} + */ + function analyse(servedCertificate) { + samples.setServiceInfo('gateway', 'servedCertificate', servedCertificate); + + return analyseGatewayCertificate(samples); + } + + /** + * @param {Object} overrides + * @return {Object} + */ + function served(overrides = {}) { + return { + state: 'served', + port: 443, + certificate: { fingerprint256: 'AA:BB', validTo: validTo(30) }, + chainVerified: true, + chainError: null, + identityVerified: true, + identityError: null, + matchesOnDisk: true, + ...overrides, + }; + } + + beforeEach(() => { + config = getBaseConfigFactory()(); + + config.set('platform.enable', true); + config.set('externalIp', EXTERNAL_IP); + + samples = new Samples(); + samples.setDashmateConfig(config); + + analyseGatewayCertificate = analyseGatewayCertificateFactory(); + }); + + it('should report no problem for a healthy certificate', () => { + expect(analyse(served())).to.be.empty(); + }); + + it('should report an expired certificate that clients cannot connect to', () => { + const problems = analyse(served({ + certificate: { fingerprint256: 'AA:BB', validTo: validTo(-158) }, + })); + + expect(problems).to.have.lengthOf(1); + expect(problems[0].getDescription()).to.include('expired'); + expect(problems[0].getSeverity()).to.equal(SEVERITY.HIGH); + expect(problems[0].getSolution()).to.include('dashmate_helper'); + }); + + it('should distinguish a certificate that was renewed but never reached the gateway', () => { + const problems = analyse(served({ + certificate: { fingerprint256: 'AA:BB', validTo: validTo(-2) }, + matchesOnDisk: false, + })); + + expect(problems).to.have.lengthOf(1); + expect(problems[0].getDescription()).to.include('newer one is already present on disk'); + expect(problems[0].getSolution()).to.include('dashmate restart --platform'); + }); + + it('should warn before the outage when a renewed certificate has not been picked up', () => { + const problems = analyse(served({ matchesOnDisk: false })); + + expect(problems).to.have.lengthOf(1); + expect(problems[0].getDescription()).to.include('older certificate than the one on disk'); + expect(problems[0].getSeverity()).to.equal(SEVERITY.HIGH); + }); + + it('should report an untrusted certificate separately from expiry', () => { + const problems = analyse(served({ + chainVerified: false, + chainError: 'UNABLE_TO_VERIFY_LEAF_SIGNATURE', + })); + + expect(problems).to.have.lengthOf(1); + expect(problems[0].getDescription()).to.include('not trusted by standard clients'); + }); + + it('should treat an identity mismatch as not having reached this node and stop there', () => { + // A different node or a proxy answering on the port returns a certificate that says nothing + // about this node, so reporting it as stale or expired would send the operator the wrong way. + const problems = analyse(served({ + identityVerified: false, + identityError: 'Host: 198.51.100.7 is not in the cert\'s altnames', + matchesOnDisk: false, + certificate: { fingerprint256: 'CC:DD', validTo: validTo(-10) }, + })); + + expect(problems).to.have.lengthOf(1); + expect(problems[0].getDescription()).to.include('not valid for 198.51.100.7'); + }); + + it('should judge expiry against the time the samples were taken, not the time of analysis', () => { + // Reports are commonly opened days after collection, and a Let's Encrypt certificate for an + // IP address lives about six days, so judging at analysis time reports healthy nodes as dead. + samples.date = new Date(Date.now() - 10 * 24 * 60 * 60 * 1000); + + const problems = analyse(served({ + certificate: { fingerprint256: 'AA:BB', validTo: validTo(-4) }, + })); + + expect(problems).to.be.empty(); + }); + + it('should report a closed port 80 as a likely cause when a certificate problem exists', () => { + samples.setServiceInfo('gateway', 'validationHttpPort', 'CLOSED'); + + const problems = analyse(served({ + certificate: { fingerprint256: 'AA:BB', validTo: validTo(-3) }, + })); + + expect(problems).to.have.lengthOf(2); + expect(problems[1].getDescription()).to.include('port 80'); + }); + + it('should not report a closed port 80 on a node whose certificate is healthy', () => { + // The port is only bound for the seconds a validation takes, so an external check finds it + // closed on actively renewing nodes too. Alone it would fire far more often than it is right. + samples.setServiceInfo('gateway', 'validationHttpPort', 'CLOSED'); + + expect(analyse(served())).to.be.empty(); + }); + + it('should report a gateway that does not answer TLS', () => { + const problems = analyse({ state: 'unreachable', reason: 'ECONNREFUSED' }); + + expect(problems).to.have.lengthOf(1); + expect(problems[0].getDescription()).to.include('did not answer'); + expect(problems[0].getSeverity()).to.equal(SEVERITY.MEDIUM); + }); + + it('should report nothing when the probe was skipped', () => { + expect(analyse({ state: 'skipped', reason: 'self-signed' })).to.be.empty(); + }); + + it('should report nothing when platform is disabled', () => { + config.set('platform.enable', false); + + expect(analyse(served({ certificate: { validTo: validTo(-100) } }))).to.be.empty(); + }); +}); diff --git a/packages/dashmate/test/unit/listr/tasks/doctor/collectSamplesTaskFactory.spec.js b/packages/dashmate/test/unit/listr/tasks/doctor/collectSamplesTaskFactory.spec.js new file mode 100644 index 00000000000..7de4901d627 --- /dev/null +++ b/packages/dashmate/test/unit/listr/tasks/doctor/collectSamplesTaskFactory.spec.js @@ -0,0 +1,233 @@ +import fs from 'fs'; +import path from 'path'; +import tls from 'node:tls'; +import { Listr } from 'listr2'; +import getBaseConfigFactory from '../../../../../configs/defaults/getBaseConfigFactory.js'; +import HomeDir from '../../../../../src/config/HomeDir.js'; +import createCertificateForTest from '../../../../../src/test/createCertificateForTest.js'; +import analyseConfigFactory from '../../../../../src/doctor/analyse/analyseConfigFactory.js'; +import { SEVERITY } from '../../../../../src/doctor/Prescription.js'; +import Samples from '../../../../../src/doctor/Samples.js'; +import collectSamplesTaskFactory from '../../../../../src/listr/tasks/doctor/collectSamplesTaskFactory.js'; +import Certificate from '../../../../../src/ssl/zerossl/Certificate.js'; +import validateZeroSslCertificateFactory, { ERRORS as ZEROSSL_ERRORS } from '../../../../../src/ssl/zerossl/validateZeroSslCertificateFactory.js'; +import providers from '../../../../../src/status/providers.js'; + +const EXTERNAL_IP = '198.51.100.7'; + +/** + * Format a date the way the ZeroSSL API reports certificate dates + * + * @param {Date} date + * @return {string} + */ +function toZeroSslDate(date) { + const pad = (number) => String(number).padStart(2, '0'); + + return `${date.getFullYear()}-${pad(date.getMonth() + 1)}-${pad(date.getDate())} ` + + `${pad(date.getHours())}:${pad(date.getMinutes())}:${pad(date.getSeconds())}`; +} + +/** + * @param {number} days + * @return {Date} + */ +function daysFromNow(days) { + const date = new Date(); + + date.setDate(date.getDate() + days); + + return date; +} + +describe('collectSamplesTaskFactory', () => { + let homeDir; + let config; + let getCertificate; + let collectSamplesTask; + let analyseConfig; + let samples; + + /** + * Run the sample collection the same way the doctor command does: as a subtask + * of a parent list, so the parent's renderer applies. + * + * @return {Promise} + */ + async function collectSamples() { + const tasks = new Listr( + [{ task: () => collectSamplesTask(config) }], + { renderer: 'silent' }, + ); + + await tasks.run({ samples }); + } + + beforeEach(function beforeEach() { + homeDir = HomeDir.createTemp(); + + config = getBaseConfigFactory()(); + + config.set('externalIp', EXTERNAL_IP); + config.set('platform.enable', true); + config.set('platform.gateway.ssl.enabled', true); + config.set('platform.gateway.ssl.provider', 'zerossl'); + config.set('platform.gateway.ssl.providerConfigs.zerossl.apiKey', 'a'.repeat(32)); + config.set('platform.gateway.ssl.providerConfigs.zerossl.id', 'b'.repeat(32)); + + // The ZeroSSL validator inspects the certificate files on disk + const sslDir = homeDir.joinPath(config.getName(), 'platform', 'gateway', 'ssl'); + + fs.mkdirSync(sslDir, { recursive: true }); + fs.writeFileSync(path.join(sslDir, 'csr.pem'), 'csr', 'utf8'); + fs.writeFileSync(path.join(sslDir, 'private.key'), 'private key', 'utf8'); + fs.writeFileSync(path.join(sslDir, 'bundle.crt'), 'bundle', 'utf8'); + + getCertificate = this.sinon.stub(); + + this.sinon.stub(providers.mnowatch, 'checkPortStatus').resolves('OPEN'); + + this.sinon.stub(global, 'fetch').resolves({ + json: async () => ({}), + text: async () => 'metrics_sample 1', + }); + + const dockerCompose = { + throwErrorIfNotInstalled: this.sinon.stub().resolves(), + inspectService: this.sinon.stub().resolves({}), + logs: this.sinon.stub().resolves({ out: '', err: '' }), + }; + + const rpcClient = { + getBestChainLock: this.sinon.stub().resolves({ result: {} }), + quorum: this.sinon.stub().resolves({ result: {} }), + getBlockchainInfo: this.sinon.stub().resolves({ result: {} }), + getPeerInfo: this.sinon.stub().resolves({ result: {} }), + mnsync: this.sinon.stub().resolves({ result: {} }), + masternode: this.sinon.stub().resolves({ result: {} }), + }; + + collectSamplesTask = collectSamplesTaskFactory( + dockerCompose, + this.sinon.stub().returns(rpcClient), + this.sinon.stub().resolves('127.0.0.1'), + this.sinon.stub().returns({ request: this.sinon.stub().resolves({}) }), + this.sinon.stub().resolves([]), + this.sinon.stub().resolves({}), + homeDir, + validateZeroSslCertificateFactory(homeDir, getCertificate), + this.sinon.stub().resolves({}), + ); + + analyseConfig = analyseConfigFactory(); + + samples = new Samples(); + }); + + afterEach(() => { + homeDir.remove(); + }); + + it('should report a problem for a ZeroSSL certificate that expired months ago', async () => { + const expiredAt = daysFromNow(-180); + + getCertificate.resolves(new Certificate({ + id: 'certificate-id', + common_name: EXTERNAL_IP, + status: 'issued', + created: toZeroSslDate(daysFromNow(-270)), + expires: toZeroSslDate(expiredAt), + })); + + await collectSamples(); + + expect(samples.getServiceInfo('gateway', 'ssl').error) + .to.equal(ZEROSSL_ERRORS.CERTIFICATE_EXPIRES_SOON); + + const problems = analyseConfig(samples); + + const sslProblem = problems + .find((problem) => problem.getDescription().includes('ZeroSSL certificate expires at')); + + expect(sslProblem).to.exist(); + expect(sslProblem.getSeverity()).to.equal(SEVERITY.HIGH); + }); + + it('should not report a problem for a valid ZeroSSL certificate', async () => { + getCertificate.resolves(new Certificate({ + id: 'certificate-id', + common_name: EXTERNAL_IP, + status: 'issued', + created: toZeroSslDate(daysFromNow(-1)), + expires: toZeroSslDate(daysFromNow(89)), + })); + + await collectSamples(); + + expect(samples.getServiceInfo('gateway', 'ssl').error).to.be.undefined(); + + expect(analyseConfig(samples)).to.be.empty(); + }); + + it('should collect the certificate the gateway actually serves', async () => { + const { cert, key } = createCertificateForTest({ ip: EXTERNAL_IP, days: 30 }); + + const server = tls.createServer({ cert, key }, (socket) => socket.end()); + const liveSockets = []; + + server.on('secureConnection', (socket) => liveSockets.push(socket)); + + await new Promise((resolve) => { + server.listen(0, '127.0.0.1', resolve); + }); + + // The gateway's own bundle is the certificate the server presents, so disk and wire agree + fs.writeFileSync( + path.join(homeDir.joinPath(config.getName(), 'platform', 'gateway', 'ssl'), 'bundle.crt'), + cert, + 'utf8', + ); + + config.set('platform.gateway.listeners.dapiAndDrive.port', server.address().port); + + getCertificate.resolves(new Certificate({ + id: 'certificate-id', + common_name: EXTERNAL_IP, + status: 'issued', + created: toZeroSslDate(daysFromNow(-1)), + expires: toZeroSslDate(daysFromNow(89)), + })); + + try { + await collectSamples(); + } finally { + liveSockets.forEach((socket) => socket.destroy()); + await new Promise((resolve) => { + server.close(resolve); + }); + } + + const servedCertificate = samples.getServiceInfo('gateway', 'servedCertificate'); + + expect(servedCertificate.state).to.equal('served'); + expect(servedCertificate.identityVerified).to.be.true(); + expect(servedCertificate.matchesOnDisk).to.be.true(); + expect(samples.getServiceInfo('gateway', 'validationHttpPort')).to.equal('OPEN'); + }); + + it('should collect metrics as text rather than an unresolved promise', async () => { + config.set('platform.gateway.metrics.enabled', true); + + getCertificate.resolves(new Certificate({ + id: 'certificate-id', + common_name: EXTERNAL_IP, + status: 'issued', + created: toZeroSslDate(daysFromNow(-1)), + expires: toZeroSslDate(daysFromNow(89)), + })); + + await collectSamples(); + + expect(samples.getServiceInfo('gateway', 'metrics')).to.equal('metrics_sample 1'); + }); +}); diff --git a/packages/dashmate/test/unit/ssl/letsencrypt/validateLetsEncryptCertificateFactory.spec.js b/packages/dashmate/test/unit/ssl/letsencrypt/validateLetsEncryptCertificateFactory.spec.js new file mode 100644 index 00000000000..a933d62595a --- /dev/null +++ b/packages/dashmate/test/unit/ssl/letsencrypt/validateLetsEncryptCertificateFactory.spec.js @@ -0,0 +1,85 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import HomeDir from '../../../../src/config/HomeDir.js'; +import createCertificateForTest from '../../../../src/test/createCertificateForTest.js'; +import validateLetsEncryptCertificateFactory, { ERRORS } from '../../../../src/ssl/letsencrypt/validateLetsEncryptCertificateFactory.js'; + +const EXTERNAL_IP = '198.51.100.7'; +const CONFIG_NAME = 'testnet'; + +describe('validateLetsEncryptCertificateFactory', () => { + let homeDir; + let legoDir; + let sslDir; + let config; + let validateLetsEncryptCertificate; + + const issueCertificate = () => createCertificateForTest({ ip: EXTERNAL_IP, days: 60 }); + + beforeEach(function beforeEach() { + homeDir = HomeDir.createTemp(); + + legoDir = homeDir.joinPath(CONFIG_NAME, 'platform', 'gateway', 'lego', 'certificates'); + sslDir = homeDir.joinPath(CONFIG_NAME, 'platform', 'gateway', 'ssl'); + + fs.mkdirSync(legoDir, { recursive: true }); + fs.mkdirSync(sslDir, { recursive: true }); + + config = { + get: this.sinon.stub().callsFake((option) => ({ + 'platform.gateway.ssl.providerConfigs.letsencrypt.email': 'operator@example.com', + externalIp: EXTERNAL_IP, + }[option])), + getName: this.sinon.stub().returns(CONFIG_NAME), + }; + + validateLetsEncryptCertificate = validateLetsEncryptCertificateFactory(homeDir); + }); + + afterEach(() => homeDir.remove()); + + it('should expose the not-installed error so callers can match on it', () => { + expect(ERRORS.CERTIFICATE_NOT_INSTALLED).to.equal('CERTIFICATE_NOT_INSTALLED'); + }); + + it('should report no problem when the issued certificate is the one the gateway uses', async () => { + const { cert, key } = issueCertificate(); + + fs.writeFileSync(path.join(legoDir, `${EXTERNAL_IP}.crt`), cert); + fs.writeFileSync(path.join(legoDir, `${EXTERNAL_IP}.key`), key); + fs.writeFileSync(path.join(sslDir, 'bundle.crt'), cert); + fs.writeFileSync(path.join(sslDir, 'private.key'), key); + + const result = await validateLetsEncryptCertificate(config); + + expect(result.error).to.be.undefined(); + }); + + it('should report a renewed certificate that was never copied to the gateway', async () => { + // Renewal writes a new certificate and then installs it for the gateway. When the second + // step does not happen the node keeps serving the previous certificate until it expires, + // and every check based on the renewed file alone still reports the node as healthy. + const renewed = issueCertificate(); + const previous = issueCertificate(); + + fs.writeFileSync(path.join(legoDir, `${EXTERNAL_IP}.crt`), renewed.cert); + fs.writeFileSync(path.join(legoDir, `${EXTERNAL_IP}.key`), renewed.key); + fs.writeFileSync(path.join(sslDir, 'bundle.crt'), previous.cert); + fs.writeFileSync(path.join(sslDir, 'private.key'), previous.key); + + const result = await validateLetsEncryptCertificate(config); + + expect(result.error).to.equal('CERTIFICATE_NOT_INSTALLED'); + }); + + it('should report a certificate that was issued but never installed at all', async () => { + const { cert, key } = issueCertificate(); + + fs.writeFileSync(path.join(legoDir, `${EXTERNAL_IP}.crt`), cert); + fs.writeFileSync(path.join(legoDir, `${EXTERNAL_IP}.key`), key); + + const result = await validateLetsEncryptCertificate(config); + + expect(result.error).to.equal('CERTIFICATE_NOT_INSTALLED'); + }); +}); diff --git a/packages/dashmate/test/unit/ssl/probeServedCertificate.spec.js b/packages/dashmate/test/unit/ssl/probeServedCertificate.spec.js new file mode 100644 index 00000000000..5fbba336b42 --- /dev/null +++ b/packages/dashmate/test/unit/ssl/probeServedCertificate.spec.js @@ -0,0 +1,169 @@ +import net from 'node:net'; +import tls from 'node:tls'; +import probeServedCertificate, { STATE } from '../../../src/ssl/probeServedCertificate.js'; +import createCertificateForTest from '../../../src/test/createCertificateForTest.js'; + +const EXTERNAL_IP = '127.0.0.1'; + +describe('probeServedCertificate', () => { + const servers = []; + const sockets = []; + + /** + * Track every accepted connection so a server can be closed without waiting on one that the + * test deliberately left open. + * + * @param {Server} server + * @return {Server} + */ + function track(server) { + server.on('connection', (socket) => sockets.push(socket)); + server.on('secureConnection', (socket) => sockets.push(socket)); + + servers.push(server); + + return server; + } + + /** + * @param {Object} tlsOptions + * @return {Promise} listening port + */ + async function listenTls(tlsOptions) { + const server = track(tls.createServer(tlsOptions, (socket) => socket.end())); + + await new Promise((resolve) => { + server.listen(0, '127.0.0.1', resolve); + }); + + return server.address().port; + } + + afterEach(async () => { + sockets.splice(0).forEach((socket) => socket.destroy()); + + await Promise.all(servers.splice(0).map((server) => new Promise((resolve) => { + server.close(resolve); + }))); + }); + + it('should report the certificate the server actually serves', async () => { + const { cert, key } = createCertificateForTest({ days: 30 }); + const port = await listenTls({ cert, key }); + + const result = await probeServedCertificate({ host: '127.0.0.1', port, externalIp: EXTERNAL_IP }); + + expect(result.state).to.equal(STATE.SERVED); + expect(result.certificate.fingerprint256).to.match(/^[0-9A-F]{2}(:[0-9A-F]{2})+$/); + expect(new Date(result.certificate.validTo).getTime()).to.be.greaterThan(Date.now()); + }); + + it('should complete the handshake and report an expired certificate', async () => { + const { cert, key } = createCertificateForTest({ days: -5 }); + const port = await listenTls({ cert, key }); + + const result = await probeServedCertificate({ host: '127.0.0.1', port, externalIp: EXTERNAL_IP }); + + expect(result.state).to.equal(STATE.SERVED); + expect(new Date(result.certificate.validTo).getTime()).to.be.lessThan(Date.now()); + }); + + it('should not fail identity for a certificate naming the external IP rather than the probed address', async () => { + // The gateway is reached on loopback but its certificate names the node's public address. + // Judging identity against the dialled address would fail every healthy node. + const { cert, key } = createCertificateForTest({ ip: '198.51.100.7' }); + const port = await listenTls({ cert, key }); + + const result = await probeServedCertificate({ + host: '127.0.0.1', + port, + externalIp: '198.51.100.7', + }); + + expect(result.state).to.equal(STATE.SERVED); + expect(result.identityVerified).to.be.true(); + }); + + it('should report an identity mismatch against the external IP', async () => { + const { cert, key } = createCertificateForTest({ ip: '203.0.113.9' }); + const port = await listenTls({ cert, key }); + + const result = await probeServedCertificate({ + host: '127.0.0.1', + port, + externalIp: '198.51.100.7', + }); + + expect(result.state).to.equal(STATE.SERVED); + expect(result.identityVerified).to.be.false(); + }); + + it('should report identity separately from the chain verdict when both fail', async () => { + // The socket surfaces only the first verification failure, so an expired certificate that + // also names the wrong address would otherwise hide the mismatch until the expiry was fixed. + const { cert, key } = createCertificateForTest({ ip: '203.0.113.9', days: -5 }); + const port = await listenTls({ cert, key }); + + const result = await probeServedCertificate({ + host: '127.0.0.1', + port, + externalIp: '198.51.100.7', + }); + + expect(result.chainVerified).to.be.false(); + expect(result.identityVerified).to.be.false(); + }); + + it('should report unreachable when nothing is listening', async () => { + const result = await probeServedCertificate({ + host: '127.0.0.1', + // Port 1 is privileged and unused, so the connection is refused rather than answered + port: 1, + externalIp: EXTERNAL_IP, + }); + + expect(result.state).to.equal(STATE.UNREACHABLE); + expect(result.certificate).to.be.undefined(); + }); + + it('should give up on a peer that accepts the connection and never completes the handshake', async () => { + const server = track(net.createServer(() => {})); + + await new Promise((resolve) => { + server.listen(0, '127.0.0.1', resolve); + }); + + const result = await probeServedCertificate({ + host: '127.0.0.1', + port: server.address().port, + externalIp: EXTERNAL_IP, + timeout: 300, + }); + + expect(result.state).to.equal(STATE.UNREACHABLE); + expect(result.reason).to.equal('ETIMEDOUT'); + }); + + it('should give up on a peer that trickles data without completing the handshake', async () => { + // The socket's own timeout resets on every byte received, so a slow drip would keep the + // probe alive forever if the deadline were not independent of it. + const server = track(net.createServer((socket) => { + const interval = setInterval(() => socket.write('\0'), 50); + socket.on('close', () => clearInterval(interval)); + socket.on('error', () => clearInterval(interval)); + })); + + await new Promise((resolve) => { + server.listen(0, '127.0.0.1', resolve); + }); + + const result = await probeServedCertificate({ + host: '127.0.0.1', + port: server.address().port, + externalIp: EXTERNAL_IP, + timeout: 400, + }); + + expect(result.state).to.equal(STATE.UNREACHABLE); + }); +});