fix(dashmate): detect and report gateway certificate problems

packages/dashmate/src/commands/ssl/obtain.js

@@ -1,5 +1,6 @@
 import { Listr } from 'listr2';
 import { Flags } from '@oclif/core';
+import ServiceIsNotRunningError from '../../docker/errors/ServiceIsNotRunningError.js';
 import ConfigBaseCommand from '../../oclif/command/ConfigBaseCommand.js';
 import MuteOneLineError from '../../oclif/errors/MuteOneLineError.js';
 import Certificate from '../../ssl/zerossl/Certificate.js';
@@ -40,6 +41,7 @@ Certificate will be renewed if it is about to expire (see 'expiration-days' flag
    * @param {obtainLetsEncryptCertificateTask} obtainLetsEncryptCertificateTask
    * @param {ConfigFileJsonRepository} configFileRepository
    * @param {ConfigFile} configFile
+   * @param {DockerCompose} dockerCompose
    * @return {Promise<void>}
    */
   async runWithDependencies(
@@ -100,15 +102,25 @@ Certificate will be renewed if it is about to expire (see 'expiration-days' flag
           // and nothing on disk reveals which certificate Envoy currently
           // holds, so an obtain that skipped the write is also how an operator
           // retries a reload that failed earlier.
+          //
+          // The gateway is signalled without asking first whether it is running.
+          // execCommand makes that check itself, and asking separately leaves a
+          // gap in which the answer can change - the certificate has already
+          // been obtained by then, so failing there would report the whole
+          // command as failed and send the operator back to a provider that may
+          // have nothing left to issue.
           title: 'Reload gateway',
-          skip: async () => {
-            if (!await dockerCompose.isServiceRunning(config, 'gateway')) {
-              return 'Gateway is not running';
-            }
+          task: async (ctx, listrTask) => {
+            try {
+              await dockerCompose.execCommand(config, 'gateway', 'kill -SIGHUP 1');
+            } catch (e) {
+              if (!(e instanceof ServiceIsNotRunningError)) {
+                throw e;
+              }
 
-            return false;
+              listrTask.skip('Gateway is not running');
+            }
           },
-          task: () => dockerCompose.execCommand(config, 'gateway', 'kill -SIGHUP 1'),
         },
       ],
       {

packages/dashmate/src/createDIContainer.js

@@ -18,6 +18,7 @@ import createConfigFileFactory from './config/configFile/createConfigFileFactory
 import migrateConfigFileFactory from './config/configFile/migrateConfigFileFactory.js';
 import DefaultConfigs from './config/DefaultConfigs.js';
 import analyseConfigFactory from './doctor/analyse/analyseConfigFactory.js';
+import analyseGatewayCertificateFactory from './doctor/analyse/analyseGatewayCertificateFactory.js';
 import analyseCoreFactory from './doctor/analyse/analyseCoreFactory.js';
 import analysePlatformFactory from './doctor/analyse/analysePlatformFactory.js';
 import analyseServiceContainersFactory from './doctor/analyse/analyseServiceContainersFactory.js';
@@ -365,6 +366,7 @@ export default async function createDIContainer(options = {}) {
     analyseSystemResources: asFunction(analyseSystemResourcesFactory).singleton(),
     analyseServiceContainers: asFunction(analyseServiceContainersFactory).singleton(),
     analyseConfig: asFunction(analyseConfigFactory).singleton(),
+    analyseGatewayCertificate: asFunction(analyseGatewayCertificateFactory).singleton(),
     analyseCore: asFunction(analyseCoreFactory).singleton(),
     analysePlatform: asFunction(analysePlatformFactory).singleton(),
     unarchiveSamples: asFunction(unarchiveSamplesFactory).singleton(),

packages/dashmate/src/doctor/analyse/analyseConfigFactory.js

@@ -5,6 +5,16 @@ import { ERRORS as ZEROSSL_ERRORS } from '../../ssl/zerossl/validateZeroSslCerti
 import { SEVERITY } from '../Prescription.js';
 import Problem from '../Problem.js';
 
+/**
+ * Whether a ZeroSSL certificate can be renewed depends on the operator's plan, which dashmate
+ * cannot see, so both routes are offered rather than assuming which one applies.
+ */
+const LETSENCRYPT_ALTERNATIVE = chalk`Or switch to Let's Encrypt, which issues certificates for IP addresses free
+of charge:
+  {bold.cyanBright dashmate config set platform.gateway.ssl.provider letsencrypt}
+  {bold.cyanBright dashmate config set platform.gateway.ssl.providerConfigs.letsencrypt.email EMAIL}
+  {bold.cyanBright dashmate ssl obtain}`;
+
 export default function analyseConfigFactory() {
   /**
    * @typedef analyseConfig
@@ -60,10 +70,7 @@ export default function analyseConfigFactory() {
             }
             break;
           default: {
-            const {
-              description,
-              solution,
-            } = {
+            const fileProblems = {
               // File provider error
               'not-valid': {
                 description: 'SSL certificate files are not valid',
@@ -82,15 +89,17 @@ Private key file path: {bold.cyanBright ${ssl?.data?.privateFilePath}}
 
 Or use ZeroSSL https://docs.dash.org/en/stable/masternodes/dashmate.html#ssl-certificate`,
               },
-              // ZeroSSL validation errors
+            };
+
+            const zeroSslProblems = {
               [ZEROSSL_ERRORS.API_KEY_IS_NOT_SET]: {
                 description: 'ZeroSSL API key is not set.',
                 solution: chalk`Please obtain your API key from {underline.cyanBright https://app.zerossl.com/developer}
-And then update your configuration with {block.cyanBright dashmate config set platform.gateway.ssl.providerConfigs.zerossl.apiKey [KEY]}`,
+And then update your configuration with {bold.cyanBright dashmate config set platform.gateway.ssl.providerConfigs.zerossl.apiKey [KEY]}`,
               },
               [ZEROSSL_ERRORS.EXTERNAL_IP_IS_NOT_SET]: {
                 description: 'External IP is not set.',
-                solution: chalk`Please update your configuration to include your external IP using {block.cyanBright dashmate config set externalIp [IP]}`,
+                solution: chalk`Please update your configuration to include your external IP using {bold.cyanBright dashmate config set externalIp [IP]}`,
               },
               [ZEROSSL_ERRORS.CERTIFICATE_ID_IS_NOT_SET]: {
                 description: 'ZeroSSL certificate is not configured',
@@ -102,7 +111,7 @@ And then update your configuration with {block.cyanBright dashmate config set pl
 and revoke the previous certificate in the ZeroSSL dashboard`,
               },
               [ZEROSSL_ERRORS.EXTERNAL_IP_MISMATCH]: {
-                description: chalk`ZeroSSL IP ${ssl?.data?.certificate.common_name} does not match external IP ${ssl?.data?.externalIp}.`,
+                description: chalk`ZeroSSL IP ${ssl?.data?.certificate?.common_name} does not match external IP ${ssl?.data?.externalIp}.`,
                 solution: chalk`Please regenerate the certificate using {bold.cyanBright dashmate ssl obtain --force}
             and revoke the previous certificate in the ZeroSSL dashboard`,
               },
@@ -113,22 +122,38 @@ This makes auto-renewal impossible.`,
 and revoke the previous certificate in the ZeroSSL dashboard`,
               },
               [ZEROSSL_ERRORS.CERTIFICATE_EXPIRES_SOON]: {
-                description: chalk`ZeroSSL certificate expires at ${ssl?.data?.certificate.expires}.`,
-                solution: chalk`Please run {bold.cyanBright dashmate ssl obtain} to get a new one`,
+                description: chalk`ZeroSSL certificate expires at ${ssl?.data?.certificate?.expires}.`,
+                solution: chalk`Please run {bold.cyanBright dashmate ssl obtain} to get a new one, which needs an
+available certificate on your ZeroSSL plan.
+
+${LETSENCRYPT_ALTERNATIVE}`,
               },
               [ZEROSSL_ERRORS.CERTIFICATE_IS_NOT_VALIDATED]: {
                 description: chalk`ZeroSSL certificate is not approved.`,
                 solution: chalk`Please run {bold.cyanBright dashmate ssl obtain} to confirm certificate`,
               },
               [ZEROSSL_ERRORS.CERTIFICATE_IS_NOT_VALID]: {
                 description: chalk`ZeroSSL certificate is not valid.`,
-                solution: chalk`Please run {bold.cyanBright dashmate ssl zerossl obtain} to get a new one.`,
+                solution: chalk`Please run {bold.cyanBright dashmate ssl obtain} to get a new one.
+
+${LETSENCRYPT_ALTERNATIVE}`,
               },
               [ZEROSSL_ERRORS.ZERO_SSL_API_ERROR]: {
-                description: ssl?.data?.error?.message,
-                solution: chalk`Please contact ZeroSSL support if needed.`,
+                // ZeroSSL's own wording is the most accurate account of what went wrong - it
+                // names an exhausted certificate limit, an unpaid invoice or a rejected key
+                // directly. The fallback keeps the problem reported when it sends none, since
+                // an empty description would otherwise drop it silently.
+                description: ssl?.data?.error?.message
+                  ? chalk`ZeroSSL rejected the request: ${ssl.data.error.message}`
+                  : chalk`The ZeroSSL API could not be reached, so the certificate cannot be checked or renewed.`,
+                solution: chalk`If this is something you can resolve with ZeroSSL, such as an expired plan or a
+rejected API key, fix it there and run {bold.cyanBright dashmate ssl obtain}.
+
+${LETSENCRYPT_ALTERNATIVE}`,
               },
-              // Let's Encrypt validation errors
+            };
+
+            const letsEncryptProblems = {
               [LETSENCRYPT_ERRORS.EMAIL_IS_NOT_SET]: {
                 description: 'Let\'s Encrypt email is not set.',
                 solution: chalk`Please update your configuration with {bold.cyanBright dashmate config set platform.gateway.ssl.providerConfigs.letsencrypt.email [EMAIL]}`,
@@ -153,10 +178,31 @@ and revoke the previous certificate in the ZeroSSL dashboard`,
                 description: chalk`Let's Encrypt certificate expires at ${ssl?.data?.certificate?.expires}.`,
                 solution: chalk`Please run {bold.cyanBright dashmate ssl obtain --provider=letsencrypt} to renew`,
               },
+              [LETSENCRYPT_ERRORS.CERTIFICATE_NOT_INSTALLED]: {
+                description: chalk`A renewed Let's Encrypt certificate has not been installed for the gateway.`,
+                solution: chalk`The gateway keeps serving the previous certificate until it is reloaded,
+and will stop accepting clients when that one expires.
+Please restart Platform: {bold.cyanBright dashmate restart --platform}`,
+              },
               [LETSENCRYPT_ERRORS.CERTIFICATE_NOT_VALID]: {
                 description: chalk`Let's Encrypt certificate is not valid.`,
                 solution: chalk`Please run {bold.cyanBright dashmate ssl obtain --provider=letsencrypt --force} to get a new one.`,
               },
+            };
+
+            // Both providers report some errors under the same name, so only the
+            // configured provider's messages are considered. Otherwise one provider's
+            // message would describe a problem found by the other one.
+            const providerProblems = config.get('platform.gateway.ssl.provider') === 'letsencrypt'
+              ? letsEncryptProblems
+              : zeroSslProblems;
+
+            const {
+              description,
+              solution,
+            } = {
+              ...fileProblems,
+              ...providerProblems,
             }[ssl.error] ?? {};
 
             if (description) {

packages/dashmate/src/doctor/analyse/analyseGatewayCertificateFactory.js

@@ -0,0 +1,143 @@
+import chalk from 'chalk';
+import { SEVERITY } from '../Prescription.js';
+import Problem from '../Problem.js';
+
+/**
+ * The manual obtain command writes certificate files but does not signal the gateway, so an
+ * operator following the advice can succeed and see no change on the wire. Every message about
+ * a certificate the gateway has not picked up has to say this.
+ */
+const RESTART_HINT = chalk`Then restart Platform so the gateway picks it up: {bold.cyanBright dashmate restart --platform}`;
+
+export default function analyseGatewayCertificateFactory() {
+  /**
+   * Analyse the certificate the gateway actually serves.
+   *
+   * @typedef analyseGatewayCertificate
+   * @param {Samples} samples
+   * @return {Problem[]}
+   */
+  function analyseGatewayCertificate(samples) {
+    const config = samples.getDashmateConfig();
+
+    if (!config?.get('platform.enable')) {
+      return [];
+    }
+
+    const served = samples.getServiceInfo('gateway', 'servedCertificate');
+
+    if (!served) {
+      return [];
+    }
+
+    const problems = [];
+
+    // Certificate validity is judged against the moment the samples were taken, not the moment
+    // they are analysed. A report is often opened days after it was collected, and the node's
+    // certificate may be renewed every few days, so judging at analysis time would report every
+    // healthy node as expired.
+    const now = samples.date?.getTime() ?? Date.now();
+
+    if (served.state === 'unreachable') {
+      problems.push(new Problem(
+        `The gateway did not answer a TLS connection (${served.reason}). Clients may not be able to connect`,
+        chalk`Please check that the gateway is running and listening: {bold.cyanBright dashmate status platform}`,
+        SEVERITY.MEDIUM,
+      ));
+
+      return problems;
+    }
+
+    if (served.state !== 'served') {
+      return problems;
+    }
+
+    const externalIp = config.get('externalIp');
+
+    // An identity mismatch is evaluated first and stops the comparisons below. It means the
+    // connection did not reach this node's gateway at all - another config or a proxy answering
+    // on the same port - and in that case the certificate it returned says nothing about this
+    // node, so reporting it as a wrong or stale certificate would be misleading.
+    if (served.identityVerified === false) {
+      problems.push(new Problem(
+        `The certificate served on port ${served.port} is not valid for ${externalIp}: ${served.identityError}`,
+        chalk`Either the certificate is issued for the wrong address, or something other than this
+node's gateway is answering on that port. Check that no other node or proxy is using it, then
+regenerate the certificate if needed: {bold.cyanBright dashmate ssl obtain --force}
+${RESTART_HINT}`,
+        SEVERITY.HIGH,
+      ));
+
+      return problems;
+    }
+
+    const servedExpiresAt = new Date(served.certificate.validTo).getTime();
+    const isServedExpired = servedExpiresAt <= now;
+    const onDiskDiffers = served.matchesOnDisk === false;
+
+    if (isServedExpired && onDiskDiffers) {
+      problems.push(new Problem(
+        `The gateway is serving a certificate that expired on ${served.certificate.validTo}, `
+        + 'while a newer one is already present on disk',
+        chalk`The certificate was renewed but never reached the gateway.
+{bold.cyanBright dashmate restart --platform}`,
+        SEVERITY.HIGH,
+      ));
+    } else if (isServedExpired) {
+      problems.push(new Problem(
+        `The gateway is serving a certificate that expired on ${served.certificate.validTo}. `
+        + 'Clients cannot connect to this node',
+        chalk`Renewal has not succeeded. Check the renewal logs:
+{bold.cyanBright dashmate logs dashmate_helper}
+Then obtain a new certificate: {bold.cyanBright dashmate ssl obtain}
+${RESTART_HINT}`,
+        SEVERITY.HIGH,
+      ));
+    } else if (onDiskDiffers) {
+      // Still serving a valid certificate, but the renewed one has not been picked up, so this
+      // node goes dark when the served certificate expires.
+      problems.push(new Problem(
+        'The gateway is serving an older certificate than the one on disk. '
+        + `It will stop accepting clients on ${served.certificate.validTo}`,
+        chalk`The certificate was renewed but never reached the gateway.
+{bold.cyanBright dashmate restart --platform}`,
+        SEVERITY.HIGH,
+      ));
+    }
+
+    // Reported separately from expiry because the connection surfaces only its first
+    // verification failure: a certificate that is both expired and untrusted reports only the
+    // expiry, and the second fault would otherwise stay hidden until the first was fixed.
+    if (!served.chainVerified && !isServedExpired) {
+      problems.push(new Problem(
+        `The certificate served by the gateway is not trusted by standard clients (${served.chainError})`,
+        chalk`Clients verifying against public certificate authorities will reject this node.
+If the certificate chain is incomplete, make sure the bundle contains the issuing
+certificates as well as the server certificate.
+${RESTART_HINT}`,
+        SEVERITY.HIGH,
+      ));
+    }
+
+    // Both obtainable providers reach this node on port 80 to validate it. Being closed is
+    // only reported alongside a certificate problem: the port is bound just for the seconds a
+    // validation takes, so an external check finds it closed on healthy nodes too and on its
+    // own would be noise.
+    const validationHttpPort = samples.getServiceInfo('gateway', 'validationHttpPort');
+
+    if (problems.length > 0 && validationHttpPort && validationHttpPort !== 'OPEN') {
+      problems.push(new Problem(
+        'Inbound port 80 is not reachable, which is how certificates are validated. '
+        + 'This may be why renewal is failing',
+        chalk`Please make sure port 80 on ${externalIp} accepts incoming connections from the
+internet. Both certificate providers connect back to it to validate this node's
+address before issuing a certificate. If you are behind NAT, forward port 80 as well.`,
+        SEVERITY.MEDIUM,
+      ));
+    }
+
+    return problems;
+  }
+
+  return analyseGatewayCertificate;
+}

packages/dashmate/src/doctor/analyseSamplesFactory.js

@@ -5,6 +5,7 @@ import Problem from './Problem.js';
  * @param {analyseSystemResources} analyseSystemResources
  * @param {analyseServiceContainers} analyseServiceContainers
  * @param {analyseConfig} analyseConfig
+ * @param {analyseGatewayCertificate} analyseGatewayCertificate
  * @param {analyseCore} analyseCore
  * @param {analysePlatform} analysePlatform
  * @return {analyseSamples}
@@ -13,6 +14,7 @@ export default function analyseSamplesFactory(
   analyseSystemResources,
   analyseServiceContainers,
   analyseConfig,
+  analyseGatewayCertificate,
   analyseCore,
   analysePlatform,
 ) {
@@ -41,6 +43,8 @@ export default function analyseSamplesFactory(
 
     problems.push(...analyseConfig(samples));
 
+    problems.push(...analyseGatewayCertificate(samples));
+
     problems.push(...analyseCore(samples));
 
     problems.push(...analysePlatform(samples));