fix(#10462): HA derives its cluster token from arcadedb.server.rootPasswordPath, and the StatefulSet mounts its own token

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>

Author: robfrank

engine/src/main/java/com/arcadedb/GlobalConfiguration.java

@@ -1682,7 +1682,8 @@ proved who it is, and the handshake timeout (arcadedb.network.socketTimeout) onl
       String.class, null),
 
   SERVER_ROOT_PASSWORD_PATH("arcadedb.server.rootPasswordPath", SCOPE.SERVER,
-      "Path to file with password for root user to use at first startup of the server. Set this to avoid asking the password to the user",
+      "Path to file with password for root user to use at first startup of the server. Set this to avoid asking the password to the user. "
+          + "With HA enabled and no arcadedb.ha.clusterToken or arcadedb.ha.clusterTokenPath, the cluster token is derived from this password",
       String.class, null),
 
   SERVER_MODE("arcadedb.server.mode", SCOPE.SERVER, "Server mode between 'development', 'test' and 'production'", String.class,

ha-raft/src/main/java/com/arcadedb/server/ha/raft/ClusterTokenProvider.java

@@ -22,6 +22,7 @@
 import com.arcadedb.GlobalConfiguration;
 import com.arcadedb.exception.ConfigurationException;
 import com.arcadedb.log.LogManager;
+import com.arcadedb.server.security.ServerSecurity;
 
 import javax.crypto.SecretKeyFactory;
 import javax.crypto.spec.PBEKeySpec;
@@ -36,7 +37,8 @@
 /**
  * Derives and caches the cluster token used for inter-node authentication.
  * The token is either explicitly configured or derived from the cluster name
- * and root password using PBKDF2-HMAC-SHA256.
+ * and root password using PBKDF2-HMAC-SHA256. The root password is taken inline or,
+ * failing that, from the file at {@code arcadedb.server.rootPasswordPath}.
  */
 class ClusterTokenProvider {
 
@@ -77,15 +79,13 @@ synchronized void initClusterToken() {
     if (clusterName == null || clusterName.isEmpty())
       throw new ConfigurationException(
           "Cannot derive cluster token: the cluster name is empty. Set arcadedb.ha.clusterName to a unique value or provide an explicit arcadedb.ha.clusterToken");
-    // Check both the server's ContextConfiguration and the global default (system property)
-    String rootPasswordStr = configuration.getValueAsString(GlobalConfiguration.SERVER_ROOT_PASSWORD);
-    if (rootPasswordStr == null || rootPasswordStr.isEmpty())
-      rootPasswordStr = GlobalConfiguration.SERVER_ROOT_PASSWORD.getValueAsString();
-    if (rootPasswordStr == null || rootPasswordStr.isEmpty())
+    final String rootPasswordStr = resolveRootPassword(configuration);
+    if (rootPasswordStr == null)
       throw new ConfigurationException(
           """
           Cannot start HA mode without authentication: the auto-derived cluster token requires a root password. \
-          Set arcadedb.server.rootPassword or provide an explicit arcadedb.ha.clusterToken""");
+          Set arcadedb.server.rootPassword or arcadedb.server.rootPasswordPath, or provide an explicit \
+          arcadedb.ha.clusterToken or arcadedb.ha.clusterTokenPath""");
     if ("production".equals(configuration.getValueAsString(GlobalConfiguration.SERVER_MODE))
         && "arcadedb".equalsIgnoreCase(clusterName))
       LogManager.instance().log(this, Level.WARNING,
@@ -120,12 +120,10 @@ static void initClusterTokenForTest(final ContextConfiguration config) {
     if (clusterName == null || clusterName.isEmpty())
       throw new ConfigurationException(
           "Cannot derive cluster token: the cluster name is empty. Set arcadedb.ha.clusterName to a unique value or provide an explicit arcadedb.ha.clusterToken");
-    String rootPasswordStr = config.getValueAsString(GlobalConfiguration.SERVER_ROOT_PASSWORD);
-    if (rootPasswordStr == null || rootPasswordStr.isEmpty())
-      rootPasswordStr = GlobalConfiguration.SERVER_ROOT_PASSWORD.getValueAsString();
-    if (rootPasswordStr == null || rootPasswordStr.isEmpty())
+    final String rootPasswordStr = resolveRootPassword(config);
+    if (rootPasswordStr == null)
       throw new ConfigurationException(
-          "Cannot derive cluster token without a root password. Set arcadedb.server.rootPassword or arcadedb.ha.clusterToken");
+          "Cannot derive cluster token without a root password. Set arcadedb.server.rootPassword, arcadedb.server.rootPasswordPath, arcadedb.ha.clusterToken or arcadedb.ha.clusterTokenPath");
 
     config.setValue(GlobalConfiguration.HA_CLUSTER_TOKEN, deriveTokenInternal(clusterName, rootPasswordStr));
   }
@@ -166,6 +164,55 @@ private static String resolveExplicitToken(final ContextConfiguration config) {
     return token;
   }
 
+  /**
+   * Resolves the root password the token is derived from, with the same precedence as
+   * {@code ServerSecurity.askForRootPassword()} uses to create the root user:
+   * <ol>
+   *   <li>{@link GlobalConfiguration#SERVER_ROOT_PASSWORD}, from the server's configuration, then from the global one;</li>
+   *   <li>otherwise the content of the file at {@link GlobalConfiguration#SERVER_ROOT_PASSWORD_PATH}, looked up the same
+   *   way, with one trailing line terminator dropped by {@link ServerSecurity#stripTrailingNewline(String)}.</li>
+   * </ol>
+   * The file is read with the very same normalization as the root user's password, so a node configured with the
+   * password inline and a node reading it from a file derive the same token (issue #10462).
+   *
+   * @return the root password, or {@code null} when neither setting is configured
+   *
+   * @throws ConfigurationException if the password file is configured but cannot be read, or is empty.
+   */
+  private static String resolveRootPassword(final ContextConfiguration config) {
+    final String inline = valueOf(config, GlobalConfiguration.SERVER_ROOT_PASSWORD);
+    if (inline != null)
+      return inline;
+
+    final String passwordPath = valueOf(config, GlobalConfiguration.SERVER_ROOT_PASSWORD_PATH);
+    if (passwordPath == null)
+      return null;
+
+    final Path path = Path.of(passwordPath);
+    if (!Files.isReadable(path))
+      throw new ConfigurationException("Error reading root password file at path '" + passwordPath + "'");
+    final String password;
+    try {
+      password = ServerSecurity.stripTrailingNewline(Files.readString(path));
+    } catch (final IOException e) {
+      throw new ConfigurationException("Error reading root password file at path '" + passwordPath + "'", e);
+    }
+    if (password.isEmpty())
+      throw new ConfigurationException("Root password file at path '" + passwordPath + "' is empty");
+    return password;
+  }
+
+  /**
+   * Returns the setting from the server's configuration, falling back to the global default (system property), or
+   * {@code null} when neither carries a non-empty value.
+   */
+  private static String valueOf(final ContextConfiguration config, final GlobalConfiguration setting) {
+    String value = config.getValueAsString(setting);
+    if (value == null || value.isEmpty())
+      value = setting.getValueAsString();
+    return value == null || value.isEmpty() ? null : value;
+  }
+
   /**
    * Converts the root password String to a char[], delegates to
    * {@link #deriveTokenFromPassword(String, char[])}, and zeros the array before returning.

ha-raft/src/test/java/com/arcadedb/server/ha/raft/ClusterTokenProviderTest.java

@@ -137,4 +137,120 @@ void emptyTokenFileThrows(@TempDir final Path tmp) throws Exception {
     assertThatThrownBy(() -> new ClusterTokenProvider(config).getClusterToken())
         .isInstanceOf(ConfigurationException.class);
   }
+
+  // Issue #10462: a server given its root password as a file (the shipped StatefulSet mounts a Secret and passes only
+  // arcadedb.server.rootPasswordPath) has a root password, so HA must derive its cluster token from that file too.
+
+  @Test
+  void tokenDerivedFromRootPasswordFile(@TempDir final Path tmp) throws Exception {
+    final Path passwordFile = tmp.resolve("rootPassword");
+    Files.writeString(passwordFile, "test-password");
+    final ContextConfiguration fromFile = new ContextConfiguration();
+    fromFile.setValue(GlobalConfiguration.HA_CLUSTER_NAME, "test-cluster");
+    fromFile.setValue(GlobalConfiguration.SERVER_ROOT_PASSWORD_PATH, passwordFile.toString());
+
+    final ContextConfiguration inline = new ContextConfiguration();
+    inline.setValue(GlobalConfiguration.HA_CLUSTER_NAME, "test-cluster");
+    inline.setValue(GlobalConfiguration.SERVER_ROOT_PASSWORD, "test-password");
+
+    assertThat(new ClusterTokenProvider(fromFile).getClusterToken())
+        .isEqualTo(new ClusterTokenProvider(inline).getClusterToken());
+  }
+
+  @Test
+  void rootPasswordFileDropsOnlyTheTrailingNewlineLikeServerSecurity(@TempDir final Path tmp) throws Exception {
+    // `kubectl create secret --from-file`, a here-doc or an editor append a newline that is not part of the password:
+    // the token must match the one a node given the same password inline derives. Trailing spaces are part of the
+    // password, exactly as ServerSecurity reads the same file into the root user.
+    final Path passwordFile = tmp.resolve("rootPassword");
+    Files.writeString(passwordFile, "test-password \r\n");
+    final ContextConfiguration fromFile = new ContextConfiguration();
+    fromFile.setValue(GlobalConfiguration.HA_CLUSTER_NAME, "test-cluster");
+    fromFile.setValue(GlobalConfiguration.SERVER_ROOT_PASSWORD_PATH, passwordFile.toString());
+
+    final ContextConfiguration inline = new ContextConfiguration();
+    inline.setValue(GlobalConfiguration.HA_CLUSTER_NAME, "test-cluster");
+    inline.setValue(GlobalConfiguration.SERVER_ROOT_PASSWORD, "test-password ");
+
+    assertThat(new ClusterTokenProvider(fromFile).getClusterToken())
+        .isEqualTo(new ClusterTokenProvider(inline).getClusterToken());
+  }
+
+  @Test
+  void inlineRootPasswordTakesPrecedenceOverRootPasswordFile(@TempDir final Path tmp) throws Exception {
+    // Same precedence as ServerSecurity.askForRootPassword(): the file is read only when no inline password is set.
+    final Path passwordFile = tmp.resolve("rootPassword");
+    Files.writeString(passwordFile, "file-password");
+    final ContextConfiguration both = new ContextConfiguration();
+    both.setValue(GlobalConfiguration.HA_CLUSTER_NAME, "test-cluster");
+    both.setValue(GlobalConfiguration.SERVER_ROOT_PASSWORD, "inline-password");
+    both.setValue(GlobalConfiguration.SERVER_ROOT_PASSWORD_PATH, passwordFile.toString());
+
+    final ContextConfiguration inline = new ContextConfiguration();
+    inline.setValue(GlobalConfiguration.HA_CLUSTER_NAME, "test-cluster");
+    inline.setValue(GlobalConfiguration.SERVER_ROOT_PASSWORD, "inline-password");
+
+    assertThat(new ClusterTokenProvider(both).getClusterToken())
+        .isEqualTo(new ClusterTokenProvider(inline).getClusterToken());
+  }
+
+  @Test
+  void explicitTokenPathTakesPrecedenceOverRootPasswordFile(@TempDir final Path tmp) throws Exception {
+    final Path tokenFile = tmp.resolve("clusterToken");
+    Files.writeString(tokenFile, "file-token\n");
+    final Path passwordFile = tmp.resolve("rootPassword");
+    Files.writeString(passwordFile, "file-password");
+    final ContextConfiguration config = new ContextConfiguration();
+    config.setValue(GlobalConfiguration.HA_CLUSTER_NAME, "test-cluster");
+    config.setValue(GlobalConfiguration.HA_CLUSTER_TOKEN_PATH, tokenFile.toString());
+    config.setValue(GlobalConfiguration.SERVER_ROOT_PASSWORD_PATH, passwordFile.toString());
+    assertThat(new ClusterTokenProvider(config).getClusterToken()).isEqualTo("file-token");
+  }
+
+  @Test
+  void unreadableRootPasswordFileThrowsNamingThePath() {
+    final ContextConfiguration config = new ContextConfiguration();
+    config.setValue(GlobalConfiguration.HA_CLUSTER_NAME, "test-cluster");
+    config.setValue(GlobalConfiguration.SERVER_ROOT_PASSWORD_PATH, "/non/existent/rootPassword");
+    assertThatThrownBy(() -> new ClusterTokenProvider(config).getClusterToken())
+        .isInstanceOf(ConfigurationException.class).hasMessageContaining("/non/existent/rootPassword");
+  }
+
+  @Test
+  void emptyRootPasswordFileThrows(@TempDir final Path tmp) throws Exception {
+    final Path passwordFile = tmp.resolve("rootPassword");
+    Files.writeString(passwordFile, "\n");
+    final ContextConfiguration config = new ContextConfiguration();
+    config.setValue(GlobalConfiguration.HA_CLUSTER_NAME, "test-cluster");
+    config.setValue(GlobalConfiguration.SERVER_ROOT_PASSWORD_PATH, passwordFile.toString());
+    assertThatThrownBy(() -> new ClusterTokenProvider(config).getClusterToken())
+        .isInstanceOf(ConfigurationException.class).hasMessageContaining("empty");
+  }
+
+  @Test
+  void missingRootPasswordErrorNamesThePasswordFileSetting() {
+    final ContextConfiguration config = new ContextConfiguration();
+    config.setValue(GlobalConfiguration.HA_CLUSTER_NAME, "test-cluster");
+    assertThatThrownBy(() -> new ClusterTokenProvider(config).getClusterToken())
+        .isInstanceOf(ConfigurationException.class)
+        .hasMessageContaining(GlobalConfiguration.SERVER_ROOT_PASSWORD_PATH.getKey())
+        .hasMessageContaining(GlobalConfiguration.HA_CLUSTER_TOKEN_PATH.getKey());
+  }
+
+  @Test
+  void testHelperDerivesFromRootPasswordFile(@TempDir final Path tmp) throws Exception {
+    final Path passwordFile = tmp.resolve("rootPassword");
+    Files.writeString(passwordFile, "test-password\n");
+    final ContextConfiguration fromFile = new ContextConfiguration();
+    fromFile.setValue(GlobalConfiguration.HA_CLUSTER_NAME, "test-cluster");
+    fromFile.setValue(GlobalConfiguration.SERVER_ROOT_PASSWORD_PATH, passwordFile.toString());
+    ClusterTokenProvider.initClusterTokenForTest(fromFile);
+
+    final ContextConfiguration inline = new ContextConfiguration();
+    inline.setValue(GlobalConfiguration.HA_CLUSTER_NAME, "test-cluster");
+    inline.setValue(GlobalConfiguration.SERVER_ROOT_PASSWORD, "test-password");
+
+    assertThat(fromFile.getValueAsString(GlobalConfiguration.HA_CLUSTER_TOKEN))
+        .isEqualTo(new ClusterTokenProvider(inline).getClusterToken());
+  }
 }

package/src/main/config/arcadedb-statefulset.yaml

@@ -12,7 +12,8 @@
 # Using ArcadeDB with k8s
 # - Create the namespace: kubectl create namespace arcadedb
 # - Create the secret   : kubectl create secret generic arcadedb-credentials \
-#                           --namespace arcadedb --from-literal=rootPassword='<choose-a-strong-password>'
+#                           --namespace arcadedb --from-literal=rootPassword='<choose-a-strong-password>' \
+#                           --from-literal=clusterToken="$(openssl rand -hex 32)"
 # - Create cluster      : kubectl apply -f arcadedb-statefulset.yaml
 # - List the pods       : kubectl get pods --namespace arcadedb
 # - Run console         : kubectl exec -it arcadedb-0 --namespace arcadedb -- bin/console.sh
@@ -28,6 +29,11 @@
 # sized for a real three-node cluster, and the Raft gRPC port is unauthenticated - restrict it to the
 # StatefulSet's own pods with a NetworkPolicy.
 #
+# The `clusterToken` key is the shared secret the nodes authenticate each other with (forwarded commands, Raft group
+# changes). Every pod mounts the same Secret, so they all read the same token. Without it the token would be derived
+# from the root password and the cluster name with a fixed public salt: a separate high-entropy value keeps a weak root
+# password from making the token guessable, and lets it be rotated without touching the root password.
+#
 # NOTE ON VARIABLE EXPANSION: Kubernetes expands only the `$(VAR)` form in `command`/`args`, and only
 # for variables declared in this container's `env`. The `${VAR}` form is NOT expanded, and since the
 # arguments are not run through a shell there is no POSIX expansion either: a `${VAR}` written below
@@ -115,9 +121,10 @@ spec:
                         - arcadedb
                 topologyKey: kubernetes.io/hostname
       volumes:
-        # The root password is mounted as a file instead of being passed on the command line: an
-        # argument is world-readable through /proc inside the pod, a mounted secret is not. ArcadeDB
-        # reads it through `arcadedb.server.rootPasswordPath`.
+        # The root password and the HA cluster token are mounted as files instead of being passed on
+        # the command line: an argument is world-readable through /proc inside the pod, a mounted secret
+        # is not. ArcadeDB reads them through `arcadedb.server.rootPasswordPath` and
+        # `arcadedb.ha.clusterTokenPath`.
         - name: root-password
           secret:
             secretName: arcadedb-credentials
@@ -131,6 +138,8 @@ spec:
             items:
               - key: rootPassword
                 path: rootPassword
+              - key: clusterToken
+                path: clusterToken
       initContainers:
         # Seeds the configuration directory on the PersistentVolume (see `configDirectory` below) with the image's
         # default files. Unlike a Docker named volume, a PersistentVolumeClaim is not populated from the image, and an
@@ -192,6 +201,8 @@ spec:
             # restart never drops the write quorum.
             - "-Darcadedb.server.readinessRequiresHA=true"
             - "-Darcadedb.ha.enabled=true"
+            # The shared secret the nodes authenticate each other with, from the same mounted Secret (see the header).
+            - "-Darcadedb.ha.clusterTokenPath=/etc/arcadedb/secrets/clusterToken"
             # arcadedb-0 lists itself, and must: this list IS the initial Raft group membership, so a
             # node absent from it is not a member. Only the first pod is listed because the remaining
             # pods are not members at boot - they synthesize their own peer entry from the pod name

server/src/main/java/com/arcadedb/server/security/ServerSecurity.java

@@ -2325,7 +2325,7 @@ public String getApiTokensJsonPayload() {
    * store a root password nobody can type. Only the final CR/LF is removed: interior characters and
    * trailing spaces are left alone, since those can legitimately belong to the password.
    */
-  static String stripTrailingNewline(final String password) {
+  public static String stripTrailingNewline(final String password) {
     if (password == null)
       return null;
     if (password.endsWith("\r\n"))

server/src/test/java/com/arcadedb/server/PackagingManifestsTest.java

@@ -168,6 +168,35 @@ void statefulSetReadsTheRootPasswordFromTheMountedSecret() {
         .contains(directory);
   }
 
+  @Test
+  void statefulSetNamesAnHaClusterTokenSource() {
+    // Issue #10462: an HA node authenticates its peers with a cluster token. Without an explicit one it is derived from
+    // the root password, and a manifest that keeps that password off the command line must not leave the cluster's
+    // authentication hanging on that fallback alone: it names the token's own secret, mounted from the same Secret.
+    final String manifest = codeOf(read(STATEFULSET));
+    assertThat(manifest).contains("-D" + GlobalConfiguration.HA_ENABLED.getKey() + "=true");
+    if (manifest.contains("-D" + GlobalConfiguration.SERVER_ROOT_PASSWORD.getKey() + "="))
+      return;
+
+    assertThat(manifest).as("the cluster token must not be passed as a command-line argument")
+        .doesNotContain("-D" + GlobalConfiguration.HA_CLUSTER_TOKEN.getKey() + "=");
+    final Matcher path = Pattern.compile("-D" + Pattern.quote(GlobalConfiguration.HA_CLUSTER_TOKEN_PATH.getKey()) + "=([^\"\\s]+)")
+        .matcher(manifest);
+    assertThat(path.find()).as("an HA StatefulSet with no inline root password must set %s",
+        GlobalConfiguration.HA_CLUSTER_TOKEN_PATH.getKey()).isTrue();
+
+    final String tokenPath = path.group(1);
+    final String directory = tokenPath.substring(0, tokenPath.lastIndexOf('/'));
+    final String file = tokenPath.substring(tokenPath.lastIndexOf('/') + 1);
+    assertThat(matches(MOUNT_PATH, manifest)).as("no volume is mounted at the configured cluster token directory")
+        .contains(directory);
+    assertThat(manifest).as("the mounted Secret projects no '%s' item, so the token file never exists", file)
+        .containsPattern("(?m)^\\s*- key: " + Pattern.quote(file) + "\\s*\\n\\s*path: " + Pattern.quote(file) + "\\s*$");
+    // The instructions at the top of the manifest create that Secret: they have to fill the key the pods mount.
+    assertThat(read(STATEFULSET)).as("the documented `kubectl create secret` does not set the '%s' key", file)
+        .contains("--from-literal=" + file + "=");
+  }
+
   @Test
   void statefulSetProbesTheEndpointTheServerRegisters() {
     final String manifest = read(STATEFULSET);