IGNITE-28867 Implement TLS certificate hot redeployment via control.sh --ssl reload

docs/_docs/tools/control-script.adoc

@@ -1688,3 +1688,24 @@ Examples:
 ----
 control.sh|bat --event list --enabled
 ----
+
+== TLS Certificate Hot Reload
+
+By default, TLS certificates are loaded once at node startup, so updating them requires a full node restart.
+The `--ssl reload` command reloads certificates at runtime on all server nodes without a restart: each node
+re-reads the key and trust stores from disk using its configured `SslContextFactory` and replaces the active
+SSL context. New connections use the updated certificates, while sessions that were already established are not
+interrupted.
+
+[source, shell]
+----
+control.sh|bat --ssl reload
+----
+
+The command applies to the communication, discovery, thin client (client connector) and binary (TCP) REST transports
+that have SSL enabled. Update the certificate files on disk (keeping the configured key and trust store paths) before
+running the command.
+
+NOTE: For the discovery transport, new outgoing connections use the reloaded certificates immediately, but the
+already-bound listening socket keeps serving the previously loaded certificate to incoming connections until it is
+recreated (for example, when the node re-enters the ring).

modules/commons/src/main/java/org/apache/ignite/ssl/AbstractSslContextFactory.java

@@ -196,4 +196,36 @@ protected void checkNullParameter(Object param, String name) throws SSLException
 
         return ctx;
     }
+
+    /**
+     * Rebuilds the SSL context from the current factory configuration, re-reading the configured key and trust
+     * stores from disk, and replaces the instance cached by {@link #create()}. Used to hot-reload TLS certificates
+     * at runtime without a node restart.
+     *
+     * @return Newly created SSL context.
+     * @throws SSLException If the new context could not be created. In this case the previously cached context is
+     *      kept intact.
+     */
+    public SSLContext reload() throws SSLException {
+        SSLContext ctx = createSslContext();
+
+        sslCtx.set(ctx);
+
+        return ctx;
+    }
+
+    /**
+     * Rebuilds the SSL context produced by the given factory. If the factory supports hot reload (is an
+     * {@link AbstractSslContextFactory}) its cached context is rebuilt from the configured key and trust stores;
+     * otherwise {@link Factory#create()} is invoked.
+     *
+     * @param factory SSL context factory.
+     * @return Reloaded SSL context.
+     * @throws SSLException If the context could not be reloaded.
+     */
+    public static SSLContext reload(Factory<SSLContext> factory) throws SSLException {
+        return factory instanceof AbstractSslContextFactory
+            ? ((AbstractSslContextFactory)factory).reload()
+            : factory.create();
+    }
 }

modules/core/src/main/java/org/apache/ignite/internal/management/IgniteCommandRegistry.java

@@ -37,6 +37,7 @@
 import org.apache.ignite.internal.management.persistence.PersistenceCommand;
 import org.apache.ignite.internal.management.property.PropertyCommand;
 import org.apache.ignite.internal.management.snapshot.SnapshotCommand;
+import org.apache.ignite.internal.management.ssl.SslCommand;
 import org.apache.ignite.internal.management.tracing.TracingConfigurationCommand;
 import org.apache.ignite.internal.management.tx.TxCommand;
 import org.apache.ignite.internal.management.wal.WalCommand;
@@ -77,7 +78,8 @@ public IgniteCommandRegistry() {
             new PerformanceStatisticsCommand(),
             new CdcCommand(),
             new ConsistencyCommand(),
-            new EventCommand()
+            new EventCommand(),
+            new SslCommand()
         );
 
         U.loadService(CommandsProvider.class).forEach(p -> p.commands().forEach(this::register));

modules/core/src/main/java/org/apache/ignite/internal/management/ssl/SslCommand.java

@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.management.ssl;
+
+import org.apache.ignite.internal.management.api.CommandRegistryImpl;
+
+/** SSL features command. */
+public class SslCommand extends CommandRegistryImpl {
+    /** */
+    public SslCommand() {
+        super(
+            new SslReloadCommand()
+        );
+    }
+}

modules/core/src/main/java/org/apache/ignite/internal/management/ssl/SslReloadCommand.java

@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.management.ssl;
+
+import java.util.Collection;
+import java.util.function.Consumer;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.internal.management.api.CommandUtils;
+import org.apache.ignite.internal.management.api.ComputeCommand;
+import org.apache.ignite.internal.management.api.NoArg;
+import org.jetbrains.annotations.Nullable;
+
+/** Reloads TLS certificates on all server nodes without a restart. */
+public class SslReloadCommand implements ComputeCommand<NoArg, String> {
+    /** {@inheritDoc} */
+    @Override public String description() {
+        return "Reload TLS certificates on all server nodes by re-reading the configured key and trust stores. " +
+            "New connections use the updated certificates while established sessions are not interrupted";
+    }
+
+    /** {@inheritDoc} */
+    @Override public Class<NoArg> argClass() {
+        return NoArg.class;
+    }
+
+    /** {@inheritDoc} */
+    @Override public Class<SslReloadTask> taskClass() {
+        return SslReloadTask.class;
+    }
+
+    /** {@inheritDoc} */
+    @Override public @Nullable Collection<ClusterNode> nodes(Collection<ClusterNode> nodes, NoArg arg) {
+        return CommandUtils.servers(nodes);
+    }
+
+    /** {@inheritDoc} */
+    @Override public void printResult(NoArg arg, String res, Consumer<String> printer) {
+        printer.accept(res);
+    }
+
+    /** {@inheritDoc} */
+    @Override public String confirmationPrompt(NoArg arg) {
+        return "Warning: the command will reload TLS certificates on all server nodes.";
+    }
+}

modules/core/src/main/java/org/apache/ignite/internal/management/ssl/SslReloadTask.java

@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.management.ssl;
+
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.compute.ComputeJobResult;
+import org.apache.ignite.internal.GridKernalContext;
+import org.apache.ignite.internal.management.api.NoArg;
+import org.apache.ignite.internal.processors.task.GridInternal;
+import org.apache.ignite.internal.util.nio.ssl.SslContextReloadable;
+import org.apache.ignite.internal.visor.VisorJob;
+import org.apache.ignite.internal.visor.VisorMultiNodeTask;
+import org.jetbrains.annotations.Nullable;
+
+/** Task that reloads TLS certificates on every mapped node. */
+@GridInternal
+public class SslReloadTask extends VisorMultiNodeTask<NoArg, String, String> {
+    /** */
+    private static final long serialVersionUID = 0L;
+
+    /** {@inheritDoc} */
+    @Override protected VisorJob<NoArg, String> job(NoArg arg) {
+        return new SslReloadJob(arg, debug);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected @Nullable String reduce0(List<ComputeJobResult> results) throws IgniteException {
+        StringBuilder res = new StringBuilder();
+
+        for (ComputeJobResult jobRes : results) {
+            if (jobRes.getException() != null)
+                throw jobRes.getException();
+
+            res.append(jobRes.getData().toString()).append('\n');
+        }
+
+        return res.toString();
+    }
+
+    /** Job that reloads TLS certificates on the local node. */
+    private static class SslReloadJob extends VisorJob<NoArg, String> {
+        /** */
+        private static final long serialVersionUID = 0L;
+
+        /** */
+        protected SslReloadJob(NoArg arg, boolean debug) {
+            super(arg, debug);
+        }
+
+        /** {@inheritDoc} */
+        @Override protected String run(NoArg arg) throws IgniteException {
+            GridKernalContext ctx = ignite.context();
+
+            List<String> reloaded = new ArrayList<>();
+
+            try {
+                reload(ctx.config().getCommunicationSpi(), "communication", reloaded);
+                reload(ctx.config().getDiscoverySpi(), "discovery", reloaded);
+                reload(ctx.clientListener(), "client connector", reloaded);
+                reload(ctx.rest(), "REST", reloaded);
+            }
+            catch (IgniteCheckedException e) {
+                throw new IgniteException("Failed to reload SSL certificates on node " +
+                    ignite.localNode().id() + ": " + e.getMessage(), e);
+            }
+
+            return ignite.localNode().id() + ": " +
+                (reloaded.isEmpty() ? "SSL is not configured, nothing to reload" : "reloaded " + reloaded);
+        }
+
+        /**
+         * Reloads the SSL context of the given component if it supports hot reload.
+         *
+         * @param comp Component to reload.
+         * @param name Human-readable component name for the result message.
+         * @param out Collects names of the components that were actually reloaded.
+         * @throws IgniteCheckedException If the reload failed.
+         */
+        private static void reload(Object comp, String name, List<String> out) throws IgniteCheckedException {
+            if (comp instanceof SslContextReloadable && ((SslContextReloadable)comp).reloadSslContext())
+                out.add(name);
+        }
+    }
+}

modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/ClientListenerProcessor.java

@@ -34,6 +34,7 @@
 import javax.management.JMException;
 import javax.management.ObjectName;
 import javax.net.ssl.SSLContext;
+import javax.net.ssl.SSLException;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteException;
 import org.apache.ignite.configuration.ClientConnectorConfiguration;
@@ -57,6 +58,7 @@
 import org.apache.ignite.internal.util.nio.GridNioServer;
 import org.apache.ignite.internal.util.nio.GridNioSession;
 import org.apache.ignite.internal.util.nio.ssl.GridNioSslFilter;
+import org.apache.ignite.internal.util.nio.ssl.SslContextReloadable;
 import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.internal.util.worker.GridWorker;
@@ -68,6 +70,7 @@
 import org.apache.ignite.spi.IgnitePortProtocol;
 import org.apache.ignite.spi.systemview.view.ClientConnectionAttributeView;
 import org.apache.ignite.spi.systemview.view.ClientConnectionView;
+import org.apache.ignite.ssl.AbstractSslContextFactory;
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
 
@@ -84,7 +87,7 @@
 /**
  * Client connector processor.
  */
-public class ClientListenerProcessor extends GridProcessorAdapter {
+public class ClientListenerProcessor extends GridProcessorAdapter implements SslContextReloadable {
     /** */
     public static final String CLI_CONN_VIEW = metricName("client", "connections");
 
@@ -118,6 +121,9 @@ public class ClientListenerProcessor extends GridProcessorAdapter {
     /** TCP Server. */
     private GridNioServer<ClientMessage> srv;
 
+    /** SSL filter of the current server, {@code null} if SSL is disabled. Used to hot-reload certificates. */
+    private volatile GridNioSslFilter sslFilter;
+
     /** Metrics. */
     private ClientListenerMetrics metrics;
 
@@ -502,6 +508,8 @@ else if (connCtx.managementClient()) {
             sslFilter.wantClientAuth(auth);
             sslFilter.needClientAuth(auth);
 
+            this.sslFilter = sslFilter;
+
             return new GridNioFilter[] {
                 openSesFilter,
                 codecFilter,
@@ -516,6 +524,32 @@ else if (connCtx.managementClient()) {
         }
     }
 
+    /** {@inheritDoc} */
+    @Override public boolean reloadSslContext() throws IgniteCheckedException {
+        ClientConnectorConfiguration cliConnCfg = ctx.config().getClientConnectorConfiguration();
+
+        GridNioSslFilter filter = sslFilter;
+
+        if (cliConnCfg == null || !cliConnCfg.isSslEnabled() || filter == null)
+            return false;
+
+        Factory<SSLContext> sslCtxFactory = cliConnCfg.isUseIgniteSslContextFactory()
+            ? ctx.config().getSslContextFactory()
+            : cliConnCfg.getSslContextFactory();
+
+        if (sslCtxFactory == null)
+            return false;
+
+        try {
+            filter.updateSslContext(AbstractSslContextFactory.reload(sslCtxFactory));
+        }
+        catch (SSLException e) {
+            throw new IgniteCheckedException("Failed to reload SSL context for client connector connections.", e);
+        }
+
+        return true;
+    }
+
     /** {@inheritDoc} */
     @Override public void onKernalStop(boolean cancel) {
         if (srv != null) {

modules/core/src/main/java/org/apache/ignite/internal/processors/rest/GridRestProcessor.java

@@ -72,6 +72,7 @@
 import org.apache.ignite.internal.thread.context.Scope;
 import org.apache.ignite.internal.util.GridSpinReadWriteLock;
 import org.apache.ignite.internal.util.future.GridFinishedFuture;
+import org.apache.ignite.internal.util.nio.ssl.SslContextReloadable;
 import org.apache.ignite.internal.util.typedef.C1;
 import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.internal.util.typedef.X;
@@ -106,7 +107,7 @@
 /**
  * Rest processor implementation.
  */
-public class GridRestProcessor extends GridProcessorAdapter implements IgniteRestProcessor {
+public class GridRestProcessor extends GridProcessorAdapter implements IgniteRestProcessor, SslContextReloadable {
     /** HTTP protocol class name. */
     private static final String HTTP_PROTO_CLS =
         "org.apache.ignite.internal.processors.rest.protocols.http.jetty.GridJettyRestProtocol";
@@ -609,6 +610,16 @@ private boolean notStartOnClient() {
         }
     }
 
+    /** {@inheritDoc} */
+    @Override public boolean reloadSslContext() throws IgniteCheckedException {
+        boolean reloaded = false;
+
+        for (GridRestProtocol proto : protos)
+            reloaded |= proto.reloadSslContext();
+
+        return reloaded;
+    }
+
     /** {@inheritDoc} */
     @SuppressWarnings("BusyWait")
     @Override public void onKernalStop(boolean cancel) {