SOLR-16743: Auto reload keystore/truststore on change

solr/bin/solr

@@ -208,9 +208,17 @@ if [ -z "${SOLR_SSL_ENABLED:-}" ]; then
 fi
 if [ "$SOLR_SSL_ENABLED" == "true" ]; then
   SOLR_JETTY_CONFIG+=("--module=https" "--lib=$DEFAULT_SERVER_DIR/solr-webapp/webapp/WEB-INF/lib/*")
+  if [ "${SOLR_SSL_RELOAD_ENABLED:-true}" == "true" ]; then
+    SOLR_JETTY_CONFIG+=("--module=ssl-reload")
+    SOLR_SSL_OPTS+=" -Dsolr.keyStoreReload.enabled=true"
+  fi
   SOLR_URL_SCHEME=https
   if [ -n "$SOLR_SSL_KEY_STORE" ]; then
     SOLR_SSL_OPTS+=" -Dsolr.jetty.keystore=$SOLR_SSL_KEY_STORE"
+    if [ "${SOLR_SSL_RELOAD_ENABLED:-true}" == "true" ] && [ "${SOLR_SECURITY_MANAGER_ENABLED:-true}" == "true"  ]; then
+      # In this case we need to allow reads from the parent directory of the keystore
+      SOLR_SSL_OPTS+=" -Dsolr.jetty.keystoreParentPath=$SOLR_SSL_KEY_STORE/.."
+    fi
   fi
   if [ -n "$SOLR_SSL_KEY_STORE_PASSWORD" ]; then
     export SOLR_SSL_KEY_STORE_PASSWORD=$SOLR_SSL_KEY_STORE_PASSWORD
@@ -249,6 +257,10 @@ if [ "$SOLR_SSL_ENABLED" == "true" ]; then
     if [ -n "$SOLR_SSL_CLIENT_KEY_STORE_TYPE" ]; then
       SOLR_SSL_OPTS+=" -Djavax.net.ssl.keyStoreType=$SOLR_SSL_CLIENT_KEY_STORE_TYPE"
     fi
+    if [ "${SOLR_SSL_RELOAD_ENABLED:-true}" == "true" ] && [ "${SOLR_SECURITY_MANAGER_ENABLED:-true}" == "true"  ]; then
+      # In this case we need to allow reads from the parent directory of the keystore
+      SOLR_SSL_OPTS+=" -Djavax.net.ssl.keyStoreParentPath=$SOLR_SSL_CLIENT_KEY_STORE/.."
+    fi
   else
     if [ -n "$SOLR_SSL_KEY_STORE" ]; then
       SOLR_SSL_OPTS+=" -Djavax.net.ssl.keyStore=$SOLR_SSL_KEY_STORE"

solr/bin/solr.cmd

@@ -84,11 +84,29 @@ IF NOT DEFINED SOLR_SSL_ENABLED (
   )
 )
 
+IF NOT DEFINED SOLR_SSL_RELOAD_ENABLED (
+  set "SOLR_SSL_RELOAD_ENABLED=true"
+)
+
+REM Enable java security manager by default (limiting filesystem access and other things)
+IF NOT DEFINED SOLR_SECURITY_MANAGER_ENABLED (
+  set SOLR_SECURITY_MANAGER_ENABLED=true
+)
+
 IF "%SOLR_SSL_ENABLED%"=="true" (
   set "SOLR_JETTY_CONFIG=--module=https --lib="%DEFAULT_SERVER_DIR%\solr-webapp\webapp\WEB-INF\lib\*""
   set SOLR_URL_SCHEME=https
+  IF "%SOLR_SSL_RELOAD_ENABLED%"=="true" (
+    set "SOLR_JETTY_CONFIG=!SOLR_JETTY_CONFIG! --module=ssl-reload"
+    set "SOLR_SSL_OPTS=!SOLR_SSL_OPTS! -Dsolr.keyStoreReload.enabled=true"
+  )
   IF DEFINED SOLR_SSL_KEY_STORE (
     set "SOLR_SSL_OPTS=!SOLR_SSL_OPTS! -Dsolr.jetty.keystore=%SOLR_SSL_KEY_STORE%"
+    IF "%SOLR_SSL_RELOAD_ENABLED%"=="true" (
+      IF "%SOLR_SECURITY_MANAGER_ENABLED%"=="true" (
+        set "SOLR_SSL_OPTS=!SOLR_SSL_OPTS! -Dsolr.jetty.keystoreParentPath=%SOLR_SSL_KEY_STORE%/.."
+      )
+    )
   )
 
   IF DEFINED SOLR_SSL_KEY_STORE_TYPE (
@@ -122,6 +140,11 @@ IF "%SOLR_SSL_ENABLED%"=="true" (
     IF DEFINED SOLR_SSL_CLIENT_KEY_STORE_TYPE (
       set "SOLR_SSL_OPTS=!SOLR_SSL_OPTS! -Djavax.net.ssl.keyStoreType=%SOLR_SSL_CLIENT_KEY_STORE_TYPE%"
     )
+    IF "%SOLR_SSL_RELOAD_ENABLED%"=="true" (
+      IF "%SOLR_SECURITY_MANAGER_ENABLED%"=="true" (
+        set "SOLR_SSL_OPTS=!SOLR_SSL_OPTS! -Djavax.net.ssl.keyStoreParentPath=%SOLR_SSL_CLIENT_KEY_STORE_TYPE%/.."
+      )
+    )
   ) ELSE (
     IF DEFINED SOLR_SSL_KEY_STORE (
       set "SOLR_SSL_OPTS=!SOLR_SSL_OPTS! -Djavax.net.ssl.keyStore=%SOLR_SSL_KEY_STORE%"
@@ -1077,11 +1100,6 @@ IF "%ENABLE_REMOTE_JMX_OPTS%"=="true" (
   set REMOTE_JMX_OPTS=
 )
 
-REM Enable java security manager by default (limiting filesystem access and other things)
-IF NOT DEFINED SOLR_SECURITY_MANAGER_ENABLED (
-  set SOLR_SECURITY_MANAGER_ENABLED=true
-)
-
 IF "%SOLR_SECURITY_MANAGER_ENABLED%"=="true" (
   set SECURITY_MANAGER_OPTS=-Djava.security.manager ^
 -Djava.security.policy="%SOLR_SERVER_DIR%\etc\security.policy" ^

solr/bin/solr.in.sh

@@ -179,6 +179,7 @@
 # Override Key/Trust Store types if necessary
 #SOLR_SSL_KEY_STORE_TYPE=PKCS12
 #SOLR_SSL_TRUST_STORE_TYPE=PKCS12
+#SOLR_SSL_RELOAD_ENABLED=true
 
 # Uncomment if you want to override previously defined SSL values for HTTP client
 # otherwise keep them commented and the above values will automatically be set for HTTP clients

solr/core/src/java/org/apache/solr/cli/CreateTool.java

@@ -202,6 +202,7 @@ protected void createCollection(CommandLine cli) throws Exception {
         new Http2SolrClient.Builder()
             .withIdleTimeout(30, TimeUnit.SECONDS)
             .withConnectionTimeout(15, TimeUnit.SECONDS)
+            .withKeyStoreReloadInterval(-1, TimeUnit.SECONDS)
             .withOptionalBasicAuthCredentials(
                 cli.getOptionValue(SolrCLI.OPTION_CREDENTIALS.getLongOpt()));
     String zkHost = SolrCLI.getZkHost(cli);

solr/core/src/java/org/apache/solr/cli/DeleteTool.java

@@ -107,6 +107,7 @@ protected void deleteCollection(CommandLine cli) throws Exception {
         new Http2SolrClient.Builder()
             .withIdleTimeout(30, TimeUnit.SECONDS)
             .withConnectionTimeout(15, TimeUnit.SECONDS)
+            .withKeyStoreReloadInterval(-1, TimeUnit.SECONDS)
             .withOptionalBasicAuthCredentials(cli.getOptionValue(("credentials")));
 
     String zkHost = SolrCLI.getZkHost(cli);

solr/core/src/java/org/apache/solr/cli/PostLogsTool.java

@@ -30,6 +30,7 @@
 import java.util.Map;
 import java.util.TreeMap;
 import java.util.UUID;
+import java.util.concurrent.TimeUnit;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 import java.util.stream.Collectors;
@@ -90,7 +91,9 @@ public void runImpl(CommandLine cli) throws Exception {
   public void runCommand(String baseUrl, String root, String credentials) throws IOException {
 
     Http2SolrClient.Builder builder =
-        new Http2SolrClient.Builder(baseUrl).withOptionalBasicAuthCredentials(credentials);
+        new Http2SolrClient.Builder(baseUrl)
+            .withKeyStoreReloadInterval(-1, TimeUnit.SECONDS)
+            .withOptionalBasicAuthCredentials(credentials);
     try (SolrClient client = builder.build()) {
       int rec = 0;
       UpdateRequest request = new UpdateRequest();

solr/core/src/java/org/apache/solr/cli/SolrCLI.java

@@ -416,6 +416,7 @@ public static SolrClient getSolrClient(String solrUrl, String credentials, boole
     Http2SolrClient.Builder builder =
         new Http2SolrClient.Builder(solrUrl)
             .withMaxConnectionsPerHost(32)
+            .withKeyStoreReloadInterval(-1, TimeUnit.SECONDS)
             .withOptionalBasicAuthCredentials(credentials);
 
     return builder.build();

solr/packaging/test/test_ssl.bats

@@ -29,6 +29,7 @@ teardown() {
   solr stop -all >/dev/null 2>&1
 }
 
+
 @test "start solr with ssl" {
   # Create a keystore
   export ssl_dir="${BATS_TEST_TMPDIR}/ssl"
@@ -151,8 +152,8 @@ teardown() {
   export SOLR_HOST=localhost
 
   solr start -c
-  solr auth enable -type basicAuth -credentials name:password
   solr assert --started https://localhost:${SOLR_PORT}/solr --timeout 5000
+  solr auth enable -type basicAuth -credentials name:password
 
   run curl -u name:password --basic --cacert "$ssl_dir/solr-ssl.pem" "https://localhost:${SOLR_PORT}/solr/admin/collections?action=CREATE&collection.configName=_default&name=test&numShards=2&replicationFactor=1&router.name=compositeId&wt=json"
   assert_output --partial '"status":0'
@@ -209,13 +210,13 @@ teardown() {
 
   run solr start -c
 
+  solr assert --started https://localhost:${SOLR_PORT}/solr --timeout 5000
+
   export SOLR_SSL_KEY_STORE=
   export SOLR_SSL_KEY_STORE_PASSWORD=
   export SOLR_SSL_TRUST_STORE=
   export SOLR_SSL_TRUST_STORE_PASSWORD=
 
-  solr assert --started https://localhost:${SOLR_PORT}/solr --timeout 5000
-
   run solr create -c test -s 2
   assert_output --partial "Created collection 'test'"
 
@@ -494,3 +495,117 @@ teardown() {
   run solr api -verbose -get "https://localhost:${SOLR_PORT}/solr/test/select?q=*:*&rows=0"
   assert_output --regexp '(unable to find valid certification path to requested target|Server refused connection)'
 }
+
+@test "test keystore reload" {
+  # Create a keystore
+  export ssl_dir="${BATS_TEST_TMPDIR}/ssl"
+  mkdir -p "$ssl_dir"
+  (
+    cd "$ssl_dir"
+    rm -f cert1.keystore.p12 cert1.pem cert2.keystore.p12 cert2.pem
+    # cert and keystore 1
+    keytool -genkeypair -alias cert1 -keyalg RSA -keysize 2048 -keypass secret -storepass secret -validity 9999 -keystore cert1.keystore.p12 -storetype PKCS12 -ext SAN=DNS:localhost,IP:127.0.0.1 -dname "CN=localhost, OU=Organizational Unit, O=Organization, L=Location, ST=State, C=Country"
+    openssl pkcs12 -in cert1.keystore.p12 -out cert1.pem -passin pass:secret -passout pass:secret
+
+    # cert and keystore 2
+    keytool -genkeypair -alias cert2 -keyalg RSA -keysize 2048 -keypass secret -storepass secret -validity 9999 -keystore cert2.keystore.p12 -storetype PKCS12 -ext SAN=DNS:localhost,IP:127.0.0.1 -dname "CN=localhost, OU=Organizational Unit, O=Organization, L=Location, ST=State, C=Country"
+    openssl pkcs12 -in cert2.keystore.p12 -out cert2.pem -passin pass:secret -passout pass:secret
+
+    cp cert1.keystore.p12 server1.keystore.p12
+    cp cert1.keystore.p12 server2.keystore.p12
+  )
+
+  # Set ENV_VARs so that Solr uses this keystore
+  export SOLR_SSL_ENABLED=true
+  export SOLR_SSL_KEY_STORE_PASSWORD=secret
+  export SOLR_SSL_TRUST_STORE_PASSWORD=secret
+  export SOLR_SSL_NEED_CLIENT_AUTH=true
+  export SOLR_SSL_WANT_CLIENT_AUTH=false
+  export SOLR_HOST=localhost
+
+  # server1 will run on $SOLR_PORT and will use server1.keystore
+  export SOLR_SSL_KEY_STORE=$ssl_dir/server1.keystore.p12
+  export SOLR_SSL_TRUST_STORE=$ssl_dir/server1.keystore.p12
+  solr start -c -a "-Dsolr.jetty.sslContext.reload.scanInterval=1 -DsocketTimeout=5000"
+  solr assert --started https://localhost:${SOLR_PORT}/solr --timeout 5000
+
+  # server2 will run on $SOLR2_PORT and will use server2.keystore. Initially, this is the same as server1.keystore
+  export SOLR_SSL_KEY_STORE=$ssl_dir/server2.keystore.p12
+  export SOLR_SSL_TRUST_STORE=$ssl_dir/server2.keystore.p12
+  solr start -c -z localhost:${ZK_PORT} -p ${SOLR2_PORT} -a "-Dsolr.jetty.sslContext.reload.scanInterval=1 -DsocketTimeout=5000"
+  solr assert --started https://localhost:${SOLR2_PORT}/solr --timeout 5000
+
+  # "test" collection is two shards, meaning there must be communication between shards for queries (handled by http shard handler factory)
+  run solr create -c test -s 2
+  assert_output --partial "Created collection 'test'"
+
+  # "test-single-shard" is one shard and one replica, this means that one of the nodes will have to forward requests to the other
+  run solr create -c test-single-shard -s 1
+  assert_output --partial "Created collection 'test-single-shard'"
+
+  run solr api -get "https://localhost:${SOLR_PORT}/solr/test/select?q=*:*"
+  assert_output --partial '"numFound":0'
+  run solr api -get "https://localhost:${SOLR2_PORT}/solr/test/select?q=*:*"
+  assert_output --partial '"numFound":0'
+
+  run solr api -get "https://localhost:${SOLR_PORT}/solr/test-single-shard/select?q=*:*"
+  assert_output --partial '"numFound":0'
+  run solr api -get "https://localhost:${SOLR2_PORT}/solr/test-single-shard/select?q=*:*"
+  assert_output --partial '"numFound":0'
+
+  run ! curl "https://localhost:${SOLR_PORT}/solr/test/select?q=*:*"
+  run ! curl "https://localhost:${SOLR2_PORT}/solr/test/select?q=*:*"
+
+  run ! curl "https://localhost:${SOLR_PORT}/solr/test-single-shard/select?q=*:*"
+  run ! curl "https://localhost:${SOLR2_PORT}/solr/test-single-shard/select?q=*:*"
+
+  export SOLR_SSL_KEY_STORE=$ssl_dir/cert2.keystore.p12
+  export SOLR_SSL_KEY_STORE_PASSWORD=secret
+  export SOLR_SSL_TRUST_STORE=$ssl_dir/cert2.keystore.p12
+  export SOLR_SSL_TRUST_STORE_PASSWORD=secret
+
+  run ! solr api -get "https://localhost:${SOLR_PORT}/solr/test/select?q=*:*"
+
+  (
+    cd "$ssl_dir"
+    # Replace server1 keystore with client's
+    cp cert2.keystore.p12 server1.keystore.p12
+  )
+  # Give some time for the server reload
+  sleep 6
+
+  run solr healthcheck -solrUrl https://localhost:${SOLR_PORT}
+
+  # Server 2 still uses the cert1, so this request should fail
+  run ! solr api -get "https://localhost:${SOLR2_PORT}/solr/test/select?q=query2"
+
+  run ! solr healthcheck -solrUrl https://localhost:${SOLR2_PORT}
+
+  (
+    cd "$ssl_dir"
+    # Replace server2 keystore with client's
+    cp cert2.keystore.p12 server2.keystore.p12
+  )
+  # Give some time for the server reload
+  sleep 6
+
+  run solr healthcheck -solrUrl https://localhost:${SOLR_PORT}
+  run solr healthcheck -solrUrl https://localhost:${SOLR2_PORT}
+
+  run solr api -get "https://localhost:${SOLR_PORT}/solr/test/select?q=query3"
+  assert_output --partial '"numFound":0'
+
+  run solr api -get "https://localhost:${SOLR2_PORT}/solr/test/select?q=query3"
+  assert_output --partial '"numFound":0'
+
+  run solr api -get "https://localhost:${SOLR_PORT}/solr/test-single-shard/select?q=query4"
+  assert_output --partial '"numFound":0'
+
+  run solr api -get "https://localhost:${SOLR2_PORT}/solr/test-single-shard/select?q=query4"
+  assert_output --partial '"numFound":0'
+
+  run solr post -url https://localhost:${SOLR_PORT}/solr/test/update -commit ${SOLR_TIP}/example/exampledocs/books.csv
+
+  run solr api -get "https://localhost:${SOLR_PORT}/solr/test/select?q=*:*"
+  assert_output --partial '"numFound":10'
+}

solr/server/etc/jetty-ssl-context-reload.xml

@@ -0,0 +1,13 @@
+<?xml version="1.0"?>
+<!DOCTYPE Configure PUBLIC "-//Jetty//Configure//EN" "http://www.eclipse.org/jetty/configure_10_0.dtd">
+
+<Configure id="Server" class="org.eclipse.jetty.server.Server">
+    <Call name="addBean">
+        <Arg>
+            <New id="keyStoreScanner" class="org.eclipse.jetty.util.ssl.KeyStoreScanner">
+                <Arg><Ref refid="sslContextFactory"/></Arg>
+                <Set name="scanInterval"><Property name="solr.jetty.sslContext.reload.scanInterval" default="30"/></Set>
+            </New>
+        </Arg>
+    </Call>
+</Configure>

solr/server/etc/security.policy

@@ -193,6 +193,9 @@ grant {
 
   permission java.io.FilePermission "${javax.net.ssl.trustStore}", "read,readlink";
 
+  permission java.io.FilePermission "${solr.jetty.keystoreParentPath}", "read,readlink";
+  permission java.io.FilePermission "${javax.net.ssl.keyStoreParentPath}", "read,readlink";
+
   permission java.io.FilePermission "${solr.install.dir}", "read,write,delete,readlink";
   permission java.io.FilePermission "${solr.install.dir}${/}-", "read,write,delete,readlink";
   permission java.io.FilePermission "${solr.install.symDir}", "read,write,delete,readlink";

solr/server/modules/ssl-reload.mod

@@ -0,0 +1,12 @@
+[description]
+Enables the KeyStore to be reloaded when the KeyStore file changes.
+
+[tags]
+connector
+ssl
+
+[depend]
+ssl
+
+[xml]
+etc/jetty-ssl-context-reload.xml