Parallell ssl cert load (#12998)

* Add parallel ssl cert loading

Author: ezelkow1

doc/admin-guide/files/records.yaml.en.rst

@@ -4089,6 +4089,13 @@ SSL Termination
    :file:`ssl_multicert.yaml` file successfully load.  If false (``0``), SSL certificate
    load failures will not prevent |TS| from starting.
 
+.. ts:cv:: CONFIG proxy.config.ssl.server.multicert.concurrency INT 1
+
+   Controls how many threads are used to load SSL certificates from :file:`ssl_multicert.yaml`
+   during configuration reloads.  On first startup, |TS| always uses all available CPU cores
+   regardless of this setting.  Set to ``0`` to automatically use the number of hardware
+   threads.  Default ``1`` (single-threaded reloads).
+
 .. ts:cv:: CONFIG proxy.config.ssl.server.cert.path STRING /config
 
    The location of the SSL certificates and chains used for accepting

include/iocore/net/SSLMultiCertConfigLoader.h

@@ -25,10 +25,12 @@
 
 #include "iocore/net/SSLTypes.h"
 #include "tsutil/DbgCtl.h"
+#include "config/ssl_multicert.h"
 
 #include <openssl/ssl.h>
 #include <swoc/Errata.h>
 
+#include <mutex>
 #include <string>
 #include <set>
 #include <vector>
@@ -51,7 +53,7 @@ class SSLMultiCertConfigLoader
   SSLMultiCertConfigLoader(const SSLConfigParams *p) : _params(p) {}
   virtual ~SSLMultiCertConfigLoader(){};
 
-  swoc::Errata load(SSLCertLookup *lookup);
+  swoc::Errata load(SSLCertLookup *lookup, bool firstLoad = false);
 
   virtual SSL_CTX *default_server_ssl_ctx();
 
@@ -88,6 +90,12 @@ class SSLMultiCertConfigLoader
   virtual bool          _store_ssl_ctx(SSLCertLookup *lookup, const shared_SSLMultiCertConfigParams &ssl_multi_cert_params);
   bool _prep_ssl_ctx(const shared_SSLMultiCertConfigParams &sslMultCertSettings, SSLMultiCertConfigLoader::CertLoadData &data,
                      std::set<std::string> &common_names, std::unordered_map<int, std::set<std::string>> &unique_names);
+
+  void _load_items(SSLCertLookup *lookup, config::SSLMultiCertConfig::const_iterator begin,
+                   config::SSLMultiCertConfig::const_iterator end, int base_index, swoc::Errata &errata);
+
+  std::mutex _loader_mutex;
+
   virtual void _set_handshake_callbacks(SSL_CTX *ctx);
   virtual bool _setup_session_cache(SSL_CTX *ctx);
   virtual bool _setup_dialog(SSL_CTX *ctx, const SSLMultiCertConfigParams *sslMultCertSettings);

src/iocore/net/P_SSLConfig.h

@@ -66,6 +66,7 @@ struct SSLConfigParams : public ConfigInfo {
   char *cipherSuite;
   char *client_cipherSuite;
   int   configExitOnLoadError;
+  int   configLoadConcurrency;
   int   clientCertLevel;
   int   verify_depth;
   int   ssl_origin_session_cache{0};

src/iocore/net/QUICMultiCertConfigLoader.cc

@@ -45,7 +45,7 @@ QUICCertConfig::reconfigure(ConfigContext ctx)
   SSLCertLookup           *lookup = new SSLCertLookup();
 
   QUICMultiCertConfigLoader loader(params);
-  auto                      errata = loader.load(lookup);
+  auto                      errata = loader.load(lookup, _config_id == 0);
   if (!lookup->is_valid || (errata.has_severity() && errata.severity() >= ERRATA_ERROR)) {
     retStatus = false;
   }

src/iocore/net/SSLConfig.cc

@@ -46,9 +46,11 @@
 #include "mgmt/config/ConfigRegistry.h"
 
 #include <openssl/pem.h>
+#include <algorithm>
 #include <array>
 #include <cstring>
 #include <cmath>
+#include <thread>
 #include <unordered_map>
 
 int                SSLConfig::config_index                           = 0;
@@ -125,6 +127,7 @@ SSLConfigParams::reset()
   ssl_ctx_options                                      = SSL_OP_NO_SSLv2 | SSL_OP_NO_SSLv3;
   ssl_client_ctx_options                               = SSL_OP_NO_SSLv2 | SSL_OP_NO_SSLv3;
   configExitOnLoadError                                = 1;
+  configLoadConcurrency                                = 1;
 }
 
 void
@@ -431,6 +434,10 @@ SSLConfigParams::initialize()
 
   configFilePath        = ats_stringdup(RecConfigReadConfigPath("proxy.config.ssl.server.multicert.filename"));
   configExitOnLoadError = RecGetRecordInt("proxy.config.ssl.server.multicert.exit_on_load_fail").value_or(0);
+  configLoadConcurrency = RecGetRecordInt("proxy.config.ssl.server.multicert.concurrency").value_or(1);
+  if (configLoadConcurrency == 0) {
+    configLoadConcurrency = std::clamp(static_cast<int>(std::thread::hardware_concurrency()), 1, 256);
+  }
 
   {
     auto rec_str{RecGetRecordStringAlloc("proxy.config.ssl.server.private_key.path")};
@@ -671,7 +678,7 @@ SSLCertificateConfig::reconfigure(ConfigContext ctx)
     ink_hrtime_sleep(HRTIME_SECONDS(secs));
   }
 
-  auto errata = SSLMultiCertConfigLoader(params).load(lookup);
+  auto errata = SSLMultiCertConfigLoader(params).load(lookup, configid == 0);
   if (!lookup->is_valid || (errata.has_severity() && errata.severity() >= ERRATA_ERROR)) {
     retStatus = false;
   }

src/iocore/net/SSLUtils.cc

@@ -69,6 +69,8 @@
 #include <openssl/ts.h>
 #endif
 
+#include <algorithm>
+#include <thread>
 #include <utility>
 #include <string>
 #include <unistd.h>
@@ -1599,11 +1601,20 @@ SSLMultiCertConfigLoader::_store_ssl_ctx(SSLCertLookup *lookup, const shared_SSL
   SSLMultiCertConfigLoader::CertLoadData         data;
 
   if (!this->_prep_ssl_ctx(sslMultCertSettings, data, common_names, unique_names)) {
-    lookup->is_valid = false;
+    {
+      std::lock_guard<std::mutex> lock(_loader_mutex);
+      lookup->is_valid = false;
+    }
     return false;
   }
 
   std::vector<SSLLoadingContext> ctxs = this->init_server_ssl_ctx(data, sslMultCertSettings.get());
+
+  // Serialize all mutations to the shared SSLCertLookup.
+  // The expensive work above (_prep_ssl_ctx + init_server_ssl_ctx) runs
+  // without the lock, allowing parallel cert loading across threads.
+  std::lock_guard<std::mutex> lock(_loader_mutex);
+
   for (const auto &loadingctx : ctxs) {
     if (!sslMultCertSettings ||
         !this->_store_single_ssl_ctx(lookup, sslMultCertSettings, shared_SSL_CTX{loadingctx.ctx, SSL_CTX_free}, loadingctx.ctx_type,
@@ -1782,7 +1793,7 @@ SSLMultiCertConfigLoader::_store_single_ssl_ctx(SSLCertLookup *lookup, const sha
 }
 
 swoc::Errata
-SSLMultiCertConfigLoader::load(SSLCertLookup *lookup)
+SSLMultiCertConfigLoader::load(SSLCertLookup *lookup, bool firstLoad)
 {
   const SSLConfigParams *params = this->_params;
 
@@ -1806,10 +1817,69 @@ SSLMultiCertConfigLoader::load(SSLCertLookup *lookup)
   }
 
   swoc::Errata errata(ERRATA_NOTE);
-  int          item_num = 0;
 
-  for (const auto &item : parse_result.value) {
+  static constexpr int MAX_LOAD_THREADS = 256;
+
+  int num_threads = params->configLoadConcurrency;
+  if (firstLoad) {
+    num_threads = std::clamp(static_cast<int>(std::thread::hardware_concurrency()), 1, MAX_LOAD_THREADS);
+  }
+  num_threads = std::min(num_threads, static_cast<int>(parse_result.value.size()));
+
+  if (num_threads > 1 && parse_result.value.size() > 1) {
+    std::size_t bucket_size = parse_result.value.size() / num_threads;
+    std::size_t remainder   = parse_result.value.size() % num_threads;
+    auto        current     = parse_result.value.cbegin();
+
+    std::vector<std::thread> threads;
+    Note("(%s) loading %zu certs with %d threads", this->_debug_tag(), parse_result.value.size(), num_threads);
+
+    for (int t = 0; t < num_threads; ++t) {
+      std::size_t this_bucket = bucket_size + (static_cast<std::size_t>(t) < remainder ? 1 : 0);
+      auto        end         = current + this_bucket;
+      int         base_index  = static_cast<int>(std::distance(parse_result.value.cbegin(), current));
+      threads.emplace_back(&SSLMultiCertConfigLoader::_load_items, this, lookup, current, end, base_index, std::ref(errata));
+      current = end;
+    }
+
+    for (auto &th : threads) {
+      th.join();
+    }
+
+    Note("(%s) loaded %zu certs in %d threads", this->_debug_tag(), parse_result.value.size(), num_threads);
+  } else {
+    _load_items(lookup, parse_result.value.cbegin(), parse_result.value.cend(), 0, errata);
+    Note("(%s) loaded %zu certs (single-threaded)", this->_debug_tag(), parse_result.value.size());
+  }
+
+  // We *must* have a default context even if it can't possibly work. The default context is used to
+  // bootstrap the SSL handshake so that we can subsequently do the SNI lookup to switch to the real
+  // context.
+  if (lookup->ssl_default == nullptr) {
+    shared_SSLMultiCertConfigParams sslMultiCertSettings(new SSLMultiCertConfigParams);
+    sslMultiCertSettings->addr = ats_strdup("*");
+    if (!this->_store_ssl_ctx(lookup, sslMultiCertSettings)) {
+      errata.note(ERRATA_ERROR, "failed set default context");
+    }
+  }
+
+  return errata;
+}
+
+void
+SSLMultiCertConfigLoader::_load_items(SSLCertLookup *lookup, config::SSLMultiCertConfig::const_iterator begin,
+                                      config::SSLMultiCertConfig::const_iterator end, int base_index, swoc::Errata &errata)
+{
+  // Each thread needs its own elevated privileges since POSIX capabilities are per-thread
+  uint32_t elevate_setting = 0;
+  elevate_setting          = RecGetRecordInt("proxy.config.ssl.cert.load_elevated").value_or(0);
+  ElevateAccess elevate_access(elevate_setting ? ElevateAccess::FILE_PRIVILEGE : 0);
+
+  int item_num = base_index;
+  for (auto it = begin; it != end; ++it) {
     item_num++;
+    const auto &item = *it;
+
     shared_SSLMultiCertConfigParams sslMultiCertSettings = std::make_shared<SSLMultiCertConfigParams>();
 
     if (!item.ssl_cert_name.empty()) {
@@ -1846,25 +1916,14 @@ SSLMultiCertConfigLoader::load(SSLCertLookup *lookup)
     // There must be a certificate specified unless the tunnel action is set.
     if (sslMultiCertSettings->cert || sslMultiCertSettings->opt == SSLCertContextOption::OPT_TUNNEL) {
       if (!this->_store_ssl_ctx(lookup, sslMultiCertSettings)) {
+        std::lock_guard<std::mutex> lock(_loader_mutex);
         errata.note(ERRATA_ERROR, "Failed to load certificate at item {}", item_num);
       }
     } else {
+      std::lock_guard<std::mutex> lock(_loader_mutex);
       errata.note(ERRATA_WARN, "No ssl_cert_name specified and no tunnel action set at item {}", item_num);
     }
   }
-
-  // We *must* have a default context even if it can't possibly work. The default context is used to
-  // bootstrap the SSL handshake so that we can subsequently do the SNI lookup to switch to the real
-  // context.
-  if (lookup->ssl_default == nullptr) {
-    shared_SSLMultiCertConfigParams sslMultiCertSettings(new SSLMultiCertConfigParams);
-    sslMultiCertSettings->addr = ats_strdup("*");
-    if (!this->_store_ssl_ctx(lookup, sslMultiCertSettings)) {
-      errata.note(ERRATA_ERROR, "failed set default context");
-    }
-  }
-
-  return errata;
 }
 
 // Release SSL_CTX and the associated data. This works for both

src/records/RecordsConfig.cc

@@ -1182,7 +1182,9 @@ static constexpr RecordElement RecordsConfig[] =
   {RECT_CONFIG, "proxy.config.ssl.server.multicert.filename", RECD_STRING, ts::filename::SSL_MULTICERT, RECU_RESTART_TS, RR_NULL, RECC_NULL, nullptr, RECA_NULL}
   ,
   {RECT_CONFIG, "proxy.config.ssl.server.multicert.exit_on_load_fail", RECD_INT, "1", RECU_RESTART_TS, RR_NULL, RECC_INT, "[0-1]", RECA_NULL}
-,
+  ,
+  {RECT_CONFIG, "proxy.config.ssl.server.multicert.concurrency", RECD_INT, "1", RECU_RESTART_TS, RR_NULL, RECC_INT, "[0-256]", RECA_NULL}
+  ,
   {RECT_CONFIG, "proxy.config.ssl.servername.filename", RECD_STRING, ts::filename::SNI, RECU_RESTART_TS, RR_NULL, RECC_NULL, nullptr, RECA_NULL}
   ,
   {RECT_CONFIG, "proxy.config.ssl.server.ticket_key.filename", RECD_STRING, nullptr, RECU_DYNAMIC, RR_NULL, RECC_NULL, nullptr, RECA_NULL}

tests/gold_tests/tls/ssl_multicert_loader.test.py

@@ -22,7 +22,7 @@
 
 ts = Test.MakeATSProcess("ts", enable_tls=True)
 server = Test.MakeOriginServer("server")
-server2 = Test.MakeOriginServer("server3")
+server2 = Test.MakeOriginServer("server2")
 request_header = {"headers": f"GET / HTTP/1.1\r\nHost: {sni_domain}\r\n\r\n", "timestamp": "1469733493.993", "body": ""}
 
 response_header = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": ""}
@@ -123,3 +123,45 @@
 ts2.Disk.traffic_out.Content = Testers.ExcludesExpression(
     'Traffic Server is fully initialized', 'process should fail when invalid certificate specified')
 ts2.Disk.diags_log.Content = Testers.IncludesExpression('EMERGENCY: failed to load SSL certificate file', 'check diags.log"')
+
+##########################################################################
+# Verify parallel cert loading on startup (firstLoad uses hardware_concurrency,
+# not the configured concurrency value, so the thread count is host-dependent)
+
+ts3 = Test.MakeATSProcess("ts3", enable_tls=True)
+server3 = Test.MakeOriginServer("server3")
+server3.addResponse("sessionlog.json", request_header, response_header)
+
+ts3.Disk.records_config.update(
+    {
+        'proxy.config.ssl.server.cert.path': f'{ts3.Variables.SSLDir}',
+        'proxy.config.ssl.server.private_key.path': f'{ts3.Variables.SSLDir}',
+    })
+
+ts3.addDefaultSSLFiles()
+
+ts3.Disk.remap_config.AddLine(f'map / http://127.0.0.1:{server3.Variables.Port}')
+
+# Need at least 2 certs for multi-threading to kick in
+ts3.Disk.ssl_multicert_yaml.AddLines(
+    """
+ssl_multicert:
+  - dest_ip: "*"
+    ssl_cert_name: server.pem
+    ssl_key_name: server.key
+  - ssl_cert_name: server.pem
+    ssl_key_name: server.key
+""".split("\n"))
+
+tr5 = Test.AddTestRun("Verify parallel cert loading")
+tr5.Processes.Default.StartBefore(ts3)
+tr5.Processes.Default.StartBefore(server3)
+tr5.StillRunningAfter = ts3
+tr5.StillRunningAfter = server3
+tr5.MakeCurlCommand(
+    f"-q -s -v -k --resolve '{sni_domain}:{ts3.Variables.ssl_port}:127.0.0.1' https://{sni_domain}:{ts3.Variables.ssl_port}",
+    ts=ts3)
+tr5.Processes.Default.ReturnCode = 0
+tr5.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could Not Connect", "Check response")
+tr5.Processes.Default.Streams.stderr = Testers.IncludesExpression(f"CN={sni_domain}", "Check response")
+ts3.Disk.diags_log.Content = Testers.IncludesExpression('loaded 2 certs', 'verify certs were loaded successfully')