Merge pull request #826 from flavio/cert-reload

feat: automatic TLS certificate reload

Author: flavio

Cargo.lock

@@ -29,7 +29,7 @@ opentelemetry = { version = "0.23.0", default-features = false, features = [
 ] }
 opentelemetry_sdk = { version = "0.23.0", features = ["rt-tokio"] }
 pprof = { version = "0.13", features = ["prost-codec"] }
-policy-evaluator = { git = "https://github.com/kubewarden/policy-evaluator", tag = "v0.18.1" }
+policy-evaluator = { git = "https://github.com/kubewarden/policy-evaluator", tag = "v0.18.2" }
 rustls-pki-types = { version = "1", features = ["alloc"] }
 rayon = "1.10"
 regex = "1.10"
@@ -55,9 +55,22 @@ jemalloc_pprof = "0.4.1"
 tikv-jemalloc-ctl = "0.5.4"
 rhai = { version = "1.19.0", features = ["sync"] }
 
+[target.'cfg(target_os = "linux")'.dependencies]
+inotify = "0.10"
+tokio-stream = "0.1.15"
+
 [dev-dependencies]
 mockall = "0.12"
 rstest = "0.21"
 tempfile = "3.10.1"
 tower = { version = "0.4", features = ["util"] }
 http-body-util = "0.1.1"
+
+[target.'cfg(target_os = "linux")'.dev-dependencies]
+rcgen = { version = "0.13", features = ["crypto"] }
+openssl = "0.10"
+reqwest = { version = "0.12", default-features = false, features = [
+  "charset",
+  "http2",
+  "rustls-tls-manual-roots",
+] }

Cargo.toml

@@ -39,14 +39,18 @@ use tokio::{
 };
 use tower_http::trace::{self, TraceLayer};
 
+// This is required by certificate hot reload when using inotify, which is available only on linux
+#[cfg(target_os = "linux")]
+use tokio_stream::StreamExt;
+
 use crate::api::handlers::{
     audit_handler, pprof_get_cpu, pprof_get_heap, readiness_handler, validate_handler,
     validate_raw_handler,
 };
 use crate::api::state::ApiServerState;
 use crate::evaluation::precompiled_policy::{PrecompiledPolicies, PrecompiledPolicy};
 use crate::policy_downloader::{Downloader, FetchedPolicies};
-use config::Config;
+use config::{Config, TlsConfig};
 
 use tikv_jemallocator::Jemalloc;
 
@@ -193,9 +197,7 @@ impl PolicyServer {
         });
 
         let tls_config = if let Some(tls_config) = config.tls_config {
-            let rustls_config =
-                RustlsConfig::from_pem_file(tls_config.cert_file, tls_config.key_file).await?;
-            Some(rustls_config)
+            Some(create_tls_config_and_watch_certificate_changes(tls_config).await?)
         } else {
             None
         };
@@ -269,6 +271,88 @@ impl PolicyServer {
     }
 }
 
+/// There's no watching of the certificate files on non-linux platforms
+/// since we rely on inotify to watch for changes
+#[cfg(not(target_os = "linux"))]
+async fn create_tls_config_and_watch_certificate_changes(
+    tls_config: TlsConfig,
+) -> Result<RustlsConfig> {
+    let cfg = RustlsConfig::from_pem_file(tls_config.cert_file, tls_config.key_file).await?;
+    Ok(cfg)
+}
+
+/// Return the RustlsConfig and watch for changes in the certificate files
+/// using inotify.
+/// When a both the certificate and its key are changed, the RustlsConfig is reloaded,
+/// causing the https server to use the new certificate.
+///
+/// Relying on inotify is only available on linux
+#[cfg(target_os = "linux")]
+async fn create_tls_config_and_watch_certificate_changes(
+    tls_config: TlsConfig,
+) -> Result<RustlsConfig> {
+    let cert_file = tls_config.cert_file.clone();
+    let key_file = tls_config.key_file.clone();
+
+    let rust_config =
+        RustlsConfig::from_pem_file(tls_config.cert_file, tls_config.key_file).await?;
+    let reloadable_rust_config = rust_config.clone();
+
+    let inotify =
+        inotify::Inotify::init().map_err(|e| anyhow!("Cannot initialize inotify: {e}"))?;
+    let cert_watch = inotify
+        .watches()
+        .add(cert_file.clone(), inotify::WatchMask::MODIFY)
+        .map_err(|e| anyhow!("Cannot watch certificate file: {e}"))?;
+    let key_watch = inotify
+        .watches()
+        .add(key_file.clone(), inotify::WatchMask::MODIFY)
+        .map_err(|e| anyhow!("Cannot watch key file: {e}"))?;
+
+    let buffer = [0; 1024];
+    let stream = inotify
+        .into_event_stream(buffer)
+        .map_err(|e| anyhow!("Cannot create inotify event stream: {e}"))?;
+
+    tokio::spawn(async move {
+        tokio::pin!(stream);
+        let mut cert_changed = false;
+        let mut key_changed = false;
+
+        while let Some(event) = stream.next().await {
+            let event = match event {
+                Ok(event) => event,
+                Err(e) => {
+                    warn!("Cannot read inotify event: {e}");
+                    continue;
+                }
+            };
+
+            if event.wd == cert_watch {
+                info!("TLS certificate file has been modified");
+                cert_changed = true;
+            }
+            if event.wd == key_watch {
+                info!("TLS key file has been modified");
+                key_changed = true;
+            }
+
+            if key_changed && cert_changed {
+                info!("reloading TLS certificate");
+
+                cert_changed = false;
+                key_changed = false;
+                reloadable_rust_config
+                    .reload_from_pem_file(cert_file.clone(), key_file.clone())
+                    .await
+                    .expect("Cannot reload TLS certificate"); //  we want to panic here
+            }
+        }
+    });
+
+    Ok(rust_config)
+}
+
 fn precompile_policies(
     engine: &wasmtime::Engine,
     fetched_policies: &FetchedPolicies,