refactor(letsencrypt): add centralized cert store and remove old main…

… storage

Cargo.toml

@@ -15,9 +15,10 @@ repository = "https://github.com/luizfonseca/proksi"
 
 [profile.dev]
 opt-level = 0
+incremental = true
 
 [profile.release]
-opt-level = "z"
+opt-level = 2
 strip = "symbols" # Automatically strip symbols from the binary.
 lto = true        # Enable link-time optimization.
 debug = false

README.md

@@ -302,9 +302,24 @@ See [the examples folder](./examples) to learn about how to use Proksi.
 
 ## Performance & Benchmarks
 
-TBA.
+Early tests are promising, but we need to do more testing to see how Proksi performs under *real* load. There are also some optimizations that can be done to improve performance in the long term, though the focus is on making it feature complete first.
 
-It's based on [Pingora](https://github.com/cloudflare/pingora), so it should be fast if cloudflare is using it.
+An sample run from the `wrk` benchmark on the simple `/ping` endpoint shows the following results:
+
+```bash
+wrk -c 50 -t 4 -d 30s http://127.0.0.1/ping
+
+Running 30s test @ http://127.0.0.1/ping
+  4 threads and 50 connections
+  Thread Stats   Avg      Stdev     Max   +/- Stdev
+    Latency   813.86us  157.65us   2.63ms   93.02%
+    Req/Sec    14.76k   276.97    15.82k    83.14%
+  1768084 requests in 30.10s, 217.52MB read
+Requests/sec:  58740.48
+Transfer/sec:      7.23MB
+```
+
+It's also based on [Pingora](https://github.com/cloudflare/pingora), so it should be fast if cloudflare is using it.
 
 
 ## Why build another proxy...?

src/main.rs

@@ -1,14 +1,14 @@
-use std::{collections::HashMap, sync::Arc};
+use std::sync::Arc;
 
 use ::pingora::{server::Server, services::background::background_service};
 use arc_swap::ArcSwap;
 use config::load_proxy_config;
-use instant_acme::KeyAuthorization;
+
 use once_cell::sync::Lazy;
 use pingora::listeners::TlsSettings;
 use pingora_load_balancing::{health_check::TcpHealthCheck, LoadBalancer};
 use pingora_proxy::http_proxy_service;
-use stores::routes::RouteStore;
+use stores::{certificates::CertificatesStore, routes::RouteStore};
 
 mod config;
 mod docker;
@@ -17,58 +17,12 @@ mod services;
 mod stores;
 mod tools;
 
-#[derive(Debug)]
-pub struct Storage {
-    orders: HashMap<String, (String, String, KeyAuthorization)>,
-    certificates: HashMap<String, String>,
-}
-
 /// Static reference to the route store that can be shared across threads
 pub static ROUTE_STORE: Lazy<ArcSwap<RouteStore>> =
-    Lazy::new(|| ArcSwap::new(Arc::new(RouteStore::new())));
-
-pub type StorageArc = Arc<tokio::sync::Mutex<Storage>>;
-
-impl Storage {
-    pub fn new() -> Self {
-        Storage {
-            orders: HashMap::new(),
-            certificates: HashMap::new(),
-        }
-    }
-
-    pub fn add_order(
-        &mut self,
-        identifier: String,
-        token: String,
-        url: String,
-        key_auth: KeyAuthorization,
-    ) {
-        self.orders.insert(identifier, (token, url, key_auth));
-    }
-
-    pub fn add_certificate(&mut self, host: String, certificate: String) {
-        self.certificates.insert(host, certificate);
-    }
+    Lazy::new(|| ArcSwap::from_pointee(RouteStore::new()));
 
-    pub fn get_certificate(&self, host: &str) -> Option<&String> {
-        self.certificates.get(host)
-    }
-
-    pub fn get_orders(&self) -> &HashMap<String, (String, String, KeyAuthorization)> {
-        &self.orders
-    }
-
-    pub fn get_order(&self, order: &str) -> Option<&(String, String, KeyAuthorization)> {
-        self.orders.get(order)
-    }
-}
-
-impl Default for Storage {
-    fn default() -> Self {
-        Self::new()
-    }
-}
+pub static CERT_STORE: Lazy<ArcSwap<CertificatesStore>> =
+    Lazy::new(|| ArcSwap::from_pointee(CertificatesStore::new()));
 
 fn main() -> Result<(), anyhow::Error> {
     // Loads configuration from command-line, YAML or TOML sources
@@ -110,9 +64,7 @@ fn main() -> Result<(), anyhow::Error> {
         pingora_server.add_service(health_check_service);
     }
 
-    let storage = Arc::new(tokio::sync::Mutex::new(Storage::new()));
-
-    let certificate_store = proxy_server::cert_store::CertStore::new(storage.clone());
+    let certificate_store = proxy_server::cert_store::CertStore::new();
 
     // Setup tls settings and Enable HTTP/2
     let mut tls_settings = TlsSettings::with_callbacks(certificate_store).unwrap();
@@ -125,9 +77,8 @@ fn main() -> Result<(), anyhow::Error> {
 
     // Service: Lets Encrypt HTTP Challenge/Certificate renewal
     let letsencrypt_http = services::letsencrypt::http01::HttpLetsencrypt::new(
-        &ROUTE_STORE.load().get_route_keys(),
+        &router_store.get_route_keys(),
         "[email protected]",
-        storage.clone(),
     );
     let le_service = background_service("letsencrypt", letsencrypt_http);
 

src/proxy_server/cert_store.rs

@@ -3,21 +3,13 @@ use pingora::listeners::TlsAccept;
 use pingora_openssl::{pkey::PKey, ssl::NameType, x509::X509};
 use tracing::debug;
 
-use crate::StorageArc;
-
 /// Provides the correct certificates when performing the SSL handshake
 #[derive(Debug)]
-pub struct CertStore {
-    /// The path to the directory containing the certificates
-    // certs: HashMap<String, CertValue>,
-    // orders: HashMap<String, OrderPayload>,
-    //
-    pub storage: StorageArc,
-}
+pub struct CertStore {}
 
 impl CertStore {
-    pub fn new(storage: StorageArc) -> Box<Self> {
-        Box::new(CertStore { storage })
+    pub fn new() -> Box<Self> {
+        Box::new(CertStore {})
     }
 }
 

src/services/letsencrypt/http01.rs

@@ -1,18 +1,48 @@
 use std::{
+    collections::HashMap,
     fs::{create_dir_all, File},
     io::{Read, Write},
     thread::sleep,
     time::Duration,
 };
 
 use async_trait::async_trait;
-use instant_acme::{AccountCredentials, ChallengeType, LetsEncrypt, Order};
+use instant_acme::{AccountCredentials, ChallengeType, KeyAuthorization, LetsEncrypt, Order};
 use pingora::{server::ShutdownWatch, services::background::BackgroundService};
 use rcgen::KeyPair;
 use serde::{Deserialize, Serialize};
 use tracing::info;
 
-use crate::StorageArc;
+#[derive(Debug)]
+struct Storage {
+    orders: HashMap<String, (String, String, KeyAuthorization)>,
+}
+
+impl Storage {
+    pub fn new() -> Self {
+        Storage {
+            orders: HashMap::new(),
+        }
+    }
+
+    pub fn add_order(
+        &mut self,
+        identifier: String,
+        token: String,
+        url: String,
+        key_auth: KeyAuthorization,
+    ) {
+        self.orders.insert(identifier, (token, url, key_auth));
+    }
+
+    pub fn get_orders(&self) -> &HashMap<String, (String, String, KeyAuthorization)> {
+        &self.orders
+    }
+
+    pub fn _get_order(&self, order: &str) -> Option<&(String, String, KeyAuthorization)> {
+        self.orders.get(order)
+    }
+}
 
 #[derive(Serialize, Deserialize)]
 struct HostCertificate {
@@ -37,22 +67,22 @@ struct HostOrder {
 }
 
 #[derive(Debug)]
-pub struct HttpLetsencrypt {
+pub struct HttpLetsencryptService {
     challenge_type: ChallengeType,
     url: String,
     contact: String,
     hosts: Vec<String>,
-    cert_store: StorageArc,
+    cert_store: Storage,
 }
 
-impl HttpLetsencrypt {
-    pub fn new(hosts: &[String], contact: &str, cert_store: StorageArc) -> Self {
-        HttpLetsencrypt {
+impl HttpLetsencryptService {
+    pub fn new(hosts: &[String], contact: &str) -> Self {
+        HttpLetsencryptService {
             challenge_type: ChallengeType::Http01,
             url: LetsEncrypt::Staging.url().to_string(),
             contact: contact.to_string(),
             hosts: hosts.to_vec(),
-            cert_store,
+            cert_store: Storage::new(),
         }
     }
 
@@ -141,7 +171,10 @@ impl HttpLetsencrypt {
     }
 
     /// Create challenges for the order
-    async fn create_challenges_from_order(&self, excluded_hosts: Vec<String>) -> Result<Order, ()> {
+    async fn create_challenges_from_order(
+        &mut self,
+        excluded_hosts: Vec<String>,
+    ) -> Result<Order, ()> {
         println!("Creating challenges from order");
         let order = self.create_order(excluded_hosts).await;
         if order.is_err() {
@@ -171,8 +204,9 @@ impl HttpLetsencrypt {
 
             let key_auth = order_result.key_authorization(challenge);
 
-            let mut lkd = self.cert_store.lock().await;
-            lkd.add_order(
+            // let mut store = self.cert_store;
+
+            self.cert_store.add_order(
                 identifier.clone(),
                 challenge.token.clone(),
                 challenge.url.clone(),
@@ -200,20 +234,37 @@ impl HttpLetsencrypt {
     }
 }
 
+#[derive(Debug)]
+pub struct HttpLetsencrypt {
+    contact: String,
+    hosts: Vec<String>,
+}
+
+impl HttpLetsencrypt {
+    pub fn new(hosts: &[String], contact: &str) -> Self {
+        HttpLetsencrypt {
+            contact: contact.to_string(),
+            hosts: hosts.to_vec(),
+        }
+    }
+}
+
 #[async_trait]
 impl BackgroundService for HttpLetsencrypt {
     async fn start(&self, _shutdown: ShutdownWatch) -> () {
         info!(service = "LetsEncrypt", "Background service started");
 
+        let mut svc = HttpLetsencryptService::new(&self.hosts, &self.contact);
+
         // create required folders if they don't exist yet
-        create_dir_all(self.challenges_path()).unwrap();
-        create_dir_all(self.certificates_path()).unwrap();
-        create_dir_all(self.account_path()).unwrap();
-        create_dir_all(self.orders_path()).unwrap();
+        create_dir_all(svc.challenges_path()).unwrap();
+        create_dir_all(svc.certificates_path()).unwrap();
+        create_dir_all(svc.account_path()).unwrap();
+        create_dir_all(svc.orders_path()).unwrap();
 
         // Check if we already have a challenge file
         let mut excluded_hosts = Vec::new();
-        for host in self.hosts.iter() {
+        for host in svc.hosts.iter() {
             let file = std::fs::File::open(format!("./data/challenges/{}/meta.csv", host));
 
             if file.is_ok() {
@@ -222,13 +273,13 @@ impl BackgroundService for HttpLetsencrypt {
             }
         }
 
-        if excluded_hosts.len() == self.hosts.len() {
+        if excluded_hosts.len() == svc.hosts.len() {
             info!("All hosts have a challenge file");
             return;
         }
 
         // Creates order if there are outstanding hosts to check
-        let order = self
+        let order = svc
             .create_challenges_from_order(excluded_hosts.clone())
             .await;
 
@@ -240,23 +291,21 @@ impl BackgroundService for HttpLetsencrypt {
         let mut order = order.unwrap();
 
         // 1. persist order to disk
-        let mut file = File::create(format!("{}/meta.txt", self.orders_path())).unwrap();
+        let mut file = File::create(format!("{}/meta.txt", svc.orders_path())).unwrap();
         let contents = format!("{:?}", order.url());
         file.write_all(contents.as_bytes()).unwrap();
         file.flush().unwrap();
 
-        let lkd = self.cert_store.lock().await;
-
-        if lkd.get_orders().is_empty() {
+        if svc.cert_store.get_orders().is_empty() {
             info!("No orders to check");
             return;
         }
 
         // write challenges to disk
-        for (key, value) in lkd.get_orders().iter() {
+        for (key, value) in svc.cert_store.get_orders().iter() {
             let (token, url, key_auth) = value;
             // Create a new folder for the challenge
-            create_dir_all(format!("{}/{}", self.challenges_path(), key)).unwrap();
+            create_dir_all(format!("{}/{}", svc.challenges_path(), key)).unwrap();
             let mut file = File::create(format!("./data/challenges/{}/meta.csv", key)).unwrap();
             let contents = format!("{};{};{}", url, key_auth.as_str(), token);
 
@@ -288,7 +337,7 @@ impl BackgroundService for HttpLetsencrypt {
             retry_delay *= 2;
         }
 
-        let non_excluded_hosts = self
+        let non_excluded_hosts = svc
             .hosts
             .iter()
             .filter(|&host| !excluded_hosts.contains(host))
@@ -331,11 +380,11 @@ impl BackgroundService for HttpLetsencrypt {
 
         for host in non_excluded_hosts.iter() {
             //create folder for the certificate
-            create_dir_all(format!("{}/{}", self.certificates_path(), host)).unwrap();
+            create_dir_all(format!("{}/{}", svc.certificates_path(), host)).unwrap();
             let mut crt_file =
-                File::create(format!("{}/{}/cert.pem", self.certificates_path(), host)).unwrap();
+                File::create(format!("{}/{}/cert.pem", svc.certificates_path(), host)).unwrap();
             let mut key_file =
-                File::create(format!("{}/{}/key.pem", self.certificates_path(), host)).unwrap();
+                File::create(format!("{}/{}/key.pem", svc.certificates_path(), host)).unwrap();
 
             crt_file.write_all(cert_chain.as_bytes()).unwrap();
             key_file.write_all(kp.serialize_pem().as_bytes()).unwrap();