feat: add tunnel cert rotation

packages/api/provision/src/route/mod.rs

@@ -6,6 +6,7 @@ use uuid::Uuid;
 
 pub mod datacenters;
 pub mod servers;
+pub mod tunnel;
 
 define_router! {
 	routes: {
@@ -28,5 +29,11 @@ define_router! {
 				internal_endpoint: true,
 			),
 		},
+
+		"tunnel" / "tls": {
+			GET: tunnel::tls(
+				internal_endpoint: true,
+			),
+		}
 	},
 }

packages/api/provision/src/route/tunnel.rs

@@ -0,0 +1,24 @@
+use api_helper::{anchor::WatchIndexQuery, ctx::Ctx};
+use rivet_api::models;
+use rivet_operation::prelude::*;
+
+use crate::auth::Auth;
+
+// MARK: GET /tunnel/tls
+pub async fn tls(
+	ctx: Ctx<Auth>,
+	_watch_index: WatchIndexQuery,
+) -> GlobalResult<models::ProvisionTunnelGetTlsResponse> {
+	ctx.auth().server()?;
+
+	let tunnel_tls_res = ctx.op(cluster::ops::tunnel::tls_get::Input {}).await?;
+
+	let tls_config = &ctx.config().server()?.tls()?;
+	let ca_cert_pem = tls_config.root_ca_cert_pem.read();
+
+	Ok(models::ProvisionTunnelGetTlsResponse {
+		cert_pem: tunnel_tls_res.cert_pem.clone(),
+		root_ca_cert_pem: ca_cert_pem.clone(),
+		private_key_pem: tunnel_tls_res.private_key_pem.clone(),
+	})
+}

packages/common/config/src/config/server/mod.rs

@@ -242,8 +242,7 @@ pub struct CloudflareZone {
 #[serde(rename_all = "snake_case", deny_unknown_fields)]
 pub struct Tls {
 	pub root_ca_cert_pem: Secret<String>,
-	pub cert_locally_signed_job_cert_pem: Secret<String>,
-	pub cert_locally_signed_job_key_pem: Secret<String>,
+	pub root_ca_key_pem: Secret<String>,
 	pub acme: TlsAcme,
 }
 

packages/common/service-manager/src/lib.rs

@@ -56,6 +56,22 @@ impl ServiceKind {
 			Cron(config) => ServiceBehavior::Cron(config.clone()),
 		}
 	}
+
+	pub fn eq(&self, other: &Self) -> bool {
+		use ServiceKind::*;
+
+		match (self, other) {
+			(ApiPublic, ApiPublic)
+			| (ApiEdge, ApiEdge)
+			| (ApiPrivate, ApiPrivate)
+			| (Standalone, Standalone)
+			| (Singleton, Singleton)
+			| (Oneshot, Oneshot)
+			| (Core, Core) => true,
+			(Cron(_), Cron(_)) => true,
+			_ => false,
+		}
+	}
 }
 
 /// Defines how a service should be ran.

packages/common/util/core/src/math.rs

@@ -1,5 +1,5 @@
 /// Divide integers of any type, rounding up. Panics on dividing by 0.
-#[deprecated(note="use x.div_ceil(y) instead")]
+#[deprecated(note = "use x.div_ceil(y) instead")]
 #[macro_export]
 macro_rules! div_up {
 	($a:expr, $b:expr) => {

packages/infra/client/actor-kv/src/lib.rs

@@ -9,13 +9,13 @@ pub use entry::Entry;
 use entry::{EntryBuilder, SubKey};
 use foundationdb::{self as fdb, directory::Directory, tuple::Subspace};
 use futures_util::{StreamExt, TryStreamExt};
+use indexmap::IndexMap;
 use key::Key;
 use list_query::ListLimitReached;
 pub use list_query::ListQuery;
 pub use metadata::Metadata;
 use pegboard::protocol;
 use prost::Message;
-use indexmap::IndexMap;
 use utils::{owner_segment, validate_entries, validate_keys, TransactionExt};
 
 mod entry;
@@ -238,7 +238,8 @@ impl ActorKv {
 
 								// Short circuit when limit is reached. This relies on data from the stream
 								// being in order.
-								if size == limit && matches!(entry, indexmap::map::Entry::Vacant(_)) {
+								if size == limit && matches!(entry, indexmap::map::Entry::Vacant(_))
+								{
 									return Err(ListLimitReached(acc).into());
 								}
 

packages/infra/client/actor-kv/src/list_query.rs

@@ -1,7 +1,7 @@
 use anyhow::*;
 use foundationdb::tuple::Subspace;
-use serde::Deserialize;
 use indexmap::IndexMap;
+use serde::Deserialize;
 
 use crate::{
 	entry::EntryBuilder,

packages/infra/client/isolate-v8-runner/src/ext/kv.rs

@@ -1,8 +1,8 @@
 use std::{collections::HashMap, future::Future, sync::Arc};
 
 use deno_core::{error::AnyError, op2, JsBuffer, OpState, ToJsBuffer};
-use serde::Serialize;
 use pegboard_actor_kv as actor_kv;
+use serde::Serialize;
 
 type FakeMap<T, U> = Box<[(T, U)]>;
 

packages/infra/client/isolate-v8-runner/src/isolate.rs

@@ -10,7 +10,6 @@ use std::{
 	thread::JoinHandle,
 };
 
-use pegboard_actor_kv::ActorKv;
 use anyhow::*;
 use deno_core::{
 	error::JsError, v8, v8::CreateParams, ModuleId, ModuleSpecifier, StaticModuleLoader,
@@ -26,6 +25,7 @@ use deno_runtime::{
 };
 use nix::{libc, unistd::pipe};
 use pegboard::protocol;
+use pegboard_actor_kv::ActorKv;
 use pegboard_config::isolate_runner as config;
 use tokio::{fs, sync::mpsc};
 use uuid::Uuid;

packages/infra/client/isolate-v8-runner/src/main.rs

@@ -7,13 +7,13 @@ use std::{
 	time::Duration,
 };
 
-use pegboard_actor_kv::ActorKv;
 use anyhow::*;
 use deno_core::{v8_set_flags, JsRuntime};
 use deno_runtime::worker::MainWorkerTerminateHandle;
 use foundationdb as fdb;
 use futures_util::{stream::SplitStream, SinkExt, StreamExt};
 use pegboard::protocol;
+use pegboard_actor_kv::ActorKv;
 use pegboard_config::{isolate_runner::Config, runner_protocol};
 use tokio::{
 	fs,

packages/infra/server/Cargo.toml

@@ -17,6 +17,7 @@ colored_json = "5.0.0"
 global-error.workspace = true
 include_dir = "0.7.4"
 indoc = "2.0.5"
+reqwest = "0.12.9"
 rivet-api.workspace = true
 rivet-migrate.workspace = true
 rivet-pools.workspace = true
@@ -58,6 +59,7 @@ workflow-gc.workspace = true
 workflow-metrics-publish.workspace = true
 
 # Cron
+cluster-tunnel-tls-renew.workspace = true
 telemetry-beacon.workspace = true
 user-delete-pending.workspace = true
 
@@ -67,16 +69,14 @@ api-monolith-public.workspace = true
 
 # Oneshot
 build-default-create.workspace = true
+chirp-client.workspace = true
+chirp-workflow.workspace = true
+cloud-default-create.workspace = true
 cluster-default-update.workspace = true
 pegboard-dc-init.workspace = true
-rivet-config.workspace = true
-reqwest = "0.12.9"
-chirp-client.workspace = true
 rivet-cache.workspace = true
-chirp-workflow.workspace = true
+rivet-config.workspace = true
 rivet-connection.workspace = true
-cloud-default-create.workspace = true
-
 
 [dependencies.sqlx]
 workspace = true

packages/infra/server/src/commands/start.rs

@@ -65,7 +65,7 @@ impl Opts {
 			run_config
 				.services
 				.iter()
-				.filter(|x| service_kinds.iter().any(|y| *y == x.kind))
+				.filter(|x| service_kinds.iter().any(|y| y.eq(&x.kind)))
 				.cloned()
 				.collect::<Vec<_>>()
 		};

packages/infra/server/src/run_config.rs

@@ -112,6 +112,15 @@ pub fn config(rivet_config: rivet_config::Config) -> Result<RunConfigData> {
 			ServiceKind::Singleton,
 			|config, pools| Box::pin(cluster_datacenter_tls_renew::start(config, pools)),
 		));
+
+		services.push(Service::new(
+			"cluster_tunnel_tls_renew",
+			ServiceKind::Cron(CronConfig {
+				run_immediately: true,
+				schedule: "0 0 0 1 * *".into(),
+			}),
+			|config, pools| Box::pin(cluster_tunnel_tls_renew::start(config, pools)),
+		));
 	}
 
 	if server_config.rivet.auth.access_kind == rivet_config::config::rivet::AccessKind::Development

packages/services/build/tests/patch_tags.rs

@@ -316,7 +316,11 @@ async fn multiple_exclusive_tags(ctx: TestCtx) {
 			("current".to_string(), Some("true".to_string())),
 			("other".to_string(), Some("tag".to_string())),
 		]),
-		exclusive_tags: Some(vec!["version".to_string(), "current".to_string(), "other".to_string()]),
+		exclusive_tags: Some(vec![
+			"version".to_string(),
+			"current".to_string(),
+			"other".to_string(),
+		]),
 	})
 	.await
 	.unwrap();

packages/services/cluster/db/cluster/migrations/20250107005643_add_tunnel_tls.up.sql

@@ -0,0 +1,7 @@
+CREATE TABLE tunnel_tls (
+	_id INT PRIMARY KEY, -- Solely to allow ON CONFLICT, there should only be 1 row in this table
+	cert_pem TEXT,
+	private_key_pem TEXT,
+	state INT NOT NULL, -- cluster::types::TlsState
+	expire_ts INT NOT NULL
+);

packages/services/cluster/src/lib.rs

@@ -1,10 +1,10 @@
 use chirp_workflow::prelude::*;
 
+pub mod metrics;
 pub mod ops;
 pub mod types;
 pub mod util;
 pub mod workflows;
-pub mod metrics;
 
 pub fn registry() -> WorkflowResult<Registry> {
 	use workflows::*;

packages/services/cluster/src/ops/mod.rs

@@ -4,3 +4,4 @@ pub mod get_for_game;
 pub mod list;
 pub mod resolve_for_name_id;
 pub mod server;
+pub mod tunnel;

packages/services/cluster/src/ops/tunnel/mod.rs

@@ -0,0 +1 @@
+pub mod tls_get;

packages/services/cluster/src/ops/tunnel/tls_get.rs

@@ -0,0 +1,32 @@
+use chirp_workflow::prelude::*;
+
+use crate::types::TlsState;
+
+#[derive(Debug)]
+pub struct Input {}
+
+#[derive(Debug)]
+pub struct Output {
+	pub cert_pem: String,
+	pub private_key_pem: String,
+}
+
+#[operation]
+pub async fn cluster_datacenter_tls_get(ctx: &OperationCtx, input: &Input) -> GlobalResult<Output> {
+	let row = sql_fetch_optional!(
+		[ctx, (String, String)]
+		"
+		SELECT cert_pem, private_key_pem
+		FROM db_cluster.tunnel_tls
+		WHERE state != $1
+		",
+		TlsState::Creating as i64,
+	)
+	.await?;
+	let (cert_pem, private_key_pem) = unwrap!(row, "tunnel tls not created yet");
+
+	Ok(Output {
+		cert_pem,
+		private_key_pem,
+	})
+}

packages/services/cluster/src/workflows/server/install/install_scripts/components/rivet.rs

@@ -22,14 +22,40 @@ pub fn fetch_info(server_token: &str) -> GlobalResult<String> {
 		))
 }
 
-pub fn fetch_tls(
+pub fn fetch_tunnel_tls(
+	initialize_immediately: bool,
+	server_token: &str,
+	traefik_instance_name: &str,
+) -> GlobalResult<String> {
+	let mut script = include_str!("../files/rivet_fetch_tunnel_tls.sh")
+		.replace("__TRAEFIK_INSTANCE_NAME__", traefik_instance_name)
+		.replace("__SERVER_TOKEN__", server_token)
+		.replace(
+			"__TUNNEL_API_EDGE_API__",
+			&format!("http://127.0.0.1:{TUNNEL_API_EDGE_PORT}"),
+		);
+
+	if initialize_immediately {
+		// Start timer & run script immediately
+		script.push_str(indoc!(
+			"
+			systemctl start rivet_fetch_tunnel_tls.timer
+			systemctl start --no-block rivet_fetch_tunnel_tls.service
+			"
+		));
+	}
+
+	Ok(script)
+}
+
+pub fn fetch_gg_tls(
 	initialize_immediately: bool,
 	server_token: &str,
 	traefik_instance_name: &str,
 	datacenter_id: Uuid,
 ) -> GlobalResult<String> {
-	let mut script = include_str!("../files/rivet_fetch_tls.sh")
-		.replace("__NAME__", traefik_instance_name)
+	let mut script = include_str!("../files/rivet_fetch_gg_tls.sh")
+		.replace("__TRAEFIK_INSTANCE_NAME__", traefik_instance_name)
 		.replace("__SERVER_TOKEN__", server_token)
 		.replace(
 			"__TUNNEL_API_EDGE_API__",
@@ -41,8 +67,8 @@ pub fn fetch_tls(
 		// Start timer & run script immediately
 		script.push_str(indoc!(
 			"
-			systemctl start rivet_fetch_tls.timer
-			systemctl start --no-block rivet_fetch_tls.service
+			systemctl start rivet_fetch_gg_tls.timer
+			systemctl start --no-block rivet_fetch_gg_tls.service
 			"
 		));
 	}