From 1bffa1406d3c2a9d48a103c4dd390971d9eb667d Mon Sep 17 00:00:00 2001 From: MasterPtato Date: Tue, 7 Jan 2025 19:27:49 +0000 Subject: [PATCH] fix: fix manager log rotation --- packages/infra/client/logs/src/lib.rs | 3 ++- packages/infra/client/manager/src/main.rs | 18 ++++++++++++++++-- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/packages/infra/client/logs/src/lib.rs b/packages/infra/client/logs/src/lib.rs index cbb7d4309..eb4ef0d67 100644 --- a/packages/infra/client/logs/src/lib.rs +++ b/packages/infra/client/logs/src/lib.rs @@ -1,7 +1,7 @@ use std::{os::fd::AsRawFd, path::PathBuf}; use anyhow::*; -use chrono::{Datelike, Duration, TimeZone, Utc}; +use chrono::{Datelike, Duration, TimeDelta, TimeZone, Utc}; use tokio::fs; pub struct Logs { @@ -40,6 +40,7 @@ impl Logs { if self.next_rotation - now > Duration::seconds(5) { tokio::time::sleep( (self.next_rotation - now - Duration::seconds(5)) + .max(TimeDelta::default()) .to_std() .expect("bad duration"), ) diff --git a/packages/infra/client/manager/src/main.rs b/packages/infra/client/manager/src/main.rs index 18e59621f..aa6d7c20e 100644 --- a/packages/infra/client/manager/src/main.rs +++ b/packages/infra/client/manager/src/main.rs @@ -40,12 +40,13 @@ fn main() -> Result<()> { init_tracing(); let init = { Runtime::new()?.block_on(init())? }; + let mut first = true; // Retry loop loop { let runtime = Builder::new_multi_thread().enable_all().build()?; - match runtime.block_on(run(init.clone())) { + match runtime.block_on(run(init.clone(), first)) { Ok(_) => return Ok(()), Err(err) => { // Only restart if its a `RuntimeError` @@ -58,6 +59,8 @@ fn main() -> Result<()> { } } + first = false; + std::thread::sleep(Duration::from_secs(2)); } } @@ -173,7 +176,18 @@ async fn init() -> Result { }) } -async fn run(init: Init) -> Result<()> { +async fn run(init: Init, first: bool) -> Result<()> { + // We have to redirect logs here as well because the entire tokio runtime gets destroyed after a runtime + // error + if !first && init.config.client.logs.redirect_logs() { + pegboard_logs::Logs::new( + init.config.client.data_dir().join("logs"), + init.config.client.logs.retention(), + ) + .start() + .await?; + } + // Start metrics server let metrics_thread = tokio::spawn(metrics::run_standalone(init.config.client.metrics.port()));