improved container wait reliability with timeouts/retries

1 files changed, 9 insertions, 3 deletions

diff --git a/src/main.rs b/src/main.rs
index 7ef6ff6..3db3169 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -565,12 +565,14 @@ async fn machine_containers_wait(
 ) -> Result<()> {
     tracing::info!("waiting for {} containers to exit", containers.len());
     let script = machine_containers_wait_script(containers);
+    let wait_timeout = Duration::from_secs(60);
     let retry_seconds = 5;
     let mut retries = 10;
     loop {
-        match machine_run_script(ctx, machine, &script).await {
-            Ok(_) => break,
-            Err(err) => {
+        let fut = tokio::time::timeout(wait_timeout, machine_run_script(ctx, machine, &script));
+        match fut.await {
+            Ok(Ok(_)) => break,
+            Ok(Err(err)) => {
                 tracing::debug!("failed to run script: {err}, {retries} left");
                 if retries == 0 {
                     return Err(err);
@@ -580,6 +582,10 @@ async fn machine_containers_wait(
                 tokio::time::sleep(Duration::from_secs(retry_seconds)).await;
                 tracing::debug!("retrying");
             }
+            Err(_) => {
+                tracing::debug!("wait timeout, retrying...");
+                tokio::time::sleep(Duration::from_secs(retry_seconds)).await;
+            }
         }
     }
     tracing::info!("all containers exited");