From e70ea8852de4e5d8dad918e13a9066949918a28e Mon Sep 17 00:00:00 2001
From: "anthony.wen" <awen01@gmail.com>
Date: Thu, 26 Mar 2026 18:49:58 -0400
Subject: [PATCH] Aggregate multi-host grouped summaries in categorized watcher
 posts

- update the categorized watcher to parse all ATVM host rows from a grouped Cloud Run Finished block instead of only the first host row
- allow a later terminal grouped summary to replace an earlier RUNNING state for the same logical group so grouped posts are emitted with the full host set
- fix multi-host same-group reporting so batches like ubuntu22.04 plus ubuntu24.04 are posted as one grouped result with both hosts included
---
 atvm/watcher-service/atvm_run_watcher.py | 49 +++++++++++++-----------
 1 file changed, 26 insertions(+), 23 deletions(-)

diff --git a/atvm/watcher-service/atvm_run_watcher.py b/atvm/watcher-service/atvm_run_watcher.py
index 40d67b9..aff180e 100644
--- a/atvm/watcher-service/atvm_run_watcher.py
+++ b/atvm/watcher-service/atvm_run_watcher.py
@@ -281,33 +281,33 @@ def extract_completed_subrun_summaries(log_text: str, inventory: Dict[str, str])
         block_text = block.group(1)
         currents_url = block.group(2)
         normalized = re.sub(r"\n\s*│\s*s\s*│", "s", block_text)
-        host_match = re.search(
+        host_results: Dict[str, HostResult] = {}
+        for host_match in re.finditer(
             r"✔\s+(atvm[^\s]+)\.ts\s+([0-9hms.\s]+?)\s+(\d+)\s+(\d+)\s+([-\d]+)\s+([-\d]+)\s+([-\d]+)",
             normalized,
             re.S,
-        )
-        if not host_match:
+        ):
+            host = host_match.group(1)
+            duration_seconds = parse_duration_seconds(host_match.group(2))
+            tests = int(host_match.group(3))
+            passing = int(host_match.group(4))
+            failing = 0 if host_match.group(5) == "-" else int(host_match.group(5))
+            detail = f"{tests} tests, {failing} failures"
+            status = "FAIL" if failing else "PASS"
+            host_results[host] = HostResult(
+                host=host,
+                kernel=inventory.get(host, "unknown"),
+                status=status,
+                detail=detail,
+                tests=tests,
+                failures=failing,
+                duration_seconds=duration_seconds,
+            )
+        if not host_results:
             continue
-        host = host_match.group(1)
-        duration_seconds = parse_duration_seconds(host_match.group(2))
-        tests = int(host_match.group(3))
-        passing = int(host_match.group(4))
-        failing = 0 if host_match.group(5) == "-" else int(host_match.group(5))
-        detail = f"{tests} tests, {failing} failures"
-        status = "FAIL" if failing else "PASS"
         summaries.append(
             {
-                "host_results": {
-                    host: HostResult(
-                        host=host,
-                        kernel=inventory.get(host, "unknown"),
-                        status=status,
-                        detail=detail,
-                        tests=tests,
-                        failures=failing,
-                        duration_seconds=duration_seconds,
-                    )
-                },
+                "host_results": host_results,
                 "currents_url": currents_url,
             }
         )
@@ -659,8 +659,11 @@ def merge_categorized_state(
     existing_state = existing["state"]
     if state == "CANCELLED" or existing_state == "CANCELLED":
         existing["state"] = "CANCELLED"
-    elif state == "RUNNING" or existing_state == "RUNNING":
-        existing["state"] = "RUNNING"
+    elif state == "RUNNING":
+        if existing_state not in {"COMPLETED", "FAILED"}:
+            existing["state"] = "RUNNING"
+    elif existing_state == "RUNNING":
+        existing["state"] = state
     elif state == "FAILED" or existing_state == "FAILED":
         existing["state"] = "FAILED"
     elif state == "COMPLETED" or existing_state == "COMPLETED":