Harden ATVM watcher grouped-run summary recovery

Tighten `Cloud Run Finished` parsing so categorized watcher recovery only starts on real summary headers, stops at the grouped run's `Recorded Run` URL, and parses only actual summary-table host rows.

Also merge grouped per-host reporter artifacts into categorized recovery so completed grouped batches keep the correct host membership and Mattermost posts remain stable for both categorized and non-categorized runs.
This commit is contained in:
2026-04-16 10:31:20 -04:00
parent 1b88e8887e
commit 37853e56a9
2 changed files with 23 additions and 6 deletions

View File

@@ -553,3 +553,13 @@ This file stores run-specific examples only when a run produced a new learning r
- Action for future runs: - Action for future runs:
- Do not stop parent summary parsing at the Recorded Run detection log line. - Do not stop parent summary parsing at the Recorded Run detection log line.
- Bound each `Cloud Run Finished` block by the next run boundary such as the next `Extracted specPattern:` or the next `Cloud Run Finished`, then parse all host rows inside that block. - Bound each `Cloud Run Finished` block by the next run boundary such as the next `Extracted specPattern:` or the next `Cloud Run Finished`, then parse all host rows inside that block.
## Run Learning: 2026-04-16 (Categorized `Cloud Run Finished` parsing must stop at the Recorded Run URL for each grouped batch)
- Observed failure mode:
- A categorized ATVM run completed its Windows batch in the Cypress launch log, but the watcher posted only the earlier grouped results and never sent a separate Windows Mattermost status.
- The watcher let one categorized `Cloud Run Finished` block run forward into the next grouped batch because the next grouped run did not present a fresh `Extracted specPattern:` boundary before the next runner output.
- That let host-row parsing drift across grouped runs, which caused the Windows batch XML to be relabeled under the wrong subrun and left the real Windows subrun stuck in `RUNNING`.
- Action for future runs:
- For categorized grouped recovery, stop each `Cloud Run Finished` block at that grouped run's `🏁 Recorded Run:` line when it is present.
- Do not let categorized summary parsing continue into the next grouped batch's runner output.
- Keep grouped host-row parsing scoped to the actual summary table rows for that grouped run only.

View File

@@ -673,11 +673,10 @@ def summarize_host_detail_with_mochawesome(detail: str, testcase: str, message:
def extract_host_results_from_run_finished_segment(segment_text: str, inventory: Dict[str, str]) -> Dict[str, HostResult]: def extract_host_results_from_run_finished_segment(segment_text: str, inventory: Dict[str, str]) -> Dict[str, HostResult]:
host_results: Dict[str, HostResult] = {} host_results: Dict[str, HostResult] = {}
normalized = re.sub(r"\n\s*│\s*s\s*│", "s", segment_text) normalized = re.sub(r"\n\s*│\s*s\s*│", "s", segment_text)
for host_match in re.finditer( for host_match in re.finditer(
r"([✔✖])\s+(atvm[^\s]+)\.ts\s+([0-9:hms.\s]+?)\s+(\d+)\s+(\d+)\s+([-\d]+)\s+([-\d]+)\s+([-\d]+)", r"(?m)^\s*│\s*([✔✖])\s+(atvm[^\s]+)\.ts\s+([0-9:hms. ]+?)\s+(\d+)\s+(\d+)\s+([-\d]+)\s+([-\d]+)\s+([-\d]+)\s*│\s*$",
normalized, normalized,
re.S,
): ):
host = host_match.group(2) host = host_match.group(2)
duration_seconds = parse_duration_seconds(host_match.group(3)) duration_seconds = parse_duration_seconds(host_match.group(3))
@@ -700,12 +699,18 @@ def extract_host_results_from_run_finished_segment(segment_text: str, inventory:
def extract_completed_subrun_summaries(log_text: str, inventory: Dict[str, str]) -> List[Dict[str, object]]: def extract_completed_subrun_summaries(log_text: str, inventory: Dict[str, str]) -> List[Dict[str, object]]:
summaries: List[Dict[str, object]] = [] summaries: List[Dict[str, object]] = []
cloud_starts = [match.start() for match in re.finditer(r"Cloud Run Finished", log_text)] cloud_starts = [match.start() for match in re.finditer(r"(?m)^\s*Cloud Run Finished\s*$", log_text)]
previous_block_end = 0 previous_block_end = 0
for index, block_start in enumerate(cloud_starts): for index, block_start in enumerate(cloud_starts):
next_cloud_start = cloud_starts[index + 1] if index + 1 < len(cloud_starts) else len(log_text) next_cloud_start = cloud_starts[index + 1] if index + 1 < len(cloud_starts) else len(log_text)
section_text = log_text[block_start:next_cloud_start]
recorded_run_match = re.search(r"🏁 Recorded Run:\s*(https://\S+)", section_text)
next_spec_match = re.search(r"^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2},\d{3} - INFO - Extracted specPattern:", log_text[block_start + 1 :], re.M) next_spec_match = re.search(r"^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2},\d{3} - INFO - Extracted specPattern:", log_text[block_start + 1 :], re.M)
block_end = next_cloud_start block_end = next_cloud_start
if recorded_run_match:
candidate_end = block_start + recorded_run_match.end()
if candidate_end < block_end:
block_end = candidate_end
if next_spec_match: if next_spec_match:
candidate_end = block_start + 1 + next_spec_match.start() candidate_end = block_start + 1 + next_spec_match.start()
if candidate_end < block_end: if candidate_end < block_end:
@@ -1826,7 +1831,7 @@ def discover_categorized_subruns(
if summary and (not host_results or all(result.host == "check-xml-files" for result in host_results.values())): if summary and (not host_results or all(result.host == "check-xml-files" for result in host_results.values())):
host_results = summary["host_results"] host_results = summary["host_results"]
completed_hosts.extend([host for host in host_results if host not in completed_hosts]) completed_hosts.extend([host for host in host_results if host not in completed_hosts])
if not host_results and check_ts: if check_ts:
group_host_results = collect_group_host_reporter_artifacts( group_host_results = collect_group_host_reporter_artifacts(
reporter_root=reporter_root, reporter_root=reporter_root,
group_label=raw_display_group, group_label=raw_display_group,
@@ -1835,7 +1840,9 @@ def discover_categorized_subruns(
run_ended_at=check_ts + timedelta(seconds=5), run_ended_at=check_ts + timedelta(seconds=5),
) )
if group_host_results: if group_host_results:
host_results = group_host_results merged_results = dict(host_results)
merged_results.update(group_host_results)
host_results = merged_results
completed_hosts.extend([host for host in host_results if host not in completed_hosts]) completed_hosts.extend([host for host in host_results if host not in completed_hosts])
if not host_results and check_ts: if not host_results and check_ts: