Harden ATVM watcher grouped-run summary recovery
Tighten `Cloud Run Finished` parsing so categorized watcher recovery only starts on real summary headers, stops at the grouped run's `Recorded Run` URL, and parses only actual summary-table host rows. Also merge grouped per-host reporter artifacts into categorized recovery so completed grouped batches keep the correct host membership and Mattermost posts remain stable for both categorized and non-categorized runs.
This commit is contained in:
@@ -553,3 +553,13 @@ This file stores run-specific examples only when a run produced a new learning r
|
|||||||
- Action for future runs:
|
- Action for future runs:
|
||||||
- Do not stop parent summary parsing at the Recorded Run detection log line.
|
- Do not stop parent summary parsing at the Recorded Run detection log line.
|
||||||
- Bound each `Cloud Run Finished` block by the next run boundary such as the next `Extracted specPattern:` or the next `Cloud Run Finished`, then parse all host rows inside that block.
|
- Bound each `Cloud Run Finished` block by the next run boundary such as the next `Extracted specPattern:` or the next `Cloud Run Finished`, then parse all host rows inside that block.
|
||||||
|
|
||||||
|
## Run Learning: 2026-04-16 (Categorized `Cloud Run Finished` parsing must stop at the Recorded Run URL for each grouped batch)
|
||||||
|
- Observed failure mode:
|
||||||
|
- A categorized ATVM run completed its Windows batch in the Cypress launch log, but the watcher posted only the earlier grouped results and never sent a separate Windows Mattermost status.
|
||||||
|
- The watcher let one categorized `Cloud Run Finished` block run forward into the next grouped batch because the next grouped run did not present a fresh `Extracted specPattern:` boundary before the next runner output.
|
||||||
|
- That let host-row parsing drift across grouped runs, which caused the Windows batch XML to be relabeled under the wrong subrun and left the real Windows subrun stuck in `RUNNING`.
|
||||||
|
- Action for future runs:
|
||||||
|
- For categorized grouped recovery, stop each `Cloud Run Finished` block at that grouped run's `🏁 Recorded Run:` line when it is present.
|
||||||
|
- Do not let categorized summary parsing continue into the next grouped batch's runner output.
|
||||||
|
- Keep grouped host-row parsing scoped to the actual summary table rows for that grouped run only.
|
||||||
|
|||||||
@@ -673,11 +673,10 @@ def summarize_host_detail_with_mochawesome(detail: str, testcase: str, message:
|
|||||||
|
|
||||||
def extract_host_results_from_run_finished_segment(segment_text: str, inventory: Dict[str, str]) -> Dict[str, HostResult]:
|
def extract_host_results_from_run_finished_segment(segment_text: str, inventory: Dict[str, str]) -> Dict[str, HostResult]:
|
||||||
host_results: Dict[str, HostResult] = {}
|
host_results: Dict[str, HostResult] = {}
|
||||||
normalized = re.sub(r"\n\s*│\s*s\s*│", "s", segment_text)
|
normalized = re.sub(r"│\n\s*│\s*s\s*│", "s │", segment_text)
|
||||||
for host_match in re.finditer(
|
for host_match in re.finditer(
|
||||||
r"([✔✖])\s+(atvm[^\s]+)\.ts\s+([0-9:hms.\s]+?)\s+(\d+)\s+(\d+)\s+([-\d]+)\s+([-\d]+)\s+([-\d]+)",
|
r"(?m)^\s*│\s*([✔✖])\s+(atvm[^\s]+)\.ts\s+([0-9:hms. ]+?)\s+(\d+)\s+(\d+)\s+([-\d]+)\s+([-\d]+)\s+([-\d]+)\s*│\s*$",
|
||||||
normalized,
|
normalized,
|
||||||
re.S,
|
|
||||||
):
|
):
|
||||||
host = host_match.group(2)
|
host = host_match.group(2)
|
||||||
duration_seconds = parse_duration_seconds(host_match.group(3))
|
duration_seconds = parse_duration_seconds(host_match.group(3))
|
||||||
@@ -700,12 +699,18 @@ def extract_host_results_from_run_finished_segment(segment_text: str, inventory:
|
|||||||
|
|
||||||
def extract_completed_subrun_summaries(log_text: str, inventory: Dict[str, str]) -> List[Dict[str, object]]:
|
def extract_completed_subrun_summaries(log_text: str, inventory: Dict[str, str]) -> List[Dict[str, object]]:
|
||||||
summaries: List[Dict[str, object]] = []
|
summaries: List[Dict[str, object]] = []
|
||||||
cloud_starts = [match.start() for match in re.finditer(r"Cloud Run Finished", log_text)]
|
cloud_starts = [match.start() for match in re.finditer(r"(?m)^\s*Cloud Run Finished\s*$", log_text)]
|
||||||
previous_block_end = 0
|
previous_block_end = 0
|
||||||
for index, block_start in enumerate(cloud_starts):
|
for index, block_start in enumerate(cloud_starts):
|
||||||
next_cloud_start = cloud_starts[index + 1] if index + 1 < len(cloud_starts) else len(log_text)
|
next_cloud_start = cloud_starts[index + 1] if index + 1 < len(cloud_starts) else len(log_text)
|
||||||
|
section_text = log_text[block_start:next_cloud_start]
|
||||||
|
recorded_run_match = re.search(r"🏁 Recorded Run:\s*(https://\S+)", section_text)
|
||||||
next_spec_match = re.search(r"^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2},\d{3} - INFO - Extracted specPattern:", log_text[block_start + 1 :], re.M)
|
next_spec_match = re.search(r"^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2},\d{3} - INFO - Extracted specPattern:", log_text[block_start + 1 :], re.M)
|
||||||
block_end = next_cloud_start
|
block_end = next_cloud_start
|
||||||
|
if recorded_run_match:
|
||||||
|
candidate_end = block_start + recorded_run_match.end()
|
||||||
|
if candidate_end < block_end:
|
||||||
|
block_end = candidate_end
|
||||||
if next_spec_match:
|
if next_spec_match:
|
||||||
candidate_end = block_start + 1 + next_spec_match.start()
|
candidate_end = block_start + 1 + next_spec_match.start()
|
||||||
if candidate_end < block_end:
|
if candidate_end < block_end:
|
||||||
@@ -1826,7 +1831,7 @@ def discover_categorized_subruns(
|
|||||||
if summary and (not host_results or all(result.host == "check-xml-files" for result in host_results.values())):
|
if summary and (not host_results or all(result.host == "check-xml-files" for result in host_results.values())):
|
||||||
host_results = summary["host_results"]
|
host_results = summary["host_results"]
|
||||||
completed_hosts.extend([host for host in host_results if host not in completed_hosts])
|
completed_hosts.extend([host for host in host_results if host not in completed_hosts])
|
||||||
if not host_results and check_ts:
|
if check_ts:
|
||||||
group_host_results = collect_group_host_reporter_artifacts(
|
group_host_results = collect_group_host_reporter_artifacts(
|
||||||
reporter_root=reporter_root,
|
reporter_root=reporter_root,
|
||||||
group_label=raw_display_group,
|
group_label=raw_display_group,
|
||||||
@@ -1835,7 +1840,9 @@ def discover_categorized_subruns(
|
|||||||
run_ended_at=check_ts + timedelta(seconds=5),
|
run_ended_at=check_ts + timedelta(seconds=5),
|
||||||
)
|
)
|
||||||
if group_host_results:
|
if group_host_results:
|
||||||
host_results = group_host_results
|
merged_results = dict(host_results)
|
||||||
|
merged_results.update(group_host_results)
|
||||||
|
host_results = merged_results
|
||||||
completed_hosts.extend([host for host in host_results if host not in completed_hosts])
|
completed_hosts.extend([host for host in host_results if host not in completed_hosts])
|
||||||
|
|
||||||
if not host_results and check_ts:
|
if not host_results and check_ts:
|
||||||
|
|||||||
Reference in New Issue
Block a user