diff --git a/atvm/docs/automation/run-learnings.md b/atvm/docs/automation/run-learnings.md index bd44cc8..3f32921 100644 --- a/atvm/docs/automation/run-learnings.md +++ b/atvm/docs/automation/run-learnings.md @@ -497,3 +497,12 @@ This file stores run-specific examples only when a run produced a new learning r - Parse mochawesome one testcase object at a time and do not cross object boundaries when matching `title`, `fullTitle`, `state`, `message`, and `estack`. - Only use mochawesome to enrich host detail when it returns a non-empty failure payload. - If mochawesome and structured reporter artifacts disagree on the step number, keep the structured reporter step as the safer fallback for the host detail. + +## Run Learning: 2026-03-31 (Generated-spec `TEST FLOW` must not depend only on log-scoped `specPattern`) +- Observed failure mode: + - A completed `e2e-redhat8.10-both` run posted the static 22-step `cmc-e2e` flow even though the generated spec for that exact run contained a longer flow. + - The watcher only extracted generated-spec flow when it could find `Extracted specPattern:` in the available log text. + - When that log-scoped `specPattern` line was unavailable at final render time, the watcher silently fell back to the static template flow. +- Action for future runs: + - Resolve generated-spec `TEST FLOW` from the active config file's `specPattern` when the required log line is missing. + - Treat the static template flow as a last-resort fallback only after both log-derived and config-derived `specPattern` resolution fail. diff --git a/atvm/watcher-service/atvm_run_watcher.py b/atvm/watcher-service/atvm_run_watcher.py index bf98993..f81c1ad 100644 --- a/atvm/watcher-service/atvm_run_watcher.py +++ b/atvm/watcher-service/atvm_run_watcher.py @@ -1134,18 +1134,44 @@ def get_test_flow(template_name: object) -> List[str]: return TEMPLATE_TEST_FLOWS.get(template_name, DEFAULT_TEST_FLOW) -def extract_test_flow_from_generated_spec(reporter_root: Path, log_text: str) -> List[str]: - spec_match = re.search(r'Extracted specPattern:\s*(\[[^\n]+\])', log_text) - if not spec_match: +def read_spec_pattern_from_config(project_root: Path, config_file: object) -> List[str]: + if not isinstance(config_file, str) or not config_file or config_file == "unknown": + return [] + config_path = project_root / config_file + if not config_path.exists(): + return [] + text = config_path.read_text(encoding="utf-8", errors="replace") + match = re.search(r'"specPattern"\s*:\s*(\[[^\n]+\])', text) + if not match: return [] try: - spec_list = ast.literal_eval(spec_match.group(1)) + spec_list = ast.literal_eval(match.group(1)) except (SyntaxError, ValueError): return [] - if not isinstance(spec_list, list): + return spec_list if isinstance(spec_list, list) else [] + + +def extract_test_flow_from_generated_spec( + reporter_root: Path, + log_text: str, + metadata: Dict[str, object], +) -> List[str]: + spec_match = re.search(r'Extracted specPattern:\s*(\[[^\n]+\])', log_text) + cypress_root = reporter_root.parent + project_root = cypress_root.parent + spec_list: List[str] = [] + if spec_match: + try: + parsed = ast.literal_eval(spec_match.group(1)) + except (SyntaxError, ValueError): + parsed = [] + if isinstance(parsed, list): + spec_list = parsed + if not spec_list: + spec_list = read_spec_pattern_from_config(project_root, metadata.get("config_file")) + if not spec_list: return [] - cypress_root = reporter_root.parent for entry in spec_list: if not isinstance(entry, str) or "check-xml-files.ts" in entry: continue @@ -1275,7 +1301,7 @@ def build_status_markdown( ] failure_notes_block = "\n".join(f"- {note}" for note in failure_notes) if failure_notes else "- none" notes_block = "\n".join(f"- {note}" for note in notes) if notes else "- none" - resolved_flow = extract_test_flow_from_generated_spec(reporter_root, log_text) or get_test_flow(metadata.get("template")) + resolved_flow = extract_test_flow_from_generated_spec(reporter_root, log_text, metadata) or get_test_flow(metadata.get("template")) test_flow_lines = [f"- {step}" for step in resolved_flow] coverage_block = coverage_lines(metadata)