From b45375dbbc399fcda091497fcb9026269aae5bd9 Mon Sep 17 00:00:00 2001 From: "anthony.wen" Date: Mon, 30 Mar 2026 17:10:25 -0400 Subject: [PATCH] Fix false ATVM failures from reporter txt fallback --- atvm/docs/automation/run-learnings.md | 9 +++++++++ atvm/watcher-service/atvm_run_watcher.py | 6 ++++-- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/atvm/docs/automation/run-learnings.md b/atvm/docs/automation/run-learnings.md index 901c13b..b815e24 100644 --- a/atvm/docs/automation/run-learnings.md +++ b/atvm/docs/automation/run-learnings.md @@ -392,3 +392,12 @@ This file stores run-specific examples only when a run produced a new learning r - Do not treat the presence of plugin-specific strings or code blocks in the generated `.ts` file as proof that those plugin steps will execute. - For plugin-specific questions, determine expected behavior from the template/runtime gate and only call it a mismatch if the runtime logic would execute the wrong plugin path. - Continue verifying that the requested VM set is present in the generated files and `specPattern`, but keep plugin-path validation separate from simple text-presence checks. + +## Run Learning: 2026-03-30 (Do not classify reporter TXT logs as failed from generic `error` words) +- Observed failure mode: + - A completed `reboot-redhat8.10-iscsi` run actually passed in the launch log and `Cloud Run Finished` table, but the watcher saved it as failed. + - The TXT fallback matched generic strings such as `auth error encountered` and treated them as proof of host failure. +- Action for future runs: + - Do not classify a reporter TXT artifact as failed just because it contains the word `error`. + - For TXT fallback, require explicit terminal failure markers such as `cy:command error`, `cy:task error`, or real `Error:`/`AssertionError:`/timeout text. + - Prefer the parent run summary when available, because it is less prone to false failure signals than raw per-step console text. diff --git a/atvm/watcher-service/atvm_run_watcher.py b/atvm/watcher-service/atvm_run_watcher.py index 343fd4b..cdfbf06 100644 --- a/atvm/watcher-service/atvm_run_watcher.py +++ b/atvm/watcher-service/atvm_run_watcher.py @@ -751,11 +751,13 @@ def parse_host_reporter_artifact(artifact_path: Path, host: str, kernels: Dict[s except OSError: text = "" - failures = 1 if re.search(r"\bTimed out!\b|\bAssertionError\b|\bFAIL\b|\berror\b", text, re.I) else 0 + failure_detail = extract_failure_detail_from_text_blob(text) + structured_failure = re.search(r"^(?:cy:command|cy:task)\terror\t", text, re.I | re.M) + failures = 1 if failure_detail or structured_failure else 0 status = "FAIL" if failures else "PASS" detail = "1 failures" if failures else "completed" if failures: - detail = append_failure_detail(detail, extract_failure_detail_from_text_blob(text)) + detail = append_failure_detail(detail, failure_detail) return HostResult( host=host, kernel=kernels.get(host, "unknown"),