From 6c7ba5212b55e0b8e3229f9e4a2d3810f387dbba Mon Sep 17 00:00:00 2001 From: "anthony.wen" Date: Fri, 27 Mar 2026 18:49:11 -0400 Subject: [PATCH] Refine categorized timing metrics and coverage output --- atvm/docs/automation/guide.md | 2 ++ atvm/docs/automation/run-learnings.md | 8 ++++++++ atvm/docs/automation/status-template.md | 2 ++ atvm/watcher-service/atvm_run_watcher.py | 23 ++++++++++++++++++++++- 4 files changed, 34 insertions(+), 1 deletion(-) diff --git a/atvm/docs/automation/guide.md b/atvm/docs/automation/guide.md index bf4c9ae..b0862ca 100644 --- a/atvm/docs/automation/guide.md +++ b/atvm/docs/automation/guide.md @@ -243,6 +243,8 @@ Status-report expectations: - Do not include internal watcher fallback notes in `NOTES:` such as `check-xml-files.ts` validation confirmations or reporter-artifact recovery details. - The `HOSTS:` table includes `Host`, `Kernel`, `Status`, and `Detail` columns in that order. - In `COVERAGE:`, describe the important `cmc-templates.py` command inputs such as template, categorize mode, datastore/config family, config filename, migration style, plugin/integration path, and other operator-relevant run options, but do not list target hosts there or include verbose prose scope descriptions. +- If `categorize mode: enabled` is already shown in `COVERAGE:`, do not also repeat `--categorize` under `run options`. +- When grouped categorized timing is reconstructed from host reporter artifacts, derive per-host quickest/longest/average durations from the sequence of recovered host timestamps and the grouped end time instead of leaving those metrics as `n/a`. - In `TEST FLOW:`, show the template-specific numbered run flow once for the whole test, not per host. - Resolve the flow from the run template name. - `cmc-e2e` currently uses the 22-step migration flow documented in `/home/aw/code/cds/atvm/docs/automation/status-template.md`. diff --git a/atvm/docs/automation/run-learnings.md b/atvm/docs/automation/run-learnings.md index a64e4ea..feef801 100644 --- a/atvm/docs/automation/run-learnings.md +++ b/atvm/docs/automation/run-learnings.md @@ -321,6 +321,14 @@ This file stores run-specific examples only when a run produced a new learning r - Store and display the exact `cmc-templates.py` command in `NOTES:`. - Omit only the outer remote-execution wrapper. +## Run Learning: 2026-03-27 (Avoid redundant categorize flags and infer grouped timing stats) +- Observed requirement: + - When `categorize mode: enabled` is already shown in `COVERAGE:`, repeating `--categorize` under `run options` is redundant. + - Grouped categorized results should still show `quickest`, `longest`, and `average` when those values can be inferred from recovered host timing. +- Action for future runs: + - Do not repeat `--categorize` under `run options` when categorize mode is already shown separately. + - When grouped host results are reconstructed from reporter artifacts, infer per-host durations from the recovered host timestamp sequence and grouped end time so grouped timing stats do not default to `n/a` unnecessarily. + ## Run Learning: 2026-03-27 (Do not auto-add blacklist excludes for explicitly specified VMs) - Observed requirement: - When the operator explicitly specifies the VM or VM list to run, they do not want the maintained `--exclude_partial_match` blacklist added automatically. diff --git a/atvm/docs/automation/status-template.md b/atvm/docs/automation/status-template.md index e605dc7..da017bd 100644 --- a/atvm/docs/automation/status-template.md +++ b/atvm/docs/automation/status-template.md @@ -80,6 +80,8 @@ Use this as the default ATVM automation run-status template for: - Do not include internal fallback notes in `NOTES:` such as "`check-xml-files.ts` validation passed" or "host details were derived from reporter artifacts." - `COVERAGE:` should describe what the run was intended to cover without listing target hosts. - `COVERAGE:` should mostly mirror the important `cmc-templates.py` command inputs such as template, categorize mode, config filename, integration/plugin path, and important flags like `--ignore_force_shutdown`. +- If `categorize mode: enabled` is shown, do not also repeat `--categorize` under `run options`. +- When grouped categorized timing is reconstructed from host reporter artifacts, still populate `quickest`, `longest`, and `average` from inferred per-host durations when possible. - `TEST FLOW:` should describe the template-specific numbered run flow once for the whole test, not per host. - The watcher resolves `TEST FLOW:` from the run template name. - `cmc-e2e` currently uses this flow: diff --git a/atvm/watcher-service/atvm_run_watcher.py b/atvm/watcher-service/atvm_run_watcher.py index e4fc25a..c194c68 100644 --- a/atvm/watcher-service/atvm_run_watcher.py +++ b/atvm/watcher-service/atvm_run_watcher.py @@ -733,6 +733,23 @@ def get_test_flow(template_name: object) -> List[str]: return TEMPLATE_TEST_FLOWS.get(template_name, DEFAULT_TEST_FLOW) +def infer_missing_host_durations(ordered_hosts: List[HostResult], end_ts: Optional[datetime]) -> None: + timed_hosts = [host for host in ordered_hosts if host.timestamp] + if not timed_hosts: + return + timed_hosts.sort(key=lambda host: host.timestamp or datetime.fromtimestamp(0, tz=timezone.utc)) + for index, host in enumerate(timed_hosts): + if host.duration_seconds is not None: + continue + current_ts = host.timestamp + if current_ts is None: + continue + next_ts = timed_hosts[index + 1].timestamp if index + 1 < len(timed_hosts) else end_ts + if next_ts is None or next_ts <= current_ts: + continue + host.duration_seconds = (next_ts - current_ts).total_seconds() + + def build_status_markdown( build_name: str, metadata: Dict[str, object], @@ -744,6 +761,7 @@ def build_status_markdown( notes: List[str], ) -> str: ordered_hosts = list(host_results.values()) + infer_missing_host_durations(ordered_hosts, end_ts) finished = len([h for h in ordered_hosts if h.status in {"PASS", "FAIL"}]) passed = len([h for h in ordered_hosts if h.status == "PASS"]) failed = len([h for h in ordered_hosts if h.status == "FAIL"]) @@ -772,6 +790,9 @@ def build_status_markdown( notes_block = "\n".join(f"- {note}" for note in notes) if notes else "- none" test_flow_lines = [f"- {step}" for step in get_test_flow(metadata.get("template"))] + coverage_options = list(metadata.get("extra_options", [])) if isinstance(metadata.get("extra_options"), list) else [] + if metadata.get("categorized"): + coverage_options = [value for value in coverage_options if value != "--categorize"] lines = [ "## ATVM Run Status", @@ -808,7 +829,7 @@ def build_status_markdown( f"- config file: `{metadata.get('config_file', 'unknown')}`", f"- migration style: {metadata['migration_style']}", f"- integration/plugin path: `{metadata['integration_plugin']}`", - f"- run options: {', '.join(f'`{value}`' for value in metadata.get('extra_options', [])) or 'none'}", + f"- run options: {', '.join(f'`{value}`' for value in coverage_options) or 'none'}", "", "**TEST FLOW:**", *test_flow_lines,