Expand ATVM coverage details with run options

This commit is contained in:
2026-03-27 17:32:22 -04:00
parent 9024d5cadb
commit 5432240525
5 changed files with 40 additions and 1 deletions

View File

@@ -241,7 +241,7 @@ Status-report expectations:
- Do not include generic watcher bookkeeping messages in `NOTES:` such as artifact-detection confirmations.
- Do not include internal watcher fallback notes in `NOTES:` such as `check-xml-files.ts` validation confirmations or reporter-artifact recovery details.
- The `HOSTS:` table includes `Host`, `Kernel`, `Status`, and `Detail` columns in that order.
- In `COVERAGE:`, describe the template, datastore/config family, migration style, and plugin/integration path, but do not list target hosts there.
- In `COVERAGE:`, describe the template, datastore/config family, config filename, migration style, plugin/integration path, and other operator-relevant run options, but do not list target hosts there.
- In `TEST FLOW:`, show the template-specific numbered run flow once for the whole test, not per host.
- Resolve the flow from the run template name.
- `cmc-e2e` currently uses the 22-step migration flow documented in `/home/aw/code/cds/atvm/docs/automation/status-template.md`.

View File

@@ -301,6 +301,16 @@ This file stores run-specific examples only when a run produced a new learning r
- Treat `approve` as approval to run and start the watcher.
- Treat `approve without watcher` as approval to run without starting the watcher.
## Run Learning: 2026-03-27 (Expand coverage details with operator-relevant run options)
- Observed requirement:
- The operator wants `COVERAGE:` to include more than template and datastore family.
- Useful additions include the config filename and important flags such as `--ignore_force_shutdown`.
- Explicit VM names do not need to be repeated there because the host listing already shows them.
- Action for future runs:
- Include the ATVM config filename in `COVERAGE:`.
- Include important operator-relevant run options such as `--ignore_force_shutdown` in `COVERAGE:`.
- Keep `COVERAGE:` focused on run intent and options, not the explicit target-host list.
## Run Learning: 2026-03-27 (Do not auto-add blacklist excludes for explicitly specified VMs)
- Observed requirement:
- When the operator explicitly specifies the VM or VM list to run, they do not want the maintained `--exclude_partial_match` blacklist added automatically.

View File

@@ -42,8 +42,10 @@ Use this as the default ATVM automation run-status template for:
**COVERAGE:**
- template: `<template-name>`
- datastore/config family: `<config family>`
- config file: `<config-file-name>`
- migration style: `<high-level test style>`
- integration/plugin path: `<integration/plugin>`
- run options: `<operator-relevant options such as --ignore_force_shutdown>`
- scope of this run: `<batch or run scope>`
**TEST FLOW:**
@@ -76,6 +78,7 @@ Use this as the default ATVM automation run-status template for:
- Do not include generic watcher bookkeeping lines in `NOTES:` such as "run artifacts were detected" or "final reporting artifacts were detected."
- Do not include internal fallback notes in `NOTES:` such as "`check-xml-files.ts` validation passed" or "host details were derived from reporter artifacts."
- `COVERAGE:` should describe what the run was intended to cover without listing target hosts.
- `COVERAGE:` should also include operator-relevant run options such as the config filename and important flags like `--ignore_force_shutdown`.
- `TEST FLOW:` should describe the template-specific numbered run flow once for the whole test, not per host.
- The watcher resolves `TEST FLOW:` from the run template name.
- `cmc-e2e` currently uses this flow:

View File

@@ -688,11 +688,20 @@ def infer_host_from_subrun_build(
def infer_metadata() -> Dict[str, object]:
try:
extra_options = json.loads(os.environ.get("ATVM_WATCHER_EXTRA_OPTIONS", "[]"))
except json.JSONDecodeError:
extra_options = []
if not isinstance(extra_options, list):
extra_options = []
extra_options = [value for value in extra_options if isinstance(value, str) and value]
return {
"template": os.environ.get("ATVM_WATCHER_TEMPLATE", "unknown"),
"config_family": os.environ.get("ATVM_WATCHER_CONFIG_FAMILY", "unknown"),
"config_file": os.environ.get("ATVM_WATCHER_CONFIG_FILE", "unknown"),
"migration_style": os.environ.get("ATVM_WATCHER_MIGRATION_STYLE", "ATVM automation validation"),
"integration_plugin": os.environ.get("ATVM_WATCHER_INTEGRATION_PLUGIN", "unknown"),
"extra_options": extra_options,
"scope_description": os.environ.get("ATVM_WATCHER_SCOPE_DESCRIPTION", "requested ATVM run scope"),
"categorized": os.environ.get("ATVM_WATCHER_CATEGORIZED", "false").lower() == "true",
}
@@ -791,8 +800,10 @@ def build_status_markdown(
"**COVERAGE:**",
f"- template: `{metadata['template']}`",
f"- datastore/config family: `{metadata['config_family']}`",
f"- config file: `{metadata.get('config_file', 'unknown')}`",
f"- migration style: {metadata['migration_style']}",
f"- integration/plugin path: `{metadata['integration_plugin']}`",
f"- run options: {', '.join(f'`{value}`' for value in metadata.get('extra_options', [])) or 'none'}",
f"- scope of this run: {metadata['scope_description']}",
"",
"**TEST FLOW:**",

View File

@@ -10,8 +10,10 @@ Options:
--build-name <name>
--template <name>
--config-family <name>
--config-file <path>
--migration-style <text>
--integration-plugin <text>
--extra-option <text> Repeatable
--scope-description <text>
--categorize
--state-root <path> Default: /var/lib/atvm-run-watcher
@@ -21,8 +23,10 @@ EOF
BUILD_NAME=""
TEMPLATE=""
CONFIG_FAMILY=""
CONFIG_FILE=""
MIGRATION_STYLE=""
INTEGRATION_PLUGIN=""
EXTRA_OPTIONS=()
SCOPE_DESCRIPTION=""
WATCHER_CATEGORIZED="false"
STATE_ROOT="/var/lib/atvm-run-watcher"
@@ -32,8 +36,10 @@ while [[ $# -gt 0 ]]; do
--build-name) BUILD_NAME="${2:-}"; shift 2 ;;
--template) TEMPLATE="${2:-}"; shift 2 ;;
--config-family) CONFIG_FAMILY="${2:-}"; shift 2 ;;
--config-file) CONFIG_FILE="${2:-}"; shift 2 ;;
--migration-style) MIGRATION_STYLE="${2:-}"; shift 2 ;;
--integration-plugin) INTEGRATION_PLUGIN="${2:-}"; shift 2 ;;
--extra-option) EXTRA_OPTIONS+=("${2:-}"); shift 2 ;;
--scope-description) SCOPE_DESCRIPTION="${2:-}"; shift 2 ;;
--categorize) WATCHER_CATEGORIZED="true"; shift ;;
--state-root) STATE_ROOT="${2:-}"; shift 2 ;;
@@ -55,11 +61,20 @@ rm -rf "$RUN_DIR"
rm -f "$RUN_LOG"
mkdir -p "$RUN_DIR"
EXTRA_OPTIONS_JSON=$(python3 - <<'PY' "${EXTRA_OPTIONS[@]}"
import json
import sys
print(json.dumps(sys.argv[1:]))
PY
)
cat >"${RUN_DIR}/watch.env" <<EOF
ATVM_WATCHER_TEMPLATE=${TEMPLATE@Q}
ATVM_WATCHER_CONFIG_FAMILY=${CONFIG_FAMILY@Q}
ATVM_WATCHER_CONFIG_FILE=${CONFIG_FILE@Q}
ATVM_WATCHER_MIGRATION_STYLE=${MIGRATION_STYLE@Q}
ATVM_WATCHER_INTEGRATION_PLUGIN=${INTEGRATION_PLUGIN@Q}
ATVM_WATCHER_EXTRA_OPTIONS=${EXTRA_OPTIONS_JSON@Q}
ATVM_WATCHER_SCOPE_DESCRIPTION=${SCOPE_DESCRIPTION@Q}
ATVM_WATCHER_CATEGORIZED=${WATCHER_CATEGORIZED@Q}
EOF