diff --git a/atvm/docs/automation/guide.md b/atvm/docs/automation/guide.md index ae1fb9e..48ff72a 100644 --- a/atvm/docs/automation/guide.md +++ b/atvm/docs/automation/guide.md @@ -293,6 +293,7 @@ Status-report expectations: - In `TEST FLOW:`, show the template-specific numbered run flow once for the whole test, not per host. - For `TEST FLOW:`, treat the generated host spec from the actual run as the source of truth whenever it exists. - Extract the numbered flow steps from the generated `.ts` spec referenced by that run's `specPattern`. +- When the generated spec contains runtime-gated plugin branches such as `if(useFCPlugin)` and `if(useIscsiPlugin)`, only include the steps for the plugin path actually selected for that run. - Do not prefer a static template flow list over a generated spec from the actual run. - Use template-level or static fallback flow only when the generated spec cannot be found or parsed. - If fallback is required, resolve it from the run template name before using any generic default flow. diff --git a/atvm/docs/automation/run-learnings.md b/atvm/docs/automation/run-learnings.md index d32353b..bd4d879 100644 --- a/atvm/docs/automation/run-learnings.md +++ b/atvm/docs/automation/run-learnings.md @@ -517,3 +517,12 @@ This file stores run-specific examples only when a run produced a new learning r - `--test_partition` - `--set_static_ip_dest` - Only omit or change those options when the operator explicitly overrides them. + +## Run Learning: 2026-04-14 (Generated-spec `TEST FLOW` must honor the selected plugin branch) +- Observed failure mode: + - A Pure FC `cmc-e2e` run posted a 39-step `TEST FLOW:` even though the actual FC path for that template uses 22 steps. + - The generated spec contained both `if(useFCPlugin)` and `if(useIscsiPlugin)` blocks, and the watcher counted every `it(...)` step without applying the runtime plugin gate. +- Action for future runs: + - When extracting `TEST FLOW:` from a generated spec, include common steps plus only the runtime-gated plugin branch selected for that run. + - Use watcher metadata such as the approved integration/plugin path to decide whether to include FC steps, iSCSI steps, or both. + - Do not count every plugin-gated branch in the generated spec just because the text is present. diff --git a/atvm/docs/automation/status-template.md b/atvm/docs/automation/status-template.md index e012fd0..bf399a0 100644 --- a/atvm/docs/automation/status-template.md +++ b/atvm/docs/automation/status-template.md @@ -90,7 +90,7 @@ Use this as the default ATVM automation run-status template for: - If `categorize mode: enabled` is shown, do not also repeat `--categorize` under `run options`. - When grouped categorized timing is reconstructed from host reporter artifacts, still populate `quickest`, `longest`, and `average` from inferred per-host durations when possible. - `TEST FLOW:` should describe the template-specific numbered run flow once for the whole test, not per host. -- The watcher resolves `TEST FLOW:` from the run template name. +- The watcher should prefer the generated spec for the actual run when it exists, and only include the plugin-gated branch that actually ran. - `cmc-e2e` currently uses this flow: - `1. Verifying set up` - `2. Power on and obtain ip address and host name` diff --git a/atvm/watcher-service/atvm_run_watcher.py b/atvm/watcher-service/atvm_run_watcher.py index 15368aa..23ef06e 100644 --- a/atvm/watcher-service/atvm_run_watcher.py +++ b/atvm/watcher-service/atvm_run_watcher.py @@ -1242,6 +1242,69 @@ def extract_test_flow_from_generated_spec( if not spec_list: return [] + runtime_settings: Dict[str, object] = {} + config_file = metadata.get("config_file") + if isinstance(config_file, str) and config_file: + config_path = project_root / config_file + if config_path.exists(): + config_text = config_path.read_text(encoding="utf-8", errors="replace") + for key in ("pure_plugin_type", "debug-type"): + match = re.search(rf'"{re.escape(key)}"\s*:\s*"([^"]*)"', config_text) + if match: + runtime_settings[key] = match.group(1) + for key in ("test-unaligned-fio", "isRegularCutover", "test-install-only"): + match = re.search(rf'"{re.escape(key)}"\s*:\s*(true|false)', config_text) + if match: + runtime_settings[key] = match.group(1) == "true" + + def selected_plugin_gates() -> Optional[set[str]]: + pure_plugin_type = runtime_settings.get("pure_plugin_type") + if isinstance(pure_plugin_type, str): + lowered = pure_plugin_type.lower() + if lowered == "both": + return {"fc", "iscsi"} + if lowered in {"fc", "iscsi"}: + return {lowered} + integration_plugin = metadata.get("integration_plugin") + if not isinstance(integration_plugin, str): + return None + lowered = integration_plugin.lower() + if not lowered or lowered == "unknown": + return None + if "both" in lowered: + return {"fc", "iscsi"} + selected: set[str] = set() + if re.search(r"\bfc\b", lowered): + selected.add("fc") + if "iscsi" in lowered: + selected.add("iscsi") + return selected or None + + def evaluate_gate_line(line: str, allowed_plugins: Optional[set[str]]) -> Tuple[bool, Optional[bool]]: + normalized = re.sub(r"\s+", "", line) + if normalized.startswith("if(useFCPlugin)"): + return True, allowed_plugins is None or "fc" in allowed_plugins + if normalized.startswith("if(useIscsiPlugin)"): + return True, allowed_plugins is None or "iscsi" in allowed_plugins + if normalized.startswith('if(Cypress.env("test-unaligned-fio")==true)'): + value = runtime_settings.get("test-unaligned-fio") + return True, value if isinstance(value, bool) else None + if normalized.startswith('if(Cypress.env("isRegularCutover")==false)'): + value = runtime_settings.get("isRegularCutover") + return True, (value is False) if isinstance(value, bool) else None + if normalized.startswith('if(Cypress.env("isRegularCutover")===true)'): + value = runtime_settings.get("isRegularCutover") + return True, (value is True) if isinstance(value, bool) else None + if normalized.startswith("if(!enabled_percpu_debug)"): + debug_type = runtime_settings.get("debug-type") + return True, debug_type != "percpu" if isinstance(debug_type, str) else None + if normalized.startswith("if(enabled_percpu_debug)"): + debug_type = runtime_settings.get("debug-type") + return True, debug_type == "percpu" if isinstance(debug_type, str) else None + return False, None + + allowed_plugins = selected_plugin_gates() + for entry in spec_list: if not isinstance(entry, str) or "check-xml-files.ts" in entry: continue @@ -1249,12 +1312,32 @@ def extract_test_flow_from_generated_spec( if not spec_path.exists(): continue steps: List[str] = [] + active_gate_blocks: List[Tuple[Optional[bool], int]] = [] + pending_gate_block: Optional[Optional[bool]] = None + current_depth = 0 for line in spec_path.read_text(encoding="utf-8", errors="replace").splitlines(): + matched_gate, gate_result = evaluate_gate_line(line, allowed_plugins) + if matched_gate: + if "{" in line: + active_gate_blocks.append((gate_result, current_depth + line.count("{"))) + else: + pending_gate_block = gate_result + elif pending_gate_block is not None and "{" in line: + active_gate_blocks.append((pending_gate_block, current_depth + line.count("{"))) + pending_gate_block = None + match = re.search(r'it\(\s*`?\$\{numStep\+\+\}\.\s*(.*?)`\s*,', line) if match: step_text = match.group(1).strip() - if step_text: + include_step = ( + step_text + and not any(gate_result is False for gate_result, _ in active_gate_blocks) + ) + if include_step: steps.append(f"{len(steps) + 1}. {step_text}") + current_depth += line.count("{") - line.count("}") + while active_gate_blocks and current_depth < active_gate_blocks[-1][1]: + active_gate_blocks.pop() if steps: return steps return []