Fix ATVM watcher test flow plugin filtering
This commit is contained in:
@@ -293,6 +293,7 @@ Status-report expectations:
|
||||
- In `TEST FLOW:`, show the template-specific numbered run flow once for the whole test, not per host.
|
||||
- For `TEST FLOW:`, treat the generated host spec from the actual run as the source of truth whenever it exists.
|
||||
- Extract the numbered flow steps from the generated `.ts` spec referenced by that run's `specPattern`.
|
||||
- When the generated spec contains runtime-gated plugin branches such as `if(useFCPlugin)` and `if(useIscsiPlugin)`, only include the steps for the plugin path actually selected for that run.
|
||||
- Do not prefer a static template flow list over a generated spec from the actual run.
|
||||
- Use template-level or static fallback flow only when the generated spec cannot be found or parsed.
|
||||
- If fallback is required, resolve it from the run template name before using any generic default flow.
|
||||
|
||||
@@ -517,3 +517,12 @@ This file stores run-specific examples only when a run produced a new learning r
|
||||
- `--test_partition`
|
||||
- `--set_static_ip_dest`
|
||||
- Only omit or change those options when the operator explicitly overrides them.
|
||||
|
||||
## Run Learning: 2026-04-14 (Generated-spec `TEST FLOW` must honor the selected plugin branch)
|
||||
- Observed failure mode:
|
||||
- A Pure FC `cmc-e2e` run posted a 39-step `TEST FLOW:` even though the actual FC path for that template uses 22 steps.
|
||||
- The generated spec contained both `if(useFCPlugin)` and `if(useIscsiPlugin)` blocks, and the watcher counted every `it(...)` step without applying the runtime plugin gate.
|
||||
- Action for future runs:
|
||||
- When extracting `TEST FLOW:` from a generated spec, include common steps plus only the runtime-gated plugin branch selected for that run.
|
||||
- Use watcher metadata such as the approved integration/plugin path to decide whether to include FC steps, iSCSI steps, or both.
|
||||
- Do not count every plugin-gated branch in the generated spec just because the text is present.
|
||||
|
||||
@@ -90,7 +90,7 @@ Use this as the default ATVM automation run-status template for:
|
||||
- If `categorize mode: enabled` is shown, do not also repeat `--categorize` under `run options`.
|
||||
- When grouped categorized timing is reconstructed from host reporter artifacts, still populate `quickest`, `longest`, and `average` from inferred per-host durations when possible.
|
||||
- `TEST FLOW:` should describe the template-specific numbered run flow once for the whole test, not per host.
|
||||
- The watcher resolves `TEST FLOW:` from the run template name.
|
||||
- The watcher should prefer the generated spec for the actual run when it exists, and only include the plugin-gated branch that actually ran.
|
||||
- `cmc-e2e` currently uses this flow:
|
||||
- `1. Verifying set up`
|
||||
- `2. Power on and obtain ip address and host name`
|
||||
|
||||
@@ -1242,6 +1242,69 @@ def extract_test_flow_from_generated_spec(
|
||||
if not spec_list:
|
||||
return []
|
||||
|
||||
runtime_settings: Dict[str, object] = {}
|
||||
config_file = metadata.get("config_file")
|
||||
if isinstance(config_file, str) and config_file:
|
||||
config_path = project_root / config_file
|
||||
if config_path.exists():
|
||||
config_text = config_path.read_text(encoding="utf-8", errors="replace")
|
||||
for key in ("pure_plugin_type", "debug-type"):
|
||||
match = re.search(rf'"{re.escape(key)}"\s*:\s*"([^"]*)"', config_text)
|
||||
if match:
|
||||
runtime_settings[key] = match.group(1)
|
||||
for key in ("test-unaligned-fio", "isRegularCutover", "test-install-only"):
|
||||
match = re.search(rf'"{re.escape(key)}"\s*:\s*(true|false)', config_text)
|
||||
if match:
|
||||
runtime_settings[key] = match.group(1) == "true"
|
||||
|
||||
def selected_plugin_gates() -> Optional[set[str]]:
|
||||
pure_plugin_type = runtime_settings.get("pure_plugin_type")
|
||||
if isinstance(pure_plugin_type, str):
|
||||
lowered = pure_plugin_type.lower()
|
||||
if lowered == "both":
|
||||
return {"fc", "iscsi"}
|
||||
if lowered in {"fc", "iscsi"}:
|
||||
return {lowered}
|
||||
integration_plugin = metadata.get("integration_plugin")
|
||||
if not isinstance(integration_plugin, str):
|
||||
return None
|
||||
lowered = integration_plugin.lower()
|
||||
if not lowered or lowered == "unknown":
|
||||
return None
|
||||
if "both" in lowered:
|
||||
return {"fc", "iscsi"}
|
||||
selected: set[str] = set()
|
||||
if re.search(r"\bfc\b", lowered):
|
||||
selected.add("fc")
|
||||
if "iscsi" in lowered:
|
||||
selected.add("iscsi")
|
||||
return selected or None
|
||||
|
||||
def evaluate_gate_line(line: str, allowed_plugins: Optional[set[str]]) -> Tuple[bool, Optional[bool]]:
|
||||
normalized = re.sub(r"\s+", "", line)
|
||||
if normalized.startswith("if(useFCPlugin)"):
|
||||
return True, allowed_plugins is None or "fc" in allowed_plugins
|
||||
if normalized.startswith("if(useIscsiPlugin)"):
|
||||
return True, allowed_plugins is None or "iscsi" in allowed_plugins
|
||||
if normalized.startswith('if(Cypress.env("test-unaligned-fio")==true)'):
|
||||
value = runtime_settings.get("test-unaligned-fio")
|
||||
return True, value if isinstance(value, bool) else None
|
||||
if normalized.startswith('if(Cypress.env("isRegularCutover")==false)'):
|
||||
value = runtime_settings.get("isRegularCutover")
|
||||
return True, (value is False) if isinstance(value, bool) else None
|
||||
if normalized.startswith('if(Cypress.env("isRegularCutover")===true)'):
|
||||
value = runtime_settings.get("isRegularCutover")
|
||||
return True, (value is True) if isinstance(value, bool) else None
|
||||
if normalized.startswith("if(!enabled_percpu_debug)"):
|
||||
debug_type = runtime_settings.get("debug-type")
|
||||
return True, debug_type != "percpu" if isinstance(debug_type, str) else None
|
||||
if normalized.startswith("if(enabled_percpu_debug)"):
|
||||
debug_type = runtime_settings.get("debug-type")
|
||||
return True, debug_type == "percpu" if isinstance(debug_type, str) else None
|
||||
return False, None
|
||||
|
||||
allowed_plugins = selected_plugin_gates()
|
||||
|
||||
for entry in spec_list:
|
||||
if not isinstance(entry, str) or "check-xml-files.ts" in entry:
|
||||
continue
|
||||
@@ -1249,12 +1312,32 @@ def extract_test_flow_from_generated_spec(
|
||||
if not spec_path.exists():
|
||||
continue
|
||||
steps: List[str] = []
|
||||
active_gate_blocks: List[Tuple[Optional[bool], int]] = []
|
||||
pending_gate_block: Optional[Optional[bool]] = None
|
||||
current_depth = 0
|
||||
for line in spec_path.read_text(encoding="utf-8", errors="replace").splitlines():
|
||||
matched_gate, gate_result = evaluate_gate_line(line, allowed_plugins)
|
||||
if matched_gate:
|
||||
if "{" in line:
|
||||
active_gate_blocks.append((gate_result, current_depth + line.count("{")))
|
||||
else:
|
||||
pending_gate_block = gate_result
|
||||
elif pending_gate_block is not None and "{" in line:
|
||||
active_gate_blocks.append((pending_gate_block, current_depth + line.count("{")))
|
||||
pending_gate_block = None
|
||||
|
||||
match = re.search(r'it\(\s*`?\$\{numStep\+\+\}\.\s*(.*?)`\s*,', line)
|
||||
if match:
|
||||
step_text = match.group(1).strip()
|
||||
if step_text:
|
||||
include_step = (
|
||||
step_text
|
||||
and not any(gate_result is False for gate_result, _ in active_gate_blocks)
|
||||
)
|
||||
if include_step:
|
||||
steps.append(f"{len(steps) + 1}. {step_text}")
|
||||
current_depth += line.count("{") - line.count("}")
|
||||
while active_gate_blocks and current_depth < active_gate_blocks[-1][1]:
|
||||
active_gate_blocks.pop()
|
||||
if steps:
|
||||
return steps
|
||||
return []
|
||||
|
||||
Reference in New Issue
Block a user