Harden ATVM watcher artifact scanning

- skip report artifacts that disappear during watcher post-processing
- use safe mtime handling for XML and mochawesome discovery
- prevent watcher crashes from transient FileNotFoundError races

atvm/watcher-service/atvm_run_watcher.py

@@ -607,8 +607,8 @@ def extract_failure_from_mochawesome(
         return None
 
     candidates = sorted(
-        mochawesome_dir.glob(f"*{build_name}*.html"),
-        key=lambda path: (path.stat().st_mtime, 0 if path.name.endswith("_001.html") else 1),
+        existing_paths(mochawesome_dir.glob(f"*{build_name}*.html")),
+        key=lambda path: (safe_mtime(path), 0 if path.name.endswith("_001.html") else 1),
         reverse=True,
     )
     host_pattern = re.escape(host)
@@ -770,8 +770,8 @@ def collect_host_results(
     results: Dict[str, HostResult] = {}
     if not xml_dir.exists():
         return results
-    for xml_path in sorted(xml_dir.glob("test-result-*.xml"), key=lambda p: p.stat().st_mtime):
-        xml_mtime = datetime.fromtimestamp(xml_path.stat().st_mtime, tz=timezone.utc)
+    for xml_path in sorted(existing_paths(xml_dir.glob("test-result-*.xml")), key=safe_mtime):
+        xml_mtime = datetime.fromtimestamp(safe_mtime(xml_path), tz=timezone.utc)
         if xml_mtime < run_started_at:
             continue
         if run_ended_at and xml_mtime >= run_ended_at:
@@ -831,6 +831,17 @@ def reporter_artifact_run_timestamp(artifact_path: Path) -> Optional[datetime]:
         return None
 
 
+def safe_mtime(path: Path) -> float:
+    try:
+        return path.stat().st_mtime
+    except FileNotFoundError:
+        return 0.0
+
+
+def existing_paths(paths) -> List[Path]:
+    return [path for path in paths if path.exists()]
+
+
 def collect_latest_host_reporter_artifact(
     reporter_root: Path,
     expected_hosts: List[str],
@@ -1095,8 +1106,8 @@ def find_check_xml_end(
     if not xml_dir.exists():
         return None
     latest: Optional[datetime] = None
-    for xml_path in sorted(xml_dir.glob("test-result-*.xml"), key=lambda p: p.stat().st_mtime):
-        xml_mtime = datetime.fromtimestamp(xml_path.stat().st_mtime, tz=timezone.utc)
+    for xml_path in sorted(existing_paths(xml_dir.glob("test-result-*.xml")), key=safe_mtime):
+        xml_mtime = datetime.fromtimestamp(safe_mtime(xml_path), tz=timezone.utc)
         if xml_mtime < started_at:
             continue
         if ended_at and xml_mtime >= ended_at:
@@ -1792,8 +1803,8 @@ def discover_categorized_subruns(
 
     if xml_dir.exists():
         prefix = f"test-result-{build_name}-"
-        for xml_path in sorted(xml_dir.glob(f"{prefix}*.xml"), key=lambda p: p.stat().st_mtime):
-            xml_mtime = datetime.fromtimestamp(xml_path.stat().st_mtime, tz=timezone.utc)
+        for xml_path in sorted(existing_paths(xml_dir.glob(f"{prefix}*.xml")), key=safe_mtime):
+            xml_mtime = datetime.fromtimestamp(safe_mtime(xml_path), tz=timezone.utc)
             if xml_mtime < started_at:
                 continue
             raw_display_name = xml_path.stem[len("test-result-"):]