Mia.Wu/test_dashboard
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix watcher for smb

Browse Source
This commit is contained in:
Mia Wu
2026-07-14 11:43:59 -04:00
parent 834ab81a94
commit b68b8f54bd
7 changed files with 190 additions and 133 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/app.py
+2 -23
View File
@@ -27,7 +27,7 @@ from db_py import (
clear_users,
update_user_password,
)
from scanner import full_scan, is_scan_in_progress, resolve_runtime_path, scan_results_only
from scanner import full_scan, get_scan_errors, is_scan_in_progress, resolve_runtime_path
from watcher import start_results_watchers, stop_results_watchers
BASE_DIR = Path(__file__).resolve().parent
@@ -334,7 +334,7 @@ def get_stats_route():
@app.get("/api/scan-status")
@require_auth
def get_scan_status_route():
return jsonify({"scanning": is_scan_in_progress()})
return jsonify({"scanning": is_scan_in_progress(), "errors": get_scan_errors()})
@app.post("/api/auth/login")
@@ -554,27 +554,6 @@ def rescan_route():
return jsonify({"ok": True, "scanning": True, "testCount": None, "completedCount": None})
@app.post("/api/config/rescan-results")
@require_auth
@require_role("admin")
def rescan_results_route():
_apply_smb_env_from_config()
results_dir = resolve_runtime_path(get_config("results_dir"))
results_dir_ref = resolve_runtime_path(get_config("results_dir_ref"))
if not results_dir:
return jsonify({"error": "Results directory not configured"}), 400
if is_scan_in_progress():
return jsonify({"ok": True, "scanning": True, "testCount": None, "completedCount": None})
threading.Thread(
target=scan_results_only,
args=(results_dir, results_dir_ref),
daemon=True,
).start()
return jsonify({"ok": True, "scanning": True, "testCount": None, "completedCount": None})
@app.get("/")
@app.get("/<path:path>")
def static_or_spa(path=""):
server/scanner.py
+38 -15
View File
@@ -3,6 +3,7 @@ import re
import threading
import tempfile
import smbclient
from pathlib import Path
from db_py import (
mark_tests_completed,
@@ -25,6 +26,8 @@ from parser import (
_SMB_SESSIONS = set()
_SCAN_STATE_LOCK = threading.Lock()
_ACTIVE_SCAN_COUNT = 0
_LAST_SCAN_ERRORS = []
_SCAN_ERRORS_LOCK = threading.Lock()
_ELAPSED_TIME_LINE_RE = re.compile(r"Elapsed\s+time\s*:\s*(\d+):(\d{1,2}):(\d{1,2}(?:\.\d+)?)", re.IGNORECASE)
@@ -158,6 +161,23 @@ def is_scan_in_progress():
return _ACTIVE_SCAN_COUNT > 0
def _clear_scan_errors():
global _LAST_SCAN_ERRORS
with _SCAN_ERRORS_LOCK:
_LAST_SCAN_ERRORS = []
def _record_scan_error(msg):
with _SCAN_ERRORS_LOCK:
if msg not in _LAST_SCAN_ERRORS:
_LAST_SCAN_ERRORS.append(msg)
def get_scan_errors():
with _SCAN_ERRORS_LOCK:
return list(_LAST_SCAN_ERRORS)
def resolve_runtime_path(path_value):
if not path_value:
return path_value
@@ -166,12 +186,15 @@ def resolve_runtime_path(path_value):
if not raw_path:
return raw_path
# If the path is already valid in the current runtime, keep it.
if os.path.exists(raw_path):
# UNC/network paths are handled separately via smbclient — check before
# any local filesystem call because backslashes are regular characters on
# Linux, so Path(unc).exists() treats the whole string as one path
# component and raises ENAMETOOLONG for paths > 255 chars.
if raw_path.startswith("\\\\") or raw_path.startswith("//"):
return raw_path
# UNC/network paths are handled separately via smbclient.
if raw_path.startswith("\\\\") or raw_path.startswith("//"):
# If the path is already valid in the current runtime, keep it.
if Path(raw_path).exists():
return raw_path
# Map host paths (Windows or Linux) to the container mount point when running in a container.
@@ -185,7 +208,7 @@ def resolve_runtime_path(path_value):
if raw_norm.lower() == host_norm.lower() or raw_norm.lower().startswith(host_norm.lower() + "/"):
relative = raw_norm[len(host_norm):].lstrip("/")
if relative:
return os.path.join(mount_root, *relative.split("/"))
return str(Path(mount_root).joinpath(*relative.split("/")))
return mount_root
return raw_path
@@ -248,14 +271,14 @@ def _iter_dir_entries(path):
if _is_unc_path(path):
_register_smb_session_if_needed(path)
return list(smbclient.scandir(path))
return list(os.scandir(path))
return list(Path(path).iterdir())
def _join_path(path, name):
if _is_unc_path(path):
base = path.rstrip("\\")
return f"{base}\\{name}"
return os.path.join(path, name)
return str(Path(path) / name)
def _read_text_lines(path):
@@ -371,6 +394,7 @@ def full_scan(target_dir, results_dir, results_dir_ref):
results_dir = _normalize_input_path(results_dir)
results_dir_ref = _normalize_input_path(results_dir_ref)
_clear_scan_errors()
_scan_started()
try:
clear_tests()
@@ -378,7 +402,7 @@ def full_scan(target_dir, results_dir, results_dir_ref):
return
print(f"[scanner] target dir : {target_dir}")
print(f"[scanner] results dir : {results_dir}")
print(f"[scanner] results dir dut : {results_dir}")
print(f"[scanner] results dir ref : {results_dir_ref}")
if not scan_targets(target_dir):
@@ -406,8 +430,9 @@ def scan_targets(target_dir):
for entry in _iter_dir_entries(target_dir)
if entry.is_dir()
]
except OSError as exc:
except (OSError, ValueError) as exc:
print(f"[scanner] Cannot read target dir: {exc}")
_record_scan_error(f"Cannot read target directory: {exc}")
return False
print(f"[scanner] subdirectories found: {len(parent_entries)}")
@@ -422,7 +447,7 @@ def scan_targets(target_dir):
and not entry.name.startswith("GLOBAL")
and entry.name.endswith(".ini")
]
except OSError as exc:
except (OSError, ValueError) as exc:
print(f"[scanner] Cannot read parent dir {parent_name}: {exc}")
continue
@@ -476,8 +501,9 @@ def scan_results(results_dir):
for entry in _iter_dir_entries(results_dir)
if entry.is_dir()
]
except OSError as exc:
except (OSError, ValueError) as exc:
print(f"[scanner] Cannot read results dir: {exc}")
_record_scan_error(f"Cannot read results directory: {exc}")
return
print(f"[scanner] results: {len(entries)} result dir(s) found")
@@ -557,9 +583,6 @@ def extract_measurement_data(db_file_path):
print(f"[scanner] Cannot read measurement db {db_file_path}: {exc}")
finally:
if local_db_path:
try:
os.remove(local_db_path)
except OSError:
pass
Path(local_db_path).unlink(missing_ok=True)
return result
server/watcher.py
+140 -51
View File
@@ -1,8 +1,10 @@
import os
import threading
import smbclient
from watchdog.events import FileSystemEventHandler
from watchdog.observers.polling import PollingObserver
from pathlib import Path
from db_py import (
mark_tests_completed,
@@ -15,24 +17,23 @@ from scanner import process_result_dir, resolve_runtime_path
from parser import parse_deleted_result_dir_name
_OBSERVER = None
_OBSERVER_LOCK = threading.RLock()
_WATCHERS: list = []
_WATCHERS_LOCK = threading.RLock()
class ResultsDirEventHandler(FileSystemEventHandler):
def __init__(self, results_root):
super().__init__()
self.results_root = str(results_root)
self.results_root = Path(results_root)
def _is_direct_child(self, path_value):
parent = os.path.dirname(path_value.rstrip("\\/"))
return os.path.normcase(parent) == os.path.normcase(self.results_root.rstrip("\\/"))
return Path(path_value).parent == self.results_root
def _mark_completed(self, dir_path):
if not self._is_direct_child(dir_path):
return
dir_name = os.path.basename(dir_path.rstrip("\\/"))
dir_name = Path(dir_path).name
completion = process_result_dir(self.results_root, dir_name)
if not completion:
return
@@ -47,19 +48,19 @@ class ResultsDirEventHandler(FileSystemEventHandler):
def _mark_completed_from_file(self, file_path):
# New measurement/log files arriving after directory creation should update that single test.
filename = os.path.basename(file_path).lower()
p = Path(file_path)
filename = p.name.lower()
is_duration_log = filename.endswith(".log") or filename.endswith(".txt") or "log" in filename
if filename != "measurement.db" and not is_duration_log:
return
parent_dir = os.path.dirname(file_path.rstrip("\\/"))
self._mark_completed(parent_dir)
self._mark_completed(p.parent)
def _clear_completed(self, dir_path):
if not self._is_direct_child(dir_path):
return
dir_name = os.path.basename(dir_path.rstrip("\\/"))
dir_name = Path(dir_path).name
test_id, device = parse_deleted_result_dir_name(dir_name)
if not test_id:
return
@@ -92,69 +93,157 @@ class ResultsDirEventHandler(FileSystemEventHandler):
self._mark_completed_from_file(event.dest_path)
def stop_results_watchers():
global _OBSERVER
class _SmbPollerThread(threading.Thread):
"""Polls a UNC results directory via smbclient and updates the DB on directory changes."""
with _OBSERVER_LOCK:
if _OBSERVER is None:
return
_INTERVAL = 5.0
def __init__(self, unc_path):
super().__init__(daemon=True, name=f"smb-poller-{unc_path}")
self._path = unc_path.rstrip("\\")
self._stop_evt = threading.Event()
def stop(self):
self._stop_evt.set()
def run(self):
prev = self._snapshot()
while not self._stop_evt.wait(self._INTERVAL):
try:
curr = self._snapshot()
for name in curr - prev:
completion = process_result_dir(self._path, name)
if completion:
mark_tests_completed([completion])
coe = update_all_p2p_coe_pairs_sql()
p3p = update_all_p3p_pairs_sql()
print(f"[watcher] SMB: marked completed {name} (coe={coe}, p3p={p3p})")
for name in prev - curr:
test_id, device = parse_deleted_result_dir_name(name)
if test_id:
reset_by_file_id_and_device(test_id, device)
coe = update_all_p2p_coe_pairs_sql()
p3p = update_all_p3p_pairs_sql()
print(f"[watcher] SMB: cleared completed {name} (coe={coe}, p3p={p3p})")
prev = curr
except Exception as exc:
print(f"[watcher] SMB poll error {self._path}: {exc}")
def _snapshot(self):
try:
_OBSERVER.stop()
_OBSERVER.join(timeout=5)
finally:
_OBSERVER = None
print("[watcher] Stopped results directory watcher")
return {e.name for e in smbclient.scandir(self._path) if e.is_dir()}
except Exception as exc:
print(f"[watcher] SMB scandir error {self._path}: {exc}")
return set()
def _register_smb_session(unc_path):
server = unc_path[2:].split("\\", 1)[0]
if not server:
return
username = os.getenv("SMB_USERNAME", "").strip()
password = os.getenv("SMB_PASSWORD", "")
domain = os.getenv("SMB_DOMAIN", "").strip()
if username and domain and "\\" not in username and "@" not in username:
username = f"{domain}\\{username}"
try:
if username:
smbclient.register_session(server, username=username, password=password)
else:
smbclient.register_session(server)
except Exception as exc:
print(f"[watcher] SMB session registration failed for {server}: {exc}")
def stop_results_watchers():
global _WATCHERS
with _WATCHERS_LOCK:
if not _WATCHERS:
return
for w in _WATCHERS:
try:
w.stop()
except Exception:
pass
for w in _WATCHERS:
try:
w.join(timeout=5)
except Exception:
pass
_WATCHERS = []
print("[watcher] Stopped results directory watcher(s)")
def start_results_watchers(results_dir, results_dir_ref):
global _OBSERVER
global _WATCHERS
with _OBSERVER_LOCK:
with _WATCHERS_LOCK:
stop_results_watchers()
resolved = []
local_paths = []
smb_paths = []
seen = set()
for raw_dir in (results_dir, results_dir_ref):
if not raw_dir:
continue
path = resolve_runtime_path(raw_dir)
if not path:
continue
normalized = os.path.normcase(str(path).rstrip("\\/"))
if normalized and normalized not in seen:
seen.add(normalized)
resolved.append(str(path))
path = str(path)
if not resolved:
if path.startswith("\\\\") or path.startswith("//"):
unc = "\\\\" + path.lstrip("/\\").replace("/", "\\")
key = unc.lower()
if key not in seen:
seen.add(key)
smb_paths.append(unc)
else:
p = Path(path)
key = os.path.normcase(str(p))
if key not in seen:
seen.add(key)
local_paths.append(p)
if not local_paths and not smb_paths:
print("[watcher] No results directories configured, watcher not started")
return
observer = PollingObserver(timeout=1.0)
watchers = []
scheduled_count = 0
for path in resolved:
try:
observer.schedule(ResultsDirEventHandler(path), path, recursive=False)
scheduled_count += 1
print(f"[watcher] Watching results directory: {path}")
except Exception as exc:
print(f"[watcher] Skipping watch path {path}: {exc}")
if local_paths:
observer = PollingObserver(timeout=1.0)
scheduled = 0
for p in local_paths:
try:
observer.schedule(ResultsDirEventHandler(p), str(p), recursive=False)
scheduled += 1
print(f"[watcher] Watching local directory: {p}")
except Exception as exc:
print(f"[watcher] Skipping local watch path {p}: {exc}")
if scheduled:
try:
observer.start()
watchers.append(observer)
except Exception as exc:
print(f"[watcher] Failed to start local watcher: {exc}")
try:
observer.stop()
observer.join(timeout=5)
except Exception:
pass
if scheduled_count == 0:
for unc in smb_paths:
_register_smb_session(unc)
poller = _SmbPollerThread(unc)
poller.start()
watchers.append(poller)
print(f"[watcher] Watching SMB directory: {unc}")
if not watchers:
print("[watcher] No valid watch paths available, watcher not started")
return
try:
observer.start()
except Exception as exc:
print(f"[watcher] Failed to start results watcher: {exc}")
try:
observer.stop()
observer.join(timeout=5)
except Exception:
pass
return
_OBSERVER = observer
print("[watcher] Results directory watcher started")
_WATCHERS = watchers
print(f"[watcher] Results directory watcher(s) started ({len(watchers)})")