implemented watcher and rerun logic
This commit is contained in:
+185
-16
@@ -1,7 +1,14 @@
|
||||
import csv
|
||||
import os
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
try:
|
||||
import smbclient # type: ignore[import-not-found]
|
||||
except ModuleNotFoundError:
|
||||
smbclient = None
|
||||
|
||||
from db import DEVICE_DUT, DEVICE_REF, TestRecord
|
||||
|
||||
@@ -81,6 +88,126 @@ class CsvValidationError(ValueError):
|
||||
pass
|
||||
|
||||
|
||||
_SMB_SESSIONS: set[str] = set()
|
||||
|
||||
|
||||
def _normalize_input_path(path_value: str | Path) -> str:
|
||||
path = str(path_value).strip()
|
||||
if not path:
|
||||
return path
|
||||
|
||||
# Accept //server/share style and normalize to UNC for smbclient.
|
||||
if path.startswith("//"):
|
||||
return "\\\\" + path.lstrip("/").replace("/", "\\")
|
||||
|
||||
# Accept /<ipv4>/<share>/... and normalize to UNC for Linux-hosted inputs.
|
||||
if re.match(r"^/\d{1,3}(?:\.\d{1,3}){3}/[^/]+", path):
|
||||
return "\\\\" + path.lstrip("/").replace("/", "\\")
|
||||
|
||||
return path
|
||||
|
||||
|
||||
def _is_unc_path(path: str) -> bool:
|
||||
return path.startswith("\\\\")
|
||||
|
||||
|
||||
def _extract_unc_server(path: str) -> str | None:
|
||||
if not _is_unc_path(path):
|
||||
return None
|
||||
rest = path[2:]
|
||||
return rest.split("\\", 1)[0] if rest else None
|
||||
|
||||
|
||||
def _normalize_smb_credentials(smb_credentials: dict[str, Any] | None) -> tuple[str, str, str]:
|
||||
username = ""
|
||||
password = ""
|
||||
domain = ""
|
||||
|
||||
if smb_credentials:
|
||||
username = str(smb_credentials.get("username", "")).strip()
|
||||
password = str(smb_credentials.get("password", ""))
|
||||
domain = str(smb_credentials.get("domain", "")).strip()
|
||||
|
||||
if not username:
|
||||
username = os.getenv("SMB_USERNAME", "").strip()
|
||||
if not password:
|
||||
password = os.getenv("SMB_PASSWORD", "")
|
||||
if not domain:
|
||||
domain = os.getenv("SMB_DOMAIN", "").strip()
|
||||
|
||||
if username and domain and "\\" not in username and "@" not in username:
|
||||
username = f"{domain}\\{username}"
|
||||
|
||||
return username, password, domain
|
||||
|
||||
|
||||
def _register_smb_session_if_needed(path: str, smb_credentials: dict[str, Any] | None) -> None:
|
||||
if not _is_unc_path(path):
|
||||
return
|
||||
|
||||
if smbclient is None:
|
||||
raise ModuleNotFoundError("smbclient is required to read CSV files from UNC paths")
|
||||
|
||||
server = _extract_unc_server(path)
|
||||
if not server or server in _SMB_SESSIONS:
|
||||
return
|
||||
|
||||
username, password, _ = _normalize_smb_credentials(smb_credentials)
|
||||
if username:
|
||||
smbclient.register_session(server, username=username, password=password)
|
||||
else:
|
||||
smbclient.register_session(server)
|
||||
|
||||
_SMB_SESSIONS.add(server)
|
||||
|
||||
|
||||
def _path_exists(path: str, smb_credentials: dict[str, Any] | None) -> bool:
|
||||
if _is_unc_path(path):
|
||||
_register_smb_session_if_needed(path, smb_credentials)
|
||||
try:
|
||||
smbclient.stat(path)
|
||||
return True
|
||||
except OSError:
|
||||
return False
|
||||
|
||||
return Path(path).exists()
|
||||
|
||||
|
||||
def _is_dir(path: str, smb_credentials: dict[str, Any] | None) -> bool:
|
||||
if _is_unc_path(path):
|
||||
_register_smb_session_if_needed(path, smb_credentials)
|
||||
try:
|
||||
entry_iter = smbclient.scandir(path)
|
||||
for _ in entry_iter:
|
||||
break
|
||||
return True
|
||||
except OSError:
|
||||
return False
|
||||
|
||||
return Path(path).is_dir()
|
||||
|
||||
|
||||
def _path_name(path: str) -> str:
|
||||
trimmed = path.rstrip("\\/")
|
||||
if not trimmed:
|
||||
return path
|
||||
parts = re.split(r"[\\/]", trimmed)
|
||||
return parts[-1] if parts else trimmed
|
||||
|
||||
|
||||
def _csv_paths_from_dir(path: str, smb_credentials: dict[str, Any] | None) -> list[str]:
|
||||
if _is_unc_path(path):
|
||||
_register_smb_session_if_needed(path, smb_credentials)
|
||||
entries = []
|
||||
for entry in smbclient.scandir(path):
|
||||
name = getattr(entry, "name", "")
|
||||
if name and name.lower().endswith(".csv") and entry.is_file():
|
||||
entries.append(path.rstrip("\\/") + "\\" + name)
|
||||
return sorted(entries)
|
||||
|
||||
return [str(p) for p in sorted(Path(path).glob("*.csv"))]
|
||||
|
||||
|
||||
def _columns_missing(normalized_fieldnames: set[str], required_columns: list[str]) -> list[str]:
|
||||
return [
|
||||
column
|
||||
@@ -104,20 +231,25 @@ def _detect_csv_format(normalized_fieldnames: set[str]) -> str:
|
||||
)
|
||||
|
||||
|
||||
def parse_target_csv(csv_path: str | Path | list[str | Path] | tuple[str | Path, ...]) -> ParseResult:
|
||||
paths = _resolve_csv_paths(csv_path)
|
||||
def parse_target_csv(
|
||||
csv_path: str | Path | list[str | Path] | tuple[str | Path, ...],
|
||||
smb_credentials: dict[str, Any] | None = None,
|
||||
runtime_overrides: dict[str, Any] | None = None,
|
||||
) -> ParseResult:
|
||||
paths = _resolve_csv_paths(csv_path, smb_credentials=smb_credentials)
|
||||
runtime_defaults = _resolve_runtime_defaults(runtime_overrides)
|
||||
all_tests: list[TestRecord] = []
|
||||
all_warnings: list[str] = []
|
||||
seen_test_keys: set[tuple[str, str]] = set()
|
||||
|
||||
for path in paths:
|
||||
parsed = _parse_single_csv(path)
|
||||
parsed = _parse_single_csv(path, smb_credentials=smb_credentials, runtime_defaults=runtime_defaults)
|
||||
all_warnings.extend(parsed.warnings)
|
||||
for record in parsed.tests:
|
||||
key = (record.test_id, record.device)
|
||||
if key in seen_test_keys:
|
||||
all_warnings.append(
|
||||
f"File {path.name}: duplicate test/device '{record.test_id}/{record.device}', record skipped."
|
||||
f"File {_path_name(path)}: duplicate test/device '{record.test_id}/{record.device}', record skipped."
|
||||
)
|
||||
continue
|
||||
seen_test_keys.add(key)
|
||||
@@ -126,17 +258,43 @@ def parse_target_csv(csv_path: str | Path | list[str | Path] | tuple[str | Path,
|
||||
return ParseResult(tests=all_tests, warnings=all_warnings)
|
||||
|
||||
|
||||
def _resolve_runtime_defaults(runtime_overrides: dict[str, Any] | None) -> dict[str, int]:
|
||||
defaults = dict(RUNTIME_DEFAULTS)
|
||||
if not runtime_overrides:
|
||||
return defaults
|
||||
|
||||
def _resolve_csv_paths(csv_path: str | Path | list[str | Path] | tuple[str | Path, ...]) -> list[Path]:
|
||||
for test_type in ("P2P", "COE", "P3P"):
|
||||
raw_value = runtime_overrides.get(test_type)
|
||||
if raw_value is None:
|
||||
continue
|
||||
if isinstance(raw_value, str):
|
||||
raw_value = raw_value.strip()
|
||||
if not raw_value:
|
||||
continue
|
||||
try:
|
||||
minutes = int(raw_value)
|
||||
except (TypeError, ValueError):
|
||||
continue
|
||||
if minutes > 0:
|
||||
defaults[test_type] = minutes
|
||||
|
||||
return defaults
|
||||
|
||||
|
||||
|
||||
def _resolve_csv_paths(
|
||||
csv_path: str | Path | list[str | Path] | tuple[str | Path, ...],
|
||||
smb_credentials: dict[str, Any] | None,
|
||||
) -> list[str]:
|
||||
if isinstance(csv_path, (str, Path)):
|
||||
paths = [Path(csv_path)]
|
||||
paths = [_normalize_input_path(csv_path)]
|
||||
else:
|
||||
paths = [Path(item) for item in csv_path]
|
||||
paths = [_normalize_input_path(item) for item in csv_path]
|
||||
|
||||
resolved_paths: list[Path] = []
|
||||
resolved_paths: list[str] = []
|
||||
for path in paths:
|
||||
if path.is_dir():
|
||||
resolved_paths.extend(sorted(path.glob("*.csv")))
|
||||
if _is_dir(path, smb_credentials=smb_credentials):
|
||||
resolved_paths.extend(_csv_paths_from_dir(path, smb_credentials=smb_credentials))
|
||||
else:
|
||||
resolved_paths.append(path)
|
||||
|
||||
@@ -144,15 +302,26 @@ def _resolve_csv_paths(csv_path: str | Path | list[str | Path] | tuple[str | Pat
|
||||
raise FileNotFoundError("No CSV files found to parse.")
|
||||
|
||||
for path in resolved_paths:
|
||||
if not path.exists():
|
||||
if not _path_exists(path, smb_credentials=smb_credentials):
|
||||
raise FileNotFoundError(f"CSV file not found: {path}")
|
||||
|
||||
return resolved_paths
|
||||
|
||||
|
||||
|
||||
def _parse_single_csv(path: Path) -> ParseResult:
|
||||
with path.open("r", encoding="utf-8-sig", newline="") as handle:
|
||||
def _open_csv_handle(path: str, smb_credentials: dict[str, Any] | None):
|
||||
if _is_unc_path(path):
|
||||
_register_smb_session_if_needed(path, smb_credentials)
|
||||
return smbclient.open_file(path, mode="r", encoding="utf-8-sig", newline="")
|
||||
return Path(path).open("r", encoding="utf-8-sig", newline="")
|
||||
|
||||
|
||||
def _parse_single_csv(
|
||||
path: str,
|
||||
smb_credentials: dict[str, Any] | None,
|
||||
runtime_defaults: dict[str, int],
|
||||
) -> ParseResult:
|
||||
with _open_csv_handle(path, smb_credentials=smb_credentials) as handle:
|
||||
reader = csv.DictReader(handle)
|
||||
if not reader.fieldnames:
|
||||
raise CsvValidationError("CSV is missing a header row.")
|
||||
@@ -169,11 +338,11 @@ def _parse_single_csv(path: Path) -> ParseResult:
|
||||
for row_num, row in enumerate(reader, start=2):
|
||||
test_id = (row.get("TC ID") or "").strip()
|
||||
if not test_id:
|
||||
warnings.append(f"File {path.name}, row {row_num}: missing TC ID, row skipped.")
|
||||
warnings.append(f"File {_path_name(path)}, row {row_num}: missing TC ID, row skipped.")
|
||||
continue
|
||||
|
||||
if test_id in seen_test_ids:
|
||||
warnings.append(f"File {path.name}, row {row_num}: duplicate TC ID '{test_id}', row skipped.")
|
||||
warnings.append(f"File {_path_name(path)}, row {row_num}: duplicate TC ID '{test_id}', row skipped.")
|
||||
continue
|
||||
seen_test_ids.add(test_id)
|
||||
|
||||
@@ -184,7 +353,7 @@ def _parse_single_csv(path: Path) -> ParseResult:
|
||||
has_coe_pair = _normalize_yes_no(_row_get(row, "COE Pair")) if csv_format == "p2p_coe" else False
|
||||
config = _build_config(row, csv_format)
|
||||
signature = _victim_band_signature(row) if csv_format == "p2p_coe" else None
|
||||
estimated_minutes = RUNTIME_DEFAULTS.get(test_type)
|
||||
estimated_minutes = runtime_defaults.get(test_type, RUNTIME_DEFAULTS[test_type])
|
||||
victim_band_source = "Victim Band" if csv_format == "p2p_coe" else "Band"
|
||||
victim_band = _normalize_victim_band(_row_get(row, victim_band_source))
|
||||
|
||||
|
||||
Reference in New Issue
Block a user