From a49772aecea5d42fc7483ee5fbe493ec04b16e6d Mon Sep 17 00:00:00 2001 From: Mia Wu Date: Fri, 17 Jul 2026 15:17:08 -0400 Subject: [PATCH] fix test bundling bug --- backend/app.py | 40 ++++++++++++++---------- backend/db.py | 41 ++++++++++++++++++------ backend/parser.py | 65 +++++++++++++++++++++++++++++--------- backend/scanner.py | 74 +++++++++++++++++++++++++++++--------------- backend/scheduler.py | 5 +++ 5 files changed, 159 insertions(+), 66 deletions(-) diff --git a/backend/app.py b/backend/app.py index 03c2cff..0e1b162 100644 --- a/backend/app.py +++ b/backend/app.py @@ -16,6 +16,7 @@ os.environ.setdefault("DUT", db.DEVICE_DUT) os.environ.setdefault("REF", db.DEVICE_REF) from scheduler import Scheduler, Test as SchedulerTest +from scanner import resolve_runtime_path from test_config import build_bundle_test_configs, build_config_rows from test_window import get_shift_capacity_for_date, get_shift_sequence, is_off_day from watcher import configure_result_watcher, stop_result_watcher @@ -79,6 +80,28 @@ def _runtime_overrides_from_settings(settings: dict[str, Any]) -> dict[str, dict return overrides +def _is_explicit_path(path_value: str) -> bool: + return ( + path_value.startswith(("/", "\\\\", "//")) + or "\\" in path_value + or ":" in path_value + ) + + +def _resolve_requested_csv_path(requested_path: str) -> str | Path: + received_path = str(requested_path).strip() + resolved_path = str(resolve_runtime_path(received_path)).strip() + + if _is_explicit_path(received_path) or _is_explicit_path(resolved_path): + return resolved_path + + data_dir_path = APP_ROOT / "data" / received_path + if data_dir_path.exists(): + return data_dir_path + + return APP_ROOT / received_path + + class CompileScheduleRequest(BaseModel): start_date: str | None = Field(default=None, description="YYYY-MM-DD") rule: str = "" @@ -314,22 +337,7 @@ def get_settings() -> dict[str, Any]: @app.post("/api/tests/load") def load_tests(request: LoadTestsRequest) -> dict[str, Any]: - received_path = request.csv_path - - # Handle absolute paths (Windows or Unix) by extracting just the filename - if "\\" in received_path or ":" in received_path or received_path.startswith("/"): - # Absolute path (Windows with backslash or drive letter, or Unix with /) - # Extract just the filename from the path - filename = received_path.replace("\\", "/").split("/")[-1] - - data_dir_path = APP_ROOT / "data" / filename - if data_dir_path.exists(): - csv_path = data_dir_path - else: - csv_path = APP_ROOT / filename - else: - # Relative paths - prepend APP_ROOT - csv_path = APP_ROOT / received_path + csv_path = _resolve_requested_csv_path(request.csv_path) settings = db.read_settings(DB_PATH) smb_credentials = _smb_credentials_from_settings(settings) diff --git a/backend/db.py b/backend/db.py index 502d430..7e48e9b 100644 --- a/backend/db.py +++ b/backend/db.py @@ -27,6 +27,7 @@ class TestRecord: test_type: str rotation: str | None rx_tx: str | None + power_mode: str | None has_coe_pair: bool coe_pairing: list[str] priority: int @@ -80,6 +81,7 @@ def init_db(db_path: str | Path = DB_PATH) -> None: test_type TEXT NOT NULL CHECK (test_type IN ('P2P', 'COE', 'P3P')), rotation TEXT, rx_tx TEXT CHECK (rx_tx IN ('RX', 'TX') OR rx_tx IS NULL), + power_mode TEXT, has_coe_pair INTEGER NOT NULL DEFAULT 0, coe_pairing_json TEXT, priority INTEGER NOT NULL CHECK (priority BETWEEN 1 AND 5), @@ -132,6 +134,7 @@ def init_db(db_path: str | Path = DB_PATH) -> None: _ensure_column(conn, "tests", "has_coe_pair", "INTEGER NOT NULL DEFAULT 0") _ensure_column(conn, "tests", "coe_pairing_json", "TEXT") _ensure_column(conn, "tests", "config_json", "TEXT") + _ensure_column(conn, "tests", "power_mode", "TEXT") _ensure_column(conn, "tests", "victim_band", "TEXT") _ensure_column(conn, "tests", "excluded", "INTEGER NOT NULL DEFAULT 0") _ensure_column(conn, "tests", "throttled", "INTEGER NOT NULL DEFAULT 0") @@ -169,6 +172,7 @@ def upsert_tests(records: list[TestRecord], db_path: str | Path = DB_PATH) -> in r.test_type, r.rotation, r.rx_tx, + r.power_mode, int(r.has_coe_pair), json.dumps(r.coe_pairing or []), json.dumps(r.config), @@ -187,15 +191,16 @@ def upsert_tests(records: list[TestRecord], db_path: str | Path = DB_PATH) -> in conn.executemany( """ INSERT INTO tests( - test_id, device, test_type, rotation, rx_tx, has_coe_pair, + test_id, device, test_type, rotation, rx_tx, power_mode, has_coe_pair, coe_pairing_json, config_json, priority, victim_band, throttled, estimated_minutes, status, excluded, raw_payload, station_testpoint_map ) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT(test_id, device) DO UPDATE SET test_type = excluded.test_type, device = excluded.device, rotation = excluded.rotation, rx_tx = excluded.rx_tx, + power_mode = excluded.power_mode, has_coe_pair = excluded.has_coe_pair, coe_pairing_json = excluded.coe_pairing_json, config_json = excluded.config_json, @@ -277,12 +282,7 @@ def _entry_value(entry: dict[str, str | None], key: str) -> str: def _record_power_mode(record: TestRecord) -> str: - power_mode = _entry_value(_band_entry(record.config, "6G"), "power_mode") - if power_mode: - return power_mode - - raw_payload = record.raw_payload or {} - return _normalize_text(raw_payload.get("6GHz Power Mode")) + return _normalize_text(record.power_mode) def _match_any_band_value(record: TestRecord, key: str, token_suffix: str) -> bool: @@ -369,7 +369,7 @@ def list_schedulable_tests(db_path: str | Path = DB_PATH, rule: str = "") -> lis """ SELECT test_id, device, test_type, rotation, rx_tx, has_coe_pair, - config_json, victim_band, + power_mode, config_json, victim_band, coe_pairing_json, priority, throttled, estimated_minutes, excluded, status, raw_payload FROM tests @@ -387,6 +387,7 @@ def list_schedulable_tests(db_path: str | Path = DB_PATH, rule: str = "") -> lis test_type=row["test_type"], rotation=row["rotation"], rx_tx=row["rx_tx"], + power_mode=row["power_mode"], has_coe_pair=bool(row["has_coe_pair"]), coe_pairing=json.loads(row["coe_pairing_json"] or "[]"), priority=int(row["priority"]), @@ -411,7 +412,7 @@ def list_tests_for_device(device: str, db_path: str | Path = DB_PATH) -> list[Te """ SELECT test_id, device, test_type, rotation, rx_tx, has_coe_pair, - config_json, victim_band, + power_mode, config_json, victim_band, coe_pairing_json, priority, throttled, estimated_minutes, excluded, status, raw_payload, station_testpoint_map FROM tests @@ -429,6 +430,7 @@ def list_tests_for_device(device: str, db_path: str | Path = DB_PATH) -> list[Te test_type=row["test_type"], rotation=row["rotation"], rx_tx=row["rx_tx"], + power_mode=row["power_mode"], has_coe_pair=bool(row["has_coe_pair"]), coe_pairing=json.loads(row["coe_pairing_json"] or "[]"), priority=int(row["priority"]), @@ -670,6 +672,19 @@ def get_schedule_rows( return result +def reset_completed_to_pending(db_path: str | Path = DB_PATH) -> int: + """Reset all 'completed' tests back to 'pending'. Used before a full directory rescan.""" + with get_connection(db_path) as conn: + cursor = conn.execute( + """ + UPDATE tests + SET status = 'pending', updated_at = CURRENT_TIMESTAMP + WHERE status = 'completed' + """ + ) + return cursor.rowcount + + def mark_tests_completed(test_ids_with_device: list[tuple[str, str]], db_path: str | Path = DB_PATH) -> int: if not test_ids_with_device: return 0 @@ -811,3 +826,9 @@ def upsert_holidays(dates: list[str], db_path: str | Path = DB_PATH) -> None: [(d,) for d in dates], ) +def reset_tests(db_path: str | Path = DB_PATH) -> int: + """Clear all tests records keep empty tests table.""" + with get_connection(db_path) as conn: + cursor = conn.execute("DELETE FROM tests") + return cursor.rowcount + diff --git a/backend/parser.py b/backend/parser.py index adf0153..56e277c 100644 --- a/backend/parser.py +++ b/backend/parser.py @@ -11,6 +11,7 @@ except ModuleNotFoundError: smbclient = None from db import DEVICE_DUT, DEVICE_REF, TestRecord +from scanner import resolve_runtime_path P2P_COE_REQUIRED_COLUMNS = [ "Priority", @@ -96,6 +97,8 @@ def _normalize_input_path(path_value: str | Path) -> str: if not path: return path + path = str(resolve_runtime_path(path)).strip() + # Accept //server/share style and normalize to UNC for smbclient. if path.startswith("//"): return "\\\\" + path.lstrip("/").replace("/", "\\") @@ -104,6 +107,12 @@ def _normalize_input_path(path_value: str | Path) -> str: if re.match(r"^/\d{1,3}(?:\.\d{1,3}){3}/[^/]+", path): return "\\\\" + path.lstrip("/").replace("/", "\\") + if os.path.exists(path): + return path + + if path.startswith("/"): + return path + return path @@ -372,9 +381,10 @@ def _parse_single_csv( test_type = _infer_test_type(test_id) rx_tx = _infer_rx_tx(test_id) rotation = _empty_to_none(_row_get(row, "Rotation")) + power_mode = _empty_to_none(_row_get(row, "6GHz Power Mode")) has_coe_pair = _normalize_yes_no(_row_get(row, "COE Pair")) if csv_format == "p2p_coe" else False config = _build_config(row, csv_format) - signature = _victim_band_signature(row) if csv_format == "p2p_coe" else None + signature = _first_populated_p2p_signature(row) if csv_format == "p2p_coe" and test_type == "P2P" else None victim_band_source = "Victim Band" if csv_format == "p2p_coe" else "Band" victim_band = _normalize_victim_band(_row_get(row, victim_band_source)) @@ -392,6 +402,7 @@ def _parse_single_csv( test_type=test_type, rotation=rotation, rx_tx=rx_tx, + power_mode=power_mode, has_coe_pair=has_coe_pair, coe_pairing=[], priority=priority, @@ -405,14 +416,18 @@ def _parse_single_csv( ) records_with_signature.append((record, signature)) - # Build COE pairing based on victim band signature and RX/TX+band suffix. + # Build COE pairing based on full signature and RX/TX+band suffix. # Example key suffixes: RXAX, TXAX, RXBE, TXBE. - coe_by_signature_and_suffix: dict[tuple, list[str]] = {} + coe_by_signature_and_suffix: dict[tuple[tuple[str, ...], str], list[str]] = {} for record, signature in records_with_signature: - if record.test_type == "COE" and signature is not None: + if record.test_type == "COE": suffix = _extract_pairing_suffix(record.test_id) - if suffix: - key = (signature, suffix) + if not suffix: + continue + + # Index all populated COE band signatures so any one can match the selected P2P signature. + for coe_signature in _all_band_signatures(record.raw_payload or {}): + key = (coe_signature, suffix) coe_by_signature_and_suffix.setdefault(key, []).append(record.test_id) for record, signature in records_with_signature: @@ -420,7 +435,7 @@ def _parse_single_csv( if record.test_type == "P2P" and signature is not None: suffix = _extract_pairing_suffix(record.test_id) key = (signature, suffix) if suffix else None - pairs = sorted(coe_by_signature_and_suffix.get(key, [])) if key else [] + pairs = sorted(set(coe_by_signature_and_suffix.get(key, []))) if key else [] for device in (DEVICE_DUT, DEVICE_REF): device_runtime_defaults = runtime_defaults_by_device.get(device, RUNTIME_DEFAULTS) @@ -431,6 +446,7 @@ def _parse_single_csv( test_type=record.test_type, rotation=record.rotation, rx_tx=record.rx_tx, + power_mode=record.power_mode, has_coe_pair=bool(pairs), coe_pairing=pairs, priority=record.priority, @@ -493,13 +509,21 @@ def _empty_to_none(value: str | None) -> str | None: return cleaned if cleaned else None -def _victim_band_signature(row: dict[str, str]) -> tuple[str, ...] | None: - band = _normalize_victim_band(_row_get(row, "Victim Band")) - if band is None: +def _first_populated_p2p_signature(row: dict[str, str]) -> tuple[str, ...] | None: + for band in ("5G", "6G", "2G"): + signature = _band_signature(row, band) + if signature is not None: + return signature + return None + + +def _band_signature(row: dict[str, str], band: str) -> tuple[str, ...] | None: + normalized_band = _normalize_victim_band(band) + if normalized_band is None: return None # Source CSVs may use either "5G_Test Point" or "5G Test Point" style headers. - prefixes = (f"{band}_", f"{band} ") + prefixes = (f"{normalized_band}_", f"{normalized_band} ") def _pick_value(suffix: str) -> str: for prefix in prefixes: @@ -512,12 +536,24 @@ def _victim_band_signature(row: dict[str, str]) -> tuple[str, ...] | None: channel = _pick_value("Channel") rssi = _pick_value("RSSI") bandwidth = _pick_value("Bandwidth") + direction = _pick_value("Direction") + sta = _pick_value("STA") - # Direction is intentionally ignored for COE pairing matching. - if not all([test_point, channel, rssi, bandwidth]): + if not all([test_point, channel, rssi, bandwidth, direction, sta]): return None - return (band, test_point, channel, rssi, bandwidth) + return (test_point, channel, bandwidth, rssi, direction, sta) + + +def _all_band_signatures(row: dict[str, str]) -> list[tuple[str, ...]]: + seen: set[tuple[str, ...]] = set() + signatures: list[tuple[str, ...]] = [] + for band in ("5G", "6G", "2G"): + signature = _band_signature(row, band) + if signature is not None and signature not in seen: + seen.add(signature) + signatures.append(signature) + return signatures def _normalize_victim_band(value: str | None) -> str | None: @@ -561,7 +597,6 @@ def _build_legacy_config(row: dict[str, str]) -> dict[str, dict[str, str | None] "sta": _empty_to_none(_row_get(row, "5G_STA")), }, "6G": { - "power_mode": _empty_to_none(_row_get(row, "6GHz Power Mode")), "test_point": _empty_to_none(_row_get(row, "6G_Test Point")), "channel": _empty_to_none(_row_get(row, "6G_Channel")), "bandwidth": _empty_to_none(_row_get(row, "6G_Bandwidth")), diff --git a/backend/scanner.py b/backend/scanner.py index d7b3d06..17fcfdc 100644 --- a/backend/scanner.py +++ b/backend/scanner.py @@ -7,7 +7,7 @@ try: except ModuleNotFoundError: smbclient = None -from db import DEVICE_DUT, DEVICE_REF, mark_tests_completed, mark_overdue_as_rerun +from db import DEVICE_DUT, DEVICE_REF, mark_tests_completed, mark_overdue_as_rerun, reset_completed_to_pending _SMB_SESSIONS = set() _SCAN_STATE_LOCK = threading.Lock() @@ -148,19 +148,28 @@ def resolve_runtime_path(path_value): if raw_path.startswith("\\\\") or raw_path.startswith("//"): return raw_path - # Map host paths (Windows or Linux) to the container mount point when running in a container. - if os.name != "nt": - mount_root = os.getenv("HOST_MOUNT_ROOT", "/host").strip() or "/host" - host_root = os.getenv("HOST_BROWSE_ROOT", "").strip() + mount_root = (os.getenv("HOST_MOUNT_ROOT", "/host") or "/host").strip() or "/host" + host_root = (os.getenv("HOST_BROWSE_ROOT", "") or "").strip() + raw_norm = raw_path.replace("\\", "/") - raw_norm = raw_path.replace("\\", "/") - if host_root: - host_norm = host_root.replace("\\", "/").rstrip("/") - if raw_norm.lower() == host_norm.lower() or raw_norm.lower().startswith(host_norm.lower() + "/"): - relative = raw_norm[len(host_norm):].lstrip("/") - if relative: - return os.path.join(mount_root, *relative.split("/")) - return mount_root + # In container/Linux runtime, translate host-browse paths into mounted container paths. + if os.name != "nt" and host_root: + host_norm = host_root.replace("\\", "/").rstrip("/") + if raw_norm.lower() == host_norm.lower() or raw_norm.lower().startswith(host_norm.lower() + "/"): + relative = raw_norm[len(host_norm):].lstrip("/") + if relative: + return os.path.join(mount_root, *relative.split("/")) + return mount_root + + # In native Windows runtime, accept /host/... paths coming from container-oriented settings + # and map them back to HOST_BROWSE_ROOT (for example C:/Users/... ). + if os.name == "nt" and host_root: + mount_norm = mount_root.replace("\\", "/").rstrip("/") + if mount_norm and (raw_norm.lower() == mount_norm.lower() or raw_norm.lower().startswith(mount_norm.lower() + "/")): + relative = raw_norm[len(mount_norm):].lstrip("/") + if relative: + return os.path.join(host_root, *relative.split("/")) + return host_root return raw_path @@ -172,25 +181,36 @@ def _normalize_input_path(path_value): path = resolve_runtime_path(path_value) path = str(path).strip() - # Normalize forward slashes to backslashes first - path = path.replace("/", "\\") + # Accept //server/share style and normalize to UNC for smbclient. + if path.startswith("//"): + path = path.lstrip("/").replace("/", "\\") + return "\\\\" + path - # Accept \\server\share style UNC paths and ensure proper escaping + # Accept \\server\share style UNC paths and ensure proper escaping. if path.startswith("\\\\"): # Clean up any doubled backslashes from replacement while "\\\\\\" in path: path = path.replace("\\\\\\", "\\\\") return path - # Accept //server/share style and normalize to UNC for smbclient. - if path.startswith("//"): - path = path.lstrip("/").replace("/", "\\") - return "\\\\" + path - # Accept ///... and normalize to UNC for Linux-hosted inputs. - if re.match(r"^\\?\d{1,3}(?:\\\.\d{1,3}){3}\\[^\\]+", path): - if path.startswith("\\"): - path = path.lstrip("\\") + if re.match(r"^/\d{1,3}(?:\.\d{1,3}){3}/[^/]+", path): + return "\\\\" + path.lstrip("/").replace("/", "\\") + + # Preserve existing slash-based local paths such as /host/... that are valid + # in the current runtime, so scanner matches watcher behavior. + if os.path.exists(path): + return path + + # Preserve slash-based local mount paths even if they are temporarily missing. + if path.startswith("/"): + return path + + # Normalize local paths after UNC checks. + path = path.replace("/", "\\") + + # Accept \\\... and normalize to UNC. + if re.match(r"^\d{1,3}(?:\.\d{1,3}){3}\\[^\\]+", path): return "\\\\" + path return path @@ -316,9 +336,13 @@ def scan_results(results_dir_dut, results_dir_ref, smb_credentials=None): for test_id, device in completed_batch: print(f" - {test_id} on {device}") + reset_count = reset_completed_to_pending() + if reset_count: + print(f"[scanner] reset {reset_count} previously-completed test(s) to pending before resync") + updated_count = mark_tests_completed(completed_batch) print(f"[scanner] {updated_count} test(s) actually updated in database") - + newly_rerun = mark_overdue_as_rerun() if newly_rerun: print(f"[scanner] {newly_rerun} test(s) marked as rerun-required (scheduled but not completed)") diff --git a/backend/scheduler.py b/backend/scheduler.py index ce3b5be..37fd098 100644 --- a/backend/scheduler.py +++ b/backend/scheduler.py @@ -61,6 +61,11 @@ class Scheduler: if not bundles: return f"Error: No test bundles could be created from the provided tests." + # Print all test bundles for debugging + print("Compiled Test Bundles:") + for bundle in bundles: + print(f"Bundle Index: {bundle.index}, Device: {bundle.device}, Priority: {bundle.priority}, Config: {bundle.config}, Total Minutes: {bundle.total_minutes}, Tests: {bundle.tests}") + active_dut_bundles = [b for b in bundles if b.device == DUT] active_ref_bundles = [b for b in bundles if b.device == REF]