read from target dir and look up metadata from csv
This commit is contained in:
@@ -0,0 +1,532 @@
|
||||
from contextlib import asynccontextmanager
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from datetime import date, datetime, timedelta
|
||||
import os
|
||||
|
||||
from fastapi import FastAPI, HTTPException
|
||||
from fastapi.middleware.cors import CORSMiddleware
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
import db as db
|
||||
from parser import CsvValidationError
|
||||
from file_manager import resolve_requested_csv_path
|
||||
|
||||
# Ensure new_scheduler resolves the same DUT/REF labels as db records.
|
||||
os.environ.setdefault("DUT", db.DUT)
|
||||
os.environ.setdefault("REF", db.REF)
|
||||
|
||||
from scheduler import Scheduler, Test as SchedulerTest
|
||||
from file_manager import resolve_runtime_path
|
||||
from test_config import build_bundle_test_configs, build_config_rows
|
||||
from test_window import get_shift_capacity_for_date, get_shift_sequence, is_off_day
|
||||
from watcher import configure_result_watcher, stop_result_watcher
|
||||
from scanner import process_targets
|
||||
|
||||
APP_ROOT = Path(__file__).resolve().parent
|
||||
DB_PATH = Path(os.getenv("DB_PATH", str(APP_ROOT.parent / "data" / "scheduler.db")))
|
||||
DUT = os.getenv("DUT", "CGW453").strip()
|
||||
REF = os.getenv("REF", "CGW452").strip()
|
||||
|
||||
class LoadTestsRequest(BaseModel):
|
||||
csv_path: str | None = Field(default=None, description="Absolute or backend-relative path to target CSV")
|
||||
csv_paths: list[str] = Field(default_factory=list, description="One or more CSV paths to load together")
|
||||
target_dir: str = Field(..., description="Absolute or backend-relative path to target directory")
|
||||
|
||||
|
||||
class SaveSettingsRequest(BaseModel):
|
||||
settings: dict[str, Any]
|
||||
|
||||
|
||||
def _smb_credentials_from_settings(settings: dict[str, Any]) -> dict[str, str]:
|
||||
return {
|
||||
"username": str(settings.get("smbUsername") or "").strip(),
|
||||
"password": str(settings.get("smbPassword") or ""),
|
||||
"domain": str(settings.get("smbDomain") or "").strip(),
|
||||
}
|
||||
|
||||
|
||||
def _runtime_overrides_from_settings(settings: dict[str, Any]) -> dict[str, dict[str, int]]:
|
||||
def _parse_positive_int(value: Any) -> int | None:
|
||||
if value is None:
|
||||
return None
|
||||
if isinstance(value, str):
|
||||
value = value.strip()
|
||||
if not value:
|
||||
return None
|
||||
try:
|
||||
parsed = int(value)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
return parsed if parsed > 0 else None
|
||||
|
||||
legacy_overrides = {
|
||||
"P2P": _parse_positive_int(settings.get("p2pRuntimeMinutes")),
|
||||
"COE": _parse_positive_int(settings.get("coeRuntimeMinutes")),
|
||||
"P3P": _parse_positive_int(settings.get("p3pRuntimeMinutes")),
|
||||
}
|
||||
|
||||
overrides: dict[str, dict[str, int]] = {}
|
||||
for device_key, field_prefix in ((db.DUT, "dut"), (db.REF, "ref")):
|
||||
device_overrides: dict[str, int] = {}
|
||||
for test_type, suffix in (("P2P", "P2p"), ("COE", "Coe"), ("P3P", "P3p")):
|
||||
minutes = _parse_positive_int(settings.get(f"{field_prefix}{suffix}RuntimeMinutes"))
|
||||
if minutes is None:
|
||||
minutes = legacy_overrides[test_type]
|
||||
if minutes is not None:
|
||||
device_overrides[test_type] = minutes
|
||||
if device_overrides:
|
||||
overrides[device_key] = device_overrides
|
||||
|
||||
return overrides
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
class CompileScheduleRequest(BaseModel):
|
||||
start_date: str | None = Field(default=None, description="YYYY-MM-DD")
|
||||
rule: str = ""
|
||||
daytime_testing_today: bool = False
|
||||
daytime_testing_hours: int = Field(default=8, ge=0, le=8)
|
||||
daytime_testing_device: str = ""
|
||||
dual_device_weekend_start_enabled: bool = False
|
||||
dual_device_weekend_start_dates: list[str] = Field(default_factory=list)
|
||||
top_priority_tests_dut: list[str] = Field(default_factory=list)
|
||||
top_priority_tests_ref: list[str] = Field(default_factory=list)
|
||||
top_priority_tests: list[str] = Field(default_factory=list)
|
||||
lowest_priority_tests: list[str] = Field(default_factory=list)
|
||||
|
||||
|
||||
class RemoveActiveTestsRequest(BaseModel):
|
||||
test_ids: list[str] = Field(default_factory=list)
|
||||
|
||||
|
||||
class SaveHolidaysRequest(BaseModel):
|
||||
dates: list[str] = Field(default_factory=list)
|
||||
|
||||
|
||||
SHIFT_LABELS = {
|
||||
1: "12AM–9AM",
|
||||
2: "9AM–5PM",
|
||||
3: "5PM–12AM",
|
||||
}
|
||||
|
||||
SHIFT_BOUNDARIES = {
|
||||
1: ((0, 0), (9, 0)),
|
||||
2: ((9, 0), (17, 0)),
|
||||
3: ((17, 0), (24, 0)),
|
||||
}
|
||||
|
||||
|
||||
def _serialize_schedule_row(row: db.ScheduleRow) -> dict[str, Any]:
|
||||
return {
|
||||
"test_id": row.test_id,
|
||||
"device": row.device,
|
||||
"scheduled_date": row.scheduled_date,
|
||||
"shift_index": row.shift_index,
|
||||
"sequence_in_shift": row.sequence_in_shift,
|
||||
"test_type": row.test_type,
|
||||
"rotation": row.rotation,
|
||||
"config": row.config,
|
||||
"status": row.status,
|
||||
"estimated_minutes": row.estimated_minutes,
|
||||
}
|
||||
|
||||
|
||||
def _window_segment_key(day: date, shift_index: int) -> str:
|
||||
return f"{day.isoformat()}::shift{shift_index}"
|
||||
|
||||
|
||||
def _combine_date_time(day: date, hour: int, minute: int) -> str:
|
||||
if hour == 24:
|
||||
return datetime.combine(day + timedelta(days=1), datetime.min.time()).isoformat(timespec="minutes")
|
||||
return datetime.combine(day, datetime.min.time()).replace(hour=hour, minute=minute).isoformat(timespec="minutes")
|
||||
|
||||
|
||||
def _window_intersects_week(segments: list[tuple[date, int]], week_start: date, week_end: date) -> bool:
|
||||
for day, _shift_index in segments:
|
||||
if week_start <= day < week_end:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _build_schedule_windows(week_start: date, all_rows: list[db.ScheduleRow], holiday_dates: set[str]) -> list[dict[str, Any]]:
|
||||
if not all_rows:
|
||||
return []
|
||||
|
||||
all_dates = [datetime.strptime(row.scheduled_date, "%Y-%m-%d").date() for row in all_rows]
|
||||
first_scheduled_date = min(all_dates)
|
||||
last_scheduled_date = max(all_dates)
|
||||
week_end = week_start + timedelta(days=7)
|
||||
|
||||
rows_by_segment: dict[tuple[str, int], list[dict[str, Any]]] = {}
|
||||
weekday_shift2_dates: set[str] = set()
|
||||
for row in all_rows:
|
||||
serialized = _serialize_schedule_row(row)
|
||||
rows_by_segment.setdefault((row.scheduled_date, row.shift_index), []).append(serialized)
|
||||
row_date = datetime.strptime(row.scheduled_date, "%Y-%m-%d").date()
|
||||
if row.shift_index == 2 and not is_off_day(row_date, holiday_dates):
|
||||
weekday_shift2_dates.add(row.scheduled_date)
|
||||
|
||||
simulation_start = min(first_scheduled_date, week_start) - timedelta(days=7)
|
||||
while is_off_day(simulation_start, holiday_dates):
|
||||
simulation_start -= timedelta(days=1)
|
||||
|
||||
simulation_end = max(last_scheduled_date, week_end) + timedelta(days=7)
|
||||
current_date = simulation_start
|
||||
windows: list[dict[str, Any]] = []
|
||||
seen_window_ids: set[str] = set()
|
||||
|
||||
while current_date <= simulation_end:
|
||||
current_iso = current_date.isoformat()
|
||||
|
||||
if current_iso in weekday_shift2_dates:
|
||||
segments = get_shift_sequence(current_date, holiday_dates, daytime_shift2_only=True)
|
||||
window_id = f"{current_iso}-shift2"
|
||||
if window_id not in seen_window_ids and _window_intersects_week(segments, week_start, week_end):
|
||||
seen_window_ids.add(window_id)
|
||||
shift_tests = [
|
||||
test
|
||||
for day, shift_index in segments
|
||||
for test in rows_by_segment.get((day.isoformat(), shift_index), [])
|
||||
]
|
||||
available_runtime_minutes = sum(
|
||||
get_shift_capacity_for_date(day, holiday_dates, daytime_shift2_only=True).get(shift_index, 0)
|
||||
for day, shift_index in segments
|
||||
)
|
||||
start_hour, start_minute = SHIFT_BOUNDARIES[segments[0][1]][0]
|
||||
end_hour, end_minute = SHIFT_BOUNDARIES[segments[-1][1]][1]
|
||||
windows.append({
|
||||
"window_id": window_id,
|
||||
"window_type": "daytime",
|
||||
"start_date": segments[0][0].isoformat(),
|
||||
"start_shift_index": segments[0][1],
|
||||
"end_date": segments[-1][0].isoformat(),
|
||||
"end_shift_index": segments[-1][1],
|
||||
"start_at": _combine_date_time(segments[0][0], start_hour, start_minute),
|
||||
"end_at": _combine_date_time(segments[-1][0], end_hour, end_minute),
|
||||
"available_runtime_minutes": available_runtime_minutes,
|
||||
"estimated_runtime_minutes": sum(test["estimated_minutes"] for test in shift_tests),
|
||||
"bundle_test_configs": build_bundle_test_configs(shift_tests),
|
||||
"segments": [
|
||||
{
|
||||
"date": day.isoformat(),
|
||||
"shift_index": shift_index,
|
||||
"label": SHIFT_LABELS[shift_index],
|
||||
"segment_key": _window_segment_key(day, shift_index),
|
||||
}
|
||||
for day, shift_index in segments
|
||||
],
|
||||
"configRows": build_config_rows(shift_tests),
|
||||
"tests": shift_tests,
|
||||
})
|
||||
|
||||
segments = get_shift_sequence(current_date, holiday_dates, daytime_shift2_only=False)
|
||||
last_sequence_date = segments[-1][0]
|
||||
last_shift_index = segments[-1][1]
|
||||
window_kind = "offday" if is_off_day(current_date, holiday_dates) else "overnight"
|
||||
window_id = f"{current_iso}-shift{segments[0][1]}"
|
||||
|
||||
if window_id not in seen_window_ids and _window_intersects_week(segments, week_start, week_end):
|
||||
seen_window_ids.add(window_id)
|
||||
window_tests = [
|
||||
test
|
||||
for day, shift_index in segments
|
||||
for test in rows_by_segment.get((day.isoformat(), shift_index), [])
|
||||
]
|
||||
available_runtime_minutes = sum(
|
||||
get_shift_capacity_for_date(day, holiday_dates, daytime_shift2_only=False).get(shift_index, 0)
|
||||
for day, shift_index in segments
|
||||
)
|
||||
start_hour, start_minute = SHIFT_BOUNDARIES[segments[0][1]][0]
|
||||
end_hour, end_minute = SHIFT_BOUNDARIES[segments[-1][1]][1]
|
||||
windows.append({
|
||||
"window_id": window_id,
|
||||
"window_type": window_kind,
|
||||
"start_date": segments[0][0].isoformat(),
|
||||
"start_shift_index": segments[0][1],
|
||||
"end_date": segments[-1][0].isoformat(),
|
||||
"end_shift_index": segments[-1][1],
|
||||
"start_at": _combine_date_time(segments[0][0], start_hour, start_minute),
|
||||
"end_at": _combine_date_time(segments[-1][0], end_hour, end_minute),
|
||||
"available_runtime_minutes": available_runtime_minutes,
|
||||
"estimated_runtime_minutes": sum(test["estimated_minutes"] for test in window_tests),
|
||||
"bundle_test_configs": build_bundle_test_configs(window_tests),
|
||||
"segments": [
|
||||
{
|
||||
"date": day.isoformat(),
|
||||
"shift_index": shift_index,
|
||||
"label": SHIFT_LABELS[shift_index],
|
||||
"segment_key": _window_segment_key(day, shift_index),
|
||||
}
|
||||
for day, shift_index in segments
|
||||
],
|
||||
"configRows": build_config_rows(window_tests),
|
||||
"tests": window_tests,
|
||||
})
|
||||
|
||||
if last_shift_index in {1, 2}:
|
||||
current_date = last_sequence_date
|
||||
else:
|
||||
current_date = last_sequence_date + timedelta(days=1)
|
||||
|
||||
windows.sort(key=lambda window: (window["start_at"], window["window_id"]))
|
||||
return windows
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(application: FastAPI):
|
||||
db.init_db(DB_PATH)
|
||||
settings = db.read_settings(DB_PATH)
|
||||
configure_result_watcher(settings)
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
stop_result_watcher()
|
||||
|
||||
|
||||
app = FastAPI(title="Scheduler API", version="0.1.0", lifespan=lifespan)
|
||||
|
||||
app.add_middleware(
|
||||
CORSMiddleware,
|
||||
allow_origins=[
|
||||
"http://localhost:5173",
|
||||
"http://127.0.0.1:5173",
|
||||
"http://localhost:8080",
|
||||
"http://127.0.0.1:8080",
|
||||
],
|
||||
allow_methods=["*"],
|
||||
allow_headers=["*"],
|
||||
)
|
||||
|
||||
|
||||
@app.get("/api/health")
|
||||
def health() -> dict[str, str]:
|
||||
return {"status": "ok"}
|
||||
|
||||
|
||||
@app.post("/api/settings")
|
||||
def save_settings(request: SaveSettingsRequest) -> dict[str, str]:
|
||||
db.save_settings(request.settings, DB_PATH)
|
||||
configure_result_watcher(db.read_settings(DB_PATH))
|
||||
return {"status": "saved"}
|
||||
|
||||
|
||||
@app.get("/api/settings")
|
||||
def get_settings() -> dict[str, Any]:
|
||||
return db.read_settings(DB_PATH)
|
||||
|
||||
|
||||
@app.post("/api/settings/restart")
|
||||
def restart_and_clear_data() -> dict[str, Any]:
|
||||
deleted_counts = db.reset_all_data(DB_PATH)
|
||||
configure_result_watcher({})
|
||||
return {
|
||||
"status": "cleared",
|
||||
**deleted_counts,
|
||||
}
|
||||
|
||||
|
||||
@app.post("/api/tests/load")
|
||||
def load_tests(request: LoadTestsRequest) -> dict[str, Any]:
|
||||
requested_paths: list[str] = []
|
||||
if request.csv_path and request.csv_path.strip():
|
||||
requested_paths.append(request.csv_path.strip())
|
||||
requested_paths.extend(path.strip() for path in request.csv_paths if path and path.strip())
|
||||
|
||||
if not requested_paths:
|
||||
raise HTTPException(status_code=400, detail="At least one CSV path is required.")
|
||||
|
||||
resolved_csv_paths = [resolve_requested_csv_path(path) for path in requested_paths]
|
||||
target_dir = resolve_runtime_path(request.target_dir)
|
||||
|
||||
settings = db.read_settings(DB_PATH)
|
||||
smb_credentials = _smb_credentials_from_settings(settings)
|
||||
runtime_overrides = _runtime_overrides_from_settings(settings)
|
||||
|
||||
count = process_targets(target_dir, resolved_csv_paths, smb_credentials=smb_credentials, runtime_overrides=runtime_overrides)
|
||||
|
||||
print(f"Loaded {count} tests from {resolved_csv_paths}.")
|
||||
return {
|
||||
"loaded_tests": count,
|
||||
}
|
||||
|
||||
@app.post("/api/schedule/active/remove")
|
||||
def remove_active_tests(request: RemoveActiveTestsRequest) -> dict[str, Any]:
|
||||
# new_scheduler does not keep global mutable active state in app lifecycle.
|
||||
# Keep endpoint for compatibility with frontend calls.
|
||||
return {"status": "ok", "removed": len(request.test_ids)}
|
||||
|
||||
|
||||
@app.post("/api/schedule/compile")
|
||||
def compile_schedule_endpoint(request: CompileScheduleRequest) -> dict[str, Any]:
|
||||
if request.start_date:
|
||||
try:
|
||||
datetime.strptime(request.start_date, "%Y-%m-%d")
|
||||
except ValueError as exc:
|
||||
raise HTTPException(status_code=400, detail="start_date must be YYYY-MM-DD") from exc
|
||||
|
||||
requested_daytime_device = str(request.daytime_testing_device or "").strip()
|
||||
if not requested_daytime_device:
|
||||
requested_daytime_device = DUT
|
||||
if requested_daytime_device not in {DUT, REF}:
|
||||
raise HTTPException(status_code=400, detail=f"daytime_testing_device must be {DUT} or {REF}")
|
||||
|
||||
all_tests = db.get_not_excluded_tests(DB_PATH, rule=request.rule)
|
||||
stored_tests = db.list_schedulable_tests(DB_PATH, rule=request.rule)
|
||||
if not stored_tests:
|
||||
version = db.create_schedule_version([], DB_PATH)
|
||||
return {
|
||||
"schedule_version": version,
|
||||
"scheduled_tests": 0,
|
||||
"completion_date": None,
|
||||
}
|
||||
|
||||
scheduler_tests = [
|
||||
SchedulerTest(
|
||||
test_id=t.test_id,
|
||||
device=t.device,
|
||||
test_type=t.test_type,
|
||||
rotation=t.rotation,
|
||||
rx_tx=t.rx_tx,
|
||||
has_coe_pair=t.has_coe_pair,
|
||||
coe_pairing=t.coe_pairing or [],
|
||||
config=t.config,
|
||||
throttled=t.throttled,
|
||||
estimated_minutes=t.estimated_minutes,
|
||||
status=t.status,
|
||||
)
|
||||
for t in stored_tests
|
||||
]
|
||||
|
||||
holiday_dates = db.list_holidays(DB_PATH)
|
||||
top_priority_pairs: set[tuple[str, str]] = set()
|
||||
dut_top_priority_ids = {item.strip() for item in request.top_priority_tests_dut if item.strip()}
|
||||
ref_top_priority_ids = {item.strip() for item in request.top_priority_tests_ref if item.strip()}
|
||||
status_lookup = {(test.test_id, test.device): test.status for test in stored_tests}
|
||||
|
||||
for test_id in dut_top_priority_ids:
|
||||
if any(t.test_id == test_id and t.device == DUT for t in stored_tests):
|
||||
top_priority_pairs.add((test_id, DUT))
|
||||
|
||||
for test_id in ref_top_priority_ids:
|
||||
if any(t.test_id == test_id and t.device == REF for t in stored_tests):
|
||||
top_priority_pairs.add((test_id, REF))
|
||||
|
||||
scheduler = Scheduler(
|
||||
all_tests=all_tests,
|
||||
schedulable_tests=scheduler_tests,
|
||||
top_priority_tests=top_priority_pairs,
|
||||
start_date=request.start_date,
|
||||
holiday_dates=holiday_dates,
|
||||
daytime_testing_today=request.daytime_testing_today,
|
||||
daytime_testing_hours=request.daytime_testing_hours,
|
||||
daytime_testing_device=requested_daytime_device,
|
||||
dual_device_weekend_start_enabled=request.dual_device_weekend_start_enabled,
|
||||
dual_device_window_start_dates=set(item.strip() for item in request.dual_device_weekend_start_dates if item.strip()),
|
||||
)
|
||||
|
||||
try:
|
||||
schedule_error = scheduler.compile_schedule()
|
||||
entries = scheduler.get_schedule()
|
||||
except (RuntimeError, ValueError) as exc:
|
||||
raise HTTPException(status_code=409, detail=str(exc)) from exc
|
||||
|
||||
completion_date = max((e.scheduled_date for e in entries), default=None)
|
||||
if schedule_error:
|
||||
raise HTTPException(status_code=409, detail=schedule_error)
|
||||
|
||||
version = db.create_schedule_version(
|
||||
[
|
||||
(
|
||||
e.test_id,
|
||||
e.device,
|
||||
e.scheduled_date,
|
||||
e.shift_index,
|
||||
e.sequence_in_shift,
|
||||
status_lookup.get((e.test_id, e.device), "pending"),
|
||||
)
|
||||
for e in entries
|
||||
],
|
||||
DB_PATH,
|
||||
)
|
||||
print(f"Schedule version {version} created with {len(entries)} entries, completion date: {completion_date}")
|
||||
return {
|
||||
"schedule_version": version,
|
||||
"scheduled_tests": len(entries),
|
||||
"completion_date": completion_date,
|
||||
}
|
||||
|
||||
|
||||
@app.get("/api/tests/rerun")
|
||||
def get_rerun_tests() -> dict[str, Any]:
|
||||
db.mark_overdue_as_rerun(DB_PATH)
|
||||
tests = db.get_rerun_tests(DB_PATH)
|
||||
total_minutes = sum(t["estimated_minutes"] for t in tests)
|
||||
return {"tests": tests, "total_estimated_minutes": total_minutes}
|
||||
|
||||
|
||||
@app.post("/api/holidays")
|
||||
def save_holidays(request: SaveHolidaysRequest) -> dict[str, Any]:
|
||||
dates = [d.strip() for d in request.dates if d.strip()]
|
||||
db.upsert_holidays(dates, DB_PATH)
|
||||
return {"status": "saved", "count": len(dates)}
|
||||
|
||||
|
||||
@app.get("/api/holidays")
|
||||
def get_holidays() -> dict[str, Any]:
|
||||
return {"dates": sorted(db.list_holidays(DB_PATH))}
|
||||
|
||||
|
||||
@app.get("/api/schedule/versions")
|
||||
def get_schedule_versions() -> dict[str, Any]:
|
||||
return {"versions": db.get_schedule_versions(DB_PATH)}
|
||||
|
||||
|
||||
@app.get("/api/schedule/week")
|
||||
def get_schedule_week(start: str | None = None, version: int | None = None) -> dict[str, Any]:
|
||||
week_start = start or date.today().isoformat()
|
||||
try:
|
||||
datetime.strptime(week_start, "%Y-%m-%d")
|
||||
except ValueError as exc:
|
||||
raise HTTPException(status_code=400, detail="start must be YYYY-MM-DD") from exc
|
||||
if version is not None and version <= 0:
|
||||
raise HTTPException(status_code=400, detail="version must be a positive integer")
|
||||
|
||||
week_start_date = datetime.strptime(week_start, "%Y-%m-%d").date()
|
||||
selected_version = db.resolve_schedule_version(version, DB_PATH)
|
||||
if version is not None and selected_version is None:
|
||||
raise HTTPException(status_code=404, detail=f"Schedule version {version} was not found")
|
||||
|
||||
rows = db.get_schedule_week(week_start, selected_version, DB_PATH)
|
||||
|
||||
completed_rows = [row for row in rows if str(row.status).strip().lower() == "completed"]
|
||||
rerun_rows = [row for row in rows if str(row.status).strip().lower() == "rerun"]
|
||||
print(
|
||||
f"[api] /schedule/week: completed={len(completed_rows)} rerun_required={len(rerun_rows)}"
|
||||
)
|
||||
|
||||
all_rows = db.get_schedule_rows(selected_version, DB_PATH)
|
||||
completion_date = max((row.scheduled_date for row in all_rows), default=None)
|
||||
start_shift = min(
|
||||
((row.scheduled_date, row.shift_index) for row in all_rows),
|
||||
default=(None, None),
|
||||
)
|
||||
schedule_start_date, schedule_start_shift_index = start_shift
|
||||
holiday_dates = db.list_holidays(DB_PATH)
|
||||
return {
|
||||
"start_date": week_start,
|
||||
"schedule_version": selected_version,
|
||||
"schedule_start_date": schedule_start_date,
|
||||
"schedule_start_shift_index": schedule_start_shift_index,
|
||||
"items": [_serialize_schedule_row(row) for row in rows],
|
||||
"total_scheduled_tests": len(all_rows),
|
||||
"completion_date": completion_date,
|
||||
"windows": _build_schedule_windows(week_start_date, all_rows, holiday_dates),
|
||||
}
|
||||
|
||||
if __name__ == "__main__":
|
||||
import uvicorn
|
||||
uvicorn.run("app:app", host="0.0.0.0", port=8000, reload=True)
|
||||
|
||||
@@ -0,0 +1,997 @@
|
||||
import json
|
||||
import sqlite3
|
||||
import os
|
||||
import re
|
||||
from contextlib import contextmanager
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any, Iterator
|
||||
from test_config import serialize_station_testpoint_map, resolve_test_config_keys
|
||||
|
||||
|
||||
APP_ROOT = Path(__file__).resolve().parent
|
||||
DB_PATH = Path(os.getenv("DB_PATH", str(APP_ROOT.parent / "data" / "scheduler.db")))
|
||||
|
||||
# Hardware device names (from environment or defaults)
|
||||
DUT = os.getenv("DUT", "CGW453").strip()
|
||||
REF = os.getenv("REF", "CGW452").strip()
|
||||
|
||||
# Device names for the test database (use hardware device names)
|
||||
MAX_SCHEDULE_VERSIONS = 50
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class TestRecord:
|
||||
test_id: str
|
||||
device: str
|
||||
test_type: str
|
||||
rotation: str | None
|
||||
rx_tx: str | None
|
||||
power_mode: str | None
|
||||
has_coe_pair: bool
|
||||
coe_pairing: list[str]
|
||||
victim_band: str | None
|
||||
config: dict[str, dict[str, str | None]]
|
||||
throttled: bool
|
||||
estimated_minutes: int
|
||||
status: str = "pending"
|
||||
excluded: bool = False
|
||||
raw_payload: dict[str, Any] | None = None
|
||||
station_testpoint_map: str | None = None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ScheduleRow:
|
||||
test_id: str
|
||||
device: str
|
||||
scheduled_date: str
|
||||
shift_index: int
|
||||
sequence_in_shift: int
|
||||
test_type: str
|
||||
rotation: str | None
|
||||
config: dict[str, dict[str, str | None]]
|
||||
status: str
|
||||
estimated_minutes: int
|
||||
|
||||
@contextmanager
|
||||
def get_connection(db_path: str | Path = DB_PATH) -> Iterator[sqlite3.Connection]:
|
||||
conn = sqlite3.connect(str(db_path))
|
||||
conn.row_factory = sqlite3.Row
|
||||
try:
|
||||
yield conn
|
||||
conn.commit()
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def init_db(db_path: str | Path = DB_PATH) -> None:
|
||||
# Validate device names are set
|
||||
if not DUT or not REF:
|
||||
raise ValueError(f"Invalid device names: DUT={DUT!r}, REF={REF!r}")
|
||||
|
||||
with get_connection(db_path) as conn:
|
||||
conn.executescript(
|
||||
"""
|
||||
CREATE TABLE IF NOT EXISTS tests (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
test_id TEXT NOT NULL,
|
||||
device TEXT NOT NULL,
|
||||
test_type TEXT NOT NULL CHECK (test_type IN ('P2P', 'COE', 'P3P')),
|
||||
rotation TEXT,
|
||||
rx_tx TEXT CHECK (rx_tx IN ('RX', 'TX') OR rx_tx IS NULL),
|
||||
power_mode TEXT,
|
||||
has_coe_pair INTEGER NOT NULL DEFAULT 0,
|
||||
coe_pairing_json TEXT,
|
||||
victim_band TEXT,
|
||||
config_json TEXT,
|
||||
throttled INTEGER NOT NULL DEFAULT 0,
|
||||
estimated_minutes INTEGER NOT NULL,
|
||||
status TEXT NOT NULL CHECK (status IN ('pending', 'completed', 'rerun')),
|
||||
excluded INTEGER NOT NULL DEFAULT 0,
|
||||
raw_payload TEXT,
|
||||
created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
UNIQUE(test_id, device)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS schedules (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
schedule_version INTEGER NOT NULL,
|
||||
test_id TEXT NOT NULL,
|
||||
device TEXT NOT NULL,
|
||||
scheduled_date TEXT NOT NULL,
|
||||
shift_index INTEGER NOT NULL CHECK (shift_index IN (1, 2, 3)),
|
||||
sequence_in_shift INTEGER NOT NULL,
|
||||
created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
FOREIGN KEY(test_id, device) REFERENCES tests(test_id, device)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS settings (
|
||||
key TEXT PRIMARY KEY,
|
||||
value_json TEXT NOT NULL,
|
||||
updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS holidays (
|
||||
date TEXT PRIMARY KEY,
|
||||
note TEXT
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS runtime_defaults (
|
||||
test_type TEXT PRIMARY KEY,
|
||||
minutes INTEGER NOT NULL
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_tests_status ON tests(status);
|
||||
CREATE INDEX IF NOT EXISTS idx_schedules_date_shift ON schedules(scheduled_date, shift_index);
|
||||
"""
|
||||
)
|
||||
|
||||
_ensure_column(conn, "tests", "has_coe_pair", "INTEGER NOT NULL DEFAULT 0")
|
||||
_ensure_column(conn, "tests", "coe_pairing_json", "TEXT")
|
||||
_ensure_column(conn, "tests", "config_json", "TEXT")
|
||||
_ensure_column(conn, "tests", "power_mode", "TEXT")
|
||||
_ensure_column(conn, "tests", "victim_band", "TEXT")
|
||||
_ensure_column(conn, "tests", "excluded", "INTEGER NOT NULL DEFAULT 0")
|
||||
_ensure_column(conn, "tests", "throttled", "INTEGER NOT NULL DEFAULT 0")
|
||||
_ensure_column(conn, "tests", "station_testpoint_map", "TEXT")
|
||||
_ensure_column(conn, "schedules", "status_snapshot", "TEXT")
|
||||
conn.execute(
|
||||
"""
|
||||
UPDATE schedules
|
||||
SET status_snapshot = COALESCE((
|
||||
SELECT t.status
|
||||
FROM tests t
|
||||
WHERE t.test_id = schedules.test_id
|
||||
AND t.device = schedules.device
|
||||
), 'pending')
|
||||
WHERE status_snapshot IS NULL
|
||||
"""
|
||||
)
|
||||
prune_schedule_versions(conn=conn)
|
||||
|
||||
# Seed runtime defaults for schedule estimation.
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO runtime_defaults(test_type, minutes) VALUES
|
||||
('P2P', 80),
|
||||
('COE', 115),
|
||||
('P3P', 105)
|
||||
ON CONFLICT(test_type) DO NOTHING
|
||||
"""
|
||||
)
|
||||
|
||||
def upsert_tests(records: list[TestRecord], db_path: str | Path = DB_PATH) -> int:
|
||||
if not records:
|
||||
return 0
|
||||
|
||||
values: list[tuple[Any, ...]] = []
|
||||
for r in records:
|
||||
station_testpoint_map = serialize_station_testpoint_map(r.config)
|
||||
|
||||
values.append(
|
||||
(
|
||||
r.test_id,
|
||||
r.device,
|
||||
r.test_type,
|
||||
r.rotation,
|
||||
r.rx_tx,
|
||||
r.power_mode,
|
||||
int(r.has_coe_pair),
|
||||
json.dumps(r.coe_pairing or []),
|
||||
json.dumps(r.config),
|
||||
r.victim_band,
|
||||
int(r.throttled),
|
||||
r.estimated_minutes,
|
||||
r.status,
|
||||
int(r.excluded),
|
||||
json.dumps(r.raw_payload or {}),
|
||||
station_testpoint_map,
|
||||
)
|
||||
)
|
||||
|
||||
with get_connection(db_path) as conn:
|
||||
conn.executemany(
|
||||
"""
|
||||
INSERT INTO tests(
|
||||
test_id, device, test_type, rotation, rx_tx, power_mode, has_coe_pair,
|
||||
coe_pairing_json, config_json, victim_band, throttled, estimated_minutes, status, excluded, raw_payload, station_testpoint_map
|
||||
)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT(test_id, device) DO UPDATE SET
|
||||
test_type = excluded.test_type,
|
||||
device = excluded.device,
|
||||
rotation = excluded.rotation,
|
||||
rx_tx = excluded.rx_tx,
|
||||
power_mode = excluded.power_mode,
|
||||
has_coe_pair = excluded.has_coe_pair,
|
||||
coe_pairing_json = excluded.coe_pairing_json,
|
||||
config_json = excluded.config_json,
|
||||
victim_band = excluded.victim_band,
|
||||
throttled = excluded.throttled,
|
||||
estimated_minutes = excluded.estimated_minutes,
|
||||
status = CASE
|
||||
WHEN tests.status IN ('completed', 'rerun') THEN tests.status
|
||||
ELSE excluded.status
|
||||
END,
|
||||
excluded = excluded.excluded,
|
||||
raw_payload = excluded.raw_payload,
|
||||
station_testpoint_map = excluded.station_testpoint_map,
|
||||
updated_at = CURRENT_TIMESTAMP
|
||||
""",
|
||||
values,
|
||||
)
|
||||
return len(records)
|
||||
|
||||
|
||||
def _ensure_column(conn: sqlite3.Connection, table: str, column: str, column_type: str) -> None:
|
||||
existing = conn.execute(f"PRAGMA table_info({table})").fetchall()
|
||||
column_names = {row[1] for row in existing}
|
||||
if column not in column_names:
|
||||
conn.execute(f"ALTER TABLE {table} ADD COLUMN {column} {column_type}")
|
||||
|
||||
|
||||
def save_settings(payload: dict[str, Any], db_path: str | Path = DB_PATH) -> None:
|
||||
with get_connection(db_path) as conn:
|
||||
for key, value in payload.items():
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO settings(key, value_json, updated_at)
|
||||
VALUES (?, ?, CURRENT_TIMESTAMP)
|
||||
ON CONFLICT(key) DO UPDATE SET
|
||||
value_json = excluded.value_json,
|
||||
updated_at = CURRENT_TIMESTAMP
|
||||
""",
|
||||
(key, json.dumps(value)),
|
||||
)
|
||||
|
||||
|
||||
def read_settings(db_path: str | Path = DB_PATH) -> dict[str, Any]:
|
||||
with get_connection(db_path) as conn:
|
||||
rows = conn.execute("SELECT key, value_json FROM settings").fetchall()
|
||||
return {row["key"]: json.loads(row["value_json"]) for row in rows}
|
||||
|
||||
|
||||
def _parse_rule_tokens(rule: str | None) -> list[str]:
|
||||
if not rule:
|
||||
return []
|
||||
return [token.strip().upper() for token in rule.split(",") if token.strip()]
|
||||
|
||||
|
||||
def _normalize_text(value: Any) -> str:
|
||||
if value is None:
|
||||
return ""
|
||||
return " ".join(str(value).strip().upper().split())
|
||||
|
||||
|
||||
def _normalize_compact(value: Any) -> str:
|
||||
return re.sub(r"[^A-Z0-9+\-]", "", _normalize_text(value))
|
||||
|
||||
|
||||
def _extract_signed_int(value: Any) -> str:
|
||||
match = re.search(r"[-+]?\d+", _normalize_text(value))
|
||||
return match.group(0) if match else ""
|
||||
|
||||
|
||||
def _band_entry(config: dict[str, dict[str, str | None]], band: str) -> dict[str, str | None]:
|
||||
return config.get(band) or config.get(band.lower()) or {}
|
||||
|
||||
|
||||
def _entry_value(entry: dict[str, str | None], key: str) -> str:
|
||||
if not entry:
|
||||
return ""
|
||||
return _normalize_text(entry.get(key) or entry.get(key.capitalize()) or entry.get(key.upper()))
|
||||
|
||||
|
||||
def _record_power_mode(record: TestRecord) -> str:
|
||||
return _normalize_text(record.power_mode)
|
||||
|
||||
|
||||
def _match_any_band_value(record: TestRecord, key: str, token_suffix: str) -> bool:
|
||||
if not token_suffix:
|
||||
return False
|
||||
|
||||
target_num = _extract_signed_int(token_suffix)
|
||||
target_compact = _normalize_compact(token_suffix)
|
||||
keys = ("STATION 1", "STATION 2", "STATION 3") if record.test_type == "P3P" else ("5G", "6G", "2G")
|
||||
|
||||
for band in keys:
|
||||
entry = _band_entry(record.config, band)
|
||||
value = _entry_value(entry, key)
|
||||
if not value:
|
||||
continue
|
||||
|
||||
if target_num:
|
||||
if _extract_signed_int(value) == target_num:
|
||||
return True
|
||||
elif target_compact and _normalize_compact(value) == target_compact:
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
|
||||
def _matches_atomic_exclusion_token(record: TestRecord, token: str) -> bool:
|
||||
if not token:
|
||||
return False
|
||||
|
||||
device = _normalize_text(record.device)
|
||||
test_type = _normalize_text(record.test_type)
|
||||
rotation = _normalize_text(record.rotation)
|
||||
victim_band = _normalize_text(record.victim_band)
|
||||
power_mode = _record_power_mode(record)
|
||||
known_devices = {_normalize_text(DUT), _normalize_text(REF)}
|
||||
|
||||
if token in {"LPI", "SP"}:
|
||||
return power_mode == token
|
||||
|
||||
if token in known_devices:
|
||||
return device == token
|
||||
|
||||
if token in {"P2P", "P3P", "COE"}:
|
||||
return test_type == token
|
||||
|
||||
if token in {"2G", "5G", "6G"}:
|
||||
return victim_band == token
|
||||
|
||||
if token.startswith("BW"):
|
||||
return _match_any_band_value(record, "bandwidth", token[2:])
|
||||
|
||||
if token.startswith("CH"):
|
||||
return _match_any_band_value(record, "channel", token[2:])
|
||||
|
||||
if token.startswith("RSSI"):
|
||||
return _match_any_band_value(record, "rssi", token[4:])
|
||||
|
||||
if token.startswith("R") and len(token) > 1:
|
||||
return rotation == token
|
||||
|
||||
return False
|
||||
|
||||
|
||||
def _matches_exclusion_token(record: TestRecord, token: str) -> bool:
|
||||
if not token:
|
||||
return False
|
||||
|
||||
parts = [part for part in token.split("_") if part]
|
||||
if len(parts) > 1:
|
||||
# Treat underscore as logical AND across sub-tokens.
|
||||
return all(_matches_atomic_exclusion_token(record, part) for part in parts)
|
||||
|
||||
return _matches_atomic_exclusion_token(record, token)
|
||||
|
||||
|
||||
def _should_exclude_by_rule(record: TestRecord, tokens: list[str]) -> bool:
|
||||
return any(_matches_exclusion_token(record, token) for token in tokens)
|
||||
|
||||
|
||||
def get_completed_test_tcs(db_path: str | Path = DB_PATH) -> set[str]:
|
||||
"""Return the set of TC names that have at least one completed test."""
|
||||
with get_connection(db_path) as conn:
|
||||
rows = conn.execute(
|
||||
"SELECT config_json FROM tests WHERE status = 'completed' AND excluded = 0"
|
||||
).fetchall()
|
||||
tcs: set[str] = set()
|
||||
for row in rows:
|
||||
config = json.loads(row["config_json"] or "{}")
|
||||
tcs.update(resolve_test_config_keys(config))
|
||||
return tcs
|
||||
|
||||
|
||||
def get_completed_dut_test_ids(db_path: str | Path = DB_PATH) -> set[str]:
|
||||
"""Return test IDs that have been completed on DUT."""
|
||||
with get_connection(db_path) as conn:
|
||||
rows = conn.execute(
|
||||
"SELECT test_id FROM tests WHERE status = 'completed' AND excluded = 0 AND device = ?",
|
||||
(DUT,)
|
||||
).fetchall()
|
||||
return {row["test_id"] for row in rows}
|
||||
|
||||
# Get tests for device that are not excluded regardless of completion status.
|
||||
def get_not_excluded_tests(db_path: str | Path = DB_PATH, rule: str = "") -> list[TestRecord]:
|
||||
tokens = _parse_rule_tokens(rule)
|
||||
with get_connection(db_path) as conn:
|
||||
rows = conn.execute(
|
||||
"""
|
||||
SELECT
|
||||
test_id, device, test_type, rotation, rx_tx, has_coe_pair,
|
||||
power_mode, config_json, victim_band,
|
||||
coe_pairing_json, throttled, estimated_minutes,
|
||||
excluded, status, raw_payload
|
||||
FROM tests
|
||||
WHERE excluded = 0
|
||||
"""
|
||||
).fetchall()
|
||||
|
||||
results: list[TestRecord] = []
|
||||
for row in rows:
|
||||
results.append(
|
||||
TestRecord(
|
||||
test_id=row["test_id"],
|
||||
device=row["device"],
|
||||
test_type=row["test_type"],
|
||||
rotation=row["rotation"],
|
||||
rx_tx=row["rx_tx"],
|
||||
power_mode=row["power_mode"],
|
||||
has_coe_pair=bool(row["has_coe_pair"]),
|
||||
coe_pairing=json.loads(row["coe_pairing_json"] or "[]"),
|
||||
victim_band=row["victim_band"],
|
||||
config=json.loads(row["config_json"] or "{}"),
|
||||
throttled=bool(row["throttled"]),
|
||||
estimated_minutes=int(row["estimated_minutes"]),
|
||||
status=row["status"],
|
||||
excluded=bool(row["excluded"]),
|
||||
raw_payload=json.loads(row["raw_payload"] or "{}"),
|
||||
)
|
||||
)
|
||||
|
||||
if not tokens:
|
||||
return results
|
||||
|
||||
return [record for record in results if not _should_exclude_by_rule(record, tokens)]
|
||||
|
||||
def list_schedulable_tests(db_path: str | Path = DB_PATH, rule: str = "") -> list[TestRecord]:
|
||||
tokens = _parse_rule_tokens(rule)
|
||||
with get_connection(db_path) as conn:
|
||||
rows = conn.execute(
|
||||
"""
|
||||
SELECT
|
||||
test_id, device, test_type, rotation, rx_tx, has_coe_pair,
|
||||
power_mode, config_json, victim_band,
|
||||
coe_pairing_json, throttled, estimated_minutes,
|
||||
excluded, status, raw_payload
|
||||
FROM tests
|
||||
WHERE excluded = 0
|
||||
AND status != 'completed'
|
||||
"""
|
||||
).fetchall()
|
||||
|
||||
results: list[TestRecord] = []
|
||||
for row in rows:
|
||||
results.append(
|
||||
TestRecord(
|
||||
test_id=row["test_id"],
|
||||
device=row["device"],
|
||||
test_type=row["test_type"],
|
||||
rotation=row["rotation"],
|
||||
rx_tx=row["rx_tx"],
|
||||
power_mode=row["power_mode"],
|
||||
has_coe_pair=bool(row["has_coe_pair"]),
|
||||
coe_pairing=json.loads(row["coe_pairing_json"] or "[]"),
|
||||
victim_band=row["victim_band"],
|
||||
config=json.loads(row["config_json"] or "{}"),
|
||||
throttled=bool(row["throttled"]),
|
||||
estimated_minutes=int(row["estimated_minutes"]),
|
||||
status=row["status"],
|
||||
excluded=bool(row["excluded"]),
|
||||
raw_payload=json.loads(row["raw_payload"] or "{}"),
|
||||
)
|
||||
)
|
||||
|
||||
if not tokens:
|
||||
return results
|
||||
|
||||
return [record for record in results if not _should_exclude_by_rule(record, tokens)]
|
||||
|
||||
# Get all tests for a specific device, regardless of exclusion or completion status. Used for schedule display and management.
|
||||
def list_tests_for_device(device: str, db_path: str | Path = DB_PATH) -> list[TestRecord]:
|
||||
with get_connection(db_path) as conn:
|
||||
rows = conn.execute(
|
||||
"""
|
||||
SELECT
|
||||
test_id, device, test_type, rotation, rx_tx, has_coe_pair,
|
||||
power_mode, config_json, victim_band,
|
||||
coe_pairing_json, throttled, estimated_minutes,
|
||||
excluded, status, raw_payload, station_testpoint_map
|
||||
FROM tests
|
||||
WHERE device = ?
|
||||
""",
|
||||
(device,),
|
||||
).fetchall()
|
||||
|
||||
results: list[TestRecord] = []
|
||||
for row in rows:
|
||||
results.append(
|
||||
TestRecord(
|
||||
test_id=row["test_id"],
|
||||
device=row["device"],
|
||||
test_type=row["test_type"],
|
||||
rotation=row["rotation"],
|
||||
rx_tx=row["rx_tx"],
|
||||
power_mode=row["power_mode"],
|
||||
has_coe_pair=bool(row["has_coe_pair"]),
|
||||
coe_pairing=json.loads(row["coe_pairing_json"] or "[]"),
|
||||
victim_band=row["victim_band"],
|
||||
config=json.loads(row["config_json"] or "{}"),
|
||||
throttled=bool(row["throttled"]),
|
||||
estimated_minutes=int(row["estimated_minutes"]),
|
||||
status=row["status"],
|
||||
excluded=bool(row["excluded"]),
|
||||
raw_payload=json.loads(row["raw_payload"] or "{}"),
|
||||
station_testpoint_map=row["station_testpoint_map"],
|
||||
)
|
||||
)
|
||||
|
||||
return results
|
||||
|
||||
def list_holidays(db_path: str | Path = DB_PATH) -> set[str]:
|
||||
with get_connection(db_path) as conn:
|
||||
rows = conn.execute("SELECT date FROM holidays").fetchall()
|
||||
return {row["date"] for row in rows}
|
||||
|
||||
|
||||
def create_schedule_version(
|
||||
entries: list[tuple[str, str, str, int, int, str]],
|
||||
db_path: str | Path = DB_PATH,
|
||||
) -> int:
|
||||
with get_connection(db_path) as conn:
|
||||
row = conn.execute("SELECT COALESCE(MAX(schedule_version), 0) AS current FROM schedules").fetchone()
|
||||
next_version = int(row["current"]) + 1
|
||||
|
||||
if entries:
|
||||
conn.executemany(
|
||||
"""
|
||||
INSERT INTO schedules(
|
||||
schedule_version,
|
||||
test_id,
|
||||
device,
|
||||
scheduled_date,
|
||||
shift_index,
|
||||
sequence_in_shift,
|
||||
status_snapshot
|
||||
)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?)
|
||||
""",
|
||||
[
|
||||
(next_version, test_id, device, scheduled_date, shift_index, sequence, status_snapshot)
|
||||
for test_id, device, scheduled_date, shift_index, sequence, status_snapshot in entries
|
||||
],
|
||||
)
|
||||
|
||||
prune_schedule_versions(conn=conn)
|
||||
|
||||
return next_version
|
||||
|
||||
|
||||
def prune_schedule_versions(
|
||||
max_versions: int = MAX_SCHEDULE_VERSIONS,
|
||||
db_path: str | Path = DB_PATH,
|
||||
conn: sqlite3.Connection | None = None,
|
||||
) -> int:
|
||||
if max_versions <= 0:
|
||||
raise ValueError("max_versions must be positive")
|
||||
|
||||
def _prune(active_conn: sqlite3.Connection) -> int:
|
||||
cursor = active_conn.execute(
|
||||
"""
|
||||
DELETE FROM schedules
|
||||
WHERE schedule_version IN (
|
||||
SELECT schedule_version
|
||||
FROM (
|
||||
SELECT schedule_version
|
||||
FROM schedules
|
||||
GROUP BY schedule_version
|
||||
ORDER BY schedule_version DESC
|
||||
LIMIT -1 OFFSET ?
|
||||
)
|
||||
)
|
||||
""",
|
||||
(max_versions,),
|
||||
)
|
||||
return cursor.rowcount
|
||||
|
||||
if conn is not None:
|
||||
return _prune(conn)
|
||||
|
||||
with get_connection(db_path) as active_conn:
|
||||
return _prune(active_conn)
|
||||
|
||||
|
||||
def _resolve_requested_schedule_version(
|
||||
conn: sqlite3.Connection,
|
||||
version: int | None,
|
||||
) -> tuple[int | None, int | None]:
|
||||
latest_row = conn.execute("SELECT MAX(schedule_version) AS latest FROM schedules").fetchone()
|
||||
latest = latest_row["latest"]
|
||||
latest_version = int(latest) if latest is not None else None
|
||||
|
||||
if version is not None:
|
||||
row = conn.execute(
|
||||
"SELECT 1 AS exists_row FROM schedules WHERE schedule_version = ? LIMIT 1",
|
||||
(version,),
|
||||
).fetchone()
|
||||
return (int(version) if row else None, latest_version)
|
||||
|
||||
return latest_version, latest_version
|
||||
|
||||
|
||||
def _hydrate_schedule_rows(rows: list[sqlite3.Row]) -> list[ScheduleRow]:
|
||||
result: list[ScheduleRow] = []
|
||||
for row in rows:
|
||||
result.append(
|
||||
ScheduleRow(
|
||||
test_id=row["test_id"],
|
||||
device=row["device"],
|
||||
scheduled_date=row["scheduled_date"],
|
||||
shift_index=int(row["shift_index"]),
|
||||
sequence_in_shift=int(row["sequence_in_shift"]),
|
||||
test_type=row["test_type"],
|
||||
rotation=row["rotation"],
|
||||
config=json.loads(row["config_json"] or "{}"),
|
||||
status=row["status"],
|
||||
estimated_minutes=int(row["estimated_minutes"]),
|
||||
)
|
||||
)
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def get_schedule_versions(db_path: str | Path = DB_PATH) -> list[dict[str, Any]]:
|
||||
with get_connection(db_path) as conn:
|
||||
rows = conn.execute(
|
||||
"""
|
||||
SELECT
|
||||
schedule_version,
|
||||
MIN(created_at) AS created_at,
|
||||
COUNT(*) AS entry_count
|
||||
FROM schedules
|
||||
GROUP BY schedule_version
|
||||
ORDER BY schedule_version DESC
|
||||
"""
|
||||
).fetchall()
|
||||
|
||||
return [
|
||||
{
|
||||
"schedule_version": int(row["schedule_version"]),
|
||||
"created_at": row["created_at"],
|
||||
"entry_count": int(row["entry_count"]),
|
||||
}
|
||||
for row in rows
|
||||
]
|
||||
|
||||
|
||||
def resolve_schedule_version(
|
||||
version: int | None = None,
|
||||
db_path: str | Path = DB_PATH,
|
||||
) -> int | None:
|
||||
with get_connection(db_path) as conn:
|
||||
selected_version, _latest_version = _resolve_requested_schedule_version(conn, version)
|
||||
return selected_version
|
||||
|
||||
|
||||
def get_schedule_week(
|
||||
start_date: str,
|
||||
version: int | None = None,
|
||||
db_path: str | Path = DB_PATH,
|
||||
) -> list[ScheduleRow]:
|
||||
with get_connection(db_path) as conn:
|
||||
selected_version, latest_version = _resolve_requested_schedule_version(conn, version)
|
||||
if selected_version is None:
|
||||
return []
|
||||
|
||||
use_live_status = latest_version is not None and selected_version == latest_version
|
||||
if use_live_status:
|
||||
# Only surface live 'rerun' for slots that are actually overdue (past).
|
||||
# Future slots fall back to status_snapshot so they don't show red.
|
||||
status_sql = """
|
||||
CASE
|
||||
WHEN t.status = 'completed' THEN 'completed'
|
||||
WHEN t.status = 'rerun'
|
||||
AND (
|
||||
s.scheduled_date < date('now')
|
||||
OR (s.scheduled_date = date('now') AND s.shift_index = 1)
|
||||
)
|
||||
THEN 'rerun'
|
||||
WHEN t.status = 'rerun' THEN COALESCE(s.status_snapshot, 'pending')
|
||||
ELSE COALESCE(t.status, s.status_snapshot, 'pending')
|
||||
END
|
||||
"""
|
||||
else:
|
||||
status_sql = "COALESCE(s.status_snapshot, 'pending')"
|
||||
|
||||
rows = conn.execute(
|
||||
f"""
|
||||
SELECT
|
||||
s.test_id,
|
||||
s.device,
|
||||
s.scheduled_date,
|
||||
s.shift_index,
|
||||
s.sequence_in_shift,
|
||||
t.test_type,
|
||||
t.rotation,
|
||||
t.config_json,
|
||||
{status_sql} AS status,
|
||||
t.estimated_minutes
|
||||
FROM schedules s
|
||||
JOIN tests t ON t.test_id = s.test_id AND t.device = s.device
|
||||
WHERE s.schedule_version = ?
|
||||
AND (
|
||||
(
|
||||
s.scheduled_date >= ?
|
||||
AND s.scheduled_date < date(?, '+7 day')
|
||||
AND s.shift_index IN (2, 3)
|
||||
)
|
||||
OR (
|
||||
s.scheduled_date > ?
|
||||
AND s.scheduled_date < date(?, '+7 day')
|
||||
AND s.shift_index = 1
|
||||
)
|
||||
OR (
|
||||
s.scheduled_date = date(?, '+7 day')
|
||||
AND s.shift_index = 1
|
||||
)
|
||||
)
|
||||
ORDER BY s.scheduled_date, s.shift_index, s.sequence_in_shift
|
||||
""",
|
||||
(
|
||||
selected_version,
|
||||
start_date,
|
||||
start_date,
|
||||
start_date,
|
||||
start_date,
|
||||
start_date,
|
||||
),
|
||||
).fetchall()
|
||||
|
||||
return _hydrate_schedule_rows(rows)
|
||||
|
||||
|
||||
def get_schedule_rows(
|
||||
version: int | None = None,
|
||||
db_path: str | Path = DB_PATH,
|
||||
) -> list[ScheduleRow]:
|
||||
with get_connection(db_path) as conn:
|
||||
selected_version, latest_version = _resolve_requested_schedule_version(conn, version)
|
||||
if selected_version is None:
|
||||
return []
|
||||
|
||||
use_live_status = latest_version is not None and selected_version == latest_version
|
||||
if use_live_status:
|
||||
status_sql = """
|
||||
CASE
|
||||
WHEN t.status = 'completed' THEN 'completed'
|
||||
WHEN t.status = 'rerun'
|
||||
AND (
|
||||
s.scheduled_date < date('now')
|
||||
OR (s.scheduled_date = date('now') AND s.shift_index = 1)
|
||||
)
|
||||
THEN 'rerun'
|
||||
WHEN t.status = 'rerun' THEN COALESCE(s.status_snapshot, 'pending')
|
||||
ELSE COALESCE(t.status, s.status_snapshot, 'pending')
|
||||
END
|
||||
"""
|
||||
else:
|
||||
status_sql = "COALESCE(s.status_snapshot, 'pending')"
|
||||
|
||||
rows = conn.execute(
|
||||
f"""
|
||||
SELECT
|
||||
s.test_id,
|
||||
s.device,
|
||||
s.scheduled_date,
|
||||
s.shift_index,
|
||||
s.sequence_in_shift,
|
||||
t.test_type,
|
||||
t.rotation,
|
||||
t.config_json,
|
||||
{status_sql} AS status,
|
||||
t.estimated_minutes
|
||||
FROM schedules s
|
||||
JOIN tests t ON t.test_id = s.test_id AND t.device = s.device
|
||||
WHERE s.schedule_version = ?
|
||||
ORDER BY s.scheduled_date, s.shift_index, s.sequence_in_shift
|
||||
""",
|
||||
(selected_version,),
|
||||
).fetchall()
|
||||
|
||||
return _hydrate_schedule_rows(rows)
|
||||
|
||||
def reset_completed_to_pending(db_path: str | Path = DB_PATH) -> int:
|
||||
"""Reset all 'completed' tests back to 'pending'. Used before a full directory rescan."""
|
||||
with get_connection(db_path) as conn:
|
||||
cursor = conn.execute(
|
||||
"""
|
||||
UPDATE tests
|
||||
SET status = 'pending', updated_at = CURRENT_TIMESTAMP
|
||||
WHERE status = 'completed'
|
||||
"""
|
||||
)
|
||||
return cursor.rowcount
|
||||
|
||||
|
||||
def mark_tests_completed(test_ids_with_device: list[tuple[str, str]], db_path: str | Path = DB_PATH) -> int:
|
||||
if not test_ids_with_device:
|
||||
return 0
|
||||
|
||||
with get_connection(db_path) as conn:
|
||||
# executemany() doesn't properly return rowcount, so we track manually
|
||||
total_updated = 0
|
||||
for test_id, device in test_ids_with_device:
|
||||
cursor = conn.execute(
|
||||
"""
|
||||
UPDATE tests
|
||||
SET status = 'completed', updated_at = CURRENT_TIMESTAMP
|
||||
WHERE test_id = ?
|
||||
AND device = ?
|
||||
AND status != 'completed'
|
||||
""",
|
||||
(test_id, device),
|
||||
)
|
||||
if cursor.rowcount > 0:
|
||||
total_updated += cursor.rowcount
|
||||
print(f"[db] marked {test_id} on {device} as completed")
|
||||
else:
|
||||
print(f"[db] {test_id} on {device}: no update (not found or already completed)")
|
||||
|
||||
return total_updated
|
||||
|
||||
|
||||
def mark_overdue_as_rerun(db_path: str | Path = DB_PATH) -> int:
|
||||
from datetime import date as _date, datetime as _datetime
|
||||
now = _datetime.now()
|
||||
today = now.date().isoformat()
|
||||
shift1_ended = now.hour >= 10 # shift 1 ends ~10am
|
||||
|
||||
with get_connection(db_path) as conn:
|
||||
# Use the most recent schedule version that has any overdue (past-due) entries.
|
||||
# If a fresh schedule was just compiled, its entries are all in the future and
|
||||
# MAX(schedule_version) would miss the overdue context from the prior version.
|
||||
if shift1_ended:
|
||||
overdue_date_clause = "s.scheduled_date < :today OR (s.scheduled_date = :today AND s.shift_index = 1)"
|
||||
else:
|
||||
overdue_date_clause = "s.scheduled_date < :today"
|
||||
|
||||
version_row = conn.execute(
|
||||
f"""
|
||||
SELECT MAX(s.schedule_version) AS v
|
||||
FROM schedules s
|
||||
JOIN tests t ON t.test_id = s.test_id AND t.device = s.device
|
||||
WHERE t.status = 'pending'
|
||||
AND ({overdue_date_clause})
|
||||
""",
|
||||
{"today": today},
|
||||
).fetchone()
|
||||
latest = version_row["v"] if version_row else None
|
||||
if latest is None:
|
||||
# No version has overdue pending tests — fall back to MAX to still run the
|
||||
# update (it will simply mark zero rows).
|
||||
fb = conn.execute("SELECT MAX(schedule_version) AS v FROM schedules").fetchone()
|
||||
latest = fb["v"] if fb else None
|
||||
if latest is None:
|
||||
return 0
|
||||
|
||||
if shift1_ended:
|
||||
rows = conn.execute(
|
||||
"""
|
||||
SELECT DISTINCT s.test_id, s.device
|
||||
FROM schedules s
|
||||
JOIN tests t ON t.test_id = s.test_id AND t.device = s.device
|
||||
WHERE s.schedule_version = ?
|
||||
AND t.status = 'pending'
|
||||
AND (
|
||||
s.scheduled_date < ?
|
||||
OR (s.scheduled_date = ? AND s.shift_index = 1)
|
||||
)
|
||||
""",
|
||||
(latest, today, today),
|
||||
).fetchall()
|
||||
else:
|
||||
rows = conn.execute(
|
||||
"""
|
||||
SELECT DISTINCT s.test_id, s.device
|
||||
FROM schedules s
|
||||
JOIN tests t ON t.test_id = s.test_id AND t.device = s.device
|
||||
WHERE s.schedule_version = ?
|
||||
AND s.scheduled_date < ?
|
||||
AND t.status = 'pending'
|
||||
""",
|
||||
(latest, today),
|
||||
).fetchall()
|
||||
|
||||
if not rows:
|
||||
return 0
|
||||
|
||||
conn.executemany(
|
||||
"""
|
||||
UPDATE tests
|
||||
SET status = 'rerun', updated_at = CURRENT_TIMESTAMP
|
||||
WHERE test_id = ? AND device = ? AND status = 'pending'
|
||||
""",
|
||||
[(r["test_id"], r["device"]) for r in rows],
|
||||
)
|
||||
return len(rows)
|
||||
|
||||
|
||||
def get_rerun_tests(db_path: str | Path = DB_PATH) -> list[dict]:
|
||||
"""Return all overdue tests that have not been completed.
|
||||
|
||||
Checks the most recent schedule version that contains overdue pending entries
|
||||
so that recompiling a fresh schedule does not erase the overdue context.
|
||||
Overdue = scheduled before today (any shift) or today shift 1 if it has ended.
|
||||
"""
|
||||
from datetime import date as _date, datetime as _datetime
|
||||
now = _datetime.now()
|
||||
today = now.date().isoformat()
|
||||
shift1_ended = now.hour >= 10
|
||||
|
||||
with get_connection(db_path) as conn:
|
||||
if shift1_ended:
|
||||
overdue_date_clause = "s.scheduled_date < :today OR (s.scheduled_date = :today AND s.shift_index = 1)"
|
||||
else:
|
||||
overdue_date_clause = "s.scheduled_date < :today"
|
||||
|
||||
version_row = conn.execute(
|
||||
f"""
|
||||
SELECT MAX(s.schedule_version) AS v
|
||||
FROM schedules s
|
||||
JOIN tests t ON t.test_id = s.test_id AND t.device = s.device
|
||||
WHERE t.status != 'completed'
|
||||
AND ({overdue_date_clause})
|
||||
""",
|
||||
{"today": today},
|
||||
).fetchone()
|
||||
latest = version_row["v"] if version_row else None
|
||||
if latest is None:
|
||||
return []
|
||||
|
||||
rows = conn.execute(
|
||||
f"""
|
||||
SELECT DISTINCT t.test_id, t.device, t.test_type, t.estimated_minutes
|
||||
FROM tests t
|
||||
JOIN schedules s ON s.test_id = t.test_id AND s.device = t.device
|
||||
WHERE s.schedule_version = :v
|
||||
AND t.status != 'completed'
|
||||
AND ({overdue_date_clause})
|
||||
ORDER BY t.test_id, t.device
|
||||
""",
|
||||
{"v": latest, "today": today},
|
||||
).fetchall()
|
||||
return [
|
||||
{
|
||||
"test_id": r["test_id"],
|
||||
"device": r["device"],
|
||||
"test_type": r["test_type"],
|
||||
"estimated_minutes": r["estimated_minutes"],
|
||||
}
|
||||
for r in rows
|
||||
]
|
||||
|
||||
|
||||
def upsert_holidays(dates: list[str], db_path: str | Path = DB_PATH) -> None:
|
||||
"""Replace all holidays with the provided list of YYYY-MM-DD date strings."""
|
||||
with get_connection(db_path) as conn:
|
||||
conn.execute("DELETE FROM holidays")
|
||||
if dates:
|
||||
conn.executemany(
|
||||
"INSERT OR IGNORE INTO holidays(date) VALUES (?)",
|
||||
[(d,) for d in dates],
|
||||
)
|
||||
|
||||
def reset_tests(db_path: str | Path = DB_PATH) -> int:
|
||||
"""Clear all tests records keep empty tests table."""
|
||||
with get_connection(db_path) as conn:
|
||||
cursor = conn.execute("DELETE FROM tests")
|
||||
return cursor.rowcount
|
||||
|
||||
|
||||
def reset_all_data(db_path: str | Path = DB_PATH) -> dict[str, int]:
|
||||
"""Clear all scheduler data from mutable tables.
|
||||
|
||||
Keeps schema and runtime default rows intact.
|
||||
"""
|
||||
with get_connection(db_path) as conn:
|
||||
schedules_deleted = conn.execute("DELETE FROM schedules").rowcount
|
||||
tests_deleted = conn.execute("DELETE FROM tests").rowcount
|
||||
settings_deleted = conn.execute("DELETE FROM settings").rowcount
|
||||
holidays_deleted = conn.execute("DELETE FROM holidays").rowcount
|
||||
|
||||
# Reset autoincrement counters for cleared tables.
|
||||
conn.execute(
|
||||
"DELETE FROM sqlite_sequence WHERE name IN ('tests', 'schedules')"
|
||||
)
|
||||
|
||||
return {
|
||||
"tests_deleted": tests_deleted,
|
||||
"schedules_deleted": schedules_deleted,
|
||||
"settings_deleted": settings_deleted,
|
||||
"holidays_deleted": holidays_deleted,
|
||||
}
|
||||
|
||||
@@ -0,0 +1,232 @@
|
||||
import os
|
||||
import re
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import smbclient # type: ignore[import-not-found]
|
||||
|
||||
_SMB_SESSIONS: set[str] = set()
|
||||
APP_ROOT = Path(__file__).resolve().parent
|
||||
|
||||
|
||||
def normalize_smb_credentials(smb_credentials: dict[str, Any] | None) -> tuple[str, str, str]:
|
||||
username = ""
|
||||
password = ""
|
||||
domain = ""
|
||||
|
||||
if smb_credentials:
|
||||
username = str(smb_credentials.get("username", "")).strip()
|
||||
password = str(smb_credentials.get("password", ""))
|
||||
domain = str(smb_credentials.get("domain", "")).strip()
|
||||
|
||||
if not username:
|
||||
username = os.getenv("SMB_USERNAME", "").strip()
|
||||
if not password:
|
||||
password = os.getenv("SMB_PASSWORD", "")
|
||||
if not domain:
|
||||
domain = os.getenv("SMB_DOMAIN", "").strip()
|
||||
|
||||
if username and domain and "\\" not in username and "@" not in username:
|
||||
username = f"{domain}\\{username}"
|
||||
|
||||
return username, password, domain
|
||||
|
||||
|
||||
def resolve_runtime_path(path_value):
|
||||
if not path_value:
|
||||
return path_value
|
||||
|
||||
raw_path = str(path_value).strip()
|
||||
if not raw_path:
|
||||
return raw_path
|
||||
|
||||
if os.path.exists(raw_path):
|
||||
return raw_path
|
||||
|
||||
if raw_path.startswith("\\\\") or raw_path.startswith("//"):
|
||||
return raw_path
|
||||
|
||||
mount_root = (os.getenv("HOST_MOUNT_ROOT", "/host") or "/host").strip() or "/host"
|
||||
host_root = (os.getenv("HOST_BROWSE_ROOT", "") or "").strip()
|
||||
raw_norm = raw_path.replace("\\", "/")
|
||||
|
||||
if os.name != "nt" and host_root:
|
||||
host_norm = host_root.replace("\\", "/").rstrip("/")
|
||||
if raw_norm.lower() == host_norm.lower() or raw_norm.lower().startswith(host_norm.lower() + "/"):
|
||||
relative = raw_norm[len(host_norm):].lstrip("/")
|
||||
if relative:
|
||||
return os.path.join(mount_root, *relative.split("/"))
|
||||
return mount_root
|
||||
|
||||
if os.name == "nt" and host_root:
|
||||
mount_norm = mount_root.replace("\\", "/").rstrip("/")
|
||||
if mount_norm and (raw_norm.lower() == mount_norm.lower() or raw_norm.lower().startswith(mount_norm.lower() + "/")):
|
||||
relative = raw_norm[len(mount_norm):].lstrip("/")
|
||||
if relative:
|
||||
return os.path.join(host_root, *relative.split("/"))
|
||||
return host_root
|
||||
|
||||
return raw_path
|
||||
|
||||
|
||||
def normalize_input_path(path_value: str | Path) -> str:
|
||||
path = str(path_value).strip()
|
||||
if not path:
|
||||
return path
|
||||
|
||||
path = str(resolve_runtime_path(path)).strip()
|
||||
|
||||
# Accept //server/share style and normalize to UNC for smbclient.
|
||||
if path.startswith("//"):
|
||||
path = path.lstrip("/").replace("/", "\\")
|
||||
return "\\\\" + path
|
||||
|
||||
# Accept \\server\share style UNC paths and ensure proper escaping.
|
||||
if path.startswith("\\\\"):
|
||||
while "\\\\\\" in path:
|
||||
path = path.replace("\\\\\\", "\\\\")
|
||||
return path
|
||||
|
||||
# Accept /<ipv4>/<share>/... and normalize to UNC for Linux-hosted inputs.
|
||||
if re.match(r"^/\d{1,3}(?:\.\d{1,3}){3}/[^/]+", path):
|
||||
return "\\\\" + path.lstrip("/").replace("/", "\\")
|
||||
|
||||
if os.path.exists(path):
|
||||
return path
|
||||
|
||||
if path.startswith("/"):
|
||||
return path
|
||||
|
||||
path = path.replace("/", "\\")
|
||||
|
||||
# Accept <ipv4>\<share>\... and normalize to UNC.
|
||||
if re.match(r"^\d{1,3}(?:\.\d{1,3}){3}\\[^\\]+", path):
|
||||
return "\\\\" + path
|
||||
|
||||
return path
|
||||
|
||||
def is_unc_path(path: str) -> bool:
|
||||
return path.startswith("\\\\")
|
||||
|
||||
def extract_unc_server(path: str) -> str | None:
|
||||
if not is_unc_path(path):
|
||||
return None
|
||||
rest = path[2:]
|
||||
return rest.split("\\", 1)[0] if rest else None
|
||||
|
||||
|
||||
def _register_smb_session_if_needed(path: str, smb_credentials: dict[str, Any] | None = None) -> None:
|
||||
if not is_unc_path(path):
|
||||
return
|
||||
|
||||
if smbclient is None:
|
||||
raise ModuleNotFoundError("smbclient is required to scan UNC result paths")
|
||||
|
||||
server = extract_unc_server(path)
|
||||
if not server or server in _SMB_SESSIONS:
|
||||
return
|
||||
|
||||
username, password, _domain = normalize_smb_credentials(smb_credentials)
|
||||
if username:
|
||||
smbclient.register_session(server, username=username, password=password)
|
||||
else:
|
||||
smbclient.register_session(server)
|
||||
|
||||
_SMB_SESSIONS.add(server)
|
||||
|
||||
def is_dir(path: str, smb_credentials: dict[str, Any] | None) -> bool:
|
||||
if is_unc_path(path):
|
||||
_register_smb_session_if_needed(path, smb_credentials)
|
||||
if smbclient is None:
|
||||
raise ModuleNotFoundError("smbclient is required to scan UNC result paths")
|
||||
try:
|
||||
entry_iter = smbclient.scandir(path)
|
||||
for _ in entry_iter:
|
||||
break
|
||||
return True
|
||||
except OSError:
|
||||
return False
|
||||
|
||||
return Path(path).is_dir()
|
||||
|
||||
def path_name(path: str) -> str:
|
||||
trimmed = path.rstrip("\\/")
|
||||
if not trimmed:
|
||||
return path
|
||||
parts = re.split(r"[\\/]", trimmed)
|
||||
return parts[-1] if parts else trimmed
|
||||
|
||||
def csv_paths_from_dir(path: str, smb_credentials: dict[str, Any] | None) -> list[str]:
|
||||
if is_unc_path(path):
|
||||
_register_smb_session_if_needed(path, smb_credentials)
|
||||
if smbclient is None:
|
||||
raise ModuleNotFoundError("smbclient is required to scan UNC result paths")
|
||||
entries = []
|
||||
for entry in smbclient.scandir(path):
|
||||
name = getattr(entry, "name", "")
|
||||
if name and name.lower().endswith(".csv") and entry.is_file():
|
||||
entries.append(path.rstrip("\\/") + "\\" + name)
|
||||
return sorted(entries)
|
||||
|
||||
return [str(p) for p in sorted(Path(path).glob("*.csv"))]
|
||||
|
||||
def path_exists_with_smb(path, smb_credentials=None):
|
||||
"""Check if a path exists, with SMB authentication for UNC paths."""
|
||||
if not path:
|
||||
return False
|
||||
|
||||
path = normalize_input_path(path)
|
||||
|
||||
# For UNC paths, use SMB to check
|
||||
if is_unc_path(path):
|
||||
try:
|
||||
_register_smb_session_if_needed(path, smb_credentials=smb_credentials)
|
||||
# Try to list entries; if successful, path exists
|
||||
iter_dir_entries(path, smb_credentials=smb_credentials)
|
||||
return True
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
# For local paths, use standard os.path.exists
|
||||
return os.path.exists(path)
|
||||
|
||||
def iter_dir_entries(path, smb_credentials=None):
|
||||
path = normalize_input_path(path)
|
||||
if is_unc_path(path):
|
||||
if smbclient is None:
|
||||
raise ModuleNotFoundError("smbclient is required to scan UNC result paths")
|
||||
_register_smb_session_if_needed(path, smb_credentials=smb_credentials)
|
||||
return list(smbclient.scandir(path))
|
||||
return list(os.scandir(path))
|
||||
|
||||
def open_csv_handle(path: str, smb_credentials: dict[str, Any] | None):
|
||||
if is_unc_path(path):
|
||||
_register_smb_session_if_needed(path, smb_credentials=smb_credentials)
|
||||
return smbclient.open_file(path, mode="r", encoding="utf-8-sig", newline="")
|
||||
return Path(path).open("r", encoding="utf-8-sig", newline="")
|
||||
|
||||
def join_path(path, name):
|
||||
if is_unc_path(path):
|
||||
base = path.rstrip("\\")
|
||||
return f"{base}\\{name}"
|
||||
return str(Path(path) / name)
|
||||
|
||||
def _is_explicit_path(path_value: str) -> bool:
|
||||
return (
|
||||
path_value.startswith(("/", "\\\\", "//"))
|
||||
or "\\" in path_value
|
||||
or ":" in path_value
|
||||
)
|
||||
|
||||
def resolve_requested_csv_path(requested_path: str) -> str | Path:
|
||||
received_path = str(requested_path).strip()
|
||||
resolved_path = str(resolve_runtime_path(received_path)).strip()
|
||||
|
||||
if _is_explicit_path(received_path) or _is_explicit_path(resolved_path):
|
||||
return resolved_path
|
||||
|
||||
data_dir_path = APP_ROOT / "data" / received_path
|
||||
if data_dir_path.exists():
|
||||
return data_dir_path
|
||||
|
||||
return APP_ROOT / received_path
|
||||
@@ -0,0 +1,520 @@
|
||||
import csv
|
||||
import os
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from db import DUT, REF, TestRecord
|
||||
from file_manager import (
|
||||
open_csv_handle
|
||||
)
|
||||
|
||||
TEST_TYPES = {"P2P", "COE", "P3P"}
|
||||
POWER_MODES = {"LPI", "SP"}
|
||||
RX_TX = {"RX", "TX"}
|
||||
|
||||
P2P_COE_REQUIRED_COLUMNS = [
|
||||
"Priority",
|
||||
"Index",
|
||||
"Interferer",
|
||||
"COE Pair",
|
||||
"Rotation",
|
||||
"TC ID",
|
||||
"Victim Band",
|
||||
"6GHz Power Mode",
|
||||
"5G_Test Point",
|
||||
"5G_Channel",
|
||||
"5G_Bandwidth",
|
||||
"5G_RSSI",
|
||||
"5G_Direction",
|
||||
"5G_STA",
|
||||
"6G_Test Point",
|
||||
"6G_Channel",
|
||||
"6G_Bandwidth",
|
||||
"6G_RSSI",
|
||||
"6G_Direction",
|
||||
"6G_STA",
|
||||
"2G_Test Point",
|
||||
"2G_Channel",
|
||||
"2G_Bandwidth",
|
||||
"2G_RSSI",
|
||||
"2G_Direction",
|
||||
"2G_STA",
|
||||
]
|
||||
|
||||
P3P_REQUIRED_COLUMNS = [
|
||||
"Priority",
|
||||
"Index",
|
||||
"Throttled",
|
||||
"Rotation",
|
||||
"TC ID",
|
||||
"Band",
|
||||
"6GHz Power Mode",
|
||||
"STATION 1_Test Point",
|
||||
"STATION 1_Channel",
|
||||
"STATION 1_Bandwidth",
|
||||
"STATION 1_RSSI",
|
||||
"STATION 1_Direction",
|
||||
"STATION 1_Rate",
|
||||
"STATION 1_STA",
|
||||
"STATION 2_Test Point",
|
||||
"STATION 2_Channel",
|
||||
"STATION 2_Bandwidth",
|
||||
"STATION 2_RSSI",
|
||||
"STATION 2_Direction",
|
||||
"STATION 2_Rate",
|
||||
"STATION 2_STA",
|
||||
"STATION 3_Test Point",
|
||||
"STATION 3_Channel",
|
||||
"STATION 3_Bandwidth",
|
||||
"STATION 3_RSSI",
|
||||
"STATION 3_Direction",
|
||||
"STATION 3_Rate",
|
||||
"STATION 3_STA",
|
||||
]
|
||||
|
||||
RUNTIME_DEFAULTS = {
|
||||
"P2P": 80,
|
||||
"COE": 115,
|
||||
"P3P": 105,
|
||||
}
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ParseResult:
|
||||
tests: list[TestRecord]
|
||||
warnings: list[str]
|
||||
|
||||
|
||||
class CsvValidationError(ValueError):
|
||||
pass
|
||||
|
||||
|
||||
_SMB_SESSIONS: set[str] = set()
|
||||
|
||||
|
||||
def _columns_missing(normalized_fieldnames: set[str], required_columns: list[str]) -> list[str]:
|
||||
return [
|
||||
column
|
||||
for column in required_columns
|
||||
if _normalize_column_name(column) not in normalized_fieldnames
|
||||
]
|
||||
|
||||
|
||||
def _detect_csv_format(normalized_fieldnames: set[str]) -> str:
|
||||
p2p_coe_missing = _columns_missing(normalized_fieldnames, P2P_COE_REQUIRED_COLUMNS)
|
||||
if not p2p_coe_missing:
|
||||
return "p2p_coe"
|
||||
|
||||
p3p_missing = _columns_missing(normalized_fieldnames, P3P_REQUIRED_COLUMNS)
|
||||
if not p3p_missing:
|
||||
return "p3p"
|
||||
|
||||
raise CsvValidationError(
|
||||
"CSV format not recognized. Missing columns for P2P/COE format: "
|
||||
f"{', '.join(p2p_coe_missing)}; missing columns for P3P format: {', '.join(p3p_missing)}"
|
||||
)
|
||||
|
||||
def parse_target_csv(
|
||||
paths: str | Path | list[str | Path] | tuple[str | Path, ...],
|
||||
smb_credentials: dict[str, Any] | None = None,
|
||||
runtime_overrides: dict[str, Any] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
runtime_defaults_by_device = _resolve_runtime_defaults(runtime_overrides)
|
||||
results = {}
|
||||
|
||||
if isinstance(paths, (str, Path)):
|
||||
paths = [paths]
|
||||
|
||||
for path in paths:
|
||||
csv_result = _parse_single_csv(
|
||||
path,
|
||||
smb_credentials=smb_credentials,
|
||||
runtime_defaults_by_device=runtime_defaults_by_device,
|
||||
)
|
||||
|
||||
results.update(csv_result)
|
||||
return results
|
||||
|
||||
def _coerce_runtime_defaults(raw_overrides: dict[str, Any] | None) -> dict[str, int]:
|
||||
defaults = dict(RUNTIME_DEFAULTS)
|
||||
if not raw_overrides:
|
||||
return defaults
|
||||
|
||||
for test_type in ("P2P", "COE", "P3P"):
|
||||
raw_value = raw_overrides.get(test_type)
|
||||
if raw_value is None:
|
||||
continue
|
||||
if isinstance(raw_value, str):
|
||||
raw_value = raw_value.strip()
|
||||
if not raw_value:
|
||||
continue
|
||||
try:
|
||||
minutes = int(raw_value)
|
||||
except (TypeError, ValueError):
|
||||
continue
|
||||
if minutes > 0:
|
||||
defaults[test_type] = minutes
|
||||
|
||||
return defaults
|
||||
|
||||
|
||||
def _resolve_runtime_defaults(runtime_overrides: dict[str, Any] | None) -> dict[str, dict[str, int]]:
|
||||
legacy_defaults = _coerce_runtime_defaults(runtime_overrides)
|
||||
if not runtime_overrides:
|
||||
return {
|
||||
DUT: dict(legacy_defaults),
|
||||
REF: dict(legacy_defaults),
|
||||
}
|
||||
|
||||
resolved: dict[str, dict[str, int]] = {}
|
||||
for device in (DUT, REF):
|
||||
device_defaults = dict(legacy_defaults)
|
||||
raw_device_overrides = runtime_overrides.get(device)
|
||||
if isinstance(raw_device_overrides, dict):
|
||||
device_defaults.update(_coerce_runtime_defaults(raw_device_overrides))
|
||||
resolved[device] = device_defaults
|
||||
|
||||
return resolved
|
||||
|
||||
def _parse_single_csv(
|
||||
path: str,
|
||||
smb_credentials: dict[str, Any] | None,
|
||||
runtime_defaults_by_device: dict[str, dict[str, int]],
|
||||
) -> dict[str, Any]:
|
||||
records: dict[str, Any] = {}
|
||||
with open_csv_handle(path, smb_credentials=smb_credentials) as handle:
|
||||
reader = csv.DictReader(handle)
|
||||
if not reader.fieldnames:
|
||||
raise CsvValidationError("CSV is missing a header row.")
|
||||
|
||||
normalized_fieldnames = {_normalize_column_name(name) for name in reader.fieldnames if name}
|
||||
csv_format = _detect_csv_format(normalized_fieldnames)
|
||||
|
||||
tests: list[TestRecord] = []
|
||||
warnings: list[str] = []
|
||||
seen_test_ids: set[str] = set()
|
||||
|
||||
records_with_signature: list[tuple[TestRecord, tuple[str, ...] | None]] = []
|
||||
|
||||
for row_num, row in enumerate(reader, start=2):
|
||||
test_id = (row.get("TC ID") or "").strip()
|
||||
if not test_id:
|
||||
warnings.append(f"File {path}, row {row_num}: missing TC ID, row skipped.")
|
||||
continue
|
||||
|
||||
if test_id in seen_test_ids:
|
||||
warnings.append(f"File {path}, row {row_num}: duplicate TC ID '{test_id}', row skipped.")
|
||||
continue
|
||||
seen_test_ids.add(test_id)
|
||||
|
||||
throttled = _normalize_yes_no(_row_get(row, "Throttled")) if csv_format == "p3p" else False
|
||||
test_type = _infer_test_type(test_id)
|
||||
rx_tx = _infer_rx_tx(test_id)
|
||||
rotation = _empty_to_none(_row_get(row, "Rotation"))
|
||||
power_mode = _empty_to_none(_row_get(row, "6GHz Power Mode"))
|
||||
has_coe_pair = _normalize_yes_no(_row_get(row, "COE Pair")) if csv_format == "p2p_coe" else False
|
||||
config = _build_config(row, csv_format)
|
||||
signature = _first_populated_p2p_signature(row) if csv_format == "p2p_coe" and test_type == "P2P" else None
|
||||
victim_band_source = "Victim Band" if csv_format == "p2p_coe" else "Band"
|
||||
victim_band = _normalize_victim_band(_row_get(row, victim_band_source))
|
||||
|
||||
records[test_id] = {
|
||||
"test_id": test_id,
|
||||
"test_type": test_type,
|
||||
"rotation": rotation,
|
||||
"rx_tx": rx_tx,
|
||||
"power_mode": power_mode,
|
||||
"has_coe_pair": has_coe_pair,
|
||||
"coe_pairing": [],
|
||||
"victim_band": victim_band,
|
||||
"config": config,
|
||||
"throttled": throttled,
|
||||
"estimated_minutes": {device: defaults[test_type] for device, defaults in runtime_defaults_by_device.items()},
|
||||
"status": "pending",
|
||||
"excluded": False,
|
||||
"raw_payload": row,
|
||||
}
|
||||
records_with_signature.append((records[test_id], signature))
|
||||
|
||||
# Build COE pairing based on full signature and RX/TX+band suffix.
|
||||
# Example key suffixes: RXAX, TXAX, RXBE, TXBE.
|
||||
coe_by_signature_and_suffix: dict[tuple[tuple[str, ...], str], list[str]] = {}
|
||||
for record, signature in records_with_signature:
|
||||
if record["test_type"] == "COE":
|
||||
suffix = _extract_pairing_suffix(record["test_id"])
|
||||
if not suffix:
|
||||
continue
|
||||
|
||||
# Index all populated COE band signatures so any one can match the selected P2P signature.
|
||||
for coe_signature in _all_band_signatures(record["raw_payload"] or {}):
|
||||
key = (coe_signature, suffix)
|
||||
coe_by_signature_and_suffix.setdefault(key, []).append(record["test_id"])
|
||||
|
||||
for record, signature in records_with_signature:
|
||||
pairs = record["coe_pairing"]
|
||||
if record["test_type"] == "P2P" and signature is not None:
|
||||
suffix = _extract_pairing_suffix(record["test_id"])
|
||||
key = (signature, suffix) if suffix else None
|
||||
print(f"[parser] P2P test {record['test_id']} with signature {signature} and suffix {suffix} has COE pairs: {coe_by_signature_and_suffix.get(key, [])}")
|
||||
pairs = sorted(set(coe_by_signature_and_suffix.get(key, []))) if key else []
|
||||
|
||||
records[record["test_id"]].update(
|
||||
has_coe_pair=bool(pairs),
|
||||
coe_pairing=pairs,
|
||||
)
|
||||
|
||||
return records
|
||||
|
||||
|
||||
def _infer_test_type(test_id: str) -> str:
|
||||
token = test_id.upper()
|
||||
if token.startswith("COE"):
|
||||
return "COE"
|
||||
if token.startswith("P3P"):
|
||||
return "P3P"
|
||||
return "P2P"
|
||||
|
||||
|
||||
def _infer_rx_tx(test_id: str) -> str | None:
|
||||
token = test_id.upper()
|
||||
if "RX" in token:
|
||||
return "RX"
|
||||
if "TX" in token:
|
||||
return "TX"
|
||||
return None
|
||||
|
||||
|
||||
def _extract_pairing_suffix(test_id: str) -> str | None:
|
||||
"""Extract the RX/TX+band suffix used for pairing.
|
||||
|
||||
Format: [COE|P2P][RX|TX][Band][Number]
|
||||
Example: P2PRXAC004 -> 'RXAC', COETXAX012 -> 'TXAX'
|
||||
"""
|
||||
token = test_id.upper()
|
||||
# Remove COE/P2P prefix
|
||||
if token.startswith("COE"):
|
||||
token = token[3:]
|
||||
elif token.startswith("P2P") or token.startswith("P3P"):
|
||||
token = token[3:]
|
||||
else:
|
||||
return None
|
||||
|
||||
# Extract RX/TX + band (for example RXAX, TXBE).
|
||||
if len(token) >= 4 and (token.startswith("RX") or token.startswith("TX")):
|
||||
return token[:4]
|
||||
return None
|
||||
|
||||
|
||||
def _empty_to_none(value: str | None) -> str | None:
|
||||
if value is None:
|
||||
return None
|
||||
cleaned = value.strip()
|
||||
return cleaned if cleaned else None
|
||||
|
||||
|
||||
def _first_populated_p2p_signature(row: dict[str, str]) -> tuple[str, ...] | None:
|
||||
for band in ("5G", "6G", "2G"):
|
||||
signature = _band_signature(row, band)
|
||||
if signature is not None:
|
||||
return signature
|
||||
return None
|
||||
|
||||
|
||||
def _band_signature(row: dict[str, str], band: str) -> tuple[str, ...] | None:
|
||||
normalized_band = _normalize_victim_band(band)
|
||||
if normalized_band is None:
|
||||
return None
|
||||
|
||||
# Source CSVs may use either "5G_Test Point" or "5G Test Point" style headers.
|
||||
prefixes = (f"{normalized_band}_", f"{normalized_band} ")
|
||||
|
||||
def _pick_value(suffix: str) -> str:
|
||||
for prefix in prefixes:
|
||||
value = _row_get(row, f"{prefix}{suffix}")
|
||||
if _normalize_value(value):
|
||||
return _normalize_value(value)
|
||||
return ""
|
||||
|
||||
test_point = _pick_value("Test Point")
|
||||
channel = _pick_value("Channel")
|
||||
rssi = _pick_value("RSSI")
|
||||
bandwidth = _pick_value("Bandwidth")
|
||||
# Pairing should ignore direction so reverse-signed COE and P2P rows still match.
|
||||
sta = _pick_value("STA")
|
||||
|
||||
if not all([test_point, channel, rssi, bandwidth, sta]):
|
||||
return None
|
||||
|
||||
return (test_point, channel, bandwidth, rssi, sta)
|
||||
|
||||
|
||||
def _all_band_signatures(row: dict[str, str]) -> list[tuple[str, ...]]:
|
||||
seen: set[tuple[str, ...]] = set()
|
||||
signatures: list[tuple[str, ...]] = []
|
||||
for band in ("5G", "6G", "2G"):
|
||||
signature = _band_signature(row, band)
|
||||
if signature is not None and signature not in seen:
|
||||
seen.add(signature)
|
||||
signatures.append(signature)
|
||||
return signatures
|
||||
|
||||
|
||||
def _normalize_victim_band(value: str | None) -> str | None:
|
||||
if value is None:
|
||||
return None
|
||||
normalized = value.strip().upper().replace("GHZ", "G")
|
||||
if normalized.startswith("5"):
|
||||
return "5G"
|
||||
if normalized.startswith("6"):
|
||||
return "6G"
|
||||
if normalized.startswith("2"):
|
||||
return "2G"
|
||||
return None
|
||||
|
||||
|
||||
def _normalize_value(value: str | None) -> str:
|
||||
if value is None:
|
||||
return ""
|
||||
return " ".join(value.strip().upper().split())
|
||||
|
||||
|
||||
def _normalize_column_name(value: str | None) -> str:
|
||||
if value is None:
|
||||
return ""
|
||||
# Accept common header variants, for example: "TC ID", "TC_ID", or "TC\u00A0ID".
|
||||
return re.sub(r"[^A-Z0-9]+", "", value.upper())
|
||||
|
||||
|
||||
def _normalize_yes_no(value: str | None) -> bool:
|
||||
return _normalize_value(value) == "YES"
|
||||
|
||||
|
||||
def _build_p2p_coe_config(row: dict[str, str]) -> dict[str, dict[str, str | None]]:
|
||||
return {
|
||||
"5G": {
|
||||
"test_point": _empty_to_none(_row_get(row, "5G_Test Point")),
|
||||
"channel": _empty_to_none(_row_get(row, "5G_Channel")),
|
||||
"bandwidth": _empty_to_none(_row_get(row, "5G_Bandwidth")),
|
||||
"rssi": _empty_to_none(_row_get(row, "5G_RSSI")),
|
||||
"direction": _empty_to_none(_row_get(row, "5G_Direction")),
|
||||
"sta": _empty_to_none(_row_get(row, "5G_STA")),
|
||||
},
|
||||
"6G": {
|
||||
"test_point": _empty_to_none(_row_get(row, "6G_Test Point")),
|
||||
"channel": _empty_to_none(_row_get(row, "6G_Channel")),
|
||||
"bandwidth": _empty_to_none(_row_get(row, "6G_Bandwidth")),
|
||||
"rssi": _empty_to_none(_row_get(row, "6G_RSSI")),
|
||||
"direction": _empty_to_none(_row_get(row, "6G_Direction")),
|
||||
"sta": _empty_to_none(_row_get(row, "6G_STA")),
|
||||
},
|
||||
"2G": {
|
||||
"test_point": _empty_to_none(_row_get(row, "2G_Test Point")),
|
||||
"channel": _empty_to_none(_row_get(row, "2G_Channel")),
|
||||
"bandwidth": _empty_to_none(_row_get(row, "2G_Bandwidth")),
|
||||
"rssi": _empty_to_none(_row_get(row, "2G_RSSI")),
|
||||
"direction": _empty_to_none(_row_get(row, "2G_Direction")),
|
||||
"sta": _empty_to_none(_row_get(row, "2G_STA")),
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _build_p3p_config(row: dict[str, str]) -> dict[str, dict[str, str | None]]:
|
||||
return {
|
||||
"STATION1": {
|
||||
"test_point": _empty_to_none(_row_get(row, "STATION 1_Test Point")),
|
||||
"channel": _empty_to_none(_row_get(row, "STATION 1_Channel")),
|
||||
"bandwidth": _empty_to_none(_row_get(row, "STATION 1_Bandwidth")),
|
||||
"rssi": _empty_to_none(_row_get(row, "STATION 1_RSSI")),
|
||||
"direction": _empty_to_none(_row_get(row, "STATION 1_Direction")),
|
||||
"rate": _empty_to_none(_row_get(row, "STATION 1_Rate")),
|
||||
"sta": _empty_to_none(_row_get(row, "STATION 1_STA")),
|
||||
},
|
||||
"STATION2": {
|
||||
"test_point": _empty_to_none(_row_get(row, "STATION 2_Test Point")),
|
||||
"channel": _empty_to_none(_row_get(row, "STATION 2_Channel")),
|
||||
"bandwidth": _empty_to_none(_row_get(row, "STATION 2_Bandwidth")),
|
||||
"rssi": _empty_to_none(_row_get(row, "STATION 2_RSSI")),
|
||||
"direction": _empty_to_none(_row_get(row, "STATION 2_Direction")),
|
||||
"rate": _empty_to_none(_row_get(row, "STATION 2_Rate")),
|
||||
"sta": _empty_to_none(_row_get(row, "STATION 2_STA")),
|
||||
},
|
||||
"STATION3": {
|
||||
"test_point": _empty_to_none(_row_get(row, "STATION 3_Test Point")),
|
||||
"channel": _empty_to_none(_row_get(row, "STATION 3_Channel")),
|
||||
"bandwidth": _empty_to_none(_row_get(row, "STATION 3_Bandwidth")),
|
||||
"rssi": _empty_to_none(_row_get(row, "STATION 3_RSSI")),
|
||||
"direction": _empty_to_none(_row_get(row, "STATION 3_Direction")),
|
||||
"rate": _empty_to_none(_row_get(row, "STATION 3_Rate")),
|
||||
"sta": _empty_to_none(_row_get(row, "STATION 3_STA")),
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _build_config(row: dict[str, str], csv_format: str) -> dict[str, dict[str, str | None]]:
|
||||
if csv_format == "p3p":
|
||||
return _build_p3p_config(row)
|
||||
return _build_p2p_coe_config(row)
|
||||
|
||||
|
||||
def _row_get(row: dict[str, str], key: str) -> str | None:
|
||||
if key in row:
|
||||
return row.get(key)
|
||||
|
||||
normalized_key = _normalize_column_name(key)
|
||||
for existing_key, value in row.items():
|
||||
if existing_key and _normalize_column_name(existing_key) == normalized_key:
|
||||
return value
|
||||
|
||||
if normalized_key == _normalize_column_name("COE PAIR"):
|
||||
for alias in ("COE PAIRING", "COE_PAIRING"):
|
||||
for existing_key, value in row.items():
|
||||
if existing_key and _normalize_column_name(existing_key) == _normalize_column_name(alias):
|
||||
return value
|
||||
|
||||
return None
|
||||
|
||||
def parse_target_filename(filename, parent_dir):
|
||||
base_name = re.sub(r"\.ini$", "", filename, flags=re.IGNORECASE).upper()
|
||||
segments = base_name.split("_")
|
||||
|
||||
test_type = next((s for s in segments if s in TEST_TYPES), None)
|
||||
test_id = next((s for s in segments if re.match(r"^R\d+[A-Z0-9]+$", s, re.IGNORECASE)), None)
|
||||
device = next((s for s in segments if s in (DUT, REF)), None)
|
||||
|
||||
if test_id is None or test_type is None or device is None:
|
||||
print(f"Warning: Could not parse test_id, test_type, or device from filename '{filename}'")
|
||||
return None
|
||||
test_id = test_id[2:] if test_id else None
|
||||
throttled = None
|
||||
if test_type == "P3P" and test_id:
|
||||
# P3P test_id carries throttle marker: TH = throttled, otherwise UT.
|
||||
throttled = True if "TH" in test_id else False
|
||||
rx_tx = "RX" if "RX" in test_id else "TX"
|
||||
band = next((s for s in segments if re.match(r"^\dGHZ$", s)), None)
|
||||
bandwidth = next((s for s in segments if re.match(r"^BW\d+$", s)), None)
|
||||
pm = next ((s for s in segments if s in POWER_MODES), None)
|
||||
|
||||
rotation = None
|
||||
if parent_dir:
|
||||
rotation = next((s for s in parent_dir.split("_") if re.match(r"^ROT\d+$", s)), None)
|
||||
|
||||
return {
|
||||
"test_id": test_id,
|
||||
"device": device,
|
||||
"test_type": test_type,
|
||||
"rotation": rotation,
|
||||
"rx_tx": rx_tx,
|
||||
"power_mode": pm,
|
||||
"has_coe_pair": None,
|
||||
"coe_pairing": [],
|
||||
"victim_band": band,
|
||||
"config_json": {},
|
||||
"throttled": throttled,
|
||||
"estimated_minutes": None,
|
||||
"status": "pending",
|
||||
"excluded": False,
|
||||
"raw_payload": {},
|
||||
"station_testpoint_map": {},
|
||||
}
|
||||
|
||||
@@ -0,0 +1,296 @@
|
||||
import os
|
||||
import re
|
||||
import threading
|
||||
import json
|
||||
|
||||
try:
|
||||
import smbclient # type: ignore[import-not-found]
|
||||
except ModuleNotFoundError:
|
||||
smbclient = None
|
||||
|
||||
from parser import parse_target_csv, parse_target_filename, RUNTIME_DEFAULTS
|
||||
from db import DUT, REF, mark_tests_completed, mark_overdue_as_rerun, reset_completed_to_pending, upsert_tests, TestRecord, get_connection
|
||||
from file_manager import (
|
||||
iter_dir_entries,
|
||||
normalize_input_path,
|
||||
join_path
|
||||
)
|
||||
|
||||
_SCAN_STATE_LOCK = threading.Lock()
|
||||
_ACTIVE_SCAN_COUNT = 0
|
||||
_RESULT_TEST_ID_PATTERN = re.compile(r"(?:COE|P2P|P3P)(?:RX|TX)?[A-Z]{2}\d{3}", re.IGNORECASE)
|
||||
|
||||
|
||||
def _extract_test_id_from_result_dir_name(dir_name):
|
||||
if not dir_name:
|
||||
return None
|
||||
|
||||
match = _RESULT_TEST_ID_PATTERN.search(str(dir_name).upper())
|
||||
if match:
|
||||
return match.group(0)
|
||||
return None
|
||||
|
||||
|
||||
def _build_match_tokens(parsed):
|
||||
tokens = set()
|
||||
|
||||
def _add(value):
|
||||
if value in (None, ""):
|
||||
return
|
||||
tokens.add(str(value).upper())
|
||||
|
||||
_add(parsed.get("interference"))
|
||||
_add(parsed.get("device"))
|
||||
_add(parsed.get("rotation"))
|
||||
_add(parsed.get("test_point"))
|
||||
_add(parsed.get("rssi"))
|
||||
_add(parsed.get("station"))
|
||||
_add(parsed.get("band"))
|
||||
_add(parsed.get("channel"))
|
||||
_add(parsed.get("bandwidth"))
|
||||
_add(parsed.get("direction"))
|
||||
_add(parsed.get("throttled"))
|
||||
_add(parsed.get("test_id"))
|
||||
|
||||
for bw in parsed.get("extra_bandwidths") or []:
|
||||
_add(bw)
|
||||
|
||||
# Add numeric alias for devices, e.g. CGW453 -> 453.
|
||||
device = (parsed.get("device") or "").upper()
|
||||
device_digits = re.sub(r"\D", "", device)
|
||||
if device_digits:
|
||||
tokens.add(device_digits)
|
||||
|
||||
# SP can appear as flag or specific token variant (e.g., SP40).
|
||||
if parsed.get("sp"):
|
||||
tokens.add("SP")
|
||||
tokens.add(str(parsed.get("sp")).upper())
|
||||
|
||||
return tokens
|
||||
|
||||
|
||||
def _rule_matches_target(rule, parsed_tokens):
|
||||
# Rule parts are AND-ed: CGW453_P3P_ROT2 => device AND interference AND rotation.
|
||||
# Support _, -, or spaces as condition separators.
|
||||
parts = [p for p in re.split(r"[_\-\s]+", rule) if p]
|
||||
if not parts:
|
||||
return False
|
||||
|
||||
def _part_matches(part):
|
||||
if part in parsed_tokens:
|
||||
return True
|
||||
|
||||
# Support tag variants (e.g., BW80 should match BW80M/BW80+80 in source names).
|
||||
# Keep this conservative for very short parts to avoid overmatching.
|
||||
if len(part) >= 3:
|
||||
for token in parsed_tokens:
|
||||
if token.startswith(part) or part in token:
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
return all(_part_matches(part) for part in parts)
|
||||
|
||||
|
||||
def _should_exclude_target(parsed, exclusions):
|
||||
parsed_tokens = _build_match_tokens(parsed)
|
||||
# Any matching rule excludes the testcase.
|
||||
return any(_rule_matches_target(rule, parsed_tokens) for rule in exclusions)
|
||||
|
||||
|
||||
def _scan_started():
|
||||
global _ACTIVE_SCAN_COUNT
|
||||
with _SCAN_STATE_LOCK:
|
||||
_ACTIVE_SCAN_COUNT += 1
|
||||
|
||||
|
||||
def _scan_finished():
|
||||
global _ACTIVE_SCAN_COUNT
|
||||
with _SCAN_STATE_LOCK:
|
||||
_ACTIVE_SCAN_COUNT = max(0, _ACTIVE_SCAN_COUNT - 1)
|
||||
|
||||
|
||||
def is_scan_in_progress():
|
||||
with _SCAN_STATE_LOCK:
|
||||
return _ACTIVE_SCAN_COUNT > 0
|
||||
|
||||
|
||||
def scan_results(results_dir_dut, results_dir_ref, smb_credentials=None):
|
||||
results_dir_dut = normalize_input_path(results_dir_dut)
|
||||
results_dir_ref = normalize_input_path(results_dir_ref)
|
||||
|
||||
if not results_dir_dut or not results_dir_ref:
|
||||
return
|
||||
|
||||
try:
|
||||
dut_entries = [
|
||||
entry.name
|
||||
for entry in iter_dir_entries(results_dir_dut, smb_credentials=smb_credentials)
|
||||
if entry.is_dir()
|
||||
]
|
||||
except OSError as exc:
|
||||
print(f"[scanner] Cannot read results dir: {exc}")
|
||||
return
|
||||
|
||||
print(f"[scanner] results: {len(dut_entries)} result dir(s) found in DUT results dir")
|
||||
|
||||
try:
|
||||
ref_entries = [
|
||||
entry.name
|
||||
for entry in iter_dir_entries(results_dir_ref, smb_credentials=smb_credentials)
|
||||
if entry.is_dir()
|
||||
and not entry.name.startswith("obsolete")
|
||||
]
|
||||
except OSError as exc:
|
||||
print(f"[scanner] Cannot read results dir: {exc}")
|
||||
return
|
||||
|
||||
print(f"[scanner] results: {len(ref_entries)} result dir(s) found in reference results dir")
|
||||
|
||||
completed_batch = []
|
||||
unmatched_entries = []
|
||||
|
||||
for entry_name in dut_entries:
|
||||
test_id = _extract_test_id_from_result_dir_name(entry_name)
|
||||
if test_id:
|
||||
completed_batch.append((test_id, DUT))
|
||||
else:
|
||||
unmatched_entries.append(entry_name)
|
||||
|
||||
for entry_name in ref_entries:
|
||||
test_id = _extract_test_id_from_result_dir_name(entry_name)
|
||||
if test_id:
|
||||
completed_batch.append((test_id, REF))
|
||||
else:
|
||||
unmatched_entries.append(entry_name)
|
||||
|
||||
if unmatched_entries:
|
||||
print(f"[scanner] skipped {len(unmatched_entries)} result dir(s) with no recognizable test id")
|
||||
|
||||
if completed_batch:
|
||||
print(f"[scanner] marking {len(completed_batch)} test(s) as completed:")
|
||||
for test_id, device in completed_batch:
|
||||
print(f" - {test_id} on {device}")
|
||||
|
||||
reset_count = reset_completed_to_pending()
|
||||
if reset_count:
|
||||
print(f"[scanner] reset {reset_count} previously-completed test(s) to pending before resync")
|
||||
|
||||
updated_count = mark_tests_completed(completed_batch)
|
||||
print(f"[scanner] {updated_count} test(s) actually updated in database")
|
||||
|
||||
newly_rerun = mark_overdue_as_rerun()
|
||||
if newly_rerun:
|
||||
print(f"[scanner] {newly_rerun} test(s) marked as rerun-required (scheduled but not completed)")
|
||||
|
||||
def _normalize_rotation(rotation: str) -> str:
|
||||
"""Normalize rotation string from R<n> to ROT<n> format."""
|
||||
if not rotation:
|
||||
return ""
|
||||
rotation = str(rotation).strip().upper()
|
||||
if rotation.startswith("R") and len(rotation) > 1 and rotation[1:].isdigit():
|
||||
return f"ROT{rotation[1:]}"
|
||||
return rotation
|
||||
|
||||
def process_targets(target_dir, csv_paths, smb_credentials=None, runtime_overrides=None, db_path=None):
|
||||
target_dir = normalize_input_path(target_dir)
|
||||
csv_results = parse_target_csv(csv_paths, smb_credentials=smb_credentials, runtime_overrides=runtime_overrides)
|
||||
|
||||
batch_tests = []
|
||||
skipped_missing_csv = 0
|
||||
existing_pairings: dict[tuple[str, str], tuple[bool, list[str]]] = {}
|
||||
|
||||
try:
|
||||
with get_connection(db_path) if db_path is not None else get_connection() as conn:
|
||||
rows = conn.execute(
|
||||
"""
|
||||
SELECT test_id, device, has_coe_pair, coe_pairing_json
|
||||
FROM tests
|
||||
"""
|
||||
).fetchall()
|
||||
for row in rows:
|
||||
existing_pairings[(row["test_id"], row["device"])] = (
|
||||
bool(row["has_coe_pair"]),
|
||||
json.loads(row["coe_pairing_json"] or "[]"),
|
||||
)
|
||||
except Exception as exc:
|
||||
print(f"[scanner] Could not read existing pairings for merge: {exc}")
|
||||
|
||||
try:
|
||||
parent_entries = [
|
||||
entry.name
|
||||
for entry in iter_dir_entries(target_dir, smb_credentials=smb_credentials)
|
||||
if entry.is_dir()
|
||||
]
|
||||
except (OSError, ValueError) as exc:
|
||||
print(f"[scanner] Cannot read target dir: {exc}")
|
||||
return 0
|
||||
|
||||
print(f"[scanner] subdirectories found: {len(parent_entries)}")
|
||||
|
||||
for parent_name in parent_entries:
|
||||
parent_path = join_path(target_dir, parent_name)
|
||||
try:
|
||||
files = [
|
||||
entry.name
|
||||
for entry in iter_dir_entries(parent_path, smb_credentials=smb_credentials)
|
||||
if entry.is_file()
|
||||
and not entry.name.startswith("GLOBAL")
|
||||
and entry.name.endswith(".ini")
|
||||
]
|
||||
except (OSError, ValueError) as exc:
|
||||
print(f"[scanner] Cannot read parent dir {parent_name}: {exc}")
|
||||
continue
|
||||
|
||||
|
||||
for filename in files:
|
||||
parsed = parse_target_filename(filename, parent_name)
|
||||
if not parsed or not parsed["test_id"]:
|
||||
continue
|
||||
|
||||
csv_entry = csv_results.get(parsed["test_id"], {})
|
||||
if not csv_entry:
|
||||
skipped_missing_csv += 1
|
||||
continue
|
||||
|
||||
em_map = csv_entry.get("estimated_minutes", {})
|
||||
if isinstance(em_map, dict):
|
||||
estimated_minutes = em_map.get(parsed["device"])
|
||||
else:
|
||||
estimated_minutes = em_map
|
||||
if estimated_minutes is None:
|
||||
estimated_minutes = RUNTIME_DEFAULTS.get(parsed["test_type"], 80)
|
||||
|
||||
incoming_pairs = csv_entry.get("coe_pairing", []) or []
|
||||
incoming_has_pair = bool(csv_entry.get("has_coe_pair", False))
|
||||
existing_has_pair, existing_pairs = existing_pairings.get((parsed["test_id"], parsed["device"]), (False, []))
|
||||
|
||||
# Keep existing non-empty pairings when a later CSV load omits them for the same test.
|
||||
if not incoming_pairs and not incoming_has_pair and existing_pairs:
|
||||
incoming_pairs = existing_pairs
|
||||
incoming_has_pair = existing_has_pair
|
||||
|
||||
batch_tests.append(TestRecord(
|
||||
test_id=parsed["test_id"],
|
||||
device=parsed["device"],
|
||||
test_type=parsed["test_type"],
|
||||
rotation=parsed["rotation"] if parsed["rotation"] else _normalize_rotation(csv_entry.get("rotation")),
|
||||
rx_tx=parsed["rx_tx"],
|
||||
power_mode=parsed["power_mode"],
|
||||
has_coe_pair=incoming_has_pair,
|
||||
coe_pairing=incoming_pairs,
|
||||
victim_band=parsed["victim_band"],
|
||||
config=csv_entry.get("config", {}),
|
||||
throttled=parsed["throttled"] or False,
|
||||
estimated_minutes=estimated_minutes,
|
||||
status=parsed["status"],
|
||||
excluded=False,
|
||||
raw_payload=csv_entry.get("raw_payload", None),
|
||||
))
|
||||
|
||||
|
||||
count = upsert_tests(batch_tests) if db_path is None else upsert_tests(batch_tests, db_path)
|
||||
if skipped_missing_csv:
|
||||
print(f"[scanner] skipped {skipped_missing_csv} target file(s) not present in provided CSV input")
|
||||
|
||||
return count
|
||||
@@ -0,0 +1,380 @@
|
||||
from __future__ import annotations
|
||||
import os
|
||||
|
||||
from dataclasses import dataclass
|
||||
from datetime import date, timedelta
|
||||
from test_window import get_shift_sequence_with_capacity, get_shift_capacity_for_date, is_off_day, next_window_start_date
|
||||
from test_bundle import Test, TestBundle, build_test_bundles, update_mirrored_bundle_priorities
|
||||
|
||||
DUT = (os.getenv("DUT") or "DUT").strip()
|
||||
REF = (os.getenv("REF") or "REF").strip()
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ScheduleEntry:
|
||||
test_id: str
|
||||
device: str
|
||||
scheduled_date: str
|
||||
shift_index: int
|
||||
sequence_in_shift: int
|
||||
|
||||
|
||||
@dataclass
|
||||
class ScheduleWindow:
|
||||
index: int
|
||||
shifts: list[tuple[date, int]]
|
||||
capacity_minutes: int
|
||||
remaining_minutes: int
|
||||
daytime_mode: bool = False
|
||||
assigned_device: str | None = None
|
||||
assigned_config: tuple[str, ...] | None = None
|
||||
|
||||
|
||||
class Scheduler:
|
||||
def __init__(
|
||||
self,
|
||||
all_tests: list[Test],
|
||||
schedulable_tests: list[Test],
|
||||
top_priority_tests: set[tuple[str, str]],
|
||||
start_date: str | None = None,
|
||||
holiday_dates: set[str] = set(),
|
||||
daytime_testing_today: bool = False,
|
||||
daytime_testing_hours: int = 8,
|
||||
daytime_testing_device: str | None = None,
|
||||
dual_device_weekend_start_enabled: bool = False,
|
||||
dual_device_window_start_dates: set[str] | None = None,
|
||||
priority_weight: int = 1000,
|
||||
):
|
||||
self.all_tests = all_tests
|
||||
self.schedulable_tests = schedulable_tests
|
||||
self.top_priority_tests = top_priority_tests
|
||||
self.active_dut: dict[str, Test] = {test.test_id: test for test in schedulable_tests if test.device == DUT}
|
||||
self.active_ref: dict[str, Test] = {test.test_id: test for test in schedulable_tests if test.device == REF}
|
||||
self.start_date = date.fromisoformat(start_date) if start_date else date.today()
|
||||
self.holiday_dates = holiday_dates
|
||||
self.daytime_testing_today = daytime_testing_today
|
||||
safe_daytime_hours = max(0, min(int(daytime_testing_hours), 8))
|
||||
self.daytime_testing_minutes = safe_daytime_hours * 60
|
||||
self.daytime_testing_device = daytime_testing_device if daytime_testing_device in {DUT, REF} else DUT
|
||||
self.dual_device_weekend_start_enabled = dual_device_weekend_start_enabled
|
||||
self.dual_device_window_start_dates = dual_device_window_start_dates or set()
|
||||
self.priority_weight = priority_weight
|
||||
self.schedule: list[ScheduleEntry] = []
|
||||
self.scheduled_bundle_keys: set[tuple[int, str]] = set()
|
||||
self.scheduled_test_ids: set[str] = set()
|
||||
|
||||
|
||||
def compile_schedule(self) -> str | None:
|
||||
bundles = build_test_bundles(self.active_dut, self.active_ref, self.top_priority_tests)
|
||||
if not bundles:
|
||||
return f"Error: No test bundles could be created from the provided tests."
|
||||
|
||||
# Print all test bundles for debugging
|
||||
print("Compiled Test Bundles:")
|
||||
for bundle in bundles:
|
||||
print(f"Bundle Index: {bundle.index}, Device: {bundle.device}, Priority: {bundle.priority}, Config: {bundle.config}, Total Minutes: {bundle.total_minutes}, Tests: {bundle.tests}")
|
||||
|
||||
# Group by individual TC values (from config tuples)
|
||||
all_tcs = set()
|
||||
for bundle in bundles:
|
||||
all_tcs.update(bundle.config if bundle.config else [None])
|
||||
all_bundles_by_tc = self.create_tc_dict(bundles, all_tcs)
|
||||
|
||||
|
||||
window_index = 0
|
||||
cursor_date = self.start_date
|
||||
daytime_window_pending = self.daytime_testing_today and self.daytime_testing_minutes > 0
|
||||
|
||||
tc_order = self.get_tc_order(self.all_tests)
|
||||
|
||||
for tc in tc_order:
|
||||
tc_bundles = all_bundles_by_tc[tc]
|
||||
if tc_bundles is None or len(tc_bundles) == 0:
|
||||
print(f"[scheduler] No bundles found for TC: {tc}. Skipping to next TC.")
|
||||
continue
|
||||
print(f"[scheduler] Scheduling bundles for TC: {tc} with {len(tc_bundles)} bundles.")
|
||||
night_window_device = DUT
|
||||
pending_dut: list[TestBundle] = []
|
||||
pending_ref: list[TestBundle] = []
|
||||
|
||||
|
||||
for bundle in tc_bundles:
|
||||
if bundle.device == DUT:
|
||||
pending_dut.append(bundle)
|
||||
|
||||
elif bundle.device == REF:
|
||||
pending_ref.append(bundle)
|
||||
|
||||
# If there is priority 0 bundle in REF bundle, device should be REF for the first window, otherwise DUT
|
||||
if any(bundle.priority == 0 for bundle in pending_ref):
|
||||
night_window_device = REF
|
||||
# If there is priority 1 bundle in REF but not in DUT, device should be REF for the first window, otherwise DUT
|
||||
elif any(bundle.priority == 1 for bundle in pending_ref) and not any(bundle.priority == 1 for bundle in pending_dut):
|
||||
night_window_device = REF
|
||||
|
||||
while pending_dut or pending_ref:
|
||||
daytime_window_active = daytime_window_pending and not is_off_day(cursor_date, self.holiday_dates)
|
||||
window_device = self.daytime_testing_device if daytime_window_active else night_window_device
|
||||
|
||||
active_pending = pending_dut if window_device == DUT else pending_ref
|
||||
if len(active_pending) == 0 and not daytime_window_active:
|
||||
# If no unscheduled bundles for the current device, switch to the other device
|
||||
night_window_device = REF if night_window_device == DUT else DUT
|
||||
window_device = night_window_device
|
||||
active_pending = pending_dut if night_window_device == DUT else pending_ref
|
||||
|
||||
shifts, capacity = get_shift_sequence_with_capacity(
|
||||
cursor_date,
|
||||
self.holiday_dates,
|
||||
daytime_window_active,
|
||||
self.daytime_testing_minutes,
|
||||
)
|
||||
cursor_date_key = cursor_date.isoformat()
|
||||
dual_device_window = cursor_date_key in self.dual_device_window_start_dates
|
||||
if not dual_device_window and self.dual_device_weekend_start_enabled:
|
||||
dual_device_window = self._is_weekend_start_day(cursor_date)
|
||||
if daytime_window_active:
|
||||
dual_device_window = False
|
||||
|
||||
selected_bundles = []
|
||||
selected_bundles = self._knapsack_select(capacity, active_pending)
|
||||
remaining_time = capacity - sum(bundle.total_minutes for bundle in selected_bundles)
|
||||
|
||||
primary_device = window_device
|
||||
|
||||
if dual_device_window and remaining_time > 0:
|
||||
secondary_device = REF if primary_device == DUT else DUT
|
||||
secondary_pending = pending_dut if secondary_device == DUT else pending_ref
|
||||
secondary_selected_bundles = self._knapsack_select(remaining_time, secondary_pending)
|
||||
selected_bundles.extend(secondary_selected_bundles)
|
||||
|
||||
# Create a mirror of the other device for the next window by updating the priorities
|
||||
pending_dut, pending_ref = update_mirrored_bundle_priorities(selected_bundles, pending_dut, pending_ref)
|
||||
|
||||
window = ScheduleWindow(
|
||||
index=window_index,
|
||||
shifts=shifts,
|
||||
capacity_minutes=capacity,
|
||||
remaining_minutes=remaining_time,
|
||||
daytime_mode=daytime_window_active,
|
||||
assigned_config=tc,
|
||||
assigned_device=primary_device,
|
||||
)
|
||||
|
||||
self._place_bundles_in_window(window, selected_bundles)
|
||||
|
||||
# Mark bundles as scheduled and remove from ALL TC buckets
|
||||
selected_keys = {(b.index, b.device) for b in selected_bundles}
|
||||
|
||||
# Remove scheduled bundles from ALL TC buckets globally
|
||||
for all_tc in all_bundles_by_tc:
|
||||
all_bundles_by_tc[all_tc] = [b for b in all_bundles_by_tc[all_tc] if (b.index, b.device) not in selected_keys]
|
||||
|
||||
# Remove from pending queues
|
||||
pending_dut = [b for b in pending_dut if (b.index, b.device) not in selected_keys]
|
||||
pending_ref = [b for b in pending_ref if (b.index, b.device) not in selected_keys]
|
||||
|
||||
window_index += 1
|
||||
if not daytime_window_active:
|
||||
night_window_device = REF if night_window_device == DUT else DUT
|
||||
cursor_date = next_window_start_date(shifts)
|
||||
if daytime_window_active:
|
||||
daytime_window_pending = False
|
||||
|
||||
return None
|
||||
|
||||
def _is_weekend_start_day(self, current_date: date) -> bool:
|
||||
if is_off_day(current_date, self.holiday_dates):
|
||||
return False
|
||||
return is_off_day(current_date + timedelta(days=1), self.holiday_dates)
|
||||
|
||||
def _select_bundles(
|
||||
self,
|
||||
device: str,
|
||||
capacity: int,
|
||||
unscheduled: list[TestBundle],
|
||||
tc,
|
||||
mutate: bool,
|
||||
) -> tuple[list[TestBundle], list[TestBundle], int]:
|
||||
# Consume pending mirrored bundles only while there is room.
|
||||
pending = self.pending_dut if device == DUT else self.pending_ref
|
||||
mirrored_bundles: list[TestBundle] = []
|
||||
still_pending: list[TestBundle] = [] # Bundles that couldn't fit in the remaining capacity
|
||||
remaining_capacity = capacity
|
||||
|
||||
for bundle in pending:
|
||||
bundle_key = (bundle.index, bundle.device)
|
||||
if bundle_key in self.scheduled_bundle_keys:
|
||||
continue
|
||||
if bundle.device != device or (tc is not None and tc not in bundle.config) or (tc is None and bundle.config):
|
||||
still_pending.append(bundle)
|
||||
continue
|
||||
if bundle.total_minutes <= remaining_capacity:
|
||||
mirrored_bundles.append(bundle)
|
||||
remaining_capacity -= bundle.total_minutes
|
||||
else:
|
||||
still_pending.append(bundle)
|
||||
|
||||
if mutate:
|
||||
pending[:] = still_pending
|
||||
|
||||
# Run knapsack selection for remaining capacity
|
||||
candidates = [
|
||||
b
|
||||
for b in unscheduled
|
||||
if b not in mirrored_bundles
|
||||
and b.device == device
|
||||
and (b.index, b.device) not in self.scheduled_bundle_keys
|
||||
]
|
||||
knapsack_bundles = self._knapsack_select(remaining_capacity, candidates)
|
||||
remaining_capacity -= sum(bundle.total_minutes for bundle in knapsack_bundles)
|
||||
|
||||
print(f"Selected {len(mirrored_bundles)} mirrored bundles and {len(knapsack_bundles)} knapsack bundles for device {device} with remaining capacity {remaining_capacity} minutes.")
|
||||
return mirrored_bundles, knapsack_bundles, remaining_capacity
|
||||
|
||||
def get_schedule(self) -> list[ScheduleEntry]:
|
||||
return self.schedule
|
||||
|
||||
def _knapsack_select(
|
||||
self,
|
||||
capacity: int,
|
||||
candidates: list[TestBundle],
|
||||
) -> list[TestBundle]:
|
||||
if capacity <= 0 or not candidates:
|
||||
return []
|
||||
|
||||
# Weights are total minutes of each bundle
|
||||
weights = [bundle.total_minutes for bundle in candidates]
|
||||
# Values strongly favor higher priority tiers without hard-forcing them.
|
||||
max_priority = max(bundle.priority for bundle in candidates)
|
||||
priority_bias = 2.5
|
||||
values = [int(self.priority_weight * (priority_bias ** (max_priority - bundle.priority))) for bundle in candidates]
|
||||
|
||||
# Implement dynamic programming knapsack algorithm to select bundles
|
||||
n = len(candidates)
|
||||
dp = [[0] * (capacity + 1) for _ in range(n + 1)]
|
||||
for i in range(1, n + 1):
|
||||
for w in range(capacity + 1):
|
||||
if weights[i - 1] <= w:
|
||||
dp[i][w] = max(dp[i - 1][w], dp[i - 1][w - weights[i - 1]] + values[i - 1])
|
||||
else:
|
||||
dp[i][w] = dp[i - 1][w]
|
||||
|
||||
# Traceback to find selected bundles
|
||||
w = capacity
|
||||
selected_indices = []
|
||||
for i in range(n, 0, -1):
|
||||
if w <= 0:
|
||||
break
|
||||
if dp[i][w] != dp[i - 1][w]:
|
||||
selected_indices.append(i - 1)
|
||||
w -= weights[i - 1]
|
||||
|
||||
# Return the selected bundles in the order they were added
|
||||
return [candidates[i] for i in reversed(selected_indices)]
|
||||
|
||||
def _place_bundles_in_window(
|
||||
self,
|
||||
window: ScheduleWindow,
|
||||
bundles: list[TestBundle],
|
||||
) -> None:
|
||||
# Build per-shift remaining capacity
|
||||
shift_remaining = [
|
||||
get_shift_capacity_for_date(
|
||||
day,
|
||||
self.holiday_dates,
|
||||
window.daytime_mode,
|
||||
self.daytime_testing_minutes,
|
||||
).get(shift_idx, 0)
|
||||
for day, shift_idx in window.shifts
|
||||
]
|
||||
current_shift_pos = 0
|
||||
sequence_in_shift = [0] * len(window.shifts)
|
||||
|
||||
for bundle in bundles:
|
||||
bundle_key = (bundle.index, bundle.device)
|
||||
if bundle_key in self.scheduled_bundle_keys:
|
||||
continue
|
||||
self.scheduled_bundle_keys.add(bundle_key)
|
||||
|
||||
for test in bundle.tests:
|
||||
schedule_key = f"{test}:{bundle.device}"
|
||||
if schedule_key in self.scheduled_test_ids:
|
||||
continue
|
||||
self.scheduled_test_ids.add(schedule_key)
|
||||
|
||||
test_minutes = self._get_test_minutes(test, bundle.device)
|
||||
|
||||
while current_shift_pos < len(window.shifts) and shift_remaining[current_shift_pos] <= 0:
|
||||
current_shift_pos += 1
|
||||
|
||||
if current_shift_pos >= len(window.shifts):
|
||||
break
|
||||
|
||||
if sum(shift_remaining[current_shift_pos:]) < test_minutes:
|
||||
break
|
||||
|
||||
start_shift_pos = current_shift_pos
|
||||
shift_day, shift_idx = window.shifts[start_shift_pos]
|
||||
sequence_in_shift[start_shift_pos] += 1
|
||||
|
||||
remaining_test_minutes = test_minutes
|
||||
consume_shift_pos = start_shift_pos
|
||||
while remaining_test_minutes > 0 and consume_shift_pos < len(window.shifts):
|
||||
consumed_minutes = min(shift_remaining[consume_shift_pos], remaining_test_minutes)
|
||||
shift_remaining[consume_shift_pos] -= consumed_minutes
|
||||
remaining_test_minutes -= consumed_minutes
|
||||
consume_shift_pos += 1
|
||||
|
||||
window.remaining_minutes -= test_minutes
|
||||
|
||||
while current_shift_pos < len(window.shifts) and shift_remaining[current_shift_pos] <= 0:
|
||||
current_shift_pos += 1
|
||||
|
||||
self.schedule.append(
|
||||
ScheduleEntry(
|
||||
test_id=test,
|
||||
device=bundle.device,
|
||||
scheduled_date=str(shift_day),
|
||||
shift_index=shift_idx,
|
||||
sequence_in_shift=sequence_in_shift[start_shift_pos],
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _get_test_minutes(self, test_id: str, device: str) -> int:
|
||||
if device == DUT and test_id in self.active_dut:
|
||||
return self.active_dut[test_id].estimated_minutes
|
||||
if device == REF and test_id in self.active_ref:
|
||||
return self.active_ref[test_id].estimated_minutes
|
||||
return 0
|
||||
|
||||
def create_tc_dict(self, bundles: list[TestBundle], all_tcs: set[str | None]) -> dict[str | None, list[TestBundle]]:
|
||||
# Create dict: TC -> bundles supporting that TC
|
||||
all_bundles_by_tc: dict[str | None, list[TestBundle]] = {}
|
||||
# Sort with None last
|
||||
sorted_tcs = sorted([tc for tc in all_tcs if tc is not None]) + ([None] if None in all_tcs else [])
|
||||
for tc in sorted_tcs:
|
||||
if tc is None:
|
||||
all_bundles_by_tc[tc] = [b for b in bundles if not b.config]
|
||||
else:
|
||||
all_bundles_by_tc[tc] = [b for b in bundles if tc in b.config]
|
||||
# Sort by priority, then index
|
||||
all_bundles_by_tc[tc].sort(key=lambda b: (b.priority, b.index))
|
||||
return all_bundles_by_tc
|
||||
|
||||
def get_tc_order(self, all_tests: list[Test]) -> list[str | None]:
|
||||
# Implement the logic for getting the test case order
|
||||
all_dut = {test.test_id: test for test in all_tests if test.device == DUT}
|
||||
all_ref = {test.test_id: test for test in all_tests if test.device == REF}
|
||||
bundles = build_test_bundles(all_dut, all_ref, [])
|
||||
|
||||
all_tcs = set()
|
||||
for bundle in bundles:
|
||||
all_tcs.update(bundle.config if bundle.config else [None])
|
||||
all_bundles_by_tc = self.create_tc_dict(bundles, all_tcs)
|
||||
|
||||
# Sort TCs by the number of bundles available for each TC (most to least)
|
||||
sorted_tcs = sorted(all_bundles_by_tc.keys(), key=lambda tc: len(all_bundles_by_tc[tc]), reverse=True)
|
||||
return sorted_tcs
|
||||
|
||||
|
||||
@@ -0,0 +1,297 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
import os
|
||||
from test_config import build_bundle_test_configs
|
||||
|
||||
@dataclass(frozen=False)
|
||||
class Test:
|
||||
test_id: str
|
||||
device: str
|
||||
test_type: str
|
||||
rotation: str | None
|
||||
rx_tx: str | None
|
||||
has_coe_pair: bool
|
||||
coe_pairing: list[str]
|
||||
config: dict[str, dict[str, str | None]]
|
||||
throttled: bool
|
||||
estimated_minutes: int
|
||||
status: str
|
||||
|
||||
# Bundle priority tiers for scheduling order (lower number = higher priority)
|
||||
BUNDLE_PRIORITY_DUT_COMPLETED_MIRROR = 0 # REF-side tests whose DUT counterpart is already completed
|
||||
BUNDLE_PRIORITY_USER_TOP = 1 # User-specified top priority tests
|
||||
BUNDLE_PRIORITY_RERUN = 2 # Rerun tests (DUT or REF)
|
||||
BUNDLE_PRIORITY_MIRROR = 3
|
||||
BUNDLE_PRIORITY_P2P_WITH_COE = 4 # P2P tests with COE pairs
|
||||
BUNDLE_PRIORITY_P2P_ONLY = 5 # P2P tests without COE pairs (RX/TX bundled)
|
||||
BUNDLE_PRIORITY_COE_ONLY = 6 # COE tests without P2P pairing (should be rare/unschedulable)
|
||||
BUNDLE_PRIORITY_P3P = 6 # P3P tests
|
||||
|
||||
DUT = (os.getenv("DUT") or "DUT").strip()
|
||||
REF = (os.getenv("REF") or "REF").strip()
|
||||
|
||||
@dataclass(frozen=False)
|
||||
class TestBundle:
|
||||
index: int
|
||||
tests: list[str]
|
||||
total_minutes: int
|
||||
device: str
|
||||
config: tuple[str, ...]
|
||||
priority: int
|
||||
completed: int
|
||||
|
||||
def build_test_bundles(active_dut: dict[str, Test], active_ref: dict[str, Test], top_priority_tests: set[tuple[str, str]]) -> list[TestBundle]:
|
||||
"""Build deterministic bundles for DP scheduling."""
|
||||
|
||||
print(f"[test_bundle] Top priority tests: {top_priority_tests}")
|
||||
print(f"[debug] Tests with coe_pairing: {[(tid, t.coe_pairing) for tid, t in active_dut.items() if t.coe_pairing]}")
|
||||
processed_dut: set[str] = set()
|
||||
processed_ref: set[str] = set()
|
||||
test_bundles: list[TestBundle] = []
|
||||
bundle_index = 0
|
||||
|
||||
def dedupe_preserve_order(test_ids: list[str]) -> list[str]:
|
||||
seen: set[str] = set()
|
||||
deduped: list[str] = []
|
||||
for test_id in test_ids:
|
||||
if test_id in seen:
|
||||
continue
|
||||
seen.add(test_id)
|
||||
deduped.append(test_id)
|
||||
return deduped
|
||||
|
||||
# Phase 1: DUT P2P bundles
|
||||
for dut_test_id, dut_test in active_dut.items():
|
||||
if dut_test_id in processed_dut or dut_test.test_type != "P2P":
|
||||
continue
|
||||
|
||||
dut_bundled_tests = [dut_test_id]
|
||||
processed_dut.add(dut_test_id)
|
||||
|
||||
active_coe_pairings = [
|
||||
test_id
|
||||
for test_id in dut_test.coe_pairing
|
||||
if test_id in active_dut and test_id not in processed_dut
|
||||
]
|
||||
|
||||
if active_coe_pairings:
|
||||
dut_bundled_tests.extend(active_coe_pairings)
|
||||
processed_dut.update(active_coe_pairings)
|
||||
priority = BUNDLE_PRIORITY_P2P_WITH_COE
|
||||
else:
|
||||
rx_tx_pair_id = get_rx_tx_pair_id(dut_test, DUT, active_dut, active_ref)
|
||||
if rx_tx_pair_id and rx_tx_pair_id not in processed_dut:
|
||||
dut_bundled_tests.append(rx_tx_pair_id)
|
||||
processed_dut.add(rx_tx_pair_id)
|
||||
priority = BUNDLE_PRIORITY_P2P_ONLY
|
||||
|
||||
dut_bundled_tests = dedupe_preserve_order(dut_bundled_tests)
|
||||
|
||||
# Check if any of the DUT tests in this bundle are in the top priority list
|
||||
if any((dut_test_id, DUT) in top_priority_tests for dut_test_id in dut_bundled_tests):
|
||||
priority = BUNDLE_PRIORITY_USER_TOP
|
||||
|
||||
# Check if any of the DUT tests in this bundle are reruns
|
||||
elif any((dut_test.status == "rerun" for dut_test_id in dut_bundled_tests if (dut_test := active_dut.get(dut_test_id)))):
|
||||
priority = BUNDLE_PRIORITY_RERUN
|
||||
|
||||
test_bundles.append(
|
||||
create_bundle(dut_bundled_tests, DUT, priority, bundle_index, active_dut, active_ref)
|
||||
)
|
||||
|
||||
ref_bundled_tests = [
|
||||
test_id
|
||||
for test_id in dut_bundled_tests
|
||||
if test_id in active_ref and test_id not in processed_ref
|
||||
]
|
||||
if ref_bundled_tests:
|
||||
ref_bundled_tests = dedupe_preserve_order(ref_bundled_tests)
|
||||
processed_ref.update(ref_bundled_tests)
|
||||
|
||||
# Check if any of the REF tests in this bundle are in the top priority list
|
||||
if any((ref_test_id, REF) in top_priority_tests for ref_test_id in ref_bundled_tests):
|
||||
priority = BUNDLE_PRIORITY_USER_TOP
|
||||
# Check if any of the REF tests in this bundle are reruns
|
||||
elif any((ref_test.status == "rerun" for ref_test_id in ref_bundled_tests if (ref_test := active_ref.get(ref_test_id)))):
|
||||
priority = BUNDLE_PRIORITY_RERUN
|
||||
|
||||
test_bundles.append(
|
||||
create_bundle(ref_bundled_tests, REF, priority, bundle_index, active_dut, active_ref)
|
||||
)
|
||||
|
||||
bundle_index += 1
|
||||
|
||||
# Phase 2: DUT P3P bundles
|
||||
for dut_test_id, dut_test in active_dut.items():
|
||||
if dut_test_id in processed_dut or dut_test.test_type != "P3P":
|
||||
continue
|
||||
|
||||
dut_bundled_tests = [dut_test_id]
|
||||
processed_dut.add(dut_test_id)
|
||||
|
||||
th_ut_pair_id = get_th_ut_pair_id(dut_test, DUT, active_dut, active_ref)
|
||||
if th_ut_pair_id and th_ut_pair_id not in processed_dut:
|
||||
dut_bundled_tests.append(th_ut_pair_id)
|
||||
processed_dut.add(th_ut_pair_id)
|
||||
|
||||
dut_bundled_tests = dedupe_preserve_order(dut_bundled_tests)
|
||||
priority = BUNDLE_PRIORITY_P3P
|
||||
|
||||
# Check if any of the DUT tests in this bundle are in the top priority list
|
||||
if any((dut_test_id, DUT) in top_priority_tests for dut_test_id in dut_bundled_tests):
|
||||
priority = BUNDLE_PRIORITY_USER_TOP
|
||||
|
||||
# Check if any of the DUT tests in this bundle are reruns
|
||||
elif any((dut_test.status == "rerun" for dut_test_id in dut_bundled_tests if (dut_test := active_dut.get(dut_test_id)))):
|
||||
priority = BUNDLE_PRIORITY_RERUN
|
||||
|
||||
test_bundles.append(
|
||||
create_bundle(dut_bundled_tests, DUT, priority, bundle_index, active_dut, active_ref)
|
||||
)
|
||||
|
||||
ref_bundled_tests = [
|
||||
test_id
|
||||
for test_id in dut_bundled_tests
|
||||
if test_id in active_ref and test_id not in processed_ref
|
||||
]
|
||||
|
||||
# Check if any of the REF tests in this bundle are in the top priority list
|
||||
if any((ref_test_id, REF) in top_priority_tests for ref_test_id in ref_bundled_tests):
|
||||
priority = BUNDLE_PRIORITY_USER_TOP
|
||||
# Check if any of the REF tests in this bundle are reruns
|
||||
elif any((ref_test.status == "rerun" for ref_test_id in ref_bundled_tests if (ref_test := active_ref.get(ref_test_id)))):
|
||||
priority = BUNDLE_PRIORITY_RERUN
|
||||
|
||||
if ref_bundled_tests:
|
||||
ref_bundled_tests = dedupe_preserve_order(ref_bundled_tests)
|
||||
processed_ref.update(ref_bundled_tests)
|
||||
test_bundles.append(
|
||||
create_bundle(ref_bundled_tests, REF, priority, bundle_index, active_dut, active_ref)
|
||||
)
|
||||
|
||||
bundle_index += 1
|
||||
|
||||
# Phase 3: DUT leftovers (COE-only)
|
||||
for dut_test_id in active_dut:
|
||||
if dut_test_id in processed_dut:
|
||||
continue
|
||||
print(f"[test_bundle] Orphan COE-only DUT test found: {dut_test_id}")
|
||||
test_bundles.append(
|
||||
create_bundle([dut_test_id], DUT, BUNDLE_PRIORITY_COE_ONLY, bundle_index, active_dut, active_ref)
|
||||
)
|
||||
bundle_index += 1
|
||||
processed_dut.add(dut_test_id)
|
||||
|
||||
# Phase 4: unmatched REF bundles, meaning DUT is completed
|
||||
for ref_test_id, ref_test in active_ref.items():
|
||||
if ref_test_id in processed_ref:
|
||||
continue
|
||||
|
||||
ref_bundled_tests = [ref_test_id]
|
||||
processed_ref.add(ref_test_id)
|
||||
|
||||
if ref_test.test_type == "P2P":
|
||||
active_coe_pairings = [
|
||||
test_id
|
||||
for test_id in ref_test.coe_pairing
|
||||
if test_id in active_ref and test_id not in processed_ref
|
||||
]
|
||||
|
||||
if active_coe_pairings:
|
||||
ref_bundled_tests.extend(active_coe_pairings)
|
||||
processed_ref.update(active_coe_pairings)
|
||||
priority = BUNDLE_PRIORITY_DUT_COMPLETED_MIRROR
|
||||
else:
|
||||
rx_tx_pair_id = get_rx_tx_pair_id(ref_test, REF, active_dut, active_ref)
|
||||
if rx_tx_pair_id and rx_tx_pair_id not in processed_ref:
|
||||
ref_bundled_tests.append(rx_tx_pair_id)
|
||||
processed_ref.add(rx_tx_pair_id)
|
||||
priority = BUNDLE_PRIORITY_DUT_COMPLETED_MIRROR
|
||||
|
||||
if ref_test.test_type == "P3P":
|
||||
th_ut_pair_id = get_th_ut_pair_id(ref_test, REF, active_dut, active_ref)
|
||||
if th_ut_pair_id and th_ut_pair_id not in processed_ref:
|
||||
ref_bundled_tests.append(th_ut_pair_id)
|
||||
processed_ref.add(th_ut_pair_id)
|
||||
priority = BUNDLE_PRIORITY_DUT_COMPLETED_MIRROR
|
||||
|
||||
ref_bundled_tests = dedupe_preserve_order(ref_bundled_tests)
|
||||
test_bundles.append(
|
||||
create_bundle(ref_bundled_tests, REF, priority, bundle_index, active_dut, active_ref)
|
||||
)
|
||||
bundle_index += 1
|
||||
|
||||
# Phase 5: REF leftovers ( COE-only)
|
||||
for ref_test_id in active_ref:
|
||||
if ref_test_id in processed_ref:
|
||||
continue
|
||||
print(f"[test-bundle]Orphan COE-only REF test found: {ref_test_id}")
|
||||
test_bundles.append(
|
||||
create_bundle([ref_test_id], REF, BUNDLE_PRIORITY_DUT_COMPLETED_MIRROR, bundle_index, active_dut, active_ref)
|
||||
)
|
||||
bundle_index += 1
|
||||
processed_ref.add(ref_test_id)
|
||||
|
||||
test_bundles.sort(key=lambda b: (b.priority, b.index, b.device, b.config))
|
||||
return test_bundles
|
||||
|
||||
def get_rx_tx_pair_id(test: Test, device: str, active_dut: dict[str, Test], active_ref: dict[str, Test]) -> str | None:
|
||||
active_tests = active_dut if device == DUT else active_ref
|
||||
test_id = test.test_id
|
||||
pair_id = test_id.replace("RX", "TX", 1) if "RX" in test_id else test_id.replace("TX", "RX", 1)
|
||||
return pair_id if pair_id in active_tests else None
|
||||
|
||||
def get_th_ut_pair_id(test: Test, device: str, active_dut: dict[str, Test], active_ref: dict[str, Test]) -> str | None:
|
||||
active_tests = active_dut if device == DUT else active_ref
|
||||
test_id = test.test_id
|
||||
pair_id = test_id.replace("TH", "UT", 1) if "TH" in test_id else test_id.replace("UT", "TH", 1)
|
||||
return pair_id if pair_id in active_tests else None
|
||||
|
||||
def create_bundle(tests: list[str], device: str, priority: int, index: int, active_dut: dict[str, Test], active_ref: dict[str, Test]) -> TestBundle:
|
||||
if not tests:
|
||||
raise ValueError("Cannot create bundle with no tests")
|
||||
|
||||
if device == DUT:
|
||||
active_tests = active_dut
|
||||
else:
|
||||
active_tests = active_ref
|
||||
|
||||
total_minutes = sum(active_tests[test_id].estimated_minutes for test_id in tests)
|
||||
|
||||
bundle_tests = [active_tests[test_id] for test_id in tests]
|
||||
config = tuple(build_bundle_test_configs(bundle_tests))
|
||||
|
||||
return TestBundle(
|
||||
index=index,
|
||||
tests=tests,
|
||||
total_minutes=total_minutes,
|
||||
device=device,
|
||||
config=config,
|
||||
priority=priority,
|
||||
completed=0
|
||||
)
|
||||
|
||||
def bundle_pair_lookup(bundles: list[TestBundle], dut_bundles: list[TestBundle], ref_bundles: list[TestBundle]) -> list[TestBundle]:
|
||||
device = bundles[0].device if bundles else None
|
||||
active_bundle_ids = {b.index for b in dut_bundles} if device == REF else {b.index for b in ref_bundles}
|
||||
|
||||
bundle_indexes = [b.index for b in bundles if b.index in active_bundle_ids]
|
||||
if device == DUT:
|
||||
return [b for b in ref_bundles if b.index in bundle_indexes]
|
||||
elif device == REF:
|
||||
return [b for b in dut_bundles if b.index in bundle_indexes]
|
||||
return []
|
||||
|
||||
def update_mirrored_bundle_priorities(knapsack_bundles: list[TestBundle], dut_pending: list[TestBundle], ref_pending: list[TestBundle]) -> tuple[list[TestBundle], list[TestBundle]]:
|
||||
bundle_pairs = bundle_pair_lookup(knapsack_bundles, dut_pending, ref_pending)
|
||||
|
||||
for bundle in dut_pending:
|
||||
if bundle.index in {b.index for b in bundle_pairs}:
|
||||
bundle.priority = BUNDLE_PRIORITY_MIRROR
|
||||
|
||||
for bundle in ref_pending:
|
||||
if bundle.index in {b.index for b in bundle_pairs}:
|
||||
bundle.priority = BUNDLE_PRIORITY_MIRROR
|
||||
|
||||
return dut_pending, ref_pending
|
||||
@@ -0,0 +1,159 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from typing import Any
|
||||
|
||||
TEST_CONFIG_DEFINITION = {
|
||||
"TC1": {"T1D": "STA5", "T1F": "STA56", "T1I": "STA58", "T1L": "STA64", "T2A":"STA63", "T2O":"STA65", "T2J": "STA59", "T2E": "STA6", "T3E": "STA4"},
|
||||
"TC2": {"T1D": "STA64", "T1F": "STA4", "T1I": "STA5", "T1L": "STA58", "T2A":"STA56", "T2O":"STA59", "T2J": "STA6", "T2E": "STA65", "T3E": "STA63"},
|
||||
"TC3": {"T1D": "STA58", "T1F": "STA63", "T1I": "STA64", "T2J": "STA65", "T2E": "STA59", "T3E": "STA56"},
|
||||
"TC4": {"T1B": "STA56", "T1C": "STA4", "T1D": "STA63"},
|
||||
"TC5": {"T1B": "STA4", "T1C": "STA63", "T1D": "STA56"},
|
||||
"TC6": {"T1B": "STA63", "T1C": "STA56", "T1D": "STA4"},
|
||||
"TC7": {"T1F": "STA4", "T2A": "STA63", "T3E": "STA56"},
|
||||
"TC8": {"T1F": "STA63", "T2A": "STA56", "T3E": "STA4"},
|
||||
"TC9": {"T1F": "STA63", "T2A": "STA64", "T3E": "STA4"},
|
||||
"TC10": {"T1F": "STA4", "T3E": "STA56", "T1K2A": "STA63"},
|
||||
"TC11": {"T1F": "STA56", "T3E": "STA63", "T1K2A": "STA4"},
|
||||
"TC12": {"T1F": "STA56", "T3E": "STA4", "T1K2A": "STA63"},
|
||||
}
|
||||
|
||||
|
||||
|
||||
def _norm(value: Any) -> str:
|
||||
if value is None:
|
||||
return ""
|
||||
return " ".join(str(value).strip().upper().split())
|
||||
|
||||
|
||||
def _get_test_config(test: Any) -> dict[str, dict[str, str | None]]:
|
||||
if isinstance(test, dict):
|
||||
config = test.get("config") or {}
|
||||
else:
|
||||
config = getattr(test, "config", None) or {}
|
||||
return config if isinstance(config, dict) else {}
|
||||
|
||||
|
||||
def _testpoint_to_station_sets(config: dict[str, dict[str, str | None]]) -> dict[str, set[str]]:
|
||||
result: dict[str, set[str]] = {}
|
||||
|
||||
def _add_entry(entry: dict[str, str | None]) -> None:
|
||||
testpoint = _norm(entry.get("test_point") or entry.get("Test Point"))
|
||||
sta_raw = _norm(entry.get("sta") or entry.get("STA"))
|
||||
if not testpoint or not sta_raw:
|
||||
return
|
||||
|
||||
stations = {_norm(sta) for sta in sta_raw.split(",") if _norm(sta)}
|
||||
if not stations:
|
||||
return
|
||||
|
||||
result.setdefault(testpoint, set()).update(stations)
|
||||
|
||||
for entry in config.values():
|
||||
if isinstance(entry, dict):
|
||||
_add_entry(entry)
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def build_station_testpoint_map(config: dict[str, dict[str, str | None]]) -> dict[str, str]:
|
||||
"""Build station->testpoint map used by legacy graph serialization."""
|
||||
station_to_testpoint: dict[str, str] = {}
|
||||
for testpoint, stations in _testpoint_to_station_sets(config).items():
|
||||
for station in stations:
|
||||
station_to_testpoint[station] = testpoint
|
||||
return station_to_testpoint
|
||||
|
||||
|
||||
def resolve_test_config_keys(config: dict[str, dict[str, str | None]]) -> list[str]:
|
||||
"""Return all TC keys whose definitions contain all testpoint->station mappings in config."""
|
||||
testpoint_stations = _testpoint_to_station_sets(config)
|
||||
if not testpoint_stations:
|
||||
return []
|
||||
|
||||
matches: list[str] = []
|
||||
for tc_key, tc_mapping in TEST_CONFIG_DEFINITION.items():
|
||||
is_match = True
|
||||
for testpoint, stations in testpoint_stations.items():
|
||||
# TC definitions map one testpoint to exactly one station.
|
||||
if len(stations) != 1:
|
||||
is_match = False
|
||||
break
|
||||
expected_station = tc_mapping.get(testpoint)
|
||||
if expected_station is None or expected_station not in stations:
|
||||
is_match = False
|
||||
break
|
||||
if is_match:
|
||||
matches.append(tc_key)
|
||||
|
||||
return matches
|
||||
|
||||
|
||||
def build_config_rows(tests: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||||
by_sta: dict[str, set[str]] = {}
|
||||
|
||||
for test in tests:
|
||||
config = test.get("config") or {}
|
||||
if not isinstance(config, dict):
|
||||
continue
|
||||
for entry in config.values():
|
||||
if not isinstance(entry, dict):
|
||||
continue
|
||||
sta_raw = entry.get("sta") or entry.get("STA")
|
||||
test_point = entry.get("test_point") or entry.get("Test Point")
|
||||
if not sta_raw or not test_point:
|
||||
continue
|
||||
|
||||
for sta_part in str(sta_raw).split(","):
|
||||
sta = sta_part.strip().upper()
|
||||
if not sta:
|
||||
continue
|
||||
by_sta.setdefault(sta, set()).add(str(test_point))
|
||||
|
||||
return [
|
||||
{
|
||||
"sta": sta,
|
||||
"testPoints": sorted(points),
|
||||
}
|
||||
for sta, points in sorted(by_sta.items())
|
||||
]
|
||||
|
||||
|
||||
def build_bundle_test_configs(tests: list[Any]) -> list[str]:
|
||||
merged_by_testpoint: dict[str, set[str]] = {}
|
||||
for test in tests:
|
||||
config = _get_test_config(test)
|
||||
for entry in config.values():
|
||||
if not isinstance(entry, dict):
|
||||
continue
|
||||
testpoint_raw = entry.get("test_point") or entry.get("Test Point")
|
||||
sta_raw = entry.get("sta") or entry.get("STA")
|
||||
if not testpoint_raw or not sta_raw:
|
||||
continue
|
||||
|
||||
testpoint = " ".join(str(testpoint_raw).strip().upper().split())
|
||||
if not testpoint:
|
||||
continue
|
||||
|
||||
stations = {
|
||||
" ".join(str(sta).strip().upper().split())
|
||||
for sta in str(sta_raw).split(",")
|
||||
if " ".join(str(sta).strip().upper().split())
|
||||
}
|
||||
if not stations:
|
||||
continue
|
||||
|
||||
merged_by_testpoint.setdefault(testpoint, set()).update(stations)
|
||||
|
||||
merged_config: dict[str, dict[str, str | None]] = {}
|
||||
for idx, (testpoint, stations) in enumerate(sorted(merged_by_testpoint.items()), start=1):
|
||||
merged_config[f"Station {idx}"] = {
|
||||
"test_point": testpoint,
|
||||
"sta": ",".join(sorted(stations)),
|
||||
}
|
||||
|
||||
return resolve_test_config_keys(merged_config)
|
||||
|
||||
|
||||
def serialize_station_testpoint_map(config: dict[str, dict[str, str | None]]) -> str:
|
||||
return json.dumps(build_station_testpoint_map(config))
|
||||
@@ -0,0 +1,70 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import date, timedelta
|
||||
|
||||
|
||||
def is_off_day(current_date: date, holiday_dates: set[str]) -> bool:
|
||||
return current_date.weekday() >= 5 or current_date.isoformat() in holiday_dates
|
||||
|
||||
def get_shift_sequence(
|
||||
current_date: date,
|
||||
holiday_dates: set[str],
|
||||
daytime_shift2_only: bool = False,
|
||||
) -> list[tuple[date, int]]:
|
||||
is_holiday = current_date.isoformat() in holiday_dates
|
||||
weekday = current_date.weekday()
|
||||
shifts: list[tuple[date, int]] = []
|
||||
|
||||
if daytime_shift2_only and weekday < 5 and not is_holiday:
|
||||
return [(current_date, 2)]
|
||||
|
||||
if is_off_day(current_date, holiday_dates):
|
||||
return [(current_date, 1), (current_date, 2), (current_date, 3)]
|
||||
|
||||
shifts.append((current_date, 3))
|
||||
next_day = current_date + timedelta(days=1)
|
||||
if is_off_day(next_day, holiday_dates):
|
||||
cursor = next_day
|
||||
while is_off_day(cursor, holiday_dates):
|
||||
shifts.extend([(cursor, 1), (cursor, 2), (cursor, 3)])
|
||||
cursor += timedelta(days=1)
|
||||
shifts.append((cursor, 1))
|
||||
return shifts
|
||||
|
||||
shifts.append((next_day, 1))
|
||||
return shifts
|
||||
|
||||
|
||||
def get_shift_capacity_for_date(
|
||||
current_date: date,
|
||||
holiday_dates: set[str],
|
||||
daytime_shift2_only: bool = False,
|
||||
daytime_shift2_minutes: int = 480,
|
||||
) -> dict[int, int]:
|
||||
if daytime_shift2_only:
|
||||
daytime_minutes = max(0, min(int(daytime_shift2_minutes), 480))
|
||||
return {1: 0, 2: daytime_minutes, 3: 0}
|
||||
if is_off_day(current_date, holiday_dates):
|
||||
return {1: 480, 2: 480, 3: 480}
|
||||
return {1: 480, 2: 0, 3: 480}
|
||||
|
||||
|
||||
def get_shift_sequence_with_capacity(
|
||||
current_date: date,
|
||||
holiday_dates: set[str],
|
||||
daytime_shift2_only: bool = False,
|
||||
daytime_shift2_minutes: int = 480,
|
||||
) -> tuple[list[tuple[date, int]], int]:
|
||||
shifts = get_shift_sequence(current_date, holiday_dates, daytime_shift2_only)
|
||||
capacity = sum(
|
||||
get_shift_capacity_for_date(day, holiday_dates, daytime_shift2_only, daytime_shift2_minutes).get(shift_index, 0)
|
||||
for day, shift_index in shifts
|
||||
)
|
||||
return shifts, capacity
|
||||
|
||||
|
||||
def next_window_start_date(shifts: list[tuple[date, int]]) -> date:
|
||||
last_date, last_shift = shifts[-1]
|
||||
if last_shift in (1, 2):
|
||||
return last_date
|
||||
return last_date + timedelta(days=1)
|
||||
@@ -0,0 +1,355 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
import threading
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
try:
|
||||
import smbclient # type: ignore[import-not-found]
|
||||
except ModuleNotFoundError:
|
||||
smbclient = None
|
||||
from watchdog.events import FileSystemEvent, FileSystemEventHandler, FileSystemMovedEvent
|
||||
from watchdog.observers.polling import PollingObserver
|
||||
|
||||
from file_manager import resolve_runtime_path
|
||||
from scanner import scan_results
|
||||
|
||||
|
||||
LOGGER = logging.getLogger("scheduler.watcher")
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class WatchConfig:
|
||||
dut_dir: str
|
||||
ref_dir: str
|
||||
smb_username: str
|
||||
smb_password: str
|
||||
smb_domain: str
|
||||
|
||||
|
||||
class _ResultDirEventHandler(FileSystemEventHandler):
|
||||
def __init__(self, on_new_directory):
|
||||
super().__init__()
|
||||
self._on_new_directory = on_new_directory
|
||||
|
||||
def on_created(self, event: FileSystemEvent) -> None:
|
||||
if event.is_directory:
|
||||
self._on_new_directory(event.src_path)
|
||||
|
||||
def on_moved(self, event: FileSystemMovedEvent) -> None:
|
||||
if event.is_directory:
|
||||
self._on_new_directory(event.dest_path)
|
||||
|
||||
|
||||
class _SmbPollerThread(threading.Thread):
|
||||
"""Poll a UNC results directory and trigger a scan when entries change."""
|
||||
|
||||
def __init__(self, unc_path: str, on_change, interval_seconds: float = 5.0):
|
||||
super().__init__(daemon=True, name=f"smb-poller-{unc_path}")
|
||||
self._path = unc_path.rstrip("\\")
|
||||
self._on_change = on_change
|
||||
self._interval_seconds = max(1.0, float(interval_seconds))
|
||||
self._stop_evt = threading.Event()
|
||||
|
||||
def stop(self) -> None:
|
||||
self._stop_evt.set()
|
||||
|
||||
def run(self) -> None:
|
||||
prev = self._snapshot()
|
||||
while not self._stop_evt.wait(self._interval_seconds):
|
||||
try:
|
||||
curr = self._snapshot()
|
||||
added = curr - prev
|
||||
removed = prev - curr
|
||||
if added or removed:
|
||||
LOGGER.info(
|
||||
"SMB directory change detected (%s): +%d -%d",
|
||||
self._path,
|
||||
len(added),
|
||||
len(removed),
|
||||
)
|
||||
self._on_change("smb-directory-change")
|
||||
prev = curr
|
||||
except Exception:
|
||||
LOGGER.exception("SMB poll error for %s", self._path)
|
||||
|
||||
def _snapshot(self) -> set[str]:
|
||||
if smbclient is None:
|
||||
LOGGER.error("smbclient is not installed; SMB polling is unavailable for %s", self._path)
|
||||
return set()
|
||||
try:
|
||||
return {entry.name for entry in smbclient.scandir(self._path) if entry.is_dir()}
|
||||
except Exception:
|
||||
LOGGER.exception("SMB scandir failed for %s", self._path)
|
||||
return set()
|
||||
|
||||
|
||||
class ResultDirectoryWatcher:
|
||||
"""Watch DUT/REF result directories and trigger scans when new result folders appear."""
|
||||
|
||||
def __init__(self, poll_interval_seconds: float = 2.0, min_scan_interval_seconds: float = 1.0):
|
||||
self._poll_interval_seconds = poll_interval_seconds
|
||||
self._min_scan_interval_seconds = min_scan_interval_seconds
|
||||
self._lock = threading.RLock()
|
||||
self._scan_lock = threading.Lock()
|
||||
self._watchers: list[Any] = []
|
||||
self._current_config: WatchConfig | None = None
|
||||
self._last_scan_monotonic = 0.0
|
||||
|
||||
def configure_from_settings(self, settings: dict[str, Any]) -> None:
|
||||
dut_dir = _normalize_watch_path(settings.get("dutResultDir"))
|
||||
ref_dir = _normalize_watch_path(settings.get("refResultDir"))
|
||||
new_config = WatchConfig(
|
||||
dut_dir=dut_dir,
|
||||
ref_dir=ref_dir,
|
||||
smb_username=str(settings.get("smbUsername") or "").strip(),
|
||||
smb_password=str(settings.get("smbPassword") or ""),
|
||||
smb_domain=str(settings.get("smbDomain") or "").strip(),
|
||||
)
|
||||
|
||||
with self._lock:
|
||||
if new_config == self._current_config and self._watchers:
|
||||
return
|
||||
|
||||
self._stop_locked()
|
||||
self._current_config = new_config
|
||||
|
||||
if not dut_dir or not ref_dir:
|
||||
LOGGER.info("result watcher disabled: DUT/REF result directories are not both set")
|
||||
return
|
||||
|
||||
smb_creds = {
|
||||
"username": new_config.smb_username,
|
||||
"password": new_config.smb_password,
|
||||
"domain": new_config.smb_domain,
|
||||
}
|
||||
|
||||
local_paths: list[Path] = []
|
||||
smb_paths: list[str] = []
|
||||
seen: set[str] = set()
|
||||
|
||||
for result_dir, label in ((dut_dir, "DUT"), (ref_dir, "REF")):
|
||||
is_watchable, reason = _validate_watch_path(result_dir, smb_creds)
|
||||
if not is_watchable:
|
||||
LOGGER.warning(
|
||||
"%s result directory is not watchable, skipping watch: %s (reason: %s)",
|
||||
label,
|
||||
result_dir,
|
||||
reason,
|
||||
)
|
||||
continue
|
||||
|
||||
if _is_unc_path(result_dir):
|
||||
unc = _to_unc_path(result_dir)
|
||||
key = unc.lower()
|
||||
if key not in seen:
|
||||
seen.add(key)
|
||||
smb_paths.append(unc)
|
||||
else:
|
||||
p = Path(result_dir)
|
||||
key = os.path.normcase(str(p))
|
||||
if key not in seen:
|
||||
seen.add(key)
|
||||
local_paths.append(p)
|
||||
|
||||
watchers: list[Any] = []
|
||||
|
||||
if local_paths:
|
||||
handler = _ResultDirEventHandler(self._on_new_directory)
|
||||
observer = PollingObserver(timeout=self._poll_interval_seconds)
|
||||
scheduled_count = 0
|
||||
scheduled_paths: list[str] = []
|
||||
for path in local_paths:
|
||||
try:
|
||||
observer.schedule(handler, str(path), recursive=False)
|
||||
scheduled_count += 1
|
||||
scheduled_paths.append(str(path))
|
||||
except Exception:
|
||||
LOGGER.exception("failed to watch local result directory: %s", path)
|
||||
|
||||
if scheduled_count:
|
||||
try:
|
||||
observer.start()
|
||||
watchers.append(observer)
|
||||
for path in scheduled_paths:
|
||||
print(f"[watcher] Watching local directory: {path}")
|
||||
LOGGER.info("watcher successfully watching local path: %s", path)
|
||||
except Exception:
|
||||
LOGGER.exception("failed to start local result observer")
|
||||
try:
|
||||
observer.stop()
|
||||
observer.join(timeout=5)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
for unc in smb_paths:
|
||||
registered, reason = _register_smb_session(unc, smb_creds)
|
||||
if not registered:
|
||||
LOGGER.warning("SMB watch skipped for %s (reason: %s)", unc, reason)
|
||||
continue
|
||||
poller = _SmbPollerThread(
|
||||
unc,
|
||||
on_change=lambda reason: self.scan_now(reason=reason),
|
||||
interval_seconds=max(2.0, self._poll_interval_seconds),
|
||||
)
|
||||
poller.start()
|
||||
watchers.append(poller)
|
||||
LOGGER.info("watcher successfully watching SMB path: %s", unc)
|
||||
print(f"[watcher] Watching SMB directory: {unc}")
|
||||
if not watchers:
|
||||
LOGGER.warning("result watcher not started: no valid result directories to watch")
|
||||
return
|
||||
|
||||
self._watchers = watchers
|
||||
LOGGER.info("result watcher started for DUT=%s REF=%s", dut_dir, ref_dir)
|
||||
|
||||
self.scan_now(reason="startup")
|
||||
|
||||
def _on_new_directory(self, directory_path: str) -> None:
|
||||
LOGGER.info("new result directory detected: %s", directory_path)
|
||||
self.scan_now(reason="directory-created")
|
||||
|
||||
def scan_now(self, reason: str = "manual") -> None:
|
||||
with self._lock:
|
||||
config = self._current_config
|
||||
if config is None or not config.dut_dir or not config.ref_dir:
|
||||
return
|
||||
|
||||
now = time.monotonic()
|
||||
if now - self._last_scan_monotonic < self._min_scan_interval_seconds:
|
||||
return
|
||||
self._last_scan_monotonic = now
|
||||
|
||||
if not self._scan_lock.acquire(blocking=False):
|
||||
return
|
||||
|
||||
def _run_scan() -> None:
|
||||
credentials = {
|
||||
"username": config.smb_username,
|
||||
"password": config.smb_password,
|
||||
"domain": config.smb_domain,
|
||||
}
|
||||
try:
|
||||
scan_results(config.dut_dir, config.ref_dir, smb_credentials=credentials)
|
||||
LOGGER.info("result scan finished (%s)", reason)
|
||||
except Exception:
|
||||
LOGGER.exception("result scan failed (%s)", reason)
|
||||
finally:
|
||||
self._scan_lock.release()
|
||||
|
||||
threading.Thread(target=_run_scan, daemon=True).start()
|
||||
|
||||
def stop(self) -> None:
|
||||
with self._lock:
|
||||
self._stop_locked()
|
||||
|
||||
def _stop_locked(self) -> None:
|
||||
watchers = self._watchers
|
||||
self._watchers = []
|
||||
for watcher in watchers:
|
||||
try:
|
||||
watcher.stop()
|
||||
except Exception:
|
||||
pass
|
||||
for watcher in watchers:
|
||||
try:
|
||||
watcher.join(timeout=5)
|
||||
except Exception:
|
||||
pass
|
||||
if watchers:
|
||||
LOGGER.info("result watcher stopped")
|
||||
|
||||
|
||||
_WATCHER = ResultDirectoryWatcher()
|
||||
|
||||
|
||||
def configure_result_watcher(settings: dict[str, Any]) -> None:
|
||||
_WATCHER.configure_from_settings(settings)
|
||||
|
||||
|
||||
def stop_result_watcher() -> None:
|
||||
_WATCHER.stop()
|
||||
|
||||
|
||||
def _normalize_watch_path(path_value: Any) -> str:
|
||||
if path_value is None:
|
||||
return ""
|
||||
cleaned = str(path_value).strip()
|
||||
if not cleaned:
|
||||
return ""
|
||||
return str(resolve_runtime_path(cleaned)).strip()
|
||||
|
||||
|
||||
def _is_unc_path(path_value: str) -> bool:
|
||||
path = str(path_value)
|
||||
return path.startswith("\\\\") or path.startswith("//")
|
||||
|
||||
|
||||
def _to_unc_path(path_value: str) -> str:
|
||||
path = str(path_value).replace("/", "\\")
|
||||
if path.startswith("\\\\"):
|
||||
return path
|
||||
if path.startswith("//"):
|
||||
return "\\\\" + path.lstrip("/\\")
|
||||
return "\\\\" + path.lstrip("/\\")
|
||||
|
||||
|
||||
def _validate_watch_path(path_value: str, smb_credentials: dict[str, str]) -> tuple[bool, str]:
|
||||
path = str(path_value).strip()
|
||||
if not path:
|
||||
return False, "path is empty"
|
||||
|
||||
if _is_unc_path(path):
|
||||
if smbclient is None:
|
||||
return False, "smbclient is not installed"
|
||||
unc = _to_unc_path(path)
|
||||
registered, reason = _register_smb_session(unc, smb_credentials)
|
||||
if not registered:
|
||||
return False, reason
|
||||
try:
|
||||
# Access one directory entry to validate permissions/connectivity.
|
||||
next(iter(smbclient.scandir(unc)), None)
|
||||
except Exception as exc:
|
||||
return False, f"{type(exc).__name__}: {exc}"
|
||||
return True, "ok"
|
||||
|
||||
if not os.path.exists(path):
|
||||
return False, "path does not exist"
|
||||
if not os.path.isdir(path):
|
||||
return False, "path is not a directory"
|
||||
try:
|
||||
with os.scandir(path):
|
||||
pass
|
||||
except Exception as exc:
|
||||
return False, f"{type(exc).__name__}: {exc}"
|
||||
return True, "ok"
|
||||
|
||||
|
||||
def _register_smb_session(unc_path: str, smb_credentials: dict[str, str]) -> tuple[bool, str]:
|
||||
if smbclient is None:
|
||||
return False, "smbclient is not installed"
|
||||
|
||||
server = unc_path[2:].split("\\", 1)[0]
|
||||
if not server:
|
||||
return False, "invalid UNC path (missing server)"
|
||||
|
||||
username = str(smb_credentials.get("username") or "").strip()
|
||||
password = str(smb_credentials.get("password") or "")
|
||||
domain = str(smb_credentials.get("domain") or "").strip()
|
||||
if username and domain and "\\" not in username and "@" not in username:
|
||||
username = f"{domain}\\{username}"
|
||||
|
||||
try:
|
||||
if username:
|
||||
smbclient.register_session(server, username=username, password=password)
|
||||
else:
|
||||
smbclient.register_session(server)
|
||||
return True, "ok"
|
||||
except Exception as exc:
|
||||
return False, f"SMB session registration failed for {server}: {type(exc).__name__}: {exc}"
|
||||
|
||||
|
||||
Reference in New Issue
Block a user