upsert tests and mark complete in bulk

server/scanner.py

@@ -5,14 +5,14 @@ import tempfile
 import smbclient
 
 from db_py import (
+    mark_tests_completed,
+    upsert_tests,
     clear_tests,
     extract_measurement_metrics,
     get_config,
-    mark_completed,
     reset_all_results_state,
     update_all_p2p_coe_pairs_sql,
     update_all_p3p_pairs_sql,
-    upsert_test,
 )
 from parser import (
     parse_result_filename,
@@ -335,6 +335,7 @@ def full_scan(target_dir, results_dir, results_dir_ref):
 def scan_targets(target_dir):
     target_dir = _normalize_input_path(target_dir)
     exclusions = _parse_scan_exclusions(get_config("scan_exclusions"))
+    batch_tests = []
 
     try:
         parent_entries = [
@@ -378,7 +379,7 @@ def scan_targets(target_dir):
                 continue
 
             #print( f"[scanner] found: {parent_name}/{filename}  ->  test_id={parsed['test_id']}")
-            upsert_test(
+            batch_tests.append(
                 {
                     "id": parsed["id"],
                     "test_id": parsed["test_id"],
@@ -398,6 +399,8 @@ def scan_targets(target_dir):
                 }
             )
 
+    upsert_tests(batch_tests)
+
     return True
 
 
@@ -415,8 +418,13 @@ def scan_results(results_dir):
         return
 
     print(f"[scanner] results: {len(entries)} result dir(s) found")
+    completed_batch = []
     for dir_name in entries:
-        process_result_dir(results_dir, dir_name)
+        completion = process_result_dir(results_dir, dir_name)
+        if completion:
+            completed_batch.append(completion)
+
+    mark_tests_completed(completed_batch)
 
 
 def process_result_dir(results_dir, result_dir_name):
@@ -426,7 +434,7 @@ def process_result_dir(results_dir, result_dir_name):
     test_id = parsed["test_id"]
     device = parsed["device"]
     if not test_id or not device:
-        return
+        return None
 
     result_dir_path = _join_path(results_dir, result_dir_name)
 
@@ -434,13 +442,18 @@ def process_result_dir(results_dir, result_dir_name):
         files = [entry.name for entry in _iter_dir_entries(result_dir_path) if entry.is_file()]
     except OSError as exc:
         print(f"[scanner] Cannot read result dir {result_dir_name}: {exc}")
-        return
+        return None
 
     measurement_db = next((name for name in files if name.lower() == "measurement.db"), None)
     if not measurement_db:
         print(f"[scanner] completed (no measurement.db yet): {test_id}")
-        mark_completed(test_id, device, None, None)
-        return
+        return {
+            "test_id": test_id,
+            "device": device,
+            "completed_at": None,
+            "duration_seconds": None,
+            "tput_results": None,
+        }
 
     db_path = _join_path(result_dir_path, measurement_db)
     completed_at = parse_timestamp(result_dir_name)
@@ -450,13 +463,13 @@ def process_result_dir(results_dir, result_dir_name):
         f"[scanner] completed: {test_id}  device={device}  "
         f"duration={data['duration_seconds']}s  stations={len(data['tputResults'])}  at={completed_at}"
     )'''
-    mark_completed(
-        test_id,
-        device,
-        completed_at,
-        data["duration_seconds"],
-        data["tputResults"],
-    )
+    return {
+        "test_id": test_id,
+        "device": device,
+        "completed_at": completed_at,
+        "duration_seconds": data["duration_seconds"],
+        "tput_results": data["tputResults"],
+    }
 
 
 def parse_deleted_result_dir_name(dir_name):