feat: refactor parsing and scanning logic for test files
- Updated `parseFilename` to `parseTargetFilename` and modified its return structure to include `test_id` instead of `file_id`. - Introduced `parseResultFilename` to extract `test_id` and `device` from result file names. - Enhanced `fullScan` to separately handle target and results directories, improving clarity and functionality. - Updated database interactions to use `test_id` instead of `file_id` across various modules. - Added a new `/rescan` endpoint to trigger a full scan of target and results directories. - Improved logging and error handling throughout the scanning process. - Introduced `parseTputRssi` to extract throughput and RSSI data from log files.
This commit is contained in:
+47
-49
@@ -2,7 +2,7 @@
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
const readline = require('readline');
|
||||
const { parseFilename, parseTimestamp, parseElapsedTime } = require('./parser');
|
||||
const { parseTargetFilename, parseResultFilename, parseTimestamp, parseElapsedTime, parseTputRssi } = require('./parser');
|
||||
const { upsertTest, markCompleted, clearTests } = require('./db');
|
||||
|
||||
/**
|
||||
@@ -16,7 +16,13 @@ async function fullScan(targetDir, resultsDir) {
|
||||
console.log(`[scanner] target dir : ${targetDir}`);
|
||||
console.log(`[scanner] results dir: ${resultsDir}`);
|
||||
|
||||
// ── 1. Walk target directory ──────────────────────────────────────────────
|
||||
await scanTargets(targetDir);
|
||||
await scanResults(resultsDir);
|
||||
}
|
||||
|
||||
async function scanTargets(targetDir) {
|
||||
// Walk the target directory, find all TC_WIFI_*.ini files, parse their names,
|
||||
// and upsert them into the DB. Ignore files without a valid test_id.
|
||||
let parentEntries;
|
||||
try {
|
||||
parentEntries = fs.readdirSync(targetDir, { withFileTypes: true }).filter(d => d.isDirectory());
|
||||
@@ -32,24 +38,24 @@ async function fullScan(targetDir, resultsDir) {
|
||||
let files;
|
||||
try {
|
||||
files = fs.readdirSync(parentDirPath, { withFileTypes: true })
|
||||
.filter(f => f.isFile() && f.name.startsWith('TC_WIFI_') && f.name.endsWith('.ini'));
|
||||
.filter(f => f.isFile() && !f.name.startsWith('GLOBAL') && f.name.endsWith('.ini'));
|
||||
} catch (e) {
|
||||
console.error(`[scanner] Cannot read parent dir ${parentEntry.name}:`, e.message);
|
||||
continue;
|
||||
}
|
||||
|
||||
console.log(`[scanner] ${parentEntry.name} → ${files.length} TC_WIFI_*.ini file(s)`);
|
||||
console.log(`[scanner] ${parentEntry.name} → ${files.length} target test file(s)`);
|
||||
|
||||
for (const file of files) {
|
||||
const parsed = parseFilename(file.name, parentEntry.name);
|
||||
if (!parsed.file_id) {
|
||||
console.log(`[scanner] skip (no file_id): ${parentEntry.name}/${file.name}`);
|
||||
const parsed = parseTargetFilename(file.name, parentEntry.name);
|
||||
if (!parsed.test_id) {
|
||||
console.log(`[scanner] skip (no test_id): ${parentEntry.name}/${file.name}`);
|
||||
continue;
|
||||
}
|
||||
console.log(`[scanner] found: ${parentEntry.name}/${file.name} → file_id=${parsed.file_id}`);
|
||||
console.log(`[scanner] found: ${parentEntry.name}/${file.name} → test_id=${parsed.test_id}`);
|
||||
upsertTest({
|
||||
id: parsed.id,
|
||||
file_id: parsed.file_id,
|
||||
test_id: parsed.test_id,
|
||||
parent_dir: parentEntry.name,
|
||||
filename: file.name,
|
||||
interference: parsed.interference,
|
||||
@@ -66,8 +72,6 @@ async function fullScan(targetDir, resultsDir) {
|
||||
}
|
||||
}
|
||||
|
||||
// ── 2. Walk results directory ─────────────────────────────────────────────
|
||||
await scanResults(resultsDir);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -88,83 +92,77 @@ async function scanResults(resultsDir) {
|
||||
}
|
||||
|
||||
/**
|
||||
* Process one result directory: extract file_id, find the latest matching
|
||||
* Process one result directory: extract test_id, find the latest matching
|
||||
* log file, parse its elapsed-time line, and mark the test as completed.
|
||||
*/
|
||||
async function processResultDir(resultsDir, resultDirName) {
|
||||
const segments = resultDirName.split(/[_\-]/);
|
||||
const file_id = segments.find(s => /^R\d+[A-Z0-9]+$/i.test(s)) || null;
|
||||
const device = segments.find(s => /^CGW\d+$/i.test(s)) || null;
|
||||
console.log(`[scanner] result dir: ${resultDirName} → file_id=${file_id} device=${device}`);
|
||||
if (!file_id) return;
|
||||
const { test_id, device } = parseResultFilename(resultDirName);
|
||||
//console.log(`[scanner] result dir: ${resultDirName} → test_id=${test_id} device=${device}`);
|
||||
if (!test_id || !device) return;
|
||||
|
||||
const resultDirPath = path.join(resultsDir, resultDirName);
|
||||
|
||||
let logFiles;
|
||||
try {
|
||||
logFiles = fs.readdirSync(resultDirPath, { withFileTypes: true })
|
||||
.filter(f => f.isFile() && f.name.endsWith('.txt') && f.name.includes(file_id))
|
||||
.filter(f => f.isFile() && f.name.endsWith('.txt') && f.name.includes(test_id))
|
||||
.map(f => f.name);
|
||||
} catch {
|
||||
} catch (e) {
|
||||
console.error(`[scanner] Cannot read result dir ${resultDirName}:`, e.message);
|
||||
return;
|
||||
}
|
||||
|
||||
if (logFiles.length === 0) {
|
||||
// Result directory exists but log not written yet — mark complete without timing
|
||||
console.log(`[scanner] completed (no log yet): ${file_id}`);
|
||||
markCompleted(file_id, device, null, null);
|
||||
console.log(`[scanner] completed (no log yet): ${test_id}`);
|
||||
markCompleted(test_id, device, null, null);
|
||||
return;
|
||||
}
|
||||
|
||||
// For re-runs, the latest log file has the latest timestamp in its name
|
||||
const latestLog = logFiles.sort().at(-1);
|
||||
const logPath = path.join(resultDirPath, latestLog);
|
||||
const logPath = path.join(resultDirPath, latestLog);
|
||||
|
||||
const completed_at = parseTimestamp(latestLog);
|
||||
const duration_seconds = await extractDuration(logPath, file_id);
|
||||
const completed_at = parseTimestamp(latestLog);
|
||||
const { duration_seconds, tputResults } = await extractLogData(logPath);
|
||||
|
||||
console.log(`[scanner] completed: ${file_id} device=${device} duration=${duration_seconds}s at=${completed_at}`);
|
||||
markCompleted(file_id, device, completed_at, duration_seconds);
|
||||
console.log(`[scanner] completed: ${test_id} device=${device} duration=${duration_seconds}s stations=${tputResults.length} at=${completed_at}`);
|
||||
markCompleted(test_id, device, completed_at, duration_seconds, tputResults);
|
||||
}
|
||||
|
||||
/**
|
||||
* Stream-read a log file and return the elapsed time in seconds,
|
||||
* or null if the expected line is not found.
|
||||
* Single-pass log file reader: extracts elapsed time and all station tput/rssi entries.
|
||||
*/
|
||||
function extractDuration(logFilePath) {
|
||||
console.log(`[scanner] extracting duration from log: ${logFilePath}`);
|
||||
const REGEX = /\[.*?INFO\]\s+Elapsed time\s*:\s*([\d]+:[\d]{2}:[\d]{2}\.[\d]+)/;
|
||||
function extractLogData(logFilePath) {
|
||||
const ELAPSED_REGEX = /\[.*?INFO\]\s+Elapsed time\s*:\s*([\d]+:[\d]{2}:[\d]{2}\.[\d]+)/;
|
||||
const TPUT_RSSI_REGEX = /\[.*?INFO\]\s+STA(\d+) over angles >> AVG IxChariot TPUT: (\d+) Mbps, AVG DL RSSI: (-?\d+) dBm, AVG UL RSSI: (-?\d+) dBm/
|
||||
|
||||
return new Promise((resolve) => {
|
||||
const result = { duration_seconds: null, tputResults: [] };
|
||||
let stream;
|
||||
try {
|
||||
stream = fs.createReadStream(logFilePath, { encoding: 'utf8' });
|
||||
} catch {
|
||||
return resolve(null);
|
||||
} catch (e) {
|
||||
console.error(`[scanner] Cannot read log file ${logFilePath}:`, e.message);
|
||||
return resolve(result);
|
||||
}
|
||||
|
||||
const rl = readline.createInterface({ input: stream, crlfDelay: Infinity });
|
||||
let done = false;
|
||||
|
||||
function finish(value) {
|
||||
if (done) return;
|
||||
done = true;
|
||||
resolve(value);
|
||||
}
|
||||
|
||||
rl.on('line', (line) => {
|
||||
if (done) return;
|
||||
const match = line.match(REGEX);
|
||||
if (match) {
|
||||
finish(parseElapsedTime(match[1]));
|
||||
rl.close();
|
||||
stream.destroy();
|
||||
const timeMatch = line.match(ELAPSED_REGEX);
|
||||
const tputMatch = line.match(TPUT_RSSI_REGEX);
|
||||
|
||||
if (timeMatch) result.duration_seconds = parseElapsedTime(timeMatch[1]);
|
||||
else if (tputMatch) {
|
||||
parsed = parseTputRssi(line);
|
||||
if (parsed) result.tputResults.push(parsed);
|
||||
}
|
||||
});
|
||||
|
||||
rl.on('close', () => finish(null));
|
||||
rl.on('error', () => finish(null));
|
||||
stream.on('error', () => finish(null));
|
||||
rl.on('close', () => resolve(result));
|
||||
rl.on('error', () => resolve(result));
|
||||
stream.on('error', () => resolve(result));
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user