matthias/dashboard
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(sync): remove debug file writing and optimize DB queries

Browse Source
This commit is contained in:
Matthias Hochmeister
2026-04-18 18:15:40 +02:00
parent 26df8b427e
commit 0a6377a64f
3 changed files with 66 additions and 129 deletions
Download Patch File Download Diff File Expand all files Collapse all files

58
sync/src/scraper.ts
View File

@@ -1,6 +1,4 @@
import { chromium, Page, Frame } from '@playwright/test';
import * as fs from 'fs';
import * as path from 'path';
import {
FdiskMember,
FdiskAusbildung,
@@ -12,27 +10,10 @@ import {
const BASE_URL = process.env.FDISK_BASE_URL ?? 'https://app.fdisk.at';
const ID_FEUERWEHREN = process.env.FDISK_ID_FEUERWEHREN ?? '164';
const ID_INSTANZEN = process.env.FDISK_ID_INSTANZEN ?? '2853';
const DEBUG_HTML = process.env.FDISK_DEBUG_HTML === '1' || process.env.FDISK_DEBUG_HTML === 'true';
const LOGIN_URL = `${BASE_URL}/fdisk/module/vws/logins/logins.aspx`;
const MEMBERS_URL = `${BASE_URL}/fdisk/module/mgvw/mitgliedschaften/meine_Mitglieder.aspx`;
/** Save frame HTML to debug/ folder when FDISK_DEBUG_HTML=1 */
async function dumpHtml(frame: Frame, label: string): Promise<void> {
if (!DEBUG_HTML) return;
try {
const debugDir = path.resolve(process.cwd(), 'debug');
fs.mkdirSync(debugDir, { recursive: true });
const html = await frame.content();
const safeName = label.replace(/[^a-zA-Z0-9_-]/g, '_');
const filePath = path.join(debugDir, `${safeName}.html`);
fs.writeFileSync(filePath, html, 'utf-8');
log(` [debug] saved HTML → ${filePath} (${(html.length / 1024).toFixed(1)} KB)`);
} catch (err: any) {
log(` [debug] failed to save HTML for "${label}": ${err.message}`);
}
}
/**
* Maps a raw FDISK status string to a dashboard status value.
* Returns null for unknown/unneeded statuses — those members should be skipped.
@@ -219,7 +200,6 @@ export async function scrapeAll(username: string, password: string): Promise<{
const members = await scrapeMembers(mainFrame);
log(`Found ${members.length} members (full scrape)`);
if (DEBUG_HTML) log(`[debug] HTML dump mode ON — saving pages to debug/`);
const ausbildungen: FdiskAusbildung[] = [];
const befoerderungen: FdiskBefoerderung[] = [];
@@ -246,9 +226,6 @@ export async function scrapeAll(username: string, password: string): Promise<{
member.wohnort = profileFields.wohnort;
member.plz = profileFields.plz;
// Debug: dump the member detail page (Ausbildungen are scraped from here)
await dumpHtml(mainFrame, `detail_StNr${member.standesbuchNr}`);
// Extract mitgliedschaft + person params from the current URL for constructing sub-section URLs.
// PersonenForm.aspx is in the personen module; sub-sections are each in their own module.
// URL pattern: ?search=1&searchid_mitgliedschaften=X&id_personen=Y&id_mitgliedschaften=X&searchid_personen=Y&searchid_maskmode=
@@ -266,15 +243,6 @@ export async function scrapeAll(username: string, password: string): Promise<{
log(` ${member.vorname} ${member.zuname}: ${quals.length} Ausbildungen`);
} catch (err: any) {
log(` WARN: Ausbildungen scrape failed for ${member.vorname} ${member.zuname} (StNr ${member.standesbuchNr}): ${err.message}`);
// Always dump HTML on failure for diagnosis
try {
const debugDir = path.resolve(process.cwd(), 'debug');
fs.mkdirSync(debugDir, { recursive: true });
const html = await mainFrame.content();
const filePath = path.join(debugDir, `ausbildungen_error_StNr${member.standesbuchNr}.html`);
fs.writeFileSync(filePath, html, 'utf-8');
log(` [debug] saved error HTML → ${filePath}`);
} catch { /* ignore dump errors */ }
}
}
@@ -769,9 +737,6 @@ async function scrapeAusbildungenFromDetailPage(
// Ensure all rows are visible (the URL param should already set this, but belt-and-suspenders)
await selectAlleAnzeige(frame);
// Dump HTML for debugging
await dumpHtml(frame, `kurse_StNr${member.standesbuchNr}`);
// Read indexed form fields — same pattern as scrapeMemberFahrgenehmigungen
const rawRows = await frame.evaluate((stNr: string) => {
const rows: Array<{
@@ -849,10 +814,6 @@ async function scrapeAusbildungenFromDetailPage(
};
});
if (results.length === 0) {
await dumpHtml(frame, `kurse_empty_StNr${member.standesbuchNr}`);
}
return results;
}
@@ -958,12 +919,6 @@ async function navigateAndGetTableRows(
log(` → ${allRows.length} total rows, ${fdcRows.length} FdcLayList rows, ${dataRows.length} data rows (date in col ${dateColIdx})`);
// Debug: dump HTML when no data rows found
if (dataRows.length === 0) {
const urlSlug = url.split('/').pop()?.split('?')[0] ?? 'unknown';
await dumpHtml(frame, `navigateAndGetTableRows_${urlSlug}`);
}
return { rows: dataRows, dateColIdx };
}
@@ -1021,16 +976,12 @@ async function scrapeMemberUntersuchungen(
const title = await frame.title().catch(() => '');
if (landed.includes('BLError') || landed.includes('support.aspx') || title.toLowerCase().includes('fehler')) {
log(` → Untersuchungen ERROR page: ${landed}`);
await dumpHtml(frame, `untersuchungen_error_StNr${standesbuchNr}`);
return [];
}
// Show all rows
await selectAlleAnzeige(frame);
// Dump HTML for diagnosis (always when debug enabled)
await dumpHtml(frame, `untersuchungen_StNr${standesbuchNr}`);
// Try to navigate to history/detail view if available
// FDISK may show only the most recent per exam type on the list page.
// Look for a "Verlauf" or "Detail" or "Alle anzeigen" link/button
@@ -1073,7 +1024,6 @@ async function scrapeMemberUntersuchungen(
await frame.waitForNavigation({ timeout: 5000 }).catch(() => {});
}
await selectAlleAnzeige(frame);
await dumpHtml(frame, `untersuchungen_history_StNr${standesbuchNr}`);
} catch (e) {
log(` → Failed to follow history link: ${e}`);
}
@@ -1135,10 +1085,6 @@ async function scrapeMemberUntersuchungen(
log(` → Untersuchungen: ${allRows.length} total rows, ${dataRows.length} data rows (date in col ${dateColIdx})`);
if (dataRows.length === 0) {
await dumpHtml(frame, `untersuchungen_empty_StNr${standesbuchNr}`);
}
const results: FdiskUntersuchung[] = [];
for (const row of dataRows) {
const valueCols: string[] = [];
@@ -1196,9 +1142,6 @@ async function scrapeMemberFahrgenehmigungen(
// Show all rows (default is 10)
await selectAlleAnzeige(frame);
// Dump HTML for diagnostics
await dumpHtml(frame, `fahrgenehmigungen_StNr${standesbuchNr}`);
// Read form fields by ID pattern: {fieldname}_{rowIndex}
const rawRows = await frame.evaluate(() => {
const rows: Array<{
@@ -1402,7 +1345,6 @@ async function scrapeMemberFahrgenehmigungen(
if (klasseIdx === -1) {
log(` Fahrgenehmigungen for StNr ${standesbuchNr}: could not determine Klasse column. Returning empty.`);
await dumpHtml(frame, `fahrgenehmigungen_fallback_StNr${standesbuchNr}`);
return [];
}