fix(sync): remove debug file writing and optimize DB queries

frontend/src/pages/MitgliedDetail.tsx

@@ -1265,7 +1265,7 @@ function MitgliedDetail() {
                         <Typography variant="body2" fontWeight={500}>
                           {f.klasse}
                         </Typography>
-                        <Box sx={{ display: 'flex', gap: 2 }}>
+                        <Box sx={{ display: 'flex', flexDirection: 'column', gap: 0.5 }}>
                           <Typography variant="caption" color="text.secondary">
                             Erhalten am: {f.ausstellungsdatum ? new Date(f.ausstellungsdatum).toLocaleDateString('de-AT') : '—'}
                           </Typography>

sync/src/db.ts

@@ -255,23 +255,50 @@ export async function syncToDatabase(
 
     log(`Members: ${updated} changed, ${unchanged} unchanged, ${forced} forced, ${created} created, ${skipped} skipped`);
 
-    // Upsert Ausbildungen
+    // Build StNr → userId lookup map (single query instead of per-record lookups)
-    let ausbildungNew = 0;
+    const lookupResult = await client.query<{ fdisk_standesbuch_nr: string; user_id: string }>(
-    let ausbildungUpdated = 0;
+      `SELECT fdisk_standesbuch_nr, user_id FROM mitglieder_profile WHERE fdisk_standesbuch_nr IS NOT NULL`
-    let ausbildungSkipped = 0;
-
-    for (const ausb of ausbildungen) {
-      const result = await client.query<{ user_id: string }>(
-        `SELECT user_id FROM mitglieder_profile WHERE fdisk_standesbuch_nr = $1`,
-        [ausb.standesbuchNr]
     );
+    const stNrToUserId = new Map<string, string>(
+      lookupResult.rows.map(r => [r.fdisk_standesbuch_nr, r.user_id])
+    );
+    log(`Lookup map: ${stNrToUserId.size} StNr→userId mappings`);
 
-      if (result.rows.length === 0) {
+    // Upsert Ausbildungen
-        ausbildungSkipped++;
+    const ausbildungStats = await syncAusbildungen(client, ausbildungen, stNrToUserId);
-        continue;
+    log(`Ausbildungen: ${ausbildungStats.neu} neu, ${ausbildungStats.updated} unverändert, ${ausbildungStats.skipped} übersprungen`);
+
+    // Upsert Beförderungen
+    const befoerderungStats = await syncBefoerderungen(client, befoerderungen, stNrToUserId);
+    log(`Beförderungen: ${befoerderungStats.neu} neu, ${befoerderungStats.updated} unverändert, ${befoerderungStats.skipped} übersprungen`);
+
+    // Upsert Untersuchungen
+    const untersuchungStats = await syncUntersuchungen(client, untersuchungen, stNrToUserId);
+    log(`Untersuchungen: ${untersuchungStats.neu} neu, ${untersuchungStats.updated} unverändert, ${untersuchungStats.skipped} übersprungen`);
+
+    // Upsert Fahrgenehmigungen
+    const fahrgenStats = await syncFahrgenehmigungen(client, fahrgenehmigungen, stNrToUserId);
+    log(`Fahrgenehmigungen: ${fahrgenStats.neu} neu, ${fahrgenStats.updated} unverändert, ${fahrgenStats.skipped} übersprungen`);
+
+    await client.query('COMMIT');
+  } catch (err) {
+    await client.query('ROLLBACK');
+    throw err;
+  } finally {
+    client.release();
+  }
 }
 
-      const userId = result.rows[0].user_id;
+async function syncAusbildungen(
+  client: PoolClient,
+  ausbildungen: FdiskAusbildung[],
+  stNrToUserId: Map<string, string>
+): Promise<{ neu: number; updated: number; skipped: number }> {
+  let neu = 0, updated = 0, skipped = 0;
+
+  for (const ausb of ausbildungen) {
+    const userId = stNrToUserId.get(ausb.standesbuchNr);
+    if (!userId) { skipped++; continue; }
 
     const upsertResult = await client.query<{ was_inserted: boolean }>(
       `INSERT INTO ausbildung (user_id, kursname, kursnummer, kurs_kurzbezeichnung, erfolgscode, kurs_datum, ablaufdatum, ort, bemerkung, fdisk_sync_key)
@@ -292,49 +319,25 @@ export async function syncToDatabase(
 
     if (upsertResult.rows[0]?.was_inserted) {
       log(`New Ausbildung: ${ausb.standesbuchNr} — ${ausb.kursname}${ausb.kursDatum ? ` (${ausb.kursDatum})` : ''}`);
-        ausbildungNew++;
+      neu++;
     } else {
-        ausbildungUpdated++;
+      updated++;
     }
   }
 
-    log(`Ausbildungen: ${ausbildungNew} neu, ${ausbildungUpdated} unverändert, ${ausbildungSkipped} übersprungen`);
+  return { neu, updated, skipped };
-
-    // Upsert Beförderungen
-    const befoerderungStats = await syncBefoerderungen(client, befoerderungen);
-    log(`Beförderungen: ${befoerderungStats.neu} neu, ${befoerderungStats.updated} unverändert, ${befoerderungStats.skipped} übersprungen`);
-
-    // Upsert Untersuchungen
-    const untersuchungStats = await syncUntersuchungen(client, untersuchungen);
-    log(`Untersuchungen: ${untersuchungStats.neu} neu, ${untersuchungStats.updated} unverändert, ${untersuchungStats.skipped} übersprungen`);
-
-    // Upsert Fahrgenehmigungen
-    const fahrgenStats = await syncFahrgenehmigungen(client, fahrgenehmigungen);
-    log(`Fahrgenehmigungen: ${fahrgenStats.neu} neu, ${fahrgenStats.updated} unverändert, ${fahrgenStats.skipped} übersprungen`);
-
-    await client.query('COMMIT');
-  } catch (err) {
-    await client.query('ROLLBACK');
-    throw err;
-  } finally {
-    client.release();
-  }
 }
 
 async function syncBefoerderungen(
   client: PoolClient,
-  befoerderungen: FdiskBefoerderung[]
+  befoerderungen: FdiskBefoerderung[],
+  stNrToUserId: Map<string, string>
 ): Promise<{ neu: number; updated: number; skipped: number }> {
   let neu = 0, updated = 0, skipped = 0;
 
   for (const b of befoerderungen) {
-    const result = await client.query<{ user_id: string }>(
+    const userId = stNrToUserId.get(b.standesbuchNr);
-      `SELECT user_id FROM mitglieder_profile WHERE fdisk_standesbuch_nr = $1`,
+    if (!userId) { skipped++; continue; }
-      [b.standesbuchNr]
-    );
-
-    if (result.rows.length === 0) { skipped++; continue; }
-    const userId = result.rows[0].user_id;
 
     const upsertResult = await client.query<{ was_inserted: boolean }>(
       `INSERT INTO befoerderungen (user_id, datum, dienstgrad, fdisk_sync_key)
@@ -360,18 +363,14 @@ async function syncBefoerderungen(
 
 async function syncUntersuchungen(
   client: PoolClient,
-  untersuchungen: FdiskUntersuchung[]
+  untersuchungen: FdiskUntersuchung[],
+  stNrToUserId: Map<string, string>
 ): Promise<{ neu: number; updated: number; skipped: number }> {
   let neu = 0, updated = 0, skipped = 0;
 
   for (const u of untersuchungen) {
-    const result = await client.query<{ user_id: string }>(
+    const userId = stNrToUserId.get(u.standesbuchNr);
-      `SELECT user_id FROM mitglieder_profile WHERE fdisk_standesbuch_nr = $1`,
+    if (!userId) { skipped++; continue; }
-      [u.standesbuchNr]
-    );
-
-    if (result.rows.length === 0) { skipped++; continue; }
-    const userId = result.rows[0].user_id;
 
     const upsertResult = await client.query<{ was_inserted: boolean }>(
       `INSERT INTO untersuchungen (user_id, datum, anmerkungen, art, ergebnis, fdisk_sync_key)
@@ -399,7 +398,8 @@ async function syncUntersuchungen(
 
 async function syncFahrgenehmigungen(
   client: PoolClient,
-  fahrgenehmigungen: FdiskFahrgenehmigung[]
+  fahrgenehmigungen: FdiskFahrgenehmigung[],
+  stNrToUserId: Map<string, string>
 ): Promise<{ neu: number; updated: number; skipped: number }> {
   let neu = 0, updated = 0, skipped = 0;
 
@@ -420,13 +420,8 @@ async function syncFahrgenehmigungen(
       continue;
     }
 
-    const result = await client.query<{ user_id: string }>(
+    const userId = stNrToUserId.get(f.standesbuchNr);
-      `SELECT user_id FROM mitglieder_profile WHERE fdisk_standesbuch_nr = $1`,
+    if (!userId) { skipped++; continue; }
-      [f.standesbuchNr]
-    );
-
-    if (result.rows.length === 0) { skipped++; continue; }
-    const userId = result.rows[0].user_id;
 
     const upsertResult = await client.query<{ was_inserted: boolean }>(
       `INSERT INTO fahrgenehmigungen (user_id, ausstellungsdatum, gueltig_bis, behoerde, nummer, klasse, fdisk_sync_key)

sync/src/scraper.ts

@@ -1,6 +1,4 @@
 import { chromium, Page, Frame } from '@playwright/test';
-import * as fs from 'fs';
-import * as path from 'path';
 import {
   FdiskMember,
   FdiskAusbildung,
@@ -12,27 +10,10 @@ import {
 const BASE_URL           = process.env.FDISK_BASE_URL        ?? 'https://app.fdisk.at';
 const ID_FEUERWEHREN     = process.env.FDISK_ID_FEUERWEHREN  ?? '164';
 const ID_INSTANZEN       = process.env.FDISK_ID_INSTANZEN    ?? '2853';
-const DEBUG_HTML         = process.env.FDISK_DEBUG_HTML === '1' || process.env.FDISK_DEBUG_HTML === 'true';
 
 const LOGIN_URL      = `${BASE_URL}/fdisk/module/vws/logins/logins.aspx`;
 const MEMBERS_URL    = `${BASE_URL}/fdisk/module/mgvw/mitgliedschaften/meine_Mitglieder.aspx`;
 
-/** Save frame HTML to debug/ folder when FDISK_DEBUG_HTML=1 */
-async function dumpHtml(frame: Frame, label: string): Promise<void> {
-  if (!DEBUG_HTML) return;
-  try {
-    const debugDir = path.resolve(process.cwd(), 'debug');
-    fs.mkdirSync(debugDir, { recursive: true });
-    const html = await frame.content();
-    const safeName = label.replace(/[^a-zA-Z0-9_-]/g, '_');
-    const filePath = path.join(debugDir, `${safeName}.html`);
-    fs.writeFileSync(filePath, html, 'utf-8');
-    log(`  [debug] saved HTML → ${filePath} (${(html.length / 1024).toFixed(1)} KB)`);
-  } catch (err: any) {
-    log(`  [debug] failed to save HTML for "${label}": ${err.message}`);
-  }
-}
-
 /**
  * Maps a raw FDISK status string to a dashboard status value.
  * Returns null for unknown/unneeded statuses — those members should be skipped.
@@ -219,7 +200,6 @@ export async function scrapeAll(username: string, password: string): Promise<{
 
     const members = await scrapeMembers(mainFrame);
     log(`Found ${members.length} members (full scrape)`);
-    if (DEBUG_HTML) log(`[debug] HTML dump mode ON — saving pages to debug/`);
 
     const ausbildungen: FdiskAusbildung[] = [];
     const befoerderungen: FdiskBefoerderung[] = [];
@@ -246,9 +226,6 @@ export async function scrapeAll(username: string, password: string): Promise<{
         member.wohnort    = profileFields.wohnort;
         member.plz        = profileFields.plz;
 
-        // Debug: dump the member detail page (Ausbildungen are scraped from here)
-        await dumpHtml(mainFrame, `detail_StNr${member.standesbuchNr}`);
-
         // Extract mitgliedschaft + person params from the current URL for constructing sub-section URLs.
         // PersonenForm.aspx is in the personen module; sub-sections are each in their own module.
         // URL pattern: ?search=1&searchid_mitgliedschaften=X&id_personen=Y&id_mitgliedschaften=X&searchid_personen=Y&searchid_maskmode=
@@ -266,15 +243,6 @@ export async function scrapeAll(username: string, password: string): Promise<{
             log(`  ${member.vorname} ${member.zuname}: ${quals.length} Ausbildungen`);
           } catch (err: any) {
             log(`  WARN: Ausbildungen scrape failed for ${member.vorname} ${member.zuname} (StNr ${member.standesbuchNr}): ${err.message}`);
-            // Always dump HTML on failure for diagnosis
-            try {
-              const debugDir = path.resolve(process.cwd(), 'debug');
-              fs.mkdirSync(debugDir, { recursive: true });
-              const html = await mainFrame.content();
-              const filePath = path.join(debugDir, `ausbildungen_error_StNr${member.standesbuchNr}.html`);
-              fs.writeFileSync(filePath, html, 'utf-8');
-              log(`  [debug] saved error HTML → ${filePath}`);
-            } catch { /* ignore dump errors */ }
           }
         }
 
@@ -769,9 +737,6 @@ async function scrapeAusbildungenFromDetailPage(
   // Ensure all rows are visible (the URL param should already set this, but belt-and-suspenders)
   await selectAlleAnzeige(frame);
 
-  // Dump HTML for debugging
-  await dumpHtml(frame, `kurse_StNr${member.standesbuchNr}`);
-
   // Read indexed form fields — same pattern as scrapeMemberFahrgenehmigungen
   const rawRows = await frame.evaluate((stNr: string) => {
     const rows: Array<{
@@ -849,10 +814,6 @@ async function scrapeAusbildungenFromDetailPage(
     };
   });
 
-  if (results.length === 0) {
-    await dumpHtml(frame, `kurse_empty_StNr${member.standesbuchNr}`);
-  }
-
   return results;
 }
 
@@ -958,12 +919,6 @@ async function navigateAndGetTableRows(
 
   log(`    → ${allRows.length} total rows, ${fdcRows.length} FdcLayList rows, ${dataRows.length} data rows (date in col ${dateColIdx})`);
 
-  // Debug: dump HTML when no data rows found
-  if (dataRows.length === 0) {
-    const urlSlug = url.split('/').pop()?.split('?')[0] ?? 'unknown';
-    await dumpHtml(frame, `navigateAndGetTableRows_${urlSlug}`);
-  }
-
   return { rows: dataRows, dateColIdx };
 }
 
@@ -1021,16 +976,12 @@ async function scrapeMemberUntersuchungen(
   const title  = await frame.title().catch(() => '');
   if (landed.includes('BLError') || landed.includes('support.aspx') || title.toLowerCase().includes('fehler')) {
     log(`    → Untersuchungen ERROR page: ${landed}`);
-    await dumpHtml(frame, `untersuchungen_error_StNr${standesbuchNr}`);
     return [];
   }
 
   // Show all rows
   await selectAlleAnzeige(frame);
 
-  // Dump HTML for diagnosis (always when debug enabled)
-  await dumpHtml(frame, `untersuchungen_StNr${standesbuchNr}`);
-
   // Try to navigate to history/detail view if available
   // FDISK may show only the most recent per exam type on the list page.
   // Look for a "Verlauf" or "Detail" or "Alle anzeigen" link/button
@@ -1073,7 +1024,6 @@ async function scrapeMemberUntersuchungen(
         await frame.waitForNavigation({ timeout: 5000 }).catch(() => {});
       }
       await selectAlleAnzeige(frame);
-      await dumpHtml(frame, `untersuchungen_history_StNr${standesbuchNr}`);
     } catch (e) {
       log(`    → Failed to follow history link: ${e}`);
     }
@@ -1135,10 +1085,6 @@ async function scrapeMemberUntersuchungen(
 
   log(`    → Untersuchungen: ${allRows.length} total rows, ${dataRows.length} data rows (date in col ${dateColIdx})`);
 
-  if (dataRows.length === 0) {
-    await dumpHtml(frame, `untersuchungen_empty_StNr${standesbuchNr}`);
-  }
-
   const results: FdiskUntersuchung[] = [];
   for (const row of dataRows) {
     const valueCols: string[] = [];
@@ -1196,9 +1142,6 @@ async function scrapeMemberFahrgenehmigungen(
   // Show all rows (default is 10)
   await selectAlleAnzeige(frame);
 
-  // Dump HTML for diagnostics
-  await dumpHtml(frame, `fahrgenehmigungen_StNr${standesbuchNr}`);
-
   // Read form fields by ID pattern: {fieldname}_{rowIndex}
   const rawRows = await frame.evaluate(() => {
     const rows: Array<{
@@ -1402,7 +1345,6 @@ async function scrapeMemberFahrgenehmigungen(
 
   if (klasseIdx === -1) {
     log(`  Fahrgenehmigungen for StNr ${standesbuchNr}: could not determine Klasse column. Returning empty.`);
-    await dumpHtml(frame, `fahrgenehmigungen_fallback_StNr${standesbuchNr}`);
     return [];
   }