From f5d1f7b06114b6819a2848677519129498dd4369 Mon Sep 17 00:00:00 2001 From: Matthias Hochmeister Date: Fri, 13 Mar 2026 20:02:46 +0100 Subject: [PATCH] update --- .../migrations/030_extend_dienstgrad.sql | 56 ++++++++++++ sync/src/db.ts | 50 ++++++----- sync/src/scraper.ts | 88 +++++++++++-------- 3 files changed, 138 insertions(+), 56 deletions(-) create mode 100644 backend/src/database/migrations/030_extend_dienstgrad.sql diff --git a/backend/src/database/migrations/030_extend_dienstgrad.sql b/backend/src/database/migrations/030_extend_dienstgrad.sql new file mode 100644 index 0000000..541db2c --- /dev/null +++ b/backend/src/database/migrations/030_extend_dienstgrad.sql @@ -0,0 +1,56 @@ +-- Extends the dienstgrad CHECK constraint to include: +-- Jugendfeuerwehrmann, Probefeuerwehrmann, Verwaltungsmeister family, Verwalter, +-- and Ehren- prefixed variants of all Dienstgrade. + +ALTER TABLE mitglieder_profile + DROP CONSTRAINT IF EXISTS mitglieder_profile_dienstgrad_check; + +ALTER TABLE mitglieder_profile + ADD CONSTRAINT mitglieder_profile_dienstgrad_check + CHECK (dienstgrad IS NULL OR dienstgrad IN ( + -- Standard Dienstgrade + 'Feuerwehranwärter', + 'Jugendfeuerwehrmann', + 'Probefeuerwehrmann', + 'Feuerwehrmann', + 'Feuerwehrfrau', + 'Oberfeuerwehrmann', + 'Oberfeuerwehrfrau', + 'Hauptfeuerwehrmann', + 'Hauptfeuerwehrfrau', + 'Löschmeister', + 'Oberlöschmeister', + 'Hauptlöschmeister', + 'Brandmeister', + 'Oberbrandmeister', + 'Hauptbrandmeister', + 'Brandinspektor', + 'Oberbrandinspektor', + 'Brandoberinspektor', + 'Brandamtmann', + 'Verwaltungsmeister', + 'Oberverwaltungsmeister', + 'Hauptverwaltungsmeister', + 'Verwalter', + -- Ehrendienstgrade + 'Ehren-Feuerwehrmann', + 'Ehren-Feuerwehrfrau', + 'Ehren-Oberfeuerwehrmann', + 'Ehren-Oberfeuerwehrfrau', + 'Ehren-Hauptfeuerwehrmann', + 'Ehren-Hauptfeuerwehrfrau', + 'Ehren-Löschmeister', + 'Ehren-Oberlöschmeister', + 'Ehren-Hauptlöschmeister', + 'Ehren-Brandmeister', + 'Ehren-Oberbrandmeister', + 'Ehren-Hauptbrandmeister', + 'Ehren-Brandinspektor', + 'Ehren-Oberbrandinspektor', + 'Ehren-Brandoberinspektor', + 'Ehren-Brandamtmann', + 'Ehren-Verwaltungsmeister', + 'Ehren-Oberverwaltungsmeister', + 'Ehren-Hauptverwaltungsmeister', + 'Ehren-Verwalter' + )); diff --git a/sync/src/db.ts b/sync/src/db.ts index 7a8def4..fcf9a2f 100644 --- a/sync/src/db.ts +++ b/sync/src/db.ts @@ -10,9 +10,10 @@ function log(msg: string) { * Returns null if no match found — the field will be left unchanged. */ function mapDienstgrad(raw: string): string | null { - const map: Record = { - // Abbreviations + const abbrevMap: Record = { 'fa': 'Feuerwehranwärter', + 'jfm': 'Jugendfeuerwehrmann', + 'pfm': 'Probefeuerwehrmann', 'fm': 'Feuerwehrmann', 'ff': 'Feuerwehrfrau', 'ofm': 'Oberfeuerwehrmann', @@ -29,26 +30,33 @@ function mapDienstgrad(raw: string): string | null { 'obi': 'Oberbrandinspektor', 'boi': 'Brandoberinspektor', 'bam': 'Brandamtmann', - // Full names (pass-through if already matching) - 'feuerwehranwärter': 'Feuerwehranwärter', - 'feuerwehrmann': 'Feuerwehrmann', - 'feuerwehrfrau': 'Feuerwehrfrau', - 'oberfeuerwehrmann': 'Oberfeuerwehrmann', - 'oberfeuerwehrfrau': 'Oberfeuerwehrfrau', - 'hauptfeuerwehrmann': 'Hauptfeuerwehrmann', - 'hauptfeuerwehrfrau': 'Hauptfeuerwehrfrau', - 'löschmeister': 'Löschmeister', - 'oberlöschmeister': 'Oberlöschmeister', - 'hauptlöschmeister': 'Hauptlöschmeister', - 'brandmeister': 'Brandmeister', - 'oberbrandmeister': 'Oberbrandmeister', - 'hauptbrandmeister': 'Hauptbrandmeister', - 'brandinspektor': 'Brandinspektor', - 'oberbrandinspektor': 'Oberbrandinspektor', - 'brandoberinspektor': 'Brandoberinspektor', - 'brandamtmann': 'Brandamtmann', + 'vm': 'Verwaltungsmeister', + 'ovm': 'Oberverwaltungsmeister', + 'hvm': 'Hauptverwaltungsmeister', + 'v': 'Verwalter', }; - return map[raw.trim().toLowerCase()] ?? null; + + const normalized = raw.trim().toLowerCase(); + + // Direct abbreviation match + if (abbrevMap[normalized]) return abbrevMap[normalized]; + + // Ehrendienstgrad: starts with 'e', rest maps to a known abbreviation + // e.g. EOLM → Ehren-Oberlöschmeister, EVM → Ehren-Verwaltungsmeister + if (normalized.startsWith('e') && normalized.length > 1) { + const base = abbrevMap[normalized.slice(1)]; + if (base) return `Ehren-${base}`; + } + + // Full name pass-through (case-insensitive) + const allValues = Object.values(abbrevMap); + const match = allValues.find(v => v.toLowerCase() === normalized); + if (match) return match; + // Also match Ehren- full names + const ehrenMatch = allValues.find(v => `ehren-${v.toLowerCase()}` === normalized); + if (ehrenMatch) return `Ehren-${ehrenMatch}`; + + return null; } export async function syncToDatabase( diff --git a/sync/src/scraper.ts b/sync/src/scraper.ts index 935f2f2..1f9ab3b 100644 --- a/sync/src/scraper.ts +++ b/sync/src/scraper.ts @@ -236,6 +236,7 @@ async function scrapeMembers(frame: Frame): Promise { // Parse "Datensatz X-Y von Z" to check if more pages exist const pagMatch = pagination.match(/(\d+)-(\d+)\s+von\s+(\d+)/i); if (pagMatch) { + const from = parseInt(pagMatch[1], 10); const to = parseInt(pagMatch[2], 10); const total = parseInt(pagMatch[3], 10); if (to >= total) { @@ -243,46 +244,63 @@ async function scrapeMembers(frame: Frame): Promise { break; } log(`Loaded ${to} of ${total} — navigating to next page`); + + // Calculate next page number to use as a fallback click target + const pageSize = to - from + 1; + const nextPageNum = Math.floor(to / pageSize) + 1; + + // Click the "next page" link in FdcLayListNav. + // Strategy 1: text ">" or ">>" (common in FDISK) + // Strategy 2: title/alt containing navigation words + // Strategy 3: link text is the next page number (e.g. "2" when on page 1) + const nextClicked = await frame.evaluate((nextPg: number) => { + const nav = document.querySelector('table.FdcLayListNav'); + if (!nav) return false; + const links = Array.from(nav.querySelectorAll('a, input[type="button"], input[type="submit"]')); + + for (const el of links) { + const text = ((el as HTMLElement).textContent ?? '').trim(); + const title = ((el as HTMLElement).getAttribute('title') ?? '').toLowerCase(); + const alt = ((el as HTMLImageElement).alt ?? '').toLowerCase(); + if (text === '>' || text === '>>' || + title.includes('nächst') || title.includes('weiter') || title.includes('next') || title.includes('vor') || + alt.includes('next') || alt.includes('weiter') || alt.includes('vor')) { + (el as HTMLElement).click(); + return true; + } + } + + // Fallback: find a link whose text is exactly the next page number + for (const el of links) { + const text = ((el as HTMLElement).textContent ?? '').trim(); + if (text === String(nextPg)) { + (el as HTMLElement).click(); + return true; + } + } + + return false; + }, nextPageNum); + + if (!nextClicked) { + // Dump nav HTML to help diagnose the missing next-page link + const navHtml = await frame.evaluate(() => { + const nav = document.querySelector('table.FdcLayListNav'); + return nav?.innerHTML?.replace(/\s+/g, ' ').trim() ?? '(not found)'; + }); + log(`WARN: could not find next-page link (tried ">" and page "${nextPageNum}") — stopping pagination`); + log(`FdcLayListNav HTML: ${navHtml}`); + break; + } + + await frame.waitForLoadState('networkidle', { timeout: 30000 }); + pageNum++; } else { // No pagination indicator found — assume single page log('No pagination indicator found — assuming single page'); break; } - - // Click the "next page" link in FdcLayListNav - // FDISK uses __doPostBack links; find an or pointing to the next page - const nextClicked = await frame.evaluate(() => { - const nav = document.querySelector('table.FdcLayListNav'); - if (!nav) return false; - const links = Array.from(nav.querySelectorAll('a, input[type="button"], input[type="submit"]')); - // Look for next-page indicator: ">" alone, ">>" alone, or title/alt "weiter"/"next" - for (const el of links) { - const text = ((el as HTMLElement).textContent ?? '').trim(); - const title = ((el as HTMLElement).getAttribute('title') ?? '').toLowerCase(); - const alt = ((el as HTMLImageElement).alt ?? '').toLowerCase(); - if (text === '>' || text === '>>' || title.includes('nächst') || title.includes('weiter') || - title.includes('next') || alt.includes('next') || alt.includes('weiter')) { - (el as HTMLElement).click(); - return true; - } - } - return false; - }); - - if (!nextClicked) { - // Dump nav HTML to help diagnose the missing next-page link - const navHtml = await frame.evaluate(() => { - const nav = document.querySelector('table.FdcLayListNav'); - return nav?.innerHTML?.replace(/\s+/g, ' ').trim() ?? '(not found)'; - }); - log(`WARN: could not find next-page link — stopping pagination`); - log(`FdcLayListNav HTML: ${navHtml}`); - break; - } - - await frame.waitForLoadState('networkidle', { timeout: 30000 }); - pageNum++; - } + } // end while log(`Parsed ${allRows.length} rows total across ${pageNum} page(s)`);