diff --git a/sync/src/index.ts b/sync/src/index.ts index c476858..06e22d0 100644 --- a/sync/src/index.ts +++ b/sync/src/index.ts @@ -67,30 +67,7 @@ async function runSync(force = false): Promise { if (force) log('Force mode: ON'); log('Starting FDISK sync'); - // Query dashboard accounts to limit detail scraping to linked members - const stNrResult = await pool.query<{ fdisk_standesbuch_nr: string }>( - `SELECT mp.fdisk_standesbuch_nr - FROM mitglieder_profile mp - JOIN users u ON u.id = mp.user_id - WHERE mp.fdisk_standesbuch_nr IS NOT NULL - AND u.last_login_at IS NOT NULL` - ); - const knownStNrs = new Set(stNrResult.rows.map(r => r.fdisk_standesbuch_nr)); - - // Also fetch names for users without standesbuchNr yet (for first-time linking) - const nameResult = await pool.query<{ given_name: string; family_name: string }>( - `SELECT u.given_name, u.family_name - FROM users u - JOIN mitglieder_profile mp ON mp.user_id = u.id - WHERE mp.fdisk_standesbuch_nr IS NULL - AND u.given_name IS NOT NULL AND u.family_name IS NOT NULL - AND u.last_login_at IS NOT NULL` - ); - const knownNames = new Set(nameResult.rows.map(r => `${r.given_name.toLowerCase()}::${r.family_name.toLowerCase()}`)); - - log(`Detail scraping for ${knownStNrs.size} linked + ${knownNames.size} name-matchable accounts`); - - const { members, ausbildungen, befoerderungen, untersuchungen, fahrgenehmigungen } = await scrapeAll(username, password, knownStNrs, knownNames); + const { members, ausbildungen, befoerderungen, untersuchungen, fahrgenehmigungen } = await scrapeAll(username, password); await syncToDatabase(pool, members, ausbildungen, befoerderungen, untersuchungen, fahrgenehmigungen, force); log(`Sync complete — ${members.length} members, ${ausbildungen.length} Ausbildungen, ${befoerderungen.length} Beförderungen, ${untersuchungen.length} Untersuchungen, ${fahrgenehmigungen.length} Fahrgenehmigungen`); } finally { diff --git a/sync/src/scraper.ts b/sync/src/scraper.ts index ddd833c..9a9a810 100644 --- a/sync/src/scraper.ts +++ b/sync/src/scraper.ts @@ -175,7 +175,7 @@ async function scrapeKnownMembers( return members; } -export async function scrapeAll(username: string, password: string, knownStNrs: Set, knownNames: Set): Promise<{ +export async function scrapeAll(username: string, password: string): Promise<{ members: FdiskMember[]; ausbildungen: FdiskAusbildung[]; befoerderungen: FdiskBefoerderung[]; @@ -200,8 +200,8 @@ export async function scrapeAll(username: string, password: string, knownStNrs: // Navigate via the menu frame (left.aspx) to set session state properly. const mainFrame = await navigateToMemberList(page); - const members = await scrapeKnownMembers(mainFrame, knownStNrs, knownNames); - log(`Found ${members.length} members (targeted query)`); + const members = await scrapeMembers(mainFrame); + log(`Found ${members.length} members (full scrape)`); if (DEBUG_HTML) log(`[debug] HTML dump mode ON — saving pages to debug/`); const ausbildungen: FdiskAusbildung[] = [];