matthias/dashboard
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(sync): switch to full member list scrape so all FDISK members are synced, not just known accounts

Browse Source
This commit is contained in:
Matthias Hochmeister
2026-04-15 14:44:39 +02:00
parent 719b7bfcdb
commit 55b2fc1cf4
2 changed files with 4 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files

25
sync/src/index.ts
View File

@@ -67,30 +67,7 @@ async function runSync(force = false): Promise<void> {
if (force) log('Force mode: ON'); if (force) log('Force mode: ON');
log('Starting FDISK sync'); log('Starting FDISK sync');
// Query dashboard accounts to limit detail scraping to linked members const { members, ausbildungen, befoerderungen, untersuchungen, fahrgenehmigungen } = await scrapeAll(username, password);
const stNrResult = await pool.query<{ fdisk_standesbuch_nr: string }>(
`SELECT mp.fdisk_standesbuch_nr
FROM mitglieder_profile mp
JOIN users u ON u.id = mp.user_id
WHERE mp.fdisk_standesbuch_nr IS NOT NULL
AND u.last_login_at IS NOT NULL`
);
const knownStNrs = new Set(stNrResult.rows.map(r => r.fdisk_standesbuch_nr));
// Also fetch names for users without standesbuchNr yet (for first-time linking)
const nameResult = await pool.query<{ given_name: string; family_name: string }>(
`SELECT u.given_name, u.family_name
FROM users u
JOIN mitglieder_profile mp ON mp.user_id = u.id
WHERE mp.fdisk_standesbuch_nr IS NULL
AND u.given_name IS NOT NULL AND u.family_name IS NOT NULL
AND u.last_login_at IS NOT NULL`
);
const knownNames = new Set(nameResult.rows.map(r => `${r.given_name.toLowerCase()}::${r.family_name.toLowerCase()}`));
log(`Detail scraping for ${knownStNrs.size} linked + ${knownNames.size} name-matchable accounts`);
const { members, ausbildungen, befoerderungen, untersuchungen, fahrgenehmigungen } = await scrapeAll(username, password, knownStNrs, knownNames);
await syncToDatabase(pool, members, ausbildungen, befoerderungen, untersuchungen, fahrgenehmigungen, force); await syncToDatabase(pool, members, ausbildungen, befoerderungen, untersuchungen, fahrgenehmigungen, force);
log(`Sync complete — ${members.length} members, ${ausbildungen.length} Ausbildungen, ${befoerderungen.length} Beförderungen, ${untersuchungen.length} Untersuchungen, ${fahrgenehmigungen.length} Fahrgenehmigungen`); log(`Sync complete — ${members.length} members, ${ausbildungen.length} Ausbildungen, ${befoerderungen.length} Beförderungen, ${untersuchungen.length} Untersuchungen, ${fahrgenehmigungen.length} Fahrgenehmigungen`);
} finally { } finally {

6
sync/src/scraper.ts
View File

@@ -175,7 +175,7 @@ async function scrapeKnownMembers(
return members; return members;
} }
export async function scrapeAll(username: string, password: string, knownStNrs: Set<string>, knownNames: Set<string>): Promise<{ export async function scrapeAll(username: string, password: string): Promise<{
members: FdiskMember[]; members: FdiskMember[];
ausbildungen: FdiskAusbildung[]; ausbildungen: FdiskAusbildung[];
befoerderungen: FdiskBefoerderung[]; befoerderungen: FdiskBefoerderung[];
@@ -200,8 +200,8 @@ export async function scrapeAll(username: string, password: string, knownStNrs:
// Navigate via the menu frame (left.aspx) to set session state properly. // Navigate via the menu frame (left.aspx) to set session state properly.
const mainFrame = await navigateToMemberList(page); const mainFrame = await navigateToMemberList(page);
const members = await scrapeKnownMembers(mainFrame, knownStNrs, knownNames); const members = await scrapeMembers(mainFrame);
log(`Found ${members.length} members (targeted query)`); log(`Found ${members.length} members (full scrape)`);
if (DEBUG_HTML) log(`[debug] HTML dump mode ON — saving pages to debug/`); if (DEBUG_HTML) log(`[debug] HTML dump mode ON — saving pages to debug/`);
const ausbildungen: FdiskAusbildung[] = []; const ausbildungen: FdiskAusbildung[] = [];