update
This commit is contained in:
@@ -66,7 +66,31 @@ async function runSync(force = false): Promise<void> {
|
|||||||
try {
|
try {
|
||||||
if (force) log('Force mode: ON');
|
if (force) log('Force mode: ON');
|
||||||
log('Starting FDISK sync');
|
log('Starting FDISK sync');
|
||||||
const { members, ausbildungen, befoerderungen, untersuchungen, fahrgenehmigungen } = await scrapeAll(username, password);
|
|
||||||
|
// Query dashboard accounts to limit detail scraping to linked members
|
||||||
|
const stNrResult = await pool.query<{ fdisk_standesbuch_nr: string }>(
|
||||||
|
`SELECT mp.fdisk_standesbuch_nr
|
||||||
|
FROM mitglieder_profile mp
|
||||||
|
JOIN users u ON u.id = mp.user_id
|
||||||
|
WHERE mp.fdisk_standesbuch_nr IS NOT NULL
|
||||||
|
AND u.last_login_at IS NOT NULL`
|
||||||
|
);
|
||||||
|
const knownStNrs = new Set(stNrResult.rows.map(r => r.fdisk_standesbuch_nr));
|
||||||
|
|
||||||
|
// Also fetch names for users without standesbuchNr yet (for first-time linking)
|
||||||
|
const nameResult = await pool.query<{ given_name: string; family_name: string }>(
|
||||||
|
`SELECT u.given_name, u.family_name
|
||||||
|
FROM users u
|
||||||
|
JOIN mitglieder_profile mp ON mp.user_id = u.id
|
||||||
|
WHERE mp.fdisk_standesbuch_nr IS NULL
|
||||||
|
AND u.given_name IS NOT NULL AND u.family_name IS NOT NULL
|
||||||
|
AND u.last_login_at IS NOT NULL`
|
||||||
|
);
|
||||||
|
const knownNames = new Set(nameResult.rows.map(r => `${r.given_name.toLowerCase()}::${r.family_name.toLowerCase()}`));
|
||||||
|
|
||||||
|
log(`Detail scraping for ${knownStNrs.size} linked + ${knownNames.size} name-matchable accounts`);
|
||||||
|
|
||||||
|
const { members, ausbildungen, befoerderungen, untersuchungen, fahrgenehmigungen } = await scrapeAll(username, password, knownStNrs, knownNames);
|
||||||
await syncToDatabase(pool, members, ausbildungen, befoerderungen, untersuchungen, fahrgenehmigungen, force);
|
await syncToDatabase(pool, members, ausbildungen, befoerderungen, untersuchungen, fahrgenehmigungen, force);
|
||||||
log(`Sync complete — ${members.length} members, ${ausbildungen.length} Ausbildungen, ${befoerderungen.length} Beförderungen, ${untersuchungen.length} Untersuchungen, ${fahrgenehmigungen.length} Fahrgenehmigungen`);
|
log(`Sync complete — ${members.length} members, ${ausbildungen.length} Ausbildungen, ${befoerderungen.length} Beförderungen, ${untersuchungen.length} Untersuchungen, ${fahrgenehmigungen.length} Fahrgenehmigungen`);
|
||||||
} finally {
|
} finally {
|
||||||
|
|||||||
@@ -39,7 +39,7 @@ function cellText(text: string | undefined | null): string | null {
|
|||||||
return t || null;
|
return t || null;
|
||||||
}
|
}
|
||||||
|
|
||||||
export async function scrapeAll(username: string, password: string): Promise<{
|
export async function scrapeAll(username: string, password: string, knownStNrs: Set<string>, knownNames: Set<string>): Promise<{
|
||||||
members: FdiskMember[];
|
members: FdiskMember[];
|
||||||
ausbildungen: FdiskAusbildung[];
|
ausbildungen: FdiskAusbildung[];
|
||||||
befoerderungen: FdiskBefoerderung[];
|
befoerderungen: FdiskBefoerderung[];
|
||||||
@@ -73,6 +73,13 @@ export async function scrapeAll(username: string, password: string): Promise<{
|
|||||||
const fahrgenehmigungen: FdiskFahrgenehmigung[] = [];
|
const fahrgenehmigungen: FdiskFahrgenehmigung[] = [];
|
||||||
|
|
||||||
for (const member of members) {
|
for (const member of members) {
|
||||||
|
// Only scrape detail pages for members with a dashboard account
|
||||||
|
// (matched by standesbuchNr or by name for first-time linking)
|
||||||
|
const nameKey = `${member.vorname.toLowerCase()}::${member.zuname.toLowerCase()}`;
|
||||||
|
if (!knownStNrs.has(member.standesbuchNr) && !knownNames.has(nameKey)) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
try {
|
try {
|
||||||
// Navigate to member detail page — use direct URL if available, else search+click fallback
|
// Navigate to member detail page — use direct URL if available, else search+click fallback
|
||||||
const onDetail = member.detailUrl
|
const onDetail = member.detailUrl
|
||||||
|
|||||||
Reference in New Issue
Block a user