matthias/dashboard
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

update

Browse Source
This commit is contained in:
Matthias Hochmeister
2026-03-13 13:06:27 +01:00
parent 618f1d4996
commit cfb70e62c7
3 changed files with 74 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files

37
sync/src/scraper.ts
View File

@@ -1,4 +1,4 @@
import { chromium, Page } from '@playwright/test';
import { chromium, Page, Frame } from '@playwright/test';
import { FdiskMember, FdiskAusbildung } from './types';
const BASE_URL = process.env.FDISK_BASE_URL ?? 'https://app.fdisk.at';
@@ -51,14 +51,22 @@ export async function scrapeAll(username: string, password: string): Promise<{
try {
await login(page, username, password);
const members = await scrapeMembers(page);
// After login, page is on Start.aspx (frameset with top.topFrame etc.).
// The member list page runs alterBreadcrumbs() on load, which accesses top.topFrame.
// Navigating the whole page away from the frameset breaks that check → NoTabsAllowed redirect.
// Instead, navigate the mainFrame (inner frame) so the frameset context stays intact.
const mainFrame = page.frame({ name: 'mainFrame' });
if (!mainFrame) throw new Error('mainFrame not found in Start.aspx frameset');
const members = await scrapeMembers(mainFrame);
log(`Found ${members.length} members`);
const ausbildungen: FdiskAusbildung[] = [];
for (const member of members) {
if (!member.detailUrl) continue;
try {
const quals = await scrapeMemberAusbildung(page, member);
const quals = await scrapeMemberAusbildung(mainFrame, member);
ausbildungen.push(...quals);
log(` ${member.vorname} ${member.zuname}: ${quals.length} Ausbildungen`);
// polite delay between requests
@@ -114,19 +122,19 @@ async function login(page: Page, username: string, password: string): Promise<vo
log(`Logged in successfully, redirected to: ${currentUrl}`);
}
async function scrapeMembers(page: Page): Promise<FdiskMember[]> {
async function scrapeMembers(frame: Frame): Promise<FdiskMember[]> {
log(`Navigating to members list: ${MEMBERS_URL}`);
await page.goto(MEMBERS_URL, { waitUntil: 'domcontentloaded' });
await page.waitForLoadState('networkidle');
await frame.goto(MEMBERS_URL, { waitUntil: 'domcontentloaded' });
await frame.waitForLoadState('networkidle');
// The member table uses class FdcLayList
await page.waitForSelector('table.FdcLayList', { timeout: 20000 });
await frame.waitForSelector('table.FdcLayList', { timeout: 20000 });
// Column layout (0-indexed td): 0=icon, 1=Status, 2=St.-Nr., 3=Dienstgrad,
// 4=Vorname, 5=Zuname, 6=Geburtsdatum, 7=SVNR, 8=Eintrittsdatum, 9=Abmeldedatum, 10=icon
// Each <td> contains an <a title="value"> — the title is the clean cell text.
// The href on each <a> is the member detail URL (same link repeated across all cells in a row).
const rows = await page.$$eval('table.FdcLayList tbody tr', (trs) =>
const rows = await frame.$$eval('table.FdcLayList tbody tr', (trs) =>
trs.map((tr) => {
const cells = Array.from(tr.querySelectorAll('td'));
const val = (i: number) => {
@@ -171,29 +179,28 @@ async function scrapeMembers(page: Page): Promise<FdiskMember[]> {
return members;
}
async function scrapeMemberAusbildung(page: Page, member: FdiskMember): Promise<FdiskAusbildung[]> {
async function scrapeMemberAusbildung(frame: Frame, member: FdiskMember): Promise<FdiskAusbildung[]> {
if (!member.detailUrl) return [];
await page.goto(member.detailUrl, { waitUntil: 'networkidle' });
await frame.goto(member.detailUrl, { waitUntil: 'networkidle' });
// Look for Ausbildungsliste section — it's likely a table or list
// Try to find it by heading text
const ausbildungSection = page.locator('text=Ausbildung, text=Ausbildungsliste').first();
const ausbildungSection = frame.locator('text=Ausbildung, text=Ausbildungsliste').first();
const hasSec = await ausbildungSection.isVisible().catch(() => false);
if (!hasSec) {
// Try navigating to an Ausbildung tab/link if present
const ausbildungLink = page.locator('a:has-text("Ausbildung")').first();
const ausbildungLink = frame.locator('a:has-text("Ausbildung")').first();
const hasLink = await ausbildungLink.isVisible().catch(() => false);
if (hasLink) {
await ausbildungLink.click();
await page.waitForLoadState('networkidle').catch(() => {});
await frame.waitForLoadState('networkidle').catch(() => {});
}
}
// Parse the qualification table
// Expected columns: Kursname, Datum, Ablaufdatum, Ort, Bemerkung (may vary)
const tables = await page.$$('table');
const tables = await frame.$$('table');
const ausbildungen: FdiskAusbildung[] = [];
for (const table of tables) {