update
This commit is contained in:
@@ -34,14 +34,20 @@ const authLimiter = rateLimit({
|
|||||||
});
|
});
|
||||||
|
|
||||||
app.use('/api/auth', authLimiter);
|
app.use('/api/auth', authLimiter);
|
||||||
// General rate limiter — skip auth routes (they have their own limiter above)
|
// General rate limiter — skip auth routes (own limiter above) and authenticated
|
||||||
|
// requests (Bearer token present). Auth middleware validates the token downstream;
|
||||||
|
// rate-limiting authenticated dashboard polling would cause 429 floods.
|
||||||
app.use('/api', rateLimit({
|
app.use('/api', rateLimit({
|
||||||
windowMs: environment.rateLimit.windowMs,
|
windowMs: environment.rateLimit.windowMs,
|
||||||
max: environment.rateLimit.max,
|
max: environment.rateLimit.max,
|
||||||
message: 'Too many requests from this IP, please try again later.',
|
message: 'Too many requests from this IP, please try again later.',
|
||||||
standardHeaders: true,
|
standardHeaders: true,
|
||||||
legacyHeaders: false,
|
legacyHeaders: false,
|
||||||
skip: (req) => req.path.startsWith('/auth'),
|
skip: (req) => {
|
||||||
|
if (req.path.startsWith('/auth')) return true;
|
||||||
|
const auth = req.headers.authorization;
|
||||||
|
return typeof auth === 'string' && auth.startsWith('Bearer ');
|
||||||
|
},
|
||||||
}));
|
}));
|
||||||
|
|
||||||
// Body parsing middleware
|
// Body parsing middleware
|
||||||
|
|||||||
@@ -0,0 +1,2 @@
|
|||||||
|
-- Remove mitglieds_nr column (replaced by fdisk_standesbuch_nr as the canonical member number)
|
||||||
|
ALTER TABLE mitglieder_profile DROP COLUMN IF EXISTS mitglieds_nr;
|
||||||
@@ -0,0 +1,6 @@
|
|||||||
|
-- Migration 032: Add FDISK-scraped profile fields to mitglieder_profile
|
||||||
|
ALTER TABLE mitglieder_profile ADD COLUMN IF NOT EXISTS geburtsort VARCHAR(128);
|
||||||
|
ALTER TABLE mitglieder_profile ADD COLUMN IF NOT EXISTS geschlecht VARCHAR(1);
|
||||||
|
ALTER TABLE mitglieder_profile ADD COLUMN IF NOT EXISTS beruf VARCHAR(255);
|
||||||
|
ALTER TABLE mitglieder_profile ADD COLUMN IF NOT EXISTS wohnort VARCHAR(128);
|
||||||
|
ALTER TABLE mitglieder_profile ADD COLUMN IF NOT EXISTS plz VARCHAR(16);
|
||||||
@@ -0,0 +1,13 @@
|
|||||||
|
-- Migration 033: Create befoerderungen table (FDISK sync)
|
||||||
|
CREATE TABLE IF NOT EXISTS befoerderungen (
|
||||||
|
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
|
||||||
|
user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE,
|
||||||
|
datum DATE,
|
||||||
|
dienstgrad VARCHAR(64) NOT NULL,
|
||||||
|
fdisk_sync_key VARCHAR(255),
|
||||||
|
created_at TIMESTAMPTZ DEFAULT NOW(),
|
||||||
|
updated_at TIMESTAMPTZ DEFAULT NOW(),
|
||||||
|
UNIQUE(user_id, fdisk_sync_key)
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_befoerderungen_user_id ON befoerderungen(user_id);
|
||||||
@@ -0,0 +1,16 @@
|
|||||||
|
-- Migration 034: Create untersuchungen table (FDISK sync)
|
||||||
|
CREATE TABLE IF NOT EXISTS untersuchungen (
|
||||||
|
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
|
||||||
|
user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE,
|
||||||
|
datum DATE,
|
||||||
|
anmerkungen TEXT,
|
||||||
|
art VARCHAR(128) NOT NULL,
|
||||||
|
ergebnis VARCHAR(128),
|
||||||
|
fdisk_sync_key VARCHAR(255),
|
||||||
|
created_at TIMESTAMPTZ DEFAULT NOW(),
|
||||||
|
updated_at TIMESTAMPTZ DEFAULT NOW(),
|
||||||
|
UNIQUE(user_id, fdisk_sync_key)
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_untersuchungen_user_id ON untersuchungen(user_id);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_untersuchungen_art ON untersuchungen(user_id, art);
|
||||||
@@ -0,0 +1,16 @@
|
|||||||
|
-- Migration 035: Create fahrgenehmigungen table (FDISK sync)
|
||||||
|
CREATE TABLE IF NOT EXISTS fahrgenehmigungen (
|
||||||
|
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
|
||||||
|
user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE,
|
||||||
|
ausstellungsdatum DATE,
|
||||||
|
gueltig_bis DATE,
|
||||||
|
behoerde VARCHAR(128),
|
||||||
|
nummer VARCHAR(64),
|
||||||
|
klasse VARCHAR(128) NOT NULL,
|
||||||
|
fdisk_sync_key VARCHAR(255),
|
||||||
|
created_at TIMESTAMPTZ DEFAULT NOW(),
|
||||||
|
updated_at TIMESTAMPTZ DEFAULT NOW(),
|
||||||
|
UNIQUE(user_id, fdisk_sync_key)
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_fahrgenehmigungen_user_id ON fahrgenehmigungen(user_id);
|
||||||
@@ -63,7 +63,6 @@ export interface MitgliederProfile {
|
|||||||
id: string;
|
id: string;
|
||||||
user_id: string;
|
user_id: string;
|
||||||
|
|
||||||
mitglieds_nr: string | null;
|
|
||||||
fdisk_standesbuch_nr: string | null;
|
fdisk_standesbuch_nr: string | null;
|
||||||
dienstgrad: DienstgradEnum | null;
|
dienstgrad: DienstgradEnum | null;
|
||||||
dienstgrad_seit: Date | null;
|
dienstgrad_seit: Date | null;
|
||||||
@@ -86,6 +85,13 @@ export interface MitgliederProfile {
|
|||||||
bemerkungen: string | null;
|
bemerkungen: string | null;
|
||||||
bild_url: string | null;
|
bild_url: string | null;
|
||||||
|
|
||||||
|
// FDISK-synced extended profile fields
|
||||||
|
geburtsort: string | null;
|
||||||
|
geschlecht: string | null;
|
||||||
|
beruf: string | null;
|
||||||
|
wohnort: string | null;
|
||||||
|
plz: string | null;
|
||||||
|
|
||||||
created_at: Date;
|
created_at: Date;
|
||||||
updated_at: Date;
|
updated_at: Date;
|
||||||
}
|
}
|
||||||
@@ -143,7 +149,6 @@ export interface MemberListItem {
|
|||||||
|
|
||||||
// profile fields (null when no profile exists)
|
// profile fields (null when no profile exists)
|
||||||
profile_id: string | null;
|
profile_id: string | null;
|
||||||
mitglieds_nr: string | null;
|
|
||||||
fdisk_standesbuch_nr: string | null;
|
fdisk_standesbuch_nr: string | null;
|
||||||
dienstgrad: DienstgradEnum | null;
|
dienstgrad: DienstgradEnum | null;
|
||||||
funktion: FunktionEnum[];
|
funktion: FunktionEnum[];
|
||||||
@@ -156,7 +161,7 @@ export interface MemberListItem {
|
|||||||
// Filter parameters for getAllMembers()
|
// Filter parameters for getAllMembers()
|
||||||
// ============================================================
|
// ============================================================
|
||||||
export interface MemberFilters {
|
export interface MemberFilters {
|
||||||
search?: string; // matches name, email, mitglieds_nr
|
search?: string; // matches name, email, fdisk_standesbuch_nr
|
||||||
status?: StatusEnum[];
|
status?: StatusEnum[];
|
||||||
dienstgrad?: DienstgradEnum[];
|
dienstgrad?: DienstgradEnum[];
|
||||||
page?: number; // 1-based
|
page?: number; // 1-based
|
||||||
@@ -173,7 +178,6 @@ export interface MemberFilters {
|
|||||||
* status has a default; every other field may be omitted.
|
* status has a default; every other field may be omitted.
|
||||||
*/
|
*/
|
||||||
export const CreateMemberProfileSchema = z.object({
|
export const CreateMemberProfileSchema = z.object({
|
||||||
mitglieds_nr: z.string().max(32).optional(),
|
|
||||||
fdisk_standesbuch_nr: z.string().max(32).optional(),
|
fdisk_standesbuch_nr: z.string().max(32).optional(),
|
||||||
dienstgrad: z.enum(DIENSTGRAD_VALUES).optional(),
|
dienstgrad: z.enum(DIENSTGRAD_VALUES).optional(),
|
||||||
dienstgrad_seit: z.coerce.date().optional(),
|
dienstgrad_seit: z.coerce.date().optional(),
|
||||||
|
|||||||
@@ -36,7 +36,6 @@ class MemberService {
|
|||||||
-- profile columns (aliased with mp_ prefix to avoid collision)
|
-- profile columns (aliased with mp_ prefix to avoid collision)
|
||||||
mp.id AS mp_id,
|
mp.id AS mp_id,
|
||||||
mp.user_id AS mp_user_id,
|
mp.user_id AS mp_user_id,
|
||||||
mp.mitglieds_nr AS mp_mitglieds_nr,
|
|
||||||
mp.fdisk_standesbuch_nr AS mp_fdisk_standesbuch_nr,
|
mp.fdisk_standesbuch_nr AS mp_fdisk_standesbuch_nr,
|
||||||
mp.dienstgrad AS mp_dienstgrad,
|
mp.dienstgrad AS mp_dienstgrad,
|
||||||
mp.dienstgrad_seit AS mp_dienstgrad_seit,
|
mp.dienstgrad_seit AS mp_dienstgrad_seit,
|
||||||
@@ -54,6 +53,11 @@ class MemberService {
|
|||||||
mp.schuhgroesse AS mp_schuhgroesse,
|
mp.schuhgroesse AS mp_schuhgroesse,
|
||||||
mp.bemerkungen AS mp_bemerkungen,
|
mp.bemerkungen AS mp_bemerkungen,
|
||||||
mp.bild_url AS mp_bild_url,
|
mp.bild_url AS mp_bild_url,
|
||||||
|
mp.geburtsort AS mp_geburtsort,
|
||||||
|
mp.geschlecht AS mp_geschlecht,
|
||||||
|
mp.beruf AS mp_beruf,
|
||||||
|
mp.wohnort AS mp_wohnort,
|
||||||
|
mp.plz AS mp_plz,
|
||||||
mp.created_at AS mp_created_at,
|
mp.created_at AS mp_created_at,
|
||||||
mp.updated_at AS mp_updated_at
|
mp.updated_at AS mp_updated_at
|
||||||
FROM users u
|
FROM users u
|
||||||
@@ -83,7 +87,6 @@ class MemberService {
|
|||||||
? {
|
? {
|
||||||
id: row.mp_id,
|
id: row.mp_id,
|
||||||
user_id: row.mp_user_id,
|
user_id: row.mp_user_id,
|
||||||
mitglieds_nr: row.mp_mitglieds_nr,
|
|
||||||
fdisk_standesbuch_nr: row.mp_fdisk_standesbuch_nr ?? null,
|
fdisk_standesbuch_nr: row.mp_fdisk_standesbuch_nr ?? null,
|
||||||
dienstgrad: row.mp_dienstgrad,
|
dienstgrad: row.mp_dienstgrad,
|
||||||
dienstgrad_seit: row.mp_dienstgrad_seit,
|
dienstgrad_seit: row.mp_dienstgrad_seit,
|
||||||
@@ -101,6 +104,11 @@ class MemberService {
|
|||||||
schuhgroesse: row.mp_schuhgroesse,
|
schuhgroesse: row.mp_schuhgroesse,
|
||||||
bemerkungen: row.mp_bemerkungen,
|
bemerkungen: row.mp_bemerkungen,
|
||||||
bild_url: row.mp_bild_url,
|
bild_url: row.mp_bild_url,
|
||||||
|
geburtsort: row.mp_geburtsort ?? null,
|
||||||
|
geschlecht: row.mp_geschlecht ?? null,
|
||||||
|
beruf: row.mp_beruf ?? null,
|
||||||
|
wohnort: row.mp_wohnort ?? null,
|
||||||
|
plz: row.mp_plz ?? null,
|
||||||
created_at: row.mp_created_at,
|
created_at: row.mp_created_at,
|
||||||
updated_at: row.mp_updated_at,
|
updated_at: row.mp_updated_at,
|
||||||
}
|
}
|
||||||
@@ -137,7 +145,6 @@ class MemberService {
|
|||||||
OR u.email ILIKE $${paramIdx}
|
OR u.email ILIKE $${paramIdx}
|
||||||
OR u.given_name ILIKE $${paramIdx}
|
OR u.given_name ILIKE $${paramIdx}
|
||||||
OR u.family_name ILIKE $${paramIdx}
|
OR u.family_name ILIKE $${paramIdx}
|
||||||
OR mp.mitglieds_nr ILIKE $${paramIdx}
|
|
||||||
)`);
|
)`);
|
||||||
values.push(`%${search}%`);
|
values.push(`%${search}%`);
|
||||||
paramIdx++;
|
paramIdx++;
|
||||||
@@ -168,7 +175,6 @@ class MemberService {
|
|||||||
u.profile_picture_url,
|
u.profile_picture_url,
|
||||||
u.is_active,
|
u.is_active,
|
||||||
mp.id AS profile_id,
|
mp.id AS profile_id,
|
||||||
mp.mitglieds_nr,
|
|
||||||
mp.fdisk_standesbuch_nr,
|
mp.fdisk_standesbuch_nr,
|
||||||
mp.dienstgrad,
|
mp.dienstgrad,
|
||||||
mp.funktion,
|
mp.funktion,
|
||||||
@@ -204,7 +210,6 @@ class MemberService {
|
|||||||
profile_picture_url: row.profile_picture_url,
|
profile_picture_url: row.profile_picture_url,
|
||||||
is_active: row.is_active,
|
is_active: row.is_active,
|
||||||
profile_id: row.profile_id ?? null,
|
profile_id: row.profile_id ?? null,
|
||||||
mitglieds_nr: row.mitglieds_nr ?? null,
|
|
||||||
fdisk_standesbuch_nr: row.fdisk_standesbuch_nr ?? null,
|
fdisk_standesbuch_nr: row.fdisk_standesbuch_nr ?? null,
|
||||||
dienstgrad: row.dienstgrad ?? null,
|
dienstgrad: row.dienstgrad ?? null,
|
||||||
funktion: row.funktion ?? [],
|
funktion: row.funktion ?? [],
|
||||||
@@ -286,7 +291,6 @@ class MemberService {
|
|||||||
const query = `
|
const query = `
|
||||||
INSERT INTO mitglieder_profile (
|
INSERT INTO mitglieder_profile (
|
||||||
user_id,
|
user_id,
|
||||||
mitglieds_nr,
|
|
||||||
fdisk_standesbuch_nr,
|
fdisk_standesbuch_nr,
|
||||||
dienstgrad,
|
dienstgrad,
|
||||||
dienstgrad_seit,
|
dienstgrad_seit,
|
||||||
@@ -306,14 +310,13 @@ class MemberService {
|
|||||||
bild_url
|
bild_url
|
||||||
) VALUES (
|
) VALUES (
|
||||||
$1, $2, $3, $4, $5, $6, $7, $8, $9, $10,
|
$1, $2, $3, $4, $5, $6, $7, $8, $9, $10,
|
||||||
$11, $12, $13, $14, $15, $16, $17, $18, $19
|
$11, $12, $13, $14, $15, $16, $17, $18
|
||||||
)
|
)
|
||||||
RETURNING *
|
RETURNING *
|
||||||
`;
|
`;
|
||||||
|
|
||||||
const values = [
|
const values = [
|
||||||
userId,
|
userId,
|
||||||
data.mitglieds_nr ?? null,
|
|
||||||
data.fdisk_standesbuch_nr ?? null,
|
data.fdisk_standesbuch_nr ?? null,
|
||||||
data.dienstgrad ?? null,
|
data.dienstgrad ?? null,
|
||||||
data.dienstgrad_seit ?? null,
|
data.dienstgrad_seit ?? null,
|
||||||
@@ -392,7 +395,6 @@ class MemberService {
|
|||||||
let paramIdx = 1;
|
let paramIdx = 1;
|
||||||
|
|
||||||
const fieldMap: Record<string, any> = {
|
const fieldMap: Record<string, any> = {
|
||||||
mitglieds_nr: rest.mitglieds_nr,
|
|
||||||
fdisk_standesbuch_nr: rest.fdisk_standesbuch_nr,
|
fdisk_standesbuch_nr: rest.fdisk_standesbuch_nr,
|
||||||
funktion: rest.funktion,
|
funktion: rest.funktion,
|
||||||
status: rest.status,
|
status: rest.status,
|
||||||
|
|||||||
@@ -28,7 +28,6 @@ import {
|
|||||||
Cancel as CancelIcon,
|
Cancel as CancelIcon,
|
||||||
Person as PersonIcon,
|
Person as PersonIcon,
|
||||||
Phone as PhoneIcon,
|
Phone as PhoneIcon,
|
||||||
Badge as BadgeIcon,
|
|
||||||
Security as SecurityIcon,
|
Security as SecurityIcon,
|
||||||
History as HistoryIcon,
|
History as HistoryIcon,
|
||||||
DriveEta as DriveEtaIcon,
|
DriveEta as DriveEtaIcon,
|
||||||
@@ -276,7 +275,6 @@ function MitgliedDetail() {
|
|||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
if (member?.profile) {
|
if (member?.profile) {
|
||||||
setFormData({
|
setFormData({
|
||||||
mitglieds_nr: member.profile.mitglieds_nr ?? undefined,
|
|
||||||
dienstgrad: member.profile.dienstgrad ?? undefined,
|
dienstgrad: member.profile.dienstgrad ?? undefined,
|
||||||
dienstgrad_seit: toGermanDate(member.profile.dienstgrad_seit) || undefined,
|
dienstgrad_seit: toGermanDate(member.profile.dienstgrad_seit) || undefined,
|
||||||
funktion: member.profile.funktion,
|
funktion: member.profile.funktion,
|
||||||
@@ -346,7 +344,6 @@ function MitgliedDetail() {
|
|||||||
// Reset form to current profile values
|
// Reset form to current profile values
|
||||||
if (member?.profile) {
|
if (member?.profile) {
|
||||||
setFormData({
|
setFormData({
|
||||||
mitglieds_nr: member.profile.mitglieds_nr ?? undefined,
|
|
||||||
dienstgrad: member.profile.dienstgrad ?? undefined,
|
dienstgrad: member.profile.dienstgrad ?? undefined,
|
||||||
dienstgrad_seit: toGermanDate(member.profile.dienstgrad_seit) || undefined,
|
dienstgrad_seit: toGermanDate(member.profile.dienstgrad_seit) || undefined,
|
||||||
funktion: member.profile.funktion,
|
funktion: member.profile.funktion,
|
||||||
@@ -435,14 +432,6 @@ function MitgliedDetail() {
|
|||||||
<Typography variant="h5" fontWeight={600}>
|
<Typography variant="h5" fontWeight={600}>
|
||||||
{displayName}
|
{displayName}
|
||||||
</Typography>
|
</Typography>
|
||||||
{profile?.mitglieds_nr && (
|
|
||||||
<Chip
|
|
||||||
icon={<BadgeIcon />}
|
|
||||||
label={`Nr. ${profile.mitglieds_nr}`}
|
|
||||||
size="small"
|
|
||||||
variant="outlined"
|
|
||||||
/>
|
|
||||||
)}
|
|
||||||
{profile?.status && (
|
{profile?.status && (
|
||||||
<Chip
|
<Chip
|
||||||
label={STATUS_LABELS[profile.status]}
|
label={STATUS_LABELS[profile.status]}
|
||||||
@@ -617,14 +606,6 @@ function MitgliedDetail() {
|
|||||||
))}
|
))}
|
||||||
</TextField>
|
</TextField>
|
||||||
|
|
||||||
<TextField
|
|
||||||
label="Mitgliedsnummer"
|
|
||||||
fullWidth
|
|
||||||
size="small"
|
|
||||||
value={formData.mitglieds_nr ?? ''}
|
|
||||||
onChange={(e) => handleFieldChange('mitglieds_nr', e.target.value || undefined)}
|
|
||||||
/>
|
|
||||||
|
|
||||||
<TextField
|
<TextField
|
||||||
label="Eintrittsdatum"
|
label="Eintrittsdatum"
|
||||||
fullWidth
|
fullWidth
|
||||||
@@ -688,7 +669,6 @@ function MitgliedDetail() {
|
|||||||
? <Chip label={STATUS_LABELS[profile.status]} size="small" color={STATUS_COLORS[profile.status]} />
|
? <Chip label={STATUS_LABELS[profile.status]} size="small" color={STATUS_COLORS[profile.status]} />
|
||||||
: null
|
: null
|
||||||
} />
|
} />
|
||||||
<FieldRow label="Mitgliedsnummer" value={profile?.mitglieds_nr ?? null} />
|
|
||||||
<FieldRow
|
<FieldRow
|
||||||
label="Eintrittsdatum"
|
label="Eintrittsdatum"
|
||||||
value={profile?.eintrittsdatum
|
value={profile?.eintrittsdatum
|
||||||
@@ -724,6 +704,14 @@ function MitgliedDetail() {
|
|||||||
) : (
|
) : (
|
||||||
<FieldRow label="Standesbuchnummer" value={profile?.fdisk_standesbuch_nr ?? null} />
|
<FieldRow label="Standesbuchnummer" value={profile?.fdisk_standesbuch_nr ?? null} />
|
||||||
)}
|
)}
|
||||||
|
<FieldRow label="Geburtsort" value={profile?.geburtsort ?? null} />
|
||||||
|
<FieldRow label="Geschlecht" value={profile?.geschlecht ?? null} />
|
||||||
|
<FieldRow label="Beruf" value={profile?.beruf ?? null} />
|
||||||
|
<FieldRow label="Wohnort" value={
|
||||||
|
profile?.wohnort
|
||||||
|
? [profile.plz, profile.wohnort].filter(Boolean).join(' ')
|
||||||
|
: null
|
||||||
|
} />
|
||||||
</>
|
</>
|
||||||
)}
|
)}
|
||||||
</CardContent>
|
</CardContent>
|
||||||
|
|||||||
@@ -56,7 +56,6 @@ export type TshirtGroesseEnum = typeof TSHIRT_GROESSE_VALUES[number];
|
|||||||
export interface MitgliederProfile {
|
export interface MitgliederProfile {
|
||||||
id: string;
|
id: string;
|
||||||
user_id: string;
|
user_id: string;
|
||||||
mitglieds_nr: string | null;
|
|
||||||
dienstgrad: DienstgradEnum | null;
|
dienstgrad: DienstgradEnum | null;
|
||||||
dienstgrad_seit: string | null; // ISO date string from API
|
dienstgrad_seit: string | null; // ISO date string from API
|
||||||
funktion: FunktionEnum[];
|
funktion: FunktionEnum[];
|
||||||
@@ -75,6 +74,12 @@ export interface MitgliederProfile {
|
|||||||
bemerkungen: string | null;
|
bemerkungen: string | null;
|
||||||
fdisk_standesbuch_nr: string | null;
|
fdisk_standesbuch_nr: string | null;
|
||||||
bild_url: string | null;
|
bild_url: string | null;
|
||||||
|
// FDISK-synced extended profile fields
|
||||||
|
geburtsort: string | null;
|
||||||
|
geschlecht: string | null;
|
||||||
|
beruf: string | null;
|
||||||
|
wohnort: string | null;
|
||||||
|
plz: string | null;
|
||||||
created_at: string;
|
created_at: string;
|
||||||
updated_at: string;
|
updated_at: string;
|
||||||
}
|
}
|
||||||
@@ -115,7 +120,6 @@ export interface MemberListItem {
|
|||||||
profile_picture_url: string | null;
|
profile_picture_url: string | null;
|
||||||
is_active: boolean;
|
is_active: boolean;
|
||||||
profile_id: string | null;
|
profile_id: string | null;
|
||||||
mitglieds_nr: string | null;
|
|
||||||
fdisk_standesbuch_nr: string | null;
|
fdisk_standesbuch_nr: string | null;
|
||||||
dienstgrad: DienstgradEnum | null;
|
dienstgrad: DienstgradEnum | null;
|
||||||
funktion: FunktionEnum[];
|
funktion: FunktionEnum[];
|
||||||
|
|||||||
164
sync/src/db.ts
164
sync/src/db.ts
@@ -1,5 +1,11 @@
|
|||||||
import { Pool } from 'pg';
|
import { Pool } from 'pg';
|
||||||
import { FdiskMember, FdiskAusbildung } from './types';
|
import {
|
||||||
|
FdiskMember,
|
||||||
|
FdiskAusbildung,
|
||||||
|
FdiskBefoerderung,
|
||||||
|
FdiskUntersuchung,
|
||||||
|
FdiskFahrgenehmigung,
|
||||||
|
} from './types';
|
||||||
|
|
||||||
function log(msg: string) {
|
function log(msg: string) {
|
||||||
console.log(`[db] ${new Date().toISOString()} ${msg}`);
|
console.log(`[db] ${new Date().toISOString()} ${msg}`);
|
||||||
@@ -63,6 +69,9 @@ export async function syncToDatabase(
|
|||||||
pool: Pool,
|
pool: Pool,
|
||||||
members: FdiskMember[],
|
members: FdiskMember[],
|
||||||
ausbildungen: FdiskAusbildung[],
|
ausbildungen: FdiskAusbildung[],
|
||||||
|
befoerderungen: FdiskBefoerderung[],
|
||||||
|
untersuchungen: FdiskUntersuchung[],
|
||||||
|
fahrgenehmigungen: FdiskFahrgenehmigung[],
|
||||||
force = false
|
force = false
|
||||||
): Promise<void> {
|
): Promise<void> {
|
||||||
const client = await pool.connect();
|
const client = await pool.connect();
|
||||||
@@ -137,7 +146,7 @@ export async function syncToDatabase(
|
|||||||
);
|
);
|
||||||
const cur = currentResult.rows[0];
|
const cur = currentResult.rows[0];
|
||||||
|
|
||||||
// Update mitglieder_profile with FDISK data
|
// Update mitglieder_profile with FDISK data (core + extended profile fields)
|
||||||
const dienstgrad = mapDienstgrad(member.dienstgrad);
|
const dienstgrad = mapDienstgrad(member.dienstgrad);
|
||||||
|
|
||||||
await client.query(
|
await client.query(
|
||||||
@@ -147,12 +156,21 @@ export async function syncToDatabase(
|
|||||||
eintrittsdatum = COALESCE($3::date, eintrittsdatum),
|
eintrittsdatum = COALESCE($3::date, eintrittsdatum),
|
||||||
austrittsdatum = $4::date,
|
austrittsdatum = $4::date,
|
||||||
geburtsdatum = COALESCE($5::date, geburtsdatum),
|
geburtsdatum = COALESCE($5::date, geburtsdatum),
|
||||||
${dienstgrad ? 'dienstgrad = $6,' : ''}
|
geburtsort = COALESCE($6, geburtsort),
|
||||||
|
geschlecht = COALESCE($7, geschlecht),
|
||||||
|
beruf = COALESCE($8, beruf),
|
||||||
|
wohnort = COALESCE($9, wohnort),
|
||||||
|
plz = COALESCE($10, plz),
|
||||||
|
${dienstgrad ? 'dienstgrad = $11,' : ''}
|
||||||
updated_at = NOW()
|
updated_at = NOW()
|
||||||
WHERE user_id = ${dienstgrad ? '$7' : '$6'}`,
|
WHERE user_id = ${dienstgrad ? '$12' : '$11'}`,
|
||||||
dienstgrad
|
dienstgrad
|
||||||
? [member.standesbuchNr, member.status, member.eintrittsdatum, member.abmeldedatum, member.geburtsdatum, dienstgrad, userId]
|
? [member.standesbuchNr, member.status, member.eintrittsdatum, member.abmeldedatum, member.geburtsdatum,
|
||||||
: [member.standesbuchNr, member.status, member.eintrittsdatum, member.abmeldedatum, member.geburtsdatum, userId]
|
member.geburtsort, member.geschlecht, member.beruf, member.wohnort, member.plz,
|
||||||
|
dienstgrad, userId]
|
||||||
|
: [member.standesbuchNr, member.status, member.eintrittsdatum, member.abmeldedatum, member.geburtsdatum,
|
||||||
|
member.geburtsort, member.geschlecht, member.beruf, member.wohnort, member.plz,
|
||||||
|
userId]
|
||||||
);
|
);
|
||||||
|
|
||||||
// Detect and log what changed
|
// Detect and log what changed
|
||||||
@@ -195,7 +213,6 @@ export async function syncToDatabase(
|
|||||||
let ausbildungSkipped = 0;
|
let ausbildungSkipped = 0;
|
||||||
|
|
||||||
for (const ausb of ausbildungen) {
|
for (const ausb of ausbildungen) {
|
||||||
// Find user_id by standesbuch_nr
|
|
||||||
const result = await client.query<{ user_id: string }>(
|
const result = await client.query<{ user_id: string }>(
|
||||||
`SELECT user_id FROM mitglieder_profile WHERE fdisk_standesbuch_nr = $1`,
|
`SELECT user_id FROM mitglieder_profile WHERE fdisk_standesbuch_nr = $1`,
|
||||||
[ausb.standesbuchNr]
|
[ausb.standesbuchNr]
|
||||||
@@ -208,7 +225,6 @@ export async function syncToDatabase(
|
|||||||
|
|
||||||
const userId = result.rows[0].user_id;
|
const userId = result.rows[0].user_id;
|
||||||
|
|
||||||
// xmax = 0 means a fresh INSERT (not an update of an existing row)
|
|
||||||
const upsertResult = await client.query<{ was_inserted: boolean }>(
|
const upsertResult = await client.query<{ was_inserted: boolean }>(
|
||||||
`INSERT INTO ausbildung (user_id, kursname, kurs_datum, ablaufdatum, ort, bemerkung, fdisk_sync_key)
|
`INSERT INTO ausbildung (user_id, kursname, kurs_datum, ablaufdatum, ort, bemerkung, fdisk_sync_key)
|
||||||
VALUES ($1, $2, $3::date, $4::date, $5, $6, $7)
|
VALUES ($1, $2, $3::date, $4::date, $5, $6, $7)
|
||||||
@@ -227,13 +243,25 @@ export async function syncToDatabase(
|
|||||||
log(`New Ausbildung: ${ausb.standesbuchNr} — ${ausb.kursname}${ausb.kursDatum ? ` (${ausb.kursDatum})` : ''}`);
|
log(`New Ausbildung: ${ausb.standesbuchNr} — ${ausb.kursname}${ausb.kursDatum ? ` (${ausb.kursDatum})` : ''}`);
|
||||||
ausbildungNew++;
|
ausbildungNew++;
|
||||||
} else {
|
} else {
|
||||||
log(`Updated Ausbildung: ${ausb.standesbuchNr} — ${ausb.kursname}${ausb.kursDatum ? ` (${ausb.kursDatum})` : ''}`);
|
|
||||||
ausbildungUpdated++;
|
ausbildungUpdated++;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
await client.query('COMMIT');
|
|
||||||
log(`Ausbildungen: ${ausbildungNew} neu, ${ausbildungUpdated} unverändert, ${ausbildungSkipped} übersprungen`);
|
log(`Ausbildungen: ${ausbildungNew} neu, ${ausbildungUpdated} unverändert, ${ausbildungSkipped} übersprungen`);
|
||||||
|
|
||||||
|
// Upsert Beförderungen
|
||||||
|
const befoerderungStats = await syncBefoerderungen(client, befoerderungen);
|
||||||
|
log(`Beförderungen: ${befoerderungStats.neu} neu, ${befoerderungStats.updated} unverändert, ${befoerderungStats.skipped} übersprungen`);
|
||||||
|
|
||||||
|
// Upsert Untersuchungen
|
||||||
|
const untersuchungStats = await syncUntersuchungen(client, untersuchungen);
|
||||||
|
log(`Untersuchungen: ${untersuchungStats.neu} neu, ${untersuchungStats.updated} unverändert, ${untersuchungStats.skipped} übersprungen`);
|
||||||
|
|
||||||
|
// Upsert Fahrgenehmigungen
|
||||||
|
const fahrgenStats = await syncFahrgenehmigungen(client, fahrgenehmigungen);
|
||||||
|
log(`Fahrgenehmigungen: ${fahrgenStats.neu} neu, ${fahrgenStats.updated} unverändert, ${fahrgenStats.skipped} übersprungen`);
|
||||||
|
|
||||||
|
await client.query('COMMIT');
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
await client.query('ROLLBACK');
|
await client.query('ROLLBACK');
|
||||||
throw err;
|
throw err;
|
||||||
@@ -241,3 +269,119 @@ export async function syncToDatabase(
|
|||||||
client.release();
|
client.release();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async function syncBefoerderungen(
|
||||||
|
client: any,
|
||||||
|
befoerderungen: FdiskBefoerderung[]
|
||||||
|
): Promise<{ neu: number; updated: number; skipped: number }> {
|
||||||
|
let neu = 0, updated = 0, skipped = 0;
|
||||||
|
|
||||||
|
for (const b of befoerderungen) {
|
||||||
|
const result = await client.query<{ user_id: string }>(
|
||||||
|
`SELECT user_id FROM mitglieder_profile WHERE fdisk_standesbuch_nr = $1`,
|
||||||
|
[b.standesbuchNr]
|
||||||
|
);
|
||||||
|
|
||||||
|
if (result.rows.length === 0) { skipped++; continue; }
|
||||||
|
const userId = result.rows[0].user_id;
|
||||||
|
|
||||||
|
const upsertResult = await client.query<{ was_inserted: boolean }>(
|
||||||
|
`INSERT INTO befoerderungen (user_id, datum, dienstgrad, fdisk_sync_key)
|
||||||
|
VALUES ($1, $2::date, $3, $4)
|
||||||
|
ON CONFLICT (user_id, fdisk_sync_key) DO UPDATE SET
|
||||||
|
datum = EXCLUDED.datum,
|
||||||
|
dienstgrad = EXCLUDED.dienstgrad,
|
||||||
|
updated_at = NOW()
|
||||||
|
RETURNING (xmax = 0) AS was_inserted`,
|
||||||
|
[userId, b.datum, b.dienstgrad, b.syncKey]
|
||||||
|
);
|
||||||
|
|
||||||
|
if (upsertResult.rows[0]?.was_inserted) {
|
||||||
|
log(`New Beförderung: ${b.standesbuchNr} — ${b.dienstgrad}${b.datum ? ` (${b.datum})` : ''}`);
|
||||||
|
neu++;
|
||||||
|
} else {
|
||||||
|
updated++;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return { neu, updated, skipped };
|
||||||
|
}
|
||||||
|
|
||||||
|
async function syncUntersuchungen(
|
||||||
|
client: any,
|
||||||
|
untersuchungen: FdiskUntersuchung[]
|
||||||
|
): Promise<{ neu: number; updated: number; skipped: number }> {
|
||||||
|
let neu = 0, updated = 0, skipped = 0;
|
||||||
|
|
||||||
|
for (const u of untersuchungen) {
|
||||||
|
const result = await client.query<{ user_id: string }>(
|
||||||
|
`SELECT user_id FROM mitglieder_profile WHERE fdisk_standesbuch_nr = $1`,
|
||||||
|
[u.standesbuchNr]
|
||||||
|
);
|
||||||
|
|
||||||
|
if (result.rows.length === 0) { skipped++; continue; }
|
||||||
|
const userId = result.rows[0].user_id;
|
||||||
|
|
||||||
|
const upsertResult = await client.query<{ was_inserted: boolean }>(
|
||||||
|
`INSERT INTO untersuchungen (user_id, datum, anmerkungen, art, ergebnis, fdisk_sync_key)
|
||||||
|
VALUES ($1, $2::date, $3, $4, $5, $6)
|
||||||
|
ON CONFLICT (user_id, fdisk_sync_key) DO UPDATE SET
|
||||||
|
datum = EXCLUDED.datum,
|
||||||
|
anmerkungen = EXCLUDED.anmerkungen,
|
||||||
|
art = EXCLUDED.art,
|
||||||
|
ergebnis = EXCLUDED.ergebnis,
|
||||||
|
updated_at = NOW()
|
||||||
|
RETURNING (xmax = 0) AS was_inserted`,
|
||||||
|
[userId, u.datum, u.anmerkungen, u.art, u.ergebnis, u.syncKey]
|
||||||
|
);
|
||||||
|
|
||||||
|
if (upsertResult.rows[0]?.was_inserted) {
|
||||||
|
log(`New Untersuchung: ${u.standesbuchNr} — [${u.art}] ${u.ergebnis ?? '—'}${u.datum ? ` (${u.datum})` : ''} | ${u.anmerkungen ?? ''}`);
|
||||||
|
neu++;
|
||||||
|
} else {
|
||||||
|
updated++;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return { neu, updated, skipped };
|
||||||
|
}
|
||||||
|
|
||||||
|
async function syncFahrgenehmigungen(
|
||||||
|
client: any,
|
||||||
|
fahrgenehmigungen: FdiskFahrgenehmigung[]
|
||||||
|
): Promise<{ neu: number; updated: number; skipped: number }> {
|
||||||
|
let neu = 0, updated = 0, skipped = 0;
|
||||||
|
|
||||||
|
for (const f of fahrgenehmigungen) {
|
||||||
|
const result = await client.query<{ user_id: string }>(
|
||||||
|
`SELECT user_id FROM mitglieder_profile WHERE fdisk_standesbuch_nr = $1`,
|
||||||
|
[f.standesbuchNr]
|
||||||
|
);
|
||||||
|
|
||||||
|
if (result.rows.length === 0) { skipped++; continue; }
|
||||||
|
const userId = result.rows[0].user_id;
|
||||||
|
|
||||||
|
const upsertResult = await client.query<{ was_inserted: boolean }>(
|
||||||
|
`INSERT INTO fahrgenehmigungen (user_id, ausstellungsdatum, gueltig_bis, behoerde, nummer, klasse, fdisk_sync_key)
|
||||||
|
VALUES ($1, $2::date, $3::date, $4, $5, $6, $7)
|
||||||
|
ON CONFLICT (user_id, fdisk_sync_key) DO UPDATE SET
|
||||||
|
ausstellungsdatum = EXCLUDED.ausstellungsdatum,
|
||||||
|
gueltig_bis = EXCLUDED.gueltig_bis,
|
||||||
|
behoerde = EXCLUDED.behoerde,
|
||||||
|
nummer = EXCLUDED.nummer,
|
||||||
|
klasse = EXCLUDED.klasse,
|
||||||
|
updated_at = NOW()
|
||||||
|
RETURNING (xmax = 0) AS was_inserted`,
|
||||||
|
[userId, f.ausstellungsdatum, f.gueltigBis, f.behoerde, f.nummer, f.klasse, f.syncKey]
|
||||||
|
);
|
||||||
|
|
||||||
|
if (upsertResult.rows[0]?.was_inserted) {
|
||||||
|
log(`New Fahrgenehmigung: ${f.standesbuchNr} — [${f.klasse}]${f.ausstellungsdatum ? ` (${f.ausstellungsdatum})` : ''}`);
|
||||||
|
neu++;
|
||||||
|
} else {
|
||||||
|
updated++;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return { neu, updated, skipped };
|
||||||
|
}
|
||||||
|
|||||||
@@ -66,9 +66,9 @@ async function runSync(force = false): Promise<void> {
|
|||||||
try {
|
try {
|
||||||
if (force) log('Force mode: ON');
|
if (force) log('Force mode: ON');
|
||||||
log('Starting FDISK sync');
|
log('Starting FDISK sync');
|
||||||
const { members, ausbildungen } = await scrapeAll(username, password);
|
const { members, ausbildungen, befoerderungen, untersuchungen, fahrgenehmigungen } = await scrapeAll(username, password);
|
||||||
await syncToDatabase(pool, members, ausbildungen, force);
|
await syncToDatabase(pool, members, ausbildungen, befoerderungen, untersuchungen, fahrgenehmigungen, force);
|
||||||
log(`Sync complete — ${members.length} members, ${ausbildungen.length} Ausbildungen`);
|
log(`Sync complete — ${members.length} members, ${ausbildungen.length} Ausbildungen, ${befoerderungen.length} Beförderungen, ${untersuchungen.length} Untersuchungen, ${fahrgenehmigungen.length} Fahrgenehmigungen`);
|
||||||
} finally {
|
} finally {
|
||||||
syncRunning = false;
|
syncRunning = false;
|
||||||
await pool.end();
|
await pool.end();
|
||||||
|
|||||||
@@ -1,5 +1,11 @@
|
|||||||
import { chromium, Page, Frame } from '@playwright/test';
|
import { chromium, Page, Frame } from '@playwright/test';
|
||||||
import { FdiskMember, FdiskAusbildung } from './types';
|
import {
|
||||||
|
FdiskMember,
|
||||||
|
FdiskAusbildung,
|
||||||
|
FdiskBefoerderung,
|
||||||
|
FdiskUntersuchung,
|
||||||
|
FdiskFahrgenehmigung,
|
||||||
|
} from './types';
|
||||||
|
|
||||||
const BASE_URL = process.env.FDISK_BASE_URL ?? 'https://app.fdisk.at';
|
const BASE_URL = process.env.FDISK_BASE_URL ?? 'https://app.fdisk.at';
|
||||||
const ID_FEUERWEHREN = process.env.FDISK_ID_FEUERWEHREN ?? '164';
|
const ID_FEUERWEHREN = process.env.FDISK_ID_FEUERWEHREN ?? '164';
|
||||||
@@ -36,6 +42,9 @@ function cellText(text: string | undefined | null): string | null {
|
|||||||
export async function scrapeAll(username: string, password: string): Promise<{
|
export async function scrapeAll(username: string, password: string): Promise<{
|
||||||
members: FdiskMember[];
|
members: FdiskMember[];
|
||||||
ausbildungen: FdiskAusbildung[];
|
ausbildungen: FdiskAusbildung[];
|
||||||
|
befoerderungen: FdiskBefoerderung[];
|
||||||
|
untersuchungen: FdiskUntersuchung[];
|
||||||
|
fahrgenehmigungen: FdiskFahrgenehmigung[];
|
||||||
}> {
|
}> {
|
||||||
const browser = await chromium.launch({
|
const browser = await chromium.launch({
|
||||||
headless: true,
|
headless: true,
|
||||||
@@ -59,24 +68,58 @@ export async function scrapeAll(username: string, password: string): Promise<{
|
|||||||
log(`Found ${members.length} members`);
|
log(`Found ${members.length} members`);
|
||||||
|
|
||||||
const ausbildungen: FdiskAusbildung[] = [];
|
const ausbildungen: FdiskAusbildung[] = [];
|
||||||
|
const befoerderungen: FdiskBefoerderung[] = [];
|
||||||
|
const untersuchungen: FdiskUntersuchung[] = [];
|
||||||
|
const fahrgenehmigungen: FdiskFahrgenehmigung[] = [];
|
||||||
|
|
||||||
for (const member of members) {
|
for (const member of members) {
|
||||||
if (!member.detailUrl) continue;
|
if (!member.detailUrl) continue;
|
||||||
try {
|
try {
|
||||||
const quals = await scrapeMemberAusbildung(mainFrame, member);
|
// Navigate to detail page and scrape all sub-sections
|
||||||
|
await frame_goto(mainFrame, member.detailUrl);
|
||||||
|
|
||||||
|
// Scrape extra profile fields from the detail form
|
||||||
|
const profileFields = await scrapeDetailProfileFields(mainFrame);
|
||||||
|
member.geburtsort = profileFields.geburtsort;
|
||||||
|
member.geschlecht = profileFields.geschlecht;
|
||||||
|
member.beruf = profileFields.beruf;
|
||||||
|
member.wohnort = profileFields.wohnort;
|
||||||
|
member.plz = profileFields.plz;
|
||||||
|
|
||||||
|
// Ausbildungen
|
||||||
|
const quals = await scrapeAusbildungenFromDetailPage(mainFrame, member);
|
||||||
ausbildungen.push(...quals);
|
ausbildungen.push(...quals);
|
||||||
log(` ${member.vorname} ${member.zuname}: ${quals.length} Ausbildungen`);
|
|
||||||
|
// Beförderungen
|
||||||
|
const befos = await scrapeMemberBefoerderungen(mainFrame, member.standesbuchNr);
|
||||||
|
befoerderungen.push(...befos);
|
||||||
|
|
||||||
|
// Untersuchungen
|
||||||
|
const unters = await scrapeMemberUntersuchungen(mainFrame, member.standesbuchNr);
|
||||||
|
untersuchungen.push(...unters);
|
||||||
|
|
||||||
|
// Fahrgenehmigungen
|
||||||
|
const fahrg = await scrapeMemberFahrgenehmigungen(mainFrame, member.standesbuchNr);
|
||||||
|
fahrgenehmigungen.push(...fahrg);
|
||||||
|
|
||||||
|
log(` ${member.vorname} ${member.zuname}: ${quals.length} Ausbildungen, ${befos.length} Beförderungen, ${unters.length} Untersuchungen, ${fahrg.length} Fahrgenehmigungen`);
|
||||||
await page.waitForTimeout(500);
|
await page.waitForTimeout(500);
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
log(` WARN: could not scrape Ausbildung for ${member.vorname} ${member.zuname}: ${err}`);
|
log(` WARN: could not scrape detail for ${member.vorname} ${member.zuname}: ${err}`);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return { members, ausbildungen };
|
return { members, ausbildungen, befoerderungen, untersuchungen, fahrgenehmigungen };
|
||||||
} finally {
|
} finally {
|
||||||
await browser.close();
|
await browser.close();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** Navigate a frame, waiting for networkidle. Wrapper to avoid repetition. */
|
||||||
|
async function frame_goto(frame: Frame, url: string): Promise<void> {
|
||||||
|
await frame.goto(url, { waitUntil: 'networkidle' });
|
||||||
|
}
|
||||||
|
|
||||||
async function login(page: Page, username: string, password: string): Promise<void> {
|
async function login(page: Page, username: string, password: string): Promise<void> {
|
||||||
log(`Navigating to ${LOGIN_URL}`);
|
log(`Navigating to ${LOGIN_URL}`);
|
||||||
await page.goto(LOGIN_URL, { waitUntil: 'domcontentloaded' });
|
await page.goto(LOGIN_URL, { waitUntil: 'domcontentloaded' });
|
||||||
@@ -307,6 +350,11 @@ async function scrapeMembers(frame: Frame): Promise<FdiskMember[]> {
|
|||||||
abmeldedatum,
|
abmeldedatum,
|
||||||
status: abmeldedatum ? 'ausgetreten' : 'aktiv',
|
status: abmeldedatum ? 'ausgetreten' : 'aktiv',
|
||||||
detailUrl: row.href,
|
detailUrl: row.href,
|
||||||
|
geburtsort: null,
|
||||||
|
geschlecht: null,
|
||||||
|
beruf: null,
|
||||||
|
wohnort: null,
|
||||||
|
plz: null,
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
return members;
|
return members;
|
||||||
@@ -343,11 +391,44 @@ async function parseRowsFromTable(frame: Frame) {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
async function scrapeMemberAusbildung(frame: Frame, member: FdiskMember): Promise<FdiskAusbildung[]> {
|
/**
|
||||||
if (!member.detailUrl) return [];
|
* Scrape additional profile fields from the member detail form.
|
||||||
|
* Called while the frame is already on the member detail page.
|
||||||
|
*/
|
||||||
|
async function scrapeDetailProfileFields(frame: Frame): Promise<{
|
||||||
|
geburtsort: string | null;
|
||||||
|
geschlecht: string | null;
|
||||||
|
beruf: string | null;
|
||||||
|
wohnort: string | null;
|
||||||
|
plz: string | null;
|
||||||
|
}> {
|
||||||
|
return frame.evaluate(() => {
|
||||||
|
const val = (selector: string): string | null => {
|
||||||
|
const el = document.querySelector(selector) as HTMLInputElement | HTMLSelectElement | null;
|
||||||
|
if (!el) return null;
|
||||||
|
if (el.tagName === 'SELECT') {
|
||||||
|
const sel = el as HTMLSelectElement;
|
||||||
|
const opt = sel.options[sel.selectedIndex];
|
||||||
|
return opt ? (opt.text || opt.value || '').trim() || null : null;
|
||||||
|
}
|
||||||
|
return (el as HTMLInputElement).value?.trim() || null;
|
||||||
|
};
|
||||||
|
|
||||||
await frame.goto(member.detailUrl, { waitUntil: 'networkidle' });
|
return {
|
||||||
|
geburtsort: val('input[name="geburtsort"]') ?? val('input[id*="geburtsort"]'),
|
||||||
|
geschlecht: val('select[name*="geschlecht"]') ?? val('select[id*="geschlecht"]'),
|
||||||
|
beruf: val('input[name="beruf"]') ?? val('input[id*="beruf"]'),
|
||||||
|
wohnort: val('input[name="ort"]') ?? val('input[id*="_ort"]') ?? val('input[name="wohnort"]'),
|
||||||
|
plz: val('input[name="plz"]') ?? val('input[id*="plz"]'),
|
||||||
|
};
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Scrape Ausbildungen from the detail page (already loaded).
|
||||||
|
* Navigates to the Ausbildung sub-page if needed.
|
||||||
|
*/
|
||||||
|
async function scrapeAusbildungenFromDetailPage(frame: Frame, member: FdiskMember): Promise<FdiskAusbildung[]> {
|
||||||
// Look for Ausbildungsliste section — it's likely a table or list
|
// Look for Ausbildungsliste section — it's likely a table or list
|
||||||
const ausbildungSection = frame.locator('text=Ausbildung, text=Ausbildungsliste').first();
|
const ausbildungSection = frame.locator('text=Ausbildung, text=Ausbildungsliste').first();
|
||||||
const hasSec = await ausbildungSection.isVisible().catch(() => false);
|
const hasSec = await ausbildungSection.isVisible().catch(() => false);
|
||||||
@@ -363,7 +444,6 @@ async function scrapeMemberAusbildung(frame: Frame, member: FdiskMember): Promis
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Parse the qualification table
|
// Parse the qualification table
|
||||||
// Expected columns: Kursname, Datum, Ablaufdatum, Ort, Bemerkung (may vary)
|
|
||||||
const tables = await frame.$$('table');
|
const tables = await frame.$$('table');
|
||||||
const ausbildungen: FdiskAusbildung[] = [];
|
const ausbildungen: FdiskAusbildung[] = [];
|
||||||
|
|
||||||
@@ -376,7 +456,6 @@ async function scrapeMemberAusbildung(frame: Frame, member: FdiskMember): Promis
|
|||||||
|
|
||||||
if (rows.length < 2) continue;
|
if (rows.length < 2) continue;
|
||||||
|
|
||||||
// Detect if this looks like an Ausbildung table
|
|
||||||
const header = rows[0].cells.map(c => c.toLowerCase());
|
const header = rows[0].cells.map(c => c.toLowerCase());
|
||||||
const isAusbildungTable =
|
const isAusbildungTable =
|
||||||
header.some(h => h.includes('kurs') || h.includes('ausbildung') || h.includes('bezeichnung'));
|
header.some(h => h.includes('kurs') || h.includes('ausbildung') || h.includes('bezeichnung'));
|
||||||
@@ -412,3 +491,197 @@ async function scrapeMemberAusbildung(frame: Frame, member: FdiskMember): Promis
|
|||||||
|
|
||||||
return ausbildungen;
|
return ausbildungen;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Navigate to the Beförderungen sub-page and scrape all promotions.
|
||||||
|
* Navigates back to the member detail page afterwards.
|
||||||
|
*/
|
||||||
|
async function scrapeMemberBefoerderungen(frame: Frame, standesbuchNr: string): Promise<FdiskBefoerderung[]> {
|
||||||
|
// Find sidebar link to Beförderungen
|
||||||
|
const link = frame.locator('a[href*="befoerderungenList.aspx"], a[href*="BefoerderungenList.aspx"]').first();
|
||||||
|
const hasLink = await link.isVisible().catch(() => false);
|
||||||
|
if (!hasLink) {
|
||||||
|
log(` No Beförderungen link for StNr ${standesbuchNr}`);
|
||||||
|
return [];
|
||||||
|
}
|
||||||
|
|
||||||
|
const href = await link.getAttribute('href');
|
||||||
|
if (!href) return [];
|
||||||
|
|
||||||
|
const url = href.startsWith('http') ? href : new URL(href, frame.url()).toString();
|
||||||
|
await frame_goto(frame, url);
|
||||||
|
|
||||||
|
const results: FdiskBefoerderung[] = [];
|
||||||
|
|
||||||
|
try {
|
||||||
|
await frame.waitForSelector('table.FdcLayList', { timeout: 10000 });
|
||||||
|
const rows = await frame.$$eval('table.FdcLayList tbody tr', (trs) =>
|
||||||
|
trs.map((tr) => {
|
||||||
|
const cells = Array.from(tr.querySelectorAll('td'));
|
||||||
|
const cell = (i: number) => (cells[i]?.textContent ?? '').trim();
|
||||||
|
return { datum: cell(0), dienstgrad: cell(1) };
|
||||||
|
})
|
||||||
|
);
|
||||||
|
|
||||||
|
for (const row of rows) {
|
||||||
|
const dienstgrad = cellText(row.dienstgrad);
|
||||||
|
if (!dienstgrad) continue;
|
||||||
|
const datum = parseDate(row.datum);
|
||||||
|
const syncKey = `${standesbuchNr}::${dienstgrad}::${datum ?? ''}`;
|
||||||
|
results.push({ standesbuchNr, datum, dienstgrad, syncKey });
|
||||||
|
}
|
||||||
|
log(` Beförderungen for StNr ${standesbuchNr}: ${results.length} rows`);
|
||||||
|
for (const b of results) {
|
||||||
|
log(` ${b.datum ?? '—'} ${b.dienstgrad}`);
|
||||||
|
}
|
||||||
|
} catch {
|
||||||
|
log(` WARN: could not parse Beförderungen table for StNr ${standesbuchNr}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
return results;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Navigate to the Untersuchungen sub-page and scrape all medical exams.
|
||||||
|
* Keeps all rows (one per art+datum); DB stores all, queries filter latest per category.
|
||||||
|
*/
|
||||||
|
async function scrapeMemberUntersuchungen(frame: Frame, standesbuchNr: string): Promise<FdiskUntersuchung[]> {
|
||||||
|
const link = frame.locator('a[href*="UntersuchungenList.aspx"]').first();
|
||||||
|
const hasLink = await link.isVisible().catch(() => false);
|
||||||
|
if (!hasLink) {
|
||||||
|
log(` No Untersuchungen link for StNr ${standesbuchNr}`);
|
||||||
|
return [];
|
||||||
|
}
|
||||||
|
|
||||||
|
const href = await link.getAttribute('href');
|
||||||
|
if (!href) return [];
|
||||||
|
|
||||||
|
const url = href.startsWith('http') ? href : new URL(href, frame.url()).toString();
|
||||||
|
await frame_goto(frame, url);
|
||||||
|
|
||||||
|
const results: FdiskUntersuchung[] = [];
|
||||||
|
|
||||||
|
try {
|
||||||
|
await frame.waitForSelector('table.FdcLayList', { timeout: 10000 });
|
||||||
|
const rows = await frame.$$eval('table.FdcLayList tbody tr', (trs) =>
|
||||||
|
trs.map((tr) => {
|
||||||
|
const cells = Array.from(tr.querySelectorAll('td'));
|
||||||
|
const cell = (i: number) => (cells[i]?.textContent ?? '').trim();
|
||||||
|
// Columns: 0=Datum, 1=Anmerkungen, 2=Untersuchungsart, 3=Tauglichkeitsstufe
|
||||||
|
return {
|
||||||
|
datum: cell(0),
|
||||||
|
anmerkungen: cell(1),
|
||||||
|
art: cell(2),
|
||||||
|
ergebnis: cell(3),
|
||||||
|
};
|
||||||
|
})
|
||||||
|
);
|
||||||
|
|
||||||
|
for (const row of rows) {
|
||||||
|
const art = cellText(row.art);
|
||||||
|
if (!art) continue;
|
||||||
|
const datum = parseDate(row.datum);
|
||||||
|
const syncKey = `${standesbuchNr}::${art}::${datum ?? ''}`;
|
||||||
|
results.push({
|
||||||
|
standesbuchNr,
|
||||||
|
datum,
|
||||||
|
anmerkungen: cellText(row.anmerkungen),
|
||||||
|
art,
|
||||||
|
ergebnis: cellText(row.ergebnis),
|
||||||
|
syncKey,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
log(` Untersuchungen for StNr ${standesbuchNr}: ${results.length} rows`);
|
||||||
|
for (const u of results) {
|
||||||
|
log(` ${u.datum ?? '—'} [${u.art}] ${u.ergebnis ?? '—'} | ${u.anmerkungen ?? ''}`);
|
||||||
|
}
|
||||||
|
} catch {
|
||||||
|
log(` WARN: could not parse Untersuchungen table for StNr ${standesbuchNr}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
return results;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Navigate to the Gesetzliche Fahrgenehmigungen sub-page and scrape all entries.
|
||||||
|
* This is an inline-edit (ListEdit) page — values are in <input> fields.
|
||||||
|
*/
|
||||||
|
async function scrapeMemberFahrgenehmigungen(frame: Frame, standesbuchNr: string): Promise<FdiskFahrgenehmigung[]> {
|
||||||
|
const link = frame.locator('a[href*="Ges_fahrgenehmigungenListEdit.aspx"], a[href*="ges_fahrgenehmigungenListEdit.aspx"]').first();
|
||||||
|
const hasLink = await link.isVisible().catch(() => false);
|
||||||
|
if (!hasLink) {
|
||||||
|
log(` No Fahrgenehmigungen link for StNr ${standesbuchNr}`);
|
||||||
|
return [];
|
||||||
|
}
|
||||||
|
|
||||||
|
const href = await link.getAttribute('href');
|
||||||
|
if (!href) return [];
|
||||||
|
|
||||||
|
const url = href.startsWith('http') ? href : new URL(href, frame.url()).toString();
|
||||||
|
await frame_goto(frame, url);
|
||||||
|
|
||||||
|
const results: FdiskFahrgenehmigung[] = [];
|
||||||
|
|
||||||
|
try {
|
||||||
|
await frame.waitForSelector('table.FdcLayList', { timeout: 10000 });
|
||||||
|
|
||||||
|
// ListEdit pages: each data row has inline <input> fields instead of plain text.
|
||||||
|
// Columns: 0=Ausstellungsdatum, 1=Gültig bis, 2=Behörde, 3=Nummer, 4=Fahrgenehmigungsklasse
|
||||||
|
const rows = await frame.$$eval('table.FdcLayList tbody tr', (trs) =>
|
||||||
|
trs.map((tr) => {
|
||||||
|
const cells = Array.from(tr.querySelectorAll('td'));
|
||||||
|
const cellVal = (i: number): string => {
|
||||||
|
const cell = cells[i];
|
||||||
|
if (!cell) return '';
|
||||||
|
// Prefer input value, then select text, then textContent
|
||||||
|
const input = cell.querySelector('input[type="text"], input:not([type])') as HTMLInputElement | null;
|
||||||
|
if (input) return input.value?.trim() ?? '';
|
||||||
|
const select = cell.querySelector('select') as HTMLSelectElement | null;
|
||||||
|
if (select) {
|
||||||
|
const opt = select.options[select.selectedIndex];
|
||||||
|
return (opt?.text || opt?.value || '').trim();
|
||||||
|
}
|
||||||
|
return cell.textContent?.trim() ?? '';
|
||||||
|
};
|
||||||
|
return {
|
||||||
|
ausstellungsdatum: cellVal(0),
|
||||||
|
gueltigBis: cellVal(1),
|
||||||
|
behoerde: cellVal(2),
|
||||||
|
nummer: cellVal(3),
|
||||||
|
klasse: cellVal(4),
|
||||||
|
};
|
||||||
|
})
|
||||||
|
);
|
||||||
|
|
||||||
|
for (const row of rows) {
|
||||||
|
const klasse = cellText(row.klasse);
|
||||||
|
if (!klasse) continue;
|
||||||
|
const ausstellungsdatum = parseDate(row.ausstellungsdatum);
|
||||||
|
const syncKey = `${standesbuchNr}::${klasse}::${ausstellungsdatum ?? ''}`;
|
||||||
|
results.push({
|
||||||
|
standesbuchNr,
|
||||||
|
ausstellungsdatum,
|
||||||
|
gueltigBis: parseDate(row.gueltigBis),
|
||||||
|
behoerde: cellText(row.behoerde),
|
||||||
|
nummer: cellText(row.nummer),
|
||||||
|
klasse,
|
||||||
|
syncKey,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
log(` Fahrgenehmigungen for StNr ${standesbuchNr}: ${results.length} rows`);
|
||||||
|
for (const f of results) {
|
||||||
|
log(` ${f.ausstellungsdatum ?? '—'} [${f.klasse}] ${f.behoerde ?? ''} ${f.nummer ?? ''}`);
|
||||||
|
}
|
||||||
|
} catch {
|
||||||
|
log(` WARN: could not parse Fahrgenehmigungen table for StNr ${standesbuchNr}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
return results;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Legacy export kept for compatibility — delegates to the new unified flow
|
||||||
|
export async function scrapeMemberAusbildung(frame: Frame, member: FdiskMember): Promise<FdiskAusbildung[]> {
|
||||||
|
if (!member.detailUrl) return [];
|
||||||
|
await frame_goto(frame, member.detailUrl);
|
||||||
|
return scrapeAusbildungenFromDetailPage(frame, member);
|
||||||
|
}
|
||||||
|
|||||||
@@ -11,6 +11,12 @@ export interface FdiskMember {
|
|||||||
status: 'aktiv' | 'ausgetreten';
|
status: 'aktiv' | 'ausgetreten';
|
||||||
/** URL or identifier to navigate to the member detail page */
|
/** URL or identifier to navigate to the member detail page */
|
||||||
detailUrl: string | null;
|
detailUrl: string | null;
|
||||||
|
/** Additional profile fields scraped from the detail form */
|
||||||
|
geburtsort: string | null;
|
||||||
|
geschlecht: string | null;
|
||||||
|
beruf: string | null;
|
||||||
|
wohnort: string | null;
|
||||||
|
plz: string | null;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface FdiskAusbildung {
|
export interface FdiskAusbildung {
|
||||||
@@ -23,3 +29,29 @@ export interface FdiskAusbildung {
|
|||||||
/** Unique key built from standesbuchNr + kursname + kursDatum for deduplication */
|
/** Unique key built from standesbuchNr + kursname + kursDatum for deduplication */
|
||||||
syncKey: string;
|
syncKey: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export interface FdiskBefoerderung {
|
||||||
|
standesbuchNr: string;
|
||||||
|
datum: string | null; // ISO date
|
||||||
|
dienstgrad: string; // abbreviation from FDISK
|
||||||
|
syncKey: string; // `${standesbuchNr}::${dienstgrad}::${datum}`
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface FdiskUntersuchung {
|
||||||
|
standesbuchNr: string;
|
||||||
|
datum: string | null; // ISO date
|
||||||
|
anmerkungen: string | null;
|
||||||
|
art: string; // Untersuchungsart (category)
|
||||||
|
ergebnis: string | null; // Tauglichkeitsstufe (result)
|
||||||
|
syncKey: string; // `${standesbuchNr}::${art}::${datum}`
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface FdiskFahrgenehmigung {
|
||||||
|
standesbuchNr: string;
|
||||||
|
ausstellungsdatum: string | null; // ISO date
|
||||||
|
gueltigBis: string | null; // ISO date
|
||||||
|
behoerde: string | null;
|
||||||
|
nummer: string | null;
|
||||||
|
klasse: string; // Fahrgenehmigungsklasse display text
|
||||||
|
syncKey: string; // `${standesbuchNr}::${klasse}::${ausstellungsdatum}`
|
||||||
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user