update

sync/src/scraper.ts

@@ -159,7 +159,9 @@ async function scrapeMembers(frame: Frame): Promise<FdiskMember[]> {
           el.value = '';
           cleared.push(el.name || el.id);
         }
-        // Maximize page size: look for a select that controls rows per page
+        // Maximize page size: look for a select AND its paired hidden input
+        // FDISK uses a custom Dd widget where <select name="xDd_dd"> is the visible dropdown
+        // but the actual POST value comes from <input type="hidden" name="xDd_id"> or similar.
         if ((name.includes('anzahl') || id.includes('anzahl') ||
              name.includes('pagesize') || id.includes('pagesize') ||
              name.includes('rows') || id.includes('rows')) &&
@@ -178,6 +180,16 @@ async function scrapeMembers(frame: Frame): Promise<FdiskMember[]> {
           if (bestOption) {
             select.value = bestOption.value;
             pageSizeSet = `${el.name || el.id}=${bestOption.value}`;
+            // Also update the paired hidden field used by the Dd custom widget.
+            // Common patterns: xDd_dd → xDd_id or xDd_hd
+            const baseName = (el.name || el.id).replace(/_dd$/i, '');
+            for (const suffix of ['_id', '_hd', '_val']) {
+              const hidden = form.elements[baseName + suffix] as HTMLInputElement | undefined;
+              if (hidden && hidden.type === 'hidden') {
+                hidden.value = bestOption.value;
+                pageSizeSet += ` (also set ${baseName + suffix})`;
+              }
+            }
           }
         }
       }
@@ -249,51 +261,76 @@ async function scrapeMembers(frame: Frame): Promise<FdiskMember[]> {
       const pageSize = to - from + 1;
       const nextPageNum = Math.floor(to / pageSize) + 1;
 
-      // Click the "next page" link in FdcLayListNav.
+      // Click the "next page" control in FdcLayListNav.
-      // Strategy 1: text ">" or ">>" (common in FDISK)
+      // FDISK renders pagination as plain <img> inside <td class="Right"> (no <a> wrappers).
-      // Strategy 2: title/alt containing navigation words
+      // Use Playwright's click() which properly triggers JS event listeners attached via addEventListener.
-      // Strategy 3: link text is the next page number (e.g. "2" when on page 1)
+      // Try in order: b_next img → b_last img → any <a> with ">" text → page-number link.
-      const nextClicked = await frame.evaluate((nextPg: number) => {
+      let nextClicked = false;
-        const nav = document.querySelector('table.FdcLayListNav');
-        if (!nav) return false;
-        const links = Array.from(nav.querySelectorAll('a, input[type="button"], input[type="submit"]'));
 
-        for (const el of links) {
+      const nextImg = frame.locator('table.FdcLayListNav td.Right img[src*="b_next"]');
-          const text  = ((el as HTMLElement).textContent ?? '').trim();
+      if (await nextImg.count() > 0) {
-          const title = ((el as HTMLElement).getAttribute('title') ?? '').toLowerCase();
+        await nextImg.first().click({ timeout: 5000 }).catch(() => {});
-          const alt   = ((el as HTMLImageElement).alt ?? '').toLowerCase();
+        nextClicked = true;
-          if (text === '>' || text === '>>' ||
+      } else {
-              title.includes('nächst') || title.includes('weiter') || title.includes('next') || title.includes('vor') ||
+        // Fallback via evaluate for text/title/page-number patterns
-              alt.includes('next') || alt.includes('weiter') || alt.includes('vor')) {
+        const clicked = await frame.evaluate((nextPg: number) => {
-            (el as HTMLElement).click();
+          const nav = document.querySelector('table.FdcLayListNav');
-            return true;
+          if (!nav) return false;
+          const clickable = Array.from(nav.querySelectorAll('a, input[type="button"], input[type="submit"], td'));
+          for (const el of clickable) {
+            const text    = ((el as HTMLElement).textContent ?? '').trim();
+            const title   = ((el as HTMLElement).getAttribute('title') ?? '').toLowerCase();
+            const onclick = ((el as HTMLElement).getAttribute('onclick') ?? '').toLowerCase();
+            if (text === '>' || text === '>>' ||
+                title.includes('nächst') || title.includes('weiter') || title.includes('next') ||
+                onclick.includes('next') || onclick.includes('weiter') ||
+                text === String(nextPg)) {
+              (el as HTMLElement).click();
+              return true;
+            }
           }
-        }
+          return false;
-
+        }, nextPageNum);
-        // Fallback: find a link whose text is exactly the next page number
+        nextClicked = clicked;
-        for (const el of links) {
+      }
-          const text = ((el as HTMLElement).textContent ?? '').trim();
-          if (text === String(nextPg)) {
-            (el as HTMLElement).click();
-            return true;
-          }
-        }
-
-        return false;
-      }, nextPageNum);
 
       if (!nextClicked) {
-        // Dump nav HTML to help diagnose the missing next-page link
         const navHtml = await frame.evaluate(() => {
           const nav = document.querySelector('table.FdcLayListNav');
           return nav?.innerHTML?.replace(/\s+/g, ' ').trim() ?? '(not found)';
         });
-        log(`WARN: could not find next-page link (tried ">" and page "${nextPageNum}") — stopping pagination`);
+        log(`WARN: could not find next-page link — stopping pagination`);
         log(`FdcLayListNav HTML: ${navHtml}`);
         break;
       }
 
       await frame.waitForLoadState('networkidle', { timeout: 30000 });
+
+      // Verify we actually moved to the next page — if pagination didn't advance, stop
+      const newPagination = await frame.evaluate(() =>
+        document.querySelector('table.FdcLayListNav')?.textContent?.trim() ?? ''
+      );
+      if (newPagination === pagination) {
+        log(`WARN: pagination did not advance after click (still "${pagination}") — stopping`);
+        // Try clicking the Right td directly as a last resort
+        const tdClicked = await frame.evaluate(() => {
+          const td = document.querySelector('table.FdcLayListNav td.Right') as HTMLElement | null;
+          if (td) { td.click(); return true; }
+          return false;
+        });
+        if (tdClicked) {
+          await frame.waitForLoadState('networkidle', { timeout: 30000 });
+          const afterTdPagination = await frame.evaluate(() =>
+            document.querySelector('table.FdcLayListNav')?.textContent?.trim() ?? ''
+          );
+          if (afterTdPagination === pagination) {
+            log('WARN: td.Right click also did not advance — pagination is disabled, stopping');
+            break;
+          }
+        } else {
+          break;
+        }
+      }
       pageNum++;
     } else {
       // No pagination indicator found — assume single page