From 243da302c73ae7e42234b898460a1f449d26f017 Mon Sep 17 00:00:00 2001 From: Matthias Hochmeister Date: Fri, 13 Mar 2026 08:30:05 +0100 Subject: [PATCH] update backend stuck/stall --- backend/src/app.ts | 4 + backend/src/config/database.ts | 13 +- backend/src/config/httpClient.ts | 11 + .../src/jobs/notification-generation.job.ts | 283 +++++++++--------- .../middleware/request-timeout.middleware.ts | 10 + backend/src/services/bookstack.service.ts | 9 +- backend/src/services/nextcloud.service.ts | 15 +- .../src/services/serviceMonitor.service.ts | 3 +- backend/src/services/vikunja.service.ts | 7 +- 9 files changed, 199 insertions(+), 156 deletions(-) create mode 100644 backend/src/config/httpClient.ts create mode 100644 backend/src/middleware/request-timeout.middleware.ts diff --git a/backend/src/app.ts b/backend/src/app.ts index f7ad65c..aad8ecc 100644 --- a/backend/src/app.ts +++ b/backend/src/app.ts @@ -5,6 +5,7 @@ import rateLimit from 'express-rate-limit'; import environment from './config/environment'; import logger from './utils/logger'; import { errorHandler, notFoundHandler } from './middleware/error.middleware'; +import { requestTimeout } from './middleware/request-timeout.middleware'; const app: Application = express(); @@ -47,6 +48,9 @@ app.use('/api', rateLimit({ app.use(express.json({ limit: '10mb' })); app.use(express.urlencoded({ extended: true, limit: '10mb' })); +// Request timeout middleware +app.use(requestTimeout); + // Request logging middleware app.use((req: Request, _res: Response, next) => { logger.info('Incoming request', { diff --git a/backend/src/config/database.ts b/backend/src/config/database.ts index 9ce0a76..1840ddf 100644 --- a/backend/src/config/database.ts +++ b/backend/src/config/database.ts @@ -14,7 +14,7 @@ const poolConfig: PoolConfig = { database: environment.database.name, user: environment.database.user, password: environment.database.password, - max: 20, // Maximum number of clients in the pool + max: 30, // Maximum number of clients in the pool idleTimeoutMillis: 30000, // Close idle clients after 30 seconds connectionTimeoutMillis: 5000, // Return an error if connection takes longer than 5 seconds }; @@ -26,6 +26,17 @@ pool.on('error', (err) => { logger.error('Unexpected error on idle database client', err); }); +// Log pool exhaustion warnings every 60s (only when requests are waiting) +setInterval(() => { + if (pool.waitingCount > 0) { + logger.warn('DB pool pressure detected', { + total: pool.totalCount, + idle: pool.idleCount, + waiting: pool.waitingCount, + }); + } +}, 60_000).unref(); + // Test database connection export const testConnection = async (): Promise => { try { diff --git a/backend/src/config/httpClient.ts b/backend/src/config/httpClient.ts new file mode 100644 index 0000000..d1611f7 --- /dev/null +++ b/backend/src/config/httpClient.ts @@ -0,0 +1,11 @@ +import axios from 'axios'; +import * as http from 'http'; +import * as https from 'https'; + +const httpClient = axios.create({ + timeout: 10_000, + httpAgent: new http.Agent({ keepAlive: true, maxSockets: 20 }), + httpsAgent: new https.Agent({ keepAlive: true, maxSockets: 20 }), +}); + +export default httpClient; diff --git a/backend/src/jobs/notification-generation.job.ts b/backend/src/jobs/notification-generation.job.ts index 87b5360..baf9b7e 100644 --- a/backend/src/jobs/notification-generation.job.ts +++ b/backend/src/jobs/notification-generation.job.ts @@ -5,6 +5,7 @@ * 1. Personal atemschutz warnings (untersuchung / leistungstest expiring within 60 days) * 2. Vehicle issues (for fahrmeister users) * 3. Equipment issues (for fahrmeister if motorised, zeugmeister if not) + * 4. Nextcloud Talk unread messages * * Deduplicates via the unique index on (user_id, quell_typ, quell_id) WHERE NOT gelesen. * Also cleans up read notifications older than 90 days. @@ -19,12 +20,18 @@ const INTERVAL_MS = 15 * 60 * 1000; // 15 minutes const ATEMSCHUTZ_THRESHOLD = 60; // days let jobInterval: ReturnType | null = null; +let isRunning = false; // --------------------------------------------------------------------------- // Core generation function // --------------------------------------------------------------------------- export async function runNotificationGeneration(): Promise { + if (isRunning) { + logger.warn('NotificationGenerationJob: previous run still in progress — skipping'); + return; + } + isRunning = true; try { await generateAtemschutzNotifications(); await generateVehicleNotifications(); @@ -35,6 +42,8 @@ export async function runNotificationGeneration(): Promise { logger.error('NotificationGenerationJob: unexpected error', { error: error instanceof Error ? error.message : String(error), }); + } finally { + isRunning = false; } } @@ -100,146 +109,133 @@ async function generateAtemschutzNotifications(): Promise { } // --------------------------------------------------------------------------- -// 2. Vehicle issues → fahrmeister users +// 2. Vehicle issues → fahrmeister users (bulk INSERT) // --------------------------------------------------------------------------- async function generateVehicleNotifications(): Promise { try { - // Find vehicles with problems (damaged or not operational, or overdue inspection) - const vehiclesResult = await pool.query(` - SELECT id, bezeichnung, kurzname, status, naechste_pruefung_tage - FROM fahrzeuge - WHERE deleted_at IS NULL - AND ( - status IN ('beschaedigt', 'ausser_dienst') - OR (naechste_pruefung_tage IS NOT NULL AND naechste_pruefung_tage::int < 0) - ) + await pool.query(` + INSERT INTO notifications (user_id, typ, titel, nachricht, schwere, link, quell_id, quell_typ) + SELECT + u.id, + 'fahrzeug_status', + 'Fahrzeug nicht einsatzbereit', + CASE WHEN f.kurzname IS NOT NULL + THEN f.bezeichnung || ' (' || f.kurzname || ') hat den Status "' || f.status || '" und ist nicht einsatzbereit.' + ELSE f.bezeichnung || ' hat den Status "' || f.status || '" und ist nicht einsatzbereit.' + END, + 'fehler', + '/fahrzeuge/' || f.id::text, + 'fahrzeug-status-' || f.id::text, + 'fahrzeug_status' + FROM fahrzeuge f + CROSS JOIN users u + WHERE f.deleted_at IS NULL + AND f.status IN ('beschaedigt', 'ausser_dienst') + AND u.is_active = TRUE + AND 'dashboard_fahrmeister' = ANY(u.authentik_groups) + ON CONFLICT (user_id, quell_typ, quell_id) + WHERE NOT gelesen AND quell_typ IS NOT NULL AND quell_id IS NOT NULL + DO NOTHING `); - if (vehiclesResult.rows.length === 0) return; - - // Get all fahrmeister users - const usersResult = await pool.query(` - SELECT id FROM users WHERE is_active = TRUE AND 'dashboard_fahrmeister' = ANY(authentik_groups) + await pool.query(` + INSERT INTO notifications (user_id, typ, titel, nachricht, schwere, link, quell_id, quell_typ) + SELECT + u.id, + 'fahrzeug_pruefung', + 'Fahrzeugprüfung überfällig', + CASE WHEN f.kurzname IS NOT NULL + THEN 'Die Prüfung von ' || f.bezeichnung || ' (' || f.kurzname || ') ist seit ' || ABS(f.naechste_pruefung_tage::int) || ' Tagen überfällig.' + ELSE 'Die Prüfung von ' || f.bezeichnung || ' ist seit ' || ABS(f.naechste_pruefung_tage::int) || ' Tagen überfällig.' + END, + 'fehler', + '/fahrzeuge/' || f.id::text, + 'fahrzeug-pruefung-' || f.id::text, + 'fahrzeug_pruefung' + FROM fahrzeuge f + CROSS JOIN users u + WHERE f.deleted_at IS NULL + AND f.naechste_pruefung_tage IS NOT NULL + AND f.naechste_pruefung_tage::int < 0 + AND u.is_active = TRUE + AND 'dashboard_fahrmeister' = ANY(u.authentik_groups) + ON CONFLICT (user_id, quell_typ, quell_id) + WHERE NOT gelesen AND quell_typ IS NOT NULL AND quell_id IS NOT NULL + DO NOTHING `); - - for (const user of usersResult.rows) { - for (const vehicle of vehiclesResult.rows) { - const label = vehicle.kurzname ? `${vehicle.bezeichnung} (${vehicle.kurzname})` : vehicle.bezeichnung; - const isOverdueInspection = vehicle.naechste_pruefung_tage != null && parseInt(vehicle.naechste_pruefung_tage, 10) < 0; - const isBroken = ['beschaedigt', 'ausser_dienst'].includes(vehicle.status); - - if (isBroken) { - await notificationService.createNotification({ - user_id: user.id, - typ: 'fahrzeug_status', - titel: `Fahrzeug nicht einsatzbereit`, - nachricht: `${label} hat den Status "${vehicle.status}" und ist nicht einsatzbereit.`, - schwere: 'fehler', - link: `/fahrzeuge/${vehicle.id}`, - quell_id: `fahrzeug-status-${vehicle.id}`, - quell_typ: 'fahrzeug_status', - }); - } - - if (isOverdueInspection) { - const tage = Math.abs(parseInt(vehicle.naechste_pruefung_tage, 10)); - await notificationService.createNotification({ - user_id: user.id, - typ: 'fahrzeug_pruefung', - titel: `Fahrzeugprüfung überfällig`, - nachricht: `Die Prüfung von ${label} ist seit ${tage} Tagen überfällig.`, - schwere: 'fehler', - link: `/fahrzeuge/${vehicle.id}`, - quell_id: `fahrzeug-pruefung-${vehicle.id}`, - quell_typ: 'fahrzeug_pruefung', - }); - } - } - } } catch (error) { logger.error('NotificationGenerationJob: generateVehicleNotifications failed', { error }); } } // --------------------------------------------------------------------------- -// 3. Equipment issues → fahrmeister (motorised) or zeugmeister (non-motorised) +// 3. Equipment issues → fahrmeister (motorised) or zeugmeister (bulk INSERT) // --------------------------------------------------------------------------- async function generateEquipmentNotifications(): Promise { try { - // Find equipment with problems (broken, overdue inspection) - const equipmentResult = await pool.query(` + await pool.query(` + INSERT INTO notifications (user_id, typ, titel, nachricht, schwere, link, quell_id, quell_typ) SELECT - a.id, a.bezeichnung, a.status, - k.motorisiert, - (a.naechste_pruefung_am::date - CURRENT_DATE) AS pruefung_tage + u.id, + 'ausruestung_status', + 'Ausrüstung nicht einsatzbereit', + a.bezeichnung || ' hat den Status "' || a.status || '" und ist nicht einsatzbereit.', + 'fehler', + '/ausruestung/' || a.id::text, + 'ausruestung-status-' || a.id::text, + 'ausruestung_status' FROM ausruestung a JOIN ausruestung_kategorien k ON k.id = a.kategorie_id + JOIN users u ON u.is_active = TRUE AND ( + (k.motorisiert = TRUE AND 'dashboard_fahrmeister' = ANY(u.authentik_groups)) + OR + (k.motorisiert = FALSE AND 'dashboard_zeugmeister' = ANY(u.authentik_groups)) + ) WHERE a.deleted_at IS NULL - AND ( - a.status IN ('beschaedigt', 'ausser_dienst') - OR (a.naechste_pruefung_am IS NOT NULL AND a.naechste_pruefung_am::date < CURRENT_DATE) - ) + AND a.status IN ('beschaedigt', 'ausser_dienst') + ON CONFLICT (user_id, quell_typ, quell_id) + WHERE NOT gelesen AND quell_typ IS NOT NULL AND quell_id IS NOT NULL + DO NOTHING `); - if (equipmentResult.rows.length === 0) return; - - // Get fahrmeister and zeugmeister users - const [fahrResult, zeugResult] = await Promise.all([ - pool.query(`SELECT id FROM users WHERE is_active = TRUE AND 'dashboard_fahrmeister' = ANY(authentik_groups)`), - pool.query(`SELECT id FROM users WHERE is_active = TRUE AND 'dashboard_zeugmeister' = ANY(authentik_groups)`), - ]); - - const fahrmeisterIds: string[] = fahrResult.rows.map((r: any) => r.id); - const zeugmeisterIds: string[] = zeugResult.rows.map((r: any) => r.id); - - for (const item of equipmentResult.rows) { - const targetUsers: string[] = item.motorisiert ? fahrmeisterIds : zeugmeisterIds; - if (targetUsers.length === 0) continue; - - const isBroken = ['beschaedigt', 'ausser_dienst'].includes(item.status); - const pruefungTage = item.pruefung_tage != null ? parseInt(item.pruefung_tage, 10) : null; - const isOverdueInspection = pruefungTage !== null && pruefungTage < 0; - - for (const userId of targetUsers) { - if (isBroken) { - await notificationService.createNotification({ - user_id: userId, - typ: 'ausruestung_status', - titel: `Ausrüstung nicht einsatzbereit`, - nachricht: `${item.bezeichnung} hat den Status "${item.status}" und ist nicht einsatzbereit.`, - schwere: 'fehler', - link: `/ausruestung/${item.id}`, - quell_id: `ausruestung-status-${item.id}`, - quell_typ: 'ausruestung_status', - }); - } - - if (isOverdueInspection) { - const tage = Math.abs(pruefungTage!); - await notificationService.createNotification({ - user_id: userId, - typ: 'ausruestung_pruefung', - titel: `Ausrüstungsprüfung überfällig`, - nachricht: `Die Prüfung von ${item.bezeichnung} ist seit ${tage} Tagen überfällig.`, - schwere: 'fehler', - link: `/ausruestung/${item.id}`, - quell_id: `ausruestung-pruefung-${item.id}`, - quell_typ: 'ausruestung_pruefung', - }); - } - } - } + await pool.query(` + INSERT INTO notifications (user_id, typ, titel, nachricht, schwere, link, quell_id, quell_typ) + SELECT + u.id, + 'ausruestung_pruefung', + 'Ausrüstungsprüfung überfällig', + 'Die Prüfung von ' || a.bezeichnung || ' ist seit ' || ABS(a.naechste_pruefung_am::date - CURRENT_DATE) || ' Tagen überfällig.', + 'fehler', + '/ausruestung/' || a.id::text, + 'ausruestung-pruefung-' || a.id::text, + 'ausruestung_pruefung' + FROM ausruestung a + JOIN ausruestung_kategorien k ON k.id = a.kategorie_id + JOIN users u ON u.is_active = TRUE AND ( + (k.motorisiert = TRUE AND 'dashboard_fahrmeister' = ANY(u.authentik_groups)) + OR + (k.motorisiert = FALSE AND 'dashboard_zeugmeister' = ANY(u.authentik_groups)) + ) + WHERE a.deleted_at IS NULL + AND a.naechste_pruefung_am IS NOT NULL + AND a.naechste_pruefung_am::date < CURRENT_DATE + ON CONFLICT (user_id, quell_typ, quell_id) + WHERE NOT gelesen AND quell_typ IS NOT NULL AND quell_id IS NOT NULL + DO NOTHING + `); } catch (error) { logger.error('NotificationGenerationJob: generateEquipmentNotifications failed', { error }); } } // --------------------------------------------------------------------------- -// 4. Nextcloud Talk unread messages → per-user notifications +// 4. Nextcloud Talk unread messages — batched concurrency (3 users at a time) // --------------------------------------------------------------------------- +const NEXTCLOUD_BATCH_SIZE = 3; + async function generateNextcloudTalkNotifications(): Promise { const usersResult = await pool.query(` SELECT id, nextcloud_login_name, nextcloud_app_password @@ -249,40 +245,47 @@ async function generateNextcloudTalkNotifications(): Promise { AND nextcloud_app_password IS NOT NULL `); - for (const user of usersResult.rows) { - try { - const { conversations } = await nextcloudService.getConversations( - user.nextcloud_login_name, - user.nextcloud_app_password, - ); + const users = usersResult.rows; - for (const conv of conversations) { - if (conv.unreadMessages <= 0) continue; - await notificationService.createNotification({ - user_id: user.id, - typ: 'nextcloud_talk', - titel: conv.displayName, - nachricht: `${conv.unreadMessages} ungelesene Nachrichten`, - schwere: 'info', - link: conv.url, - quell_id: conv.token, - quell_typ: 'nextcloud_talk', - }); - } - } catch (error: any) { - if (error?.code === 'NEXTCLOUD_AUTH_INVALID') { - await pool.query( - `UPDATE users SET nextcloud_login_name = NULL, nextcloud_app_password = NULL WHERE id = $1`, - [user.id], - ); - logger.warn('NotificationGenerationJob: cleared invalid Nextcloud credentials', { userId: user.id }); - continue; - } - logger.error('NotificationGenerationJob: generateNextcloudTalkNotifications failed for user', { - userId: user.id, - error, + for (let i = 0; i < users.length; i += NEXTCLOUD_BATCH_SIZE) { + const batch = users.slice(i, i + NEXTCLOUD_BATCH_SIZE); + await Promise.allSettled(batch.map((user) => processNextcloudUser(user))); + } +} + +async function processNextcloudUser(user: { id: string; nextcloud_login_name: string; nextcloud_app_password: string }): Promise { + try { + const { conversations } = await nextcloudService.getConversations( + user.nextcloud_login_name, + user.nextcloud_app_password, + ); + + for (const conv of conversations) { + if (conv.unreadMessages <= 0) continue; + await notificationService.createNotification({ + user_id: user.id, + typ: 'nextcloud_talk', + titel: conv.displayName, + nachricht: `${conv.unreadMessages} ungelesene Nachrichten`, + schwere: 'info', + link: conv.url, + quell_id: conv.token, + quell_typ: 'nextcloud_talk', }); } + } catch (error: any) { + if (error?.code === 'NEXTCLOUD_AUTH_INVALID') { + await pool.query( + `UPDATE users SET nextcloud_login_name = NULL, nextcloud_app_password = NULL WHERE id = $1`, + [user.id], + ); + logger.warn('NotificationGenerationJob: cleared invalid Nextcloud credentials', { userId: user.id }); + return; + } + logger.error('NotificationGenerationJob: generateNextcloudTalkNotifications failed for user', { + userId: user.id, + error, + }); } } diff --git a/backend/src/middleware/request-timeout.middleware.ts b/backend/src/middleware/request-timeout.middleware.ts new file mode 100644 index 0000000..8a7ba17 --- /dev/null +++ b/backend/src/middleware/request-timeout.middleware.ts @@ -0,0 +1,10 @@ +import { Request, Response, NextFunction } from 'express'; + +export function requestTimeout(req: Request, res: Response, next: NextFunction): void { + req.setTimeout(15_000, () => { + if (!res.headersSent) { + res.status(503).json({ error: 'Request timeout' }); + } + }); + next(); +} diff --git a/backend/src/services/bookstack.service.ts b/backend/src/services/bookstack.service.ts index 170de6f..9ec8ede 100644 --- a/backend/src/services/bookstack.service.ts +++ b/backend/src/services/bookstack.service.ts @@ -1,4 +1,5 @@ import axios from 'axios'; +import httpClient from '../config/httpClient'; import environment from '../config/environment'; import logger from '../utils/logger'; @@ -89,7 +90,7 @@ function buildHeaders(): Record { async function getBookSlugMap(): Promise> { const { bookstack } = environment; try { - const response = await axios.get( + const response = await httpClient.get( `${bookstack.url}/api/books`, { params: { count: 500 }, headers: buildHeaders() }, ); @@ -108,7 +109,7 @@ async function getRecentPages(): Promise { try { const [response, bookSlugMap] = await Promise.all([ - axios.get( + httpClient.get( `${bookstack.url}/api/pages`, { params: { sort: '-updated_at', count: 20 }, @@ -141,7 +142,7 @@ async function searchPages(query: string): Promise { } try { - const response = await axios.get( + const response = await httpClient.get( `${bookstack.url}/api/search`, { params: { query, count: 50 }, @@ -197,7 +198,7 @@ async function getPageById(id: number): Promise { try { const [response, bookSlugMap] = await Promise.all([ - axios.get( + httpClient.get( `${bookstack.url}/api/pages/${id}`, { headers: buildHeaders() }, ), diff --git a/backend/src/services/nextcloud.service.ts b/backend/src/services/nextcloud.service.ts index 1d450ba..00d71ee 100644 --- a/backend/src/services/nextcloud.service.ts +++ b/backend/src/services/nextcloud.service.ts @@ -1,4 +1,5 @@ import axios from 'axios'; +import httpClient from '../config/httpClient'; import environment from '../config/environment'; import logger from '../utils/logger'; @@ -82,7 +83,7 @@ async function initiateLoginFlow(): Promise { } try { - const response = await axios.post(`${baseUrl}/index.php/login/v2`); + const response = await httpClient.post(`${baseUrl}/index.php/login/v2`); return { loginUrl: response.data.login, pollToken: response.data.poll.token, @@ -105,7 +106,7 @@ async function pollLoginFlow(pollEndpoint: string, pollToken: string): Promise): Promise { const start = Date.now(); try { - await axios.get(url, { timeout: 5000, headers }); + await httpClient.get(url, { timeout: 5000, headers }); return { name: '', url, diff --git a/backend/src/services/vikunja.service.ts b/backend/src/services/vikunja.service.ts index 499937a..2f378a6 100644 --- a/backend/src/services/vikunja.service.ts +++ b/backend/src/services/vikunja.service.ts @@ -1,4 +1,5 @@ import axios from 'axios'; +import httpClient from '../config/httpClient'; import environment from '../config/environment'; import logger from '../utils/logger'; @@ -71,7 +72,7 @@ async function getMyTasks(): Promise { } try { - const response = await axios.get( + const response = await httpClient.get( `${vikunja.url}/api/v1/tasks/all`, { headers: buildHeaders() }, ); @@ -104,7 +105,7 @@ async function getProjects(): Promise { } try { - const response = await axios.get( + const response = await httpClient.get( `${vikunja.url}/api/v1/projects`, { headers: buildHeaders() }, ); @@ -132,7 +133,7 @@ async function createTask(projectId: number, title: string, dueDate?: string): P if (dueDate) { body.due_date = dueDate; } - const response = await axios.put( + const response = await httpClient.put( `${vikunja.url}/api/v1/projects/${projectId}/tasks`, body, { headers: buildHeaders() },