From c91294cd69a4fb737d29eeffe23e35b6190f3af1 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 22 Jul 2026 14:02:13 +0200 Subject: [PATCH] Add statistics status dashboard --- src/api/StatusApi.test.ts | 181 ++++++++++++++ src/api/StatusApi.ts | 246 ++++++++++++++++++-- src/db/StatusRepository.test.ts | 114 +++++---- src/db/StatusRepository.ts | 113 ++++----- src/index.ts | 13 +- src/types/hbs.d.ts | 7 + views/partials/failed-podcasts.hbs | 19 ++ views/partials/statistics-endpoint-card.hbs | 31 +++ views/statistics.hbs | 244 +++++++++++++++++++ 9 files changed, 835 insertions(+), 133 deletions(-) create mode 100644 src/api/StatusApi.test.ts create mode 100644 src/types/hbs.d.ts create mode 100644 views/partials/failed-podcasts.hbs create mode 100644 views/partials/statistics-endpoint-card.hbs create mode 100644 views/statistics.hbs diff --git a/src/api/StatusApi.test.ts b/src/api/StatusApi.test.ts new file mode 100644 index 0000000..8482061 --- /dev/null +++ b/src/api/StatusApi.test.ts @@ -0,0 +1,181 @@ +import { StatusApi } from './StatusApi' +import { StatusRepository } from '../db/StatusRepository' + +const formatDate = (date: Date) => { + const year = date.getFullYear() + const month = `${date.getMonth() + 1}`.padStart(2, '0') + const day = `${date.getDate()}`.padStart(2, '0') + + return `${year}-${month}-${day}` +} + +const getDateRange = (periodDays: number) => { + const endDate = new Date() + const dates: string[] = [] + + for (let offset = periodDays - 1; offset >= 0; offset--) { + const date = new Date(endDate) + date.setDate(date.getDate() - offset) + dates.push(formatDate(date)) + } + + return dates +} + +describe('StatusApi', () => { + it('returns endpoint statistics from the same status data as the status endpoint', async () => { + const periodDays = 90 + const dates = getDateRange(periodDays) + const firstAvailableDate = dates[1] + const today = dates[dates.length - 1] + const recentDate = new Date() + recentDate.setDate(recentDate.getDate() - 1) + + const oldDate = new Date() + oldDate.setDate(oldDate.getDate() - 100) + + const statusRepo = { + getStatus: jest.fn().mockResolvedValue({ + 1: { + latestUpdates: { + 'spotify/aggregate': recentDate, + 'apple/trends': oldDate, + }, + }, + 2: { + latestUpdates: { + 'spotify/aggregate': oldDate, + }, + }, + 3: { + latestUpdates: { + 'spotify/aggregate': recentDate, + }, + }, + }), + getDailyEndpointUpdates: jest.fn().mockResolvedValue([ + { + accountId: 1, + provider: 'spotify', + endpoint: 'aggregate', + date: today, + }, + { + accountId: 3, + provider: 'spotify', + endpoint: 'aggregate', + date: today, + }, + ]), + getEndpointFirstUpdates: jest.fn().mockResolvedValue([ + { + accountId: 1, + provider: 'spotify', + endpoint: 'aggregate', + date: firstAvailableDate, + }, + { + accountId: 2, + provider: 'spotify', + endpoint: 'aggregate', + date: firstAvailableDate, + }, + { + accountId: 3, + provider: 'spotify', + endpoint: 'aggregate', + date: firstAvailableDate, + }, + { + accountId: 1, + provider: 'apple', + endpoint: 'trends', + date: today, + }, + ]), + } as unknown as StatusRepository + + const statusApi = new StatusApi(statusRepo) + + const statistics = await statusApi.getEndpointStatistics(periodDays) + + expect(statusRepo.getDailyEndpointUpdates).toHaveBeenCalledWith( + periodDays + ) + expect(statusRepo.getEndpointFirstUpdates).toHaveBeenCalledTimes(1) + expect(statistics).toHaveLength(2) + expect(statistics[0]).toEqual( + expect.objectContaining({ + provider: 'spotify', + endpoint: 'aggregate', + requests: { + total: 3, + successful: 2, + failed: 1, + successRate: 2 / 3, + }, + failedPodcasts: [ + { + podcastId: 2, + podcastName: null, + }, + ], + }) + ) + expect(statistics[0].days).toHaveLength(periodDays) + expect(statistics[0].days[0]).toEqual({ + date: dates[0], + title: `${dates[0]}: no data available`, + isUnavailable: true, + isDegraded: false, + requests: { + total: 0, + successful: 0, + failed: 0, + successRate: 0, + }, + }) + expect(statistics[0].days[1]).toEqual({ + date: firstAvailableDate, + title: `${firstAvailableDate}: 0/3 successful, 3 failed`, + isUnavailable: false, + isDegraded: true, + requests: { + total: 3, + successful: 0, + failed: 3, + successRate: 0, + }, + }) + expect(statistics[0].days[periodDays - 1]).toEqual({ + date: today, + title: `${today}: 2/3 successful, 1 failed`, + isUnavailable: false, + isDegraded: true, + requests: { + total: 3, + successful: 2, + failed: 1, + successRate: 2 / 3, + }, + }) + expect(statistics[1]).toEqual( + expect.objectContaining({ + provider: 'apple', + endpoint: 'trends', + requests: { + total: 1, + successful: 0, + failed: 1, + successRate: 0, + }, + failedPodcasts: [ + { + podcastId: 1, + podcastName: null, + }, + ], + }) + ) + }) +}) diff --git a/src/api/StatusApi.ts b/src/api/StatusApi.ts index b3de774..8a8f02f 100644 --- a/src/api/StatusApi.ts +++ b/src/api/StatusApi.ts @@ -1,6 +1,107 @@ -import { EndpointStatistics, StatusRepository } from '../db/StatusRepository' +import { StatusRepository } from '../db/StatusRepository' import { StatusPayload } from '../types/api' +type AlertData = { + podcastId: number + endpoint: string + ageHours: number +} + +type FailedPodcast = { + podcastId: number + podcastName: string | null +} + +type DailyEndpointStatistics = { + date: string + title: string + isUnavailable: boolean + isDegraded: boolean + requests: { + total: number + successful: number + failed: number + successRate: number + } +} + +type EndpointStatistics = { + provider: string + endpoint: string + requests: { + total: number + successful: number + failed: number + successRate: number + } + failedPodcasts: FailedPodcast[] + days: DailyEndpointStatistics[] +} + +const getEndpointAgeHours = (endpointDate: Date) => { + return Math.round( + (new Date().getTime() - new Date(endpointDate).getTime()) / + (1000 * 60 * 60) + ) +} + +const splitEndpointIdentifier = (identifier: string) => { + const [provider, ...endpointParts] = identifier.split('/') + + return { + provider, + endpoint: endpointParts.join('/'), + } +} + +const getEndpointIdentifier = (provider: string, endpoint: string) => { + return `${provider}/${endpoint}` +} + +const formatDate = (date: Date) => { + const year = date.getFullYear() + const month = `${date.getMonth() + 1}`.padStart(2, '0') + const day = `${date.getDate()}`.padStart(2, '0') + + return `${year}-${month}-${day}` +} + +const getDateRange = (periodDays: number) => { + const endDate = new Date() + const dates: string[] = [] + + for (let offset = periodDays - 1; offset >= 0; offset--) { + const date = new Date(endDate) + date.setDate(date.getDate() - offset) + dates.push(formatDate(date)) + } + + return dates +} + +const emptyRequests = () => ({ + total: 0, + successful: 0, + failed: 0, + successRate: 0, +}) + +const updateSuccessRate = (requests: EndpointStatistics['requests']) => { + requests.successRate = + requests.total > 0 ? requests.successful / requests.total : 0 +} + +const getDayTitle = ( + date: string, + requests: EndpointStatistics['requests'] +) => { + if (requests.total === 0) { + return `${date}: no data available` + } + + return `${date}: ${requests.successful}/${requests.total} successful, ${requests.failed} failed` +} + class StatusApi { statusRepo: StatusRepository @@ -24,11 +125,6 @@ class StatusApi { yellowAgeHours: number, redAgeHours: number ) { - type AlertData = { - podcastId: number - endpoint: string - ageHours: number - } const yellowAlerts: AlertData[] = [] const redAlerts: AlertData[] = [] Object.entries(statusData).forEach(([podcastIdStr, podcastData]) => { @@ -36,11 +132,7 @@ class StatusApi { const podcastId = parseInt(podcastIdStr) Object.entries(podcastData.latestUpdates).forEach( ([endpointName, endpointDate]) => { - const ageHours = Math.round( - (new Date().getTime() - - new Date(endpointDate).getTime()) / - (1000 * 60 * 60) - ) + const ageHours = getEndpointAgeHours(endpointDate) if (ageHours >= redAgeHours) { redAlerts.push({ podcastId: podcastId, @@ -74,7 +166,135 @@ class StatusApi { async getEndpointStatistics( periodDays: number ): Promise { - return await this.statusRepo.getEndpointStatistics(periodDays) + const status = await this.getStatus() + const dailyUpdates = await this.statusRepo.getDailyEndpointUpdates( + periodDays + ) + const firstUpdates = await this.statusRepo.getEndpointFirstUpdates() + const maxAgeHours = periodDays * 24 + const dates = getDateRange(periodDays) + const statistics: EndpointStatistics[] = [] + const statisticsByEndpoint: { [key: string]: EndpointStatistics } = {} + const successfulPodcastsByEndpointAndDate: { + [endpointIdentifier: string]: { [date: string]: Set } + } = {} + const firstUpdateByEndpointAndPodcast: { + [endpointIdentifier: string]: { [podcastId: number]: string } + } = {} + + Object.entries(status).forEach(([podcastIdStr, podcastData]) => { + const podcastId = parseInt(podcastIdStr) + + Object.entries(podcastData.latestUpdates).forEach( + ([endpointIdentifier, endpointDate]) => { + if (!statisticsByEndpoint[endpointIdentifier]) { + const { provider, endpoint } = + splitEndpointIdentifier(endpointIdentifier) + + statisticsByEndpoint[endpointIdentifier] = { + provider, + endpoint, + requests: emptyRequests(), + failedPodcasts: [], + days: dates.map((date) => ({ + date, + title: `${date}: no data available`, + isUnavailable: true, + isDegraded: false, + requests: emptyRequests(), + })), + } + statistics.push( + statisticsByEndpoint[endpointIdentifier] + ) + } + + const endpointStatistics = + statisticsByEndpoint[endpointIdentifier] + const ageHours = getEndpointAgeHours(endpointDate) + const hasRecentSuccessfulUpdate = ageHours < maxAgeHours + + endpointStatistics.requests.total += 1 + if (hasRecentSuccessfulUpdate) { + endpointStatistics.requests.successful += 1 + } else { + endpointStatistics.requests.failed += 1 + endpointStatistics.failedPodcasts.push({ + podcastId, + podcastName: null, + }) + } + } + ) + }) + + firstUpdates.forEach((update) => { + const endpointIdentifier = getEndpointIdentifier( + update.provider, + update.endpoint + ) + if (!firstUpdateByEndpointAndPodcast[endpointIdentifier]) { + firstUpdateByEndpointAndPodcast[endpointIdentifier] = {} + } + + firstUpdateByEndpointAndPodcast[endpointIdentifier][ + update.accountId + ] = update.date + }) + + dailyUpdates.forEach((update) => { + const endpointIdentifier = getEndpointIdentifier( + update.provider, + update.endpoint + ) + if (!successfulPodcastsByEndpointAndDate[endpointIdentifier]) { + successfulPodcastsByEndpointAndDate[endpointIdentifier] = {} + } + if ( + !successfulPodcastsByEndpointAndDate[endpointIdentifier][ + update.date + ] + ) { + successfulPodcastsByEndpointAndDate[endpointIdentifier][ + update.date + ] = new Set() + } + + successfulPodcastsByEndpointAndDate[endpointIdentifier][ + update.date + ].add(update.accountId) + }) + + statistics.forEach((endpointStatistics) => { + updateSuccessRate(endpointStatistics.requests) + const endpointIdentifier = getEndpointIdentifier( + endpointStatistics.provider, + endpointStatistics.endpoint + ) + const successfulPodcastsByDate = + successfulPodcastsByEndpointAndDate[endpointIdentifier] ?? {} + const firstUpdateByPodcast = + firstUpdateByEndpointAndPodcast[endpointIdentifier] ?? {} + + endpointStatistics.days.forEach((day) => { + day.requests.total = Object.values(firstUpdateByPodcast).filter( + (firstUpdateDate) => firstUpdateDate <= day.date + ).length + day.requests.successful = + successfulPodcastsByDate[day.date]?.size ?? 0 + day.requests.failed = Math.max( + day.requests.total - day.requests.successful, + 0 + ) + updateSuccessRate(day.requests) + day.isUnavailable = day.requests.total === 0 + day.isDegraded = + day.requests.total > 0 && day.requests.failed > 0 + day.title = getDayTitle(day.date, day.requests) + }) + }) + + return statistics } async updateStatus(accountId: number, update: StatusPayload): Promise { @@ -82,4 +302,4 @@ class StatusApi { } } -export { StatusApi } +export { StatusApi, EndpointStatistics } diff --git a/src/db/StatusRepository.test.ts b/src/db/StatusRepository.test.ts index b846cfa..887dd94 100644 --- a/src/db/StatusRepository.test.ts +++ b/src/db/StatusRepository.test.ts @@ -17,76 +17,104 @@ describe('StatusRepository', () => { jest.clearAllMocks() }) - it('returns endpoint statistics with failed podcasts', async () => { - const recentDate = new Date() - recentDate.setDate(recentDate.getDate() - 1) - - const oldDate = new Date() - oldDate.setDate(oldDate.getDate() - 10) - + it('returns latest updates grouped by podcast and endpoint', async () => { const rows = [ { account_id: 1, provider: 'spotify', endpoint: 'aggregate', - last_successful_update: recentDate.toISOString(), + latest_update: '2026-07-22 10:00:00', + }, + { + account_id: 1, + provider: 'apple', + endpoint: 'trends', + latest_update: '2026-07-22 11:00:00', }, { account_id: 2, provider: 'spotify', endpoint: 'aggregate', - last_successful_update: oldDate.toISOString(), + latest_update: '2026-07-21 10:00:00', }, + ] as RowDataPacket[] + + mockPool.query.mockResolvedValue([rows, []] as any) + + const status = await statusRepository.getStatus() + + expect(status).toEqual({ + 1: { + latestUpdates: { + 'spotify/aggregate': new Date('2026-07-22 10:00:00'), + 'apple/trends': new Date('2026-07-22 11:00:00'), + }, + }, + 2: { + latestUpdates: { + 'spotify/aggregate': new Date('2026-07-21 10:00:00'), + }, + }, + }) + }) + + it('returns daily endpoint updates for the requested period', async () => { + const rows = [ { - account_id: 3, + account_id: 1, provider: 'spotify', endpoint: 'aggregate', - last_successful_update: recentDate.toISOString(), + update_date: '2026-07-21', }, { - account_id: 4, - provider: 'apple', - endpoint: 'trends', - last_successful_update: oldDate.toISOString(), + account_id: 2, + provider: 'spotify', + endpoint: 'aggregate', + update_date: '2026-07-21', }, ] as RowDataPacket[] mockPool.query.mockResolvedValue([rows, []] as any) - const statistics = await statusRepository.getEndpointStatistics(7) + const updates = await statusRepository.getDailyEndpointUpdates(90) - expect(statistics).toEqual([ + expect(mockPool.query).toHaveBeenCalledWith(expect.any(String), [89]) + expect(updates).toEqual([ { + accountId: 1, provider: 'spotify', endpoint: 'aggregate', - requests: { - total: 3, - successful: 2, - failed: 1, - successRate: 2 / 3, - }, - failedPodcasts: [ - { - podcastId: 2, - podcastName: null, - }, - ], + date: '2026-07-21', }, { - provider: 'apple', - endpoint: 'trends', - requests: { - total: 1, - successful: 0, - failed: 1, - successRate: 0, - }, - failedPodcasts: [ - { - podcastId: 4, - podcastName: null, - }, - ], + accountId: 2, + provider: 'spotify', + endpoint: 'aggregate', + date: '2026-07-21', + }, + ]) + }) + + it('returns the first update date per podcast and endpoint', async () => { + const rows = [ + { + account_id: 1, + provider: 'spotify', + endpoint: 'aggregate', + first_update_date: '2026-07-01', + }, + ] as RowDataPacket[] + + mockPool.query.mockResolvedValue([rows, []] as any) + + const updates = await statusRepository.getEndpointFirstUpdates() + + expect(updates).toEqual([ + { + accountId: 1, + provider: 'spotify', + endpoint: 'aggregate', + date: '2026-07-01', }, ]) }) diff --git a/src/db/StatusRepository.ts b/src/db/StatusRepository.ts index a2db18d..450f4e1 100644 --- a/src/db/StatusRepository.ts +++ b/src/db/StatusRepository.ts @@ -12,21 +12,18 @@ type Status = { } } -type FailedPodcast = { - podcastId: number - podcastName: string | null +type DailyEndpointUpdate = { + accountId: number + provider: string + endpoint: string + date: string } -type EndpointStatistics = { +type EndpointFirstUpdate = { + accountId: number provider: string endpoint: string - requests: { - total: number - successful: number - failed: number - successRate: number - } - failedPodcasts: FailedPodcast[] + date: string } class StatusRepository { @@ -85,75 +82,51 @@ class StatusRepository { return status } - // Returns endpoint-level update statistics for the given period. Since the - // updates table only records successful connector runs, a failed request here - // means a podcast has not had a successful update for this endpoint within - // the given period. - async getEndpointStatistics( + async getDailyEndpointUpdates( periodDays: number - ): Promise { + ): Promise { + const query = ` + SELECT + account_id, + provider, + endpoint, + DATE(created) AS update_date + FROM updates + WHERE created >= DATE_SUB(CURDATE(), INTERVAL ? DAY) + GROUP BY account_id, provider, endpoint, update_date + ORDER BY update_date, provider, endpoint, account_id + ` + const response = (await this.pool.query(query, [ + Math.max(periodDays - 1, 0), + ])) as RowDataPacket[] + + return response[0].map((row: RowDataPacket) => ({ + accountId: row.account_id, + provider: row.provider, + endpoint: row.endpoint, + date: row.update_date, + })) + } + + async getEndpointFirstUpdates(): Promise { const query = ` SELECT account_id, provider, endpoint, - MAX(created) AS last_successful_update + DATE(MIN(created)) AS first_update_date FROM updates GROUP BY account_id, provider, endpoint - ORDER BY provider, endpoint, account_id + ORDER BY first_update_date, provider, endpoint, account_id ` const response = (await this.pool.query(query)) as RowDataPacket[] - const statistics: EndpointStatistics[] = [] - const statisticsByEndpoint: { - [key: string]: EndpointStatistics - } = {} - const startDate = new Date() - startDate.setDate(startDate.getDate() - periodDays) - - response[0].forEach((row: RowDataPacket) => { - const identifier = `${row.provider}/${row.endpoint}` - - if (!statisticsByEndpoint[identifier]) { - statisticsByEndpoint[identifier] = { - provider: row.provider, - endpoint: row.endpoint, - requests: { - total: 0, - successful: 0, - failed: 0, - successRate: 0, - }, - failedPodcasts: [], - } - statistics.push(statisticsByEndpoint[identifier]) - } - - const endpointStatistics = statisticsByEndpoint[identifier] - const lastSuccessfulUpdate = new Date(row.last_successful_update) - const hasRecentSuccessfulUpdate = lastSuccessfulUpdate >= startDate - - endpointStatistics.requests.total += 1 - if (hasRecentSuccessfulUpdate) { - endpointStatistics.requests.successful += 1 - } else { - endpointStatistics.requests.failed += 1 - endpointStatistics.failedPodcasts.push({ - podcastId: row.account_id, - podcastName: null, - }) - } - }) - - statistics.forEach((endpointStatistics) => { - endpointStatistics.requests.successRate = - endpointStatistics.requests.total > 0 - ? endpointStatistics.requests.successful / - endpointStatistics.requests.total - : 0 - }) - - return statistics + return response[0].map((row: RowDataPacket) => ({ + accountId: row.account_id, + provider: row.provider, + endpoint: row.endpoint, + date: row.first_update_date, + })) } // Write the raw JSON status data into the `updates` table @@ -169,4 +142,4 @@ class StatusRepository { } } -export { StatusRepository, EndpointStatistics } +export { StatusRepository, Status, DailyEndpointUpdate, EndpointFirstUpdate } diff --git a/src/index.ts b/src/index.ts index 2567b28..6812634 100644 --- a/src/index.ts +++ b/src/index.ts @@ -40,6 +40,7 @@ import fs from 'fs' import path from 'path' import swaggerUi from 'swagger-ui-express' import { swaggerSpec } from './config/swagger' +import hbs from 'hbs' const config = new Config() const ERROR_LOG_ID_PATTERN = /^[a-f0-9]{32}$/ @@ -221,6 +222,7 @@ app.disable('x-powered-by') app.use(bodyParser.json({ limit: '5mb' })) app.use(express.static('public')) +hbs.registerPartials(path.join(__dirname, '../views/partials')) app.set('view engine', 'handlebars') app.use(bodyParser.urlencoded({ extended: false })) @@ -494,15 +496,12 @@ app.get( '/statistics', async (req: Request, res: Response, next: NextFunction) => { try { - const periodDays = 7 + const periodDays = 90 const endpoints = await statusApi.getEndpointStatistics(periodDays) - res.status(200).json({ - meta: { - generatedAt: nowString(), - periodDays, - result: 'success', - }, + res.status(200).render('statistics.hbs', { + generatedAt: nowString(), + periodDays, endpoints, }) } catch (err) { diff --git a/src/types/hbs.d.ts b/src/types/hbs.d.ts new file mode 100644 index 0000000..027eb00 --- /dev/null +++ b/src/types/hbs.d.ts @@ -0,0 +1,7 @@ +declare module 'hbs' { + const hbs: { + registerPartials: (directory: string) => void + } + + export default hbs +} diff --git a/views/partials/failed-podcasts.hbs b/views/partials/failed-podcasts.hbs new file mode 100644 index 0000000..0b82991 --- /dev/null +++ b/views/partials/failed-podcasts.hbs @@ -0,0 +1,19 @@ +
+
Failed podcasts
+ {{#if failedPodcasts}} +
    + {{#each failedPodcasts}} +
  • + {{#if podcastName}} + {{podcastName}} + (#{{podcastId}}) + {{else}} + #{{podcastId}} + {{/if}} +
    • + {{/each}} +
+ {{else}} +
None
+ {{/if}} +
\ No newline at end of file diff --git a/views/partials/statistics-endpoint-card.hbs b/views/partials/statistics-endpoint-card.hbs new file mode 100644 index 0000000..4d67a38 --- /dev/null +++ b/views/partials/statistics-endpoint-card.hbs @@ -0,0 +1,31 @@ +
+
+

{{provider}}/{{endpoint}}

+ {{#if failedPodcasts}} + ! + {{else}} + ✓ + {{/if}} +
+ +
+ {{#each days}} + {{#if isUnavailable}} +
+ {{else}} + {{#if isDegraded}} +
+ {{else}} +
+ {{/if}} + {{/if}} + {{/each}} +
+ +
+ {{requests.successful}}/{{requests.total}} successful + {{requests.failed}} failed +
+ + {{> failed-podcasts}} +
diff --git a/views/statistics.hbs b/views/statistics.hbs new file mode 100644 index 0000000..ca76ab8 --- /dev/null +++ b/views/statistics.hbs @@ -0,0 +1,244 @@ + + + + Open Podcast Status + + + + + +
+
+ ✓Import status +
+ +
+

Current Status: Open Podcast Imports

+
+
Endpoint imports over the past {{periodDays}} days.
+
Generated at {{generatedAt}}
+
+
+ +
+ {{#each endpoints}} + {{> statistics-endpoint-card}} + {{/each}} +
+
+ +