Add public endpoint statistics report

src/api/StatusApi.ts

@@ -1,4 +1,4 @@
-import { StatusRepository } from '../db/StatusRepository'
+import { EndpointStatistics, StatusRepository } from '../db/StatusRepository'
 import { StatusPayload } from '../types/api'
 
 class StatusApi {
@@ -71,6 +71,12 @@ class StatusApi {
         return alerts
     }
 
+    async getEndpointStatistics(
+        periodDays: number
+    ): Promise<EndpointStatistics[]> {
+        return await this.statusRepo.getEndpointStatistics(periodDays)
+    }
+
     async updateStatus(accountId: number, update: StatusPayload): Promise<any> {
         return await this.statusRepo.updateStatus(accountId, update)
     }

src/db/StatusRepository.test.ts

@@ -0,0 +1,93 @@
+import { Pool, RowDataPacket } from 'mysql2/promise'
+import { StatusRepository } from './StatusRepository'
+
+describe('StatusRepository', () => {
+    let mockPool: jest.Mocked<Pool>
+    let statusRepository: StatusRepository
+
+    beforeEach(() => {
+        mockPool = {
+            query: jest.fn(),
+        } as any
+
+        statusRepository = new StatusRepository(mockPool)
+    })
+
+    afterEach(() => {
+        jest.clearAllMocks()
+    })
+
+    it('returns endpoint statistics with failed podcasts', async () => {
+        const recentDate = new Date()
+        recentDate.setDate(recentDate.getDate() - 1)
+
+        const oldDate = new Date()
+        oldDate.setDate(oldDate.getDate() - 10)
+
+        const rows = [
+            {
+                account_id: 1,
+                provider: 'spotify',
+                endpoint: 'aggregate',
+                last_successful_update: recentDate.toISOString(),
+            },
+            {
+                account_id: 2,
+                provider: 'spotify',
+                endpoint: 'aggregate',
+                last_successful_update: oldDate.toISOString(),
+            },
+            {
+                account_id: 3,
+                provider: 'spotify',
+                endpoint: 'aggregate',
+                last_successful_update: recentDate.toISOString(),
+            },
+            {
+                account_id: 4,
+                provider: 'apple',
+                endpoint: 'trends',
+                last_successful_update: oldDate.toISOString(),
+            },
+        ] as RowDataPacket[]
+
+        mockPool.query.mockResolvedValue([rows, []] as any)
+
+        const statistics = await statusRepository.getEndpointStatistics(7)
+
+        expect(statistics).toEqual([
+            {
+                provider: 'spotify',
+                endpoint: 'aggregate',
+                requests: {
+                    total: 3,
+                    successful: 2,
+                    failed: 1,
+                    successRate: 2 / 3,
+                },
+                failedPodcasts: [
+                    {
+                        podcastId: 2,
+                        podcastName: null,
+                    },
+                ],
+            },
+            {
+                provider: 'apple',
+                endpoint: 'trends',
+                requests: {
+                    total: 1,
+                    successful: 0,
+                    failed: 1,
+                    successRate: 0,
+                },
+                failedPodcasts: [
+                    {
+                        podcastId: 4,
+                        podcastName: null,
+                    },
+                ],
+            },
+        ])
+    })
+})

src/db/StatusRepository.ts

@@ -12,6 +12,23 @@ type Status = {
     }
 }
 
+type FailedPodcast = {
+    podcastId: number
+    podcastName: string | null
+}
+
+type EndpointStatistics = {
+    provider: string
+    endpoint: string
+    requests: {
+        total: number
+        successful: number
+        failed: number
+        successRate: number
+    }
+    failedPodcasts: FailedPodcast[]
+}
+
 class StatusRepository {
     pool: Pool
 
@@ -68,6 +85,77 @@ class StatusRepository {
         return status
     }
 
+    // Returns endpoint-level update statistics for the given period. Since the
+    // updates table only records successful connector runs, a failed request here
+    // means a podcast has not had a successful update for this endpoint within
+    // the given period.
+    async getEndpointStatistics(
+        periodDays: number
+    ): Promise<EndpointStatistics[]> {
+        const query = `
+            SELECT
+                account_id,
+                provider,
+                endpoint,
+                MAX(created) AS last_successful_update
+            FROM updates
+            GROUP BY account_id, provider, endpoint
+            ORDER BY provider, endpoint, account_id
+        `
+        const response = (await this.pool.query(query)) as RowDataPacket[]
+
+        const statistics: EndpointStatistics[] = []
+        const statisticsByEndpoint: {
+            [key: string]: EndpointStatistics
+        } = {}
+        const startDate = new Date()
+        startDate.setDate(startDate.getDate() - periodDays)
+
+        response[0].forEach((row: RowDataPacket) => {
+            const identifier = `${row.provider}/${row.endpoint}`
+
+            if (!statisticsByEndpoint[identifier]) {
+                statisticsByEndpoint[identifier] = {
+                    provider: row.provider,
+                    endpoint: row.endpoint,
+                    requests: {
+                        total: 0,
+                        successful: 0,
+                        failed: 0,
+                        successRate: 0,
+                    },
+                    failedPodcasts: [],
+                }
+                statistics.push(statisticsByEndpoint[identifier])
+            }
+
+            const endpointStatistics = statisticsByEndpoint[identifier]
+            const lastSuccessfulUpdate = new Date(row.last_successful_update)
+            const hasRecentSuccessfulUpdate = lastSuccessfulUpdate >= startDate
+
+            endpointStatistics.requests.total += 1
+            if (hasRecentSuccessfulUpdate) {
+                endpointStatistics.requests.successful += 1
+            } else {
+                endpointStatistics.requests.failed += 1
+                endpointStatistics.failedPodcasts.push({
+                    podcastId: row.account_id,
+                    podcastName: null,
+                })
+            }
+        })
+
+        statistics.forEach((endpointStatistics) => {
+            endpointStatistics.requests.successRate =
+                endpointStatistics.requests.total > 0
+                    ? endpointStatistics.requests.successful /
+                      endpointStatistics.requests.total
+                    : 0
+        })
+
+        return statistics
+    }
+
     // Write the raw JSON status data into the `updates` table
     // Use the current timestamp as the update time
     async updateStatus(accountId: number, update: StatusPayload): Promise<any> {
@@ -81,4 +169,4 @@ class StatusRepository {
     }
 }
 
-export { StatusRepository }
+export { StatusRepository, EndpointStatistics }

src/index.ts

@@ -193,6 +193,7 @@ const publicEndpoints = [
     '^/images/*',
     '^/health',
     '^/status',
+    '^/statistics',
     '^/feedback/*',
     '^/comments/*',
     '^/api-docs',
@@ -228,9 +229,7 @@ app.get('/errorlog/:password/:id', (req: Request, res: Response) => {
     const password = req.params.password
     const id = req.params.id
 
-    if (
-        !passwordsMatch(config.getErrorLogPassword(), password)
-    ) {
+    if (!passwordsMatch(config.getErrorLogPassword(), password)) {
         sendErrorLogResponse(res, 401)
         return
     }
@@ -490,6 +489,28 @@ app.get(
     }
 )
 
+// Endpoint for update statistics.
+app.get(
+    '/statistics',
+    async (req: Request, res: Response, next: NextFunction) => {
+        try {
+            const periodDays = 7
+            const endpoints = await statusApi.getEndpointStatistics(periodDays)
+
+            res.status(200).json({
+                meta: {
+                    generatedAt: nowString(),
+                    periodDays,
+                    result: 'success',
+                },
+                endpoints,
+            })
+        } catch (err) {
+            next(err)
+        }
+    }
+)
+
 // Status endpoint, which returns a JSON of the last import time per endpoint.
 // This uses our internal event sourcing to determine the last imports.
 // Additionally, it returns a list of alerts if any of the imports are too old.