Skip to content

Infer daily statistics failures from expected podcasts #230

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
mre merged 1 commit into from
Jul 22, 2026
Merged

Infer daily statistics failures from expected podcasts #230

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
44 changes: 1 addition & 43 deletions src/api/StatusApi.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,6 @@ describe('StatusApi', () => {
it('returns endpoint statistics from the same status data as the status endpoint', async () => {
const periodDays = 90
const dates = getDateRange(periodDays)
const firstAvailableDate = dates[1]
const today = dates[dates.length - 1]
const recentDate = new Date()
recentDate.setDate(recentDate.getDate() - 1)
Expand Down Expand Up @@ -67,32 +66,6 @@ describe('StatusApi', () => {
date: today,
},
]),
getEndpointFirstUpdates: jest.fn().mockResolvedValue([
{
accountId: 1,
provider: 'spotify',
endpoint: 'aggregate',
date: firstAvailableDate,
},
{
accountId: 2,
provider: 'spotify',
endpoint: 'aggregate',
date: firstAvailableDate,
},
{
accountId: 3,
provider: 'spotify',
endpoint: 'aggregate',
date: firstAvailableDate,
},
{
accountId: 1,
provider: 'apple',
endpoint: 'trends',
date: today,
},
]),
} as unknown as StatusRepository

const statusApi = new StatusApi(statusRepo)
Expand All @@ -102,7 +75,6 @@ describe('StatusApi', () => {
expect(statusRepo.getDailyEndpointUpdates).toHaveBeenCalledWith(
periodDays
)
expect(statusRepo.getEndpointFirstUpdates).toHaveBeenCalledTimes(1)
expect(statistics).toHaveLength(2)
expect(statistics[0]).toEqual(
expect.objectContaining({
Expand All @@ -125,20 +97,7 @@ describe('StatusApi', () => {
expect(statistics[0].days).toHaveLength(periodDays)
expect(statistics[0].days[0]).toEqual({
date: dates[0],
title: `${dates[0]}: no data available`,
isUnavailable: true,
isDegraded: false,
requests: {
total: 0,
successful: 0,
failed: 0,
successRate: 0,
},
})
expect(statistics[0].days[1]).toEqual({
date: firstAvailableDate,
title: `${firstAvailableDate}: 0/3 successful, 3 failed`,
isUnavailable: false,
title: `${dates[0]}: 0/3 successful, 3 failed`,
isDegraded: true,
requests: {
total: 3,
Expand All @@ -150,7 +109,6 @@ describe('StatusApi', () => {
expect(statistics[0].days[periodDays - 1]).toEqual({
date: today,
title: `${today}: 2/3 successful, 1 failed`,
isUnavailable: false,
isDegraded: true,
requests: {
total: 3,
Expand Down
54 changes: 21 additions & 33 deletions src/api/StatusApi.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,6 @@ type FailedPodcast = {
type DailyEndpointStatistics = {
date: string
title: string
isUnavailable: boolean
isDegraded: boolean
requests: {
total: number
Expand Down Expand Up @@ -95,10 +94,6 @@ const getDayTitle = (
date: string,
requests: EndpointStatistics['requests']
) => {
if (requests.total === 0) {
return `${date}: no data available`
}

return `${date}: ${requests.successful}/${requests.total} successful, ${requests.failed} failed`
}

Expand Down Expand Up @@ -170,16 +165,15 @@ class StatusApi {
const dailyUpdates = await this.statusRepo.getDailyEndpointUpdates(
periodDays
)
const firstUpdates = await this.statusRepo.getEndpointFirstUpdates()
const maxAgeHours = periodDays * 24
const dates = getDateRange(periodDays)
const statistics: EndpointStatistics[] = []
const statisticsByEndpoint: { [key: string]: EndpointStatistics } = {}
const successfulPodcastsByEndpointAndDate: {
[endpointIdentifier: string]: { [date: string]: Set<number> }
} = {}
const firstUpdateByEndpointAndPodcast: {
[endpointIdentifier: string]: { [podcastId: number]: string }
const expectedPodcastsByEndpoint: {
[endpointIdentifier: string]: Set<number>
} = {}

Object.entries(status).forEach(([podcastIdStr, podcastData]) => {
Expand All @@ -198,8 +192,7 @@ class StatusApi {
failedPodcasts: [],
days: dates.map((date) => ({
date,
title: `${date}: no data available`,
isUnavailable: true,
title: '',
isDegraded: false,
requests: emptyRequests(),
})),
Expand All @@ -209,6 +202,14 @@ class StatusApi {
)
}

if (!expectedPodcastsByEndpoint[endpointIdentifier]) {
expectedPodcastsByEndpoint[endpointIdentifier] =
new Set()
}
expectedPodcastsByEndpoint[endpointIdentifier].add(
podcastId
)

const endpointStatistics =
statisticsByEndpoint[endpointIdentifier]
const ageHours = getEndpointAgeHours(endpointDate)
Expand All @@ -228,20 +229,6 @@ class StatusApi {
)
})

firstUpdates.forEach((update) => {
const endpointIdentifier = getEndpointIdentifier(
update.provider,
update.endpoint
)
if (!firstUpdateByEndpointAndPodcast[endpointIdentifier]) {
firstUpdateByEndpointAndPodcast[endpointIdentifier] = {}
}

firstUpdateByEndpointAndPodcast[endpointIdentifier][
update.accountId
] = update.date
})

dailyUpdates.forEach((update) => {
const endpointIdentifier = getEndpointIdentifier(
update.provider,
Expand Down Expand Up @@ -273,23 +260,24 @@ class StatusApi {
)
const successfulPodcastsByDate =
successfulPodcastsByEndpointAndDate[endpointIdentifier] ?? {}
const firstUpdateByPodcast =
firstUpdateByEndpointAndPodcast[endpointIdentifier] ?? {}

const expectedPodcasts =
expectedPodcastsByEndpoint[endpointIdentifier] ?? new Set()

// The updates table only records successful imports. Until failed
// attempts are stored explicitly, daily failures are inferred as
// expected podcasts without a successful update for that endpoint on
// that day. Expected podcasts are all podcasts that have ever
// reported for the endpoint.
endpointStatistics.days.forEach((day) => {
day.requests.total = Object.values(firstUpdateByPodcast).filter(
(firstUpdateDate) => firstUpdateDate <= day.date
).length
day.requests.total = expectedPodcasts.size
day.requests.successful =
successfulPodcastsByDate[day.date]?.size ?? 0
day.requests.failed = Math.max(
day.requests.total - day.requests.successful,
0
)
updateSuccessRate(day.requests)
day.isUnavailable = day.requests.total === 0
day.isDegraded =
day.requests.total > 0 && day.requests.failed > 0
day.isDegraded = day.requests.failed > 0
day.title = getDayTitle(day.date, day.requests)
})
})
Expand Down
24 changes: 0 additions & 24 deletions src/db/StatusRepository.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -94,28 +94,4 @@ describe('StatusRepository', () => {
},
])
})

it('returns the first update date per podcast and endpoint', async () => {
const rows = [
{
account_id: 1,
provider: 'spotify',
endpoint: 'aggregate',
first_update_date: '2026-07-01',
},
] as RowDataPacket[]

mockPool.query.mockResolvedValue([rows, []] as any)

const updates = await statusRepository.getEndpointFirstUpdates()

expect(updates).toEqual([
{
accountId: 1,
provider: 'spotify',
endpoint: 'aggregate',
date: '2026-07-01',
},
])
})
})
30 changes: 1 addition & 29 deletions src/db/StatusRepository.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,13 +19,6 @@ type DailyEndpointUpdate = {
date: string
}

type EndpointFirstUpdate = {
accountId: number
provider: string
endpoint: string
date: string
}

class StatusRepository {
pool: Pool

Expand Down Expand Up @@ -108,27 +101,6 @@ class StatusRepository {
}))
}

async getEndpointFirstUpdates(): Promise<EndpointFirstUpdate[]> {
const query = `
SELECT
account_id,
provider,
endpoint,
DATE(MIN(created)) AS first_update_date
FROM updates
GROUP BY account_id, provider, endpoint
ORDER BY first_update_date, provider, endpoint, account_id
`
const response = (await this.pool.query(query)) as RowDataPacket[]

return response[0].map((row: RowDataPacket) => ({
accountId: row.account_id,
provider: row.provider,
endpoint: row.endpoint,
date: row.first_update_date,
}))
}

// Write the raw JSON status data into the `updates` table
// Use the current timestamp as the update time
async updateStatus(accountId: number, update: StatusPayload): Promise<any> {
Expand All @@ -142,4 +114,4 @@ class StatusRepository {
}
}

export { StatusRepository, Status, DailyEndpointUpdate, EndpointFirstUpdate }
export { StatusRepository, Status, DailyEndpointUpdate }
10 changes: 3 additions & 7 deletions views/partials/statistics_endpoint_card.hbs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,14 +10,10 @@

<div class="daily-bars" aria-label="Daily endpoint imports">
{{#each days}}
{{#if isUnavailable}}
<div class="daily-bar unavailable" title="{{title}}"></div>
{{#if isDegraded}}
<div class="daily-bar degraded" title="{{title}}"></div>
{{else}}
{{#if isDegraded}}
<div class="daily-bar degraded" title="{{title}}"></div>
{{else}}
<div class="daily-bar" title="{{title}}"></div>
{{/if}}
<div class="daily-bar" title="{{title}}"></div>
{{/if}}
{{/each}}
</div>
Expand Down
4 changes: 1 addition & 3 deletions views/statistics.hbs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -129,9 +129,7 @@
background: #d4a72c;
}

.daily-bar.unavailable {
background: #d1d5db;
}


.stats-row {
align-items: center;
Expand Down
Loading