diff --git a/docs/spooty-operations-snapshot.md b/docs/spooty-operations-snapshot.md index f112690..2816c53 100644 --- a/docs/spooty-operations-snapshot.md +++ b/docs/spooty-operations-snapshot.md @@ -9,16 +9,16 @@ The MVP reports truth Spooty already owns without a schema migration: - selected YouTube candidate title, author, score, and reason from `TrackEntity` - raw failure evidence from `TrackEntity.error` - download retry count from `TrackEntity.downloadAttemptCount` -- rejected YouTube candidates from `TrackEntity.rejectedYoutubeUrls` +- rejected YouTube candidate explanations from `TrackEntity.rejectedYoutubeCandidatesJson` +- legacy rejected YouTube candidate URLs from `TrackEntity.rejectedYoutubeUrls` - playlist and single-song counts from `PlaylistEntity.isTrack` - queue depth and active jobs from BullMQ queues `track-search-processor` and `track-download-processor` - Spotify connection presence from the stored Spotify user token -The endpoint intentionally does not infer candidate scoring history. Selected candidate metadata is persisted when Spooty chooses a candidate. Rejected candidates are still persisted only as URL strings. +The endpoint intentionally does not infer candidate scoring history. Selected candidate metadata is persisted when Spooty chooses a candidate. Rejected candidate metadata is persisted when a selected candidate is rejected by manual retry or download failure. Older rows and URL-only cases fall back to rejected candidate objects containing only `url`. Future persistence work is required for: - full searched candidate history -- structured rejected candidate metadata beyond URL - structured error class fields stored at write time instead of classified from the persisted raw error string - operation attempt history across search and download phases diff --git a/src/backend/src/operations/operations.service.ts b/src/backend/src/operations/operations.service.ts index 858e746..e7af4cf 100644 --- a/src/backend/src/operations/operations.service.ts +++ b/src/backend/src/operations/operations.service.ts @@ -10,6 +10,7 @@ import { ErrorClass, FailureTruth, OperationsSnapshot, + RejectedCandidateTruth, SpootyOperation, TrackTruth, } from './operations.types'; @@ -204,7 +205,15 @@ export class OperationsService { }; } - private parseRejectedCandidates(track: TrackEntity): string[] { + private parseRejectedCandidates( + track: TrackEntity, + ): RejectedCandidateTruth[] { + const structured = this.parseRejectedCandidateJson(track); + + if (structured.length) { + return structured; + } + if (!track.rejectedYoutubeUrls) { return []; } @@ -212,7 +221,52 @@ export class OperationsService { try { const parsed = JSON.parse(track.rejectedYoutubeUrls); return Array.isArray(parsed) - ? parsed.filter((item) => typeof item === 'string') + ? parsed + .filter((item) => typeof item === 'string') + .map((url) => ({ url })) + : []; + } catch { + return []; + } + } + + private parseRejectedCandidateJson( + track: TrackEntity, + ): RejectedCandidateTruth[] { + if (!track.rejectedYoutubeCandidatesJson) { + return []; + } + + try { + const parsed = JSON.parse(track.rejectedYoutubeCandidatesJson); + return Array.isArray(parsed) + ? parsed + .filter( + (item) => + !!item && + typeof item === 'object' && + typeof item.url === 'string', + ) + .map((item) => ({ + url: item.url, + ...(typeof item.title === 'string' ? { title: item.title } : {}), + ...(typeof item.author === 'string' + ? { author: item.author } + : {}), + ...(typeof item.score === 'number' ? { score: item.score } : {}), + ...(typeof item.reason === 'string' + ? { reason: item.reason } + : {}), + ...(typeof item.rejectionClass === 'string' + ? { rejectionClass: item.rejectionClass } + : {}), + ...(typeof item.rejectionSummary === 'string' + ? { rejectionSummary: item.rejectionSummary } + : {}), + ...(typeof item.rejectedAt === 'string' + ? { rejectedAt: item.rejectedAt } + : {}), + })) : []; } catch { return []; diff --git a/src/backend/src/operations/operations.types.ts b/src/backend/src/operations/operations.types.ts index 40932aa..2e831a6 100644 --- a/src/backend/src/operations/operations.types.ts +++ b/src/backend/src/operations/operations.types.ts @@ -16,6 +16,27 @@ export type ErrorClass = | 'FILE_WRITE_FAILED' | 'UNKNOWN_DOWNLOAD_ERROR'; +export type RejectionClass = + | 'DOWNLOAD_FAILED' + | 'YOUTUBE_VIDEO_UNAVAILABLE' + | 'YOUTUBE_AGE_GATED' + | 'YOUTUBE_NO_FORMATS' + | 'YOUTUBE_PRIVATE_VIDEO' + | 'YOUTUBE_EXTRACTION_FAILURE' + | 'UNKNOWN_DOWNLOAD_ERROR' + | 'MANUAL_RETRY'; + +export interface RejectedCandidateTruth { + url: string; + title?: string; + author?: string; + score?: number; + reason?: string; + rejectionClass?: RejectionClass; + rejectionSummary?: string; + rejectedAt?: string; +} + export interface TrackTruth { id: number; artist: string; @@ -35,7 +56,7 @@ export interface TrackTruth { errorSummary?: string; errorDetail?: string; rejectedCandidateCount: number; - rejectedCandidates: string[]; + rejectedCandidates: RejectedCandidateTruth[]; } export interface FailureTruth extends TrackTruth {} diff --git a/src/backend/src/track/track.entity.ts b/src/backend/src/track/track.entity.ts index 46a4ead..2141eb5 100644 --- a/src/backend/src/track/track.entity.ts +++ b/src/backend/src/track/track.entity.ts @@ -42,6 +42,9 @@ export class TrackEntity { @Column({ nullable: true, type: 'text' }) rejectedYoutubeUrls?: string; + @Column({ nullable: true, type: 'text' }) + rejectedYoutubeCandidatesJson?: string; + @Column({ default: 0 }) downloadAttemptCount?: number; diff --git a/src/backend/src/track/track.service.ts b/src/backend/src/track/track.service.ts index 3dea333..8a939a1 100644 --- a/src/backend/src/track/track.service.ts +++ b/src/backend/src/track/track.service.ts @@ -24,6 +24,32 @@ type ClientTrack = Omit, 'rejectedYoutubeUrls'> & { rejectedYoutubeUrls?: string[]; }; +type RejectionClass = + | 'DOWNLOAD_FAILED' + | 'YOUTUBE_VIDEO_UNAVAILABLE' + | 'YOUTUBE_AGE_GATED' + | 'YOUTUBE_NO_FORMATS' + | 'YOUTUBE_PRIVATE_VIDEO' + | 'YOUTUBE_EXTRACTION_FAILURE' + | 'UNKNOWN_DOWNLOAD_ERROR' + | 'MANUAL_RETRY'; + +interface RejectedYoutubeCandidate { + url: string; + title?: string; + author?: string; + score?: number; + reason?: string; + rejectionClass?: RejectionClass; + rejectionSummary?: string; + rejectedAt: string; +} + +interface RejectionReason { + rejectionClass: RejectionClass; + rejectionSummary: string; +} + @WebSocketGateway() @Injectable() export class TrackService { @@ -110,6 +136,161 @@ export class TrackService { return JSON.stringify([...new Set(urls)].slice(0, 20)); } + private parseRejectedYoutubeCandidates( + track: TrackEntity, + ): RejectedYoutubeCandidate[] { + if (!track.rejectedYoutubeCandidatesJson) { + return []; + } + + try { + const parsed = JSON.parse(track.rejectedYoutubeCandidatesJson); + return Array.isArray(parsed) + ? parsed.filter( + (item): item is RejectedYoutubeCandidate => + !!item && + typeof item === 'object' && + typeof item.url === 'string' && + typeof item.rejectedAt === 'string', + ) + : []; + } catch { + return []; + } + } + + private stringifyRejectedYoutubeCandidates( + candidates: RejectedYoutubeCandidate[], + ): string { + const byUrl = new Map(); + + for (const candidate of candidates) { + if (candidate.url) { + byUrl.set(candidate.url, candidate); + } + } + + return JSON.stringify([...byUrl.values()].slice(-20)); + } + + private buildRejectedYoutubeCandidate( + track: TrackEntity, + reason: RejectionReason, + ): RejectedYoutubeCandidate | null { + if (!track.youtubeUrl) { + return null; + } + + return { + url: track.youtubeUrl, + ...(track.selectedYoutubeTitle + ? { title: track.selectedYoutubeTitle } + : {}), + ...(track.selectedYoutubeAuthor + ? { author: track.selectedYoutubeAuthor } + : {}), + ...(typeof track.selectedYoutubeScore === 'number' + ? { score: track.selectedYoutubeScore } + : {}), + ...(track.selectedYoutubeReason + ? { reason: track.selectedYoutubeReason } + : {}), + ...reason, + rejectedAt: new Date().toISOString(), + }; + } + + private rejectCurrentYoutubeCandidate( + track: TrackEntity, + reason: RejectionReason, + ): { + rejectedYoutubeUrls: string; + rejectedYoutubeCandidatesJson: string; + } { + const rejectedUrls = this.parseRejectedYoutubeUrls(track); + const rejectedCandidates = this.parseRejectedYoutubeCandidates(track); + const rejectedCandidate = this.buildRejectedYoutubeCandidate(track, reason); + + if (rejectedCandidate) { + if (!rejectedUrls.includes(rejectedCandidate.url)) { + rejectedUrls.push(rejectedCandidate.url); + } + + rejectedCandidates.push(rejectedCandidate); + } + + return { + rejectedYoutubeUrls: this.stringifyRejectedYoutubeUrls(rejectedUrls), + rejectedYoutubeCandidatesJson: + this.stringifyRejectedYoutubeCandidates(rejectedCandidates), + }; + } + + private classifyRejection(error?: string): RejectionReason { + if (!error) { + return { + rejectionClass: 'UNKNOWN_DOWNLOAD_ERROR', + rejectionSummary: 'Unknown download error', + }; + } + + const normalized = error.toLowerCase(); + + if (normalized.includes('sign in to confirm your age')) { + return { + rejectionClass: 'YOUTUBE_AGE_GATED', + rejectionSummary: 'YouTube video is age-gated', + }; + } + + if (normalized.includes('private video')) { + return { + rejectionClass: 'YOUTUBE_PRIVATE_VIDEO', + rejectionSummary: 'YouTube video is private', + }; + } + + if ( + normalized.includes('only images are available') || + normalized.includes('requested format is not available') || + normalized.includes('no downloadable audio/video formats') + ) { + return { + rejectionClass: 'YOUTUBE_NO_FORMATS', + rejectionSummary: 'No downloadable YouTube audio format', + }; + } + + if ( + normalized.includes('video unavailable') || + normalized.includes('this video is not available') || + normalized.includes('selected video unavailable') + ) { + return { + rejectionClass: 'YOUTUBE_VIDEO_UNAVAILABLE', + rejectionSummary: 'YouTube video is unavailable', + }; + } + + if ( + normalized.includes('failed to parse yt-dlp search output') || + normalized.includes('yt-dlp exited with code') || + normalized.includes('failed to start yt-dlp') || + normalized.includes('yt-dlp exceeded maximum runtime') || + normalized.includes('unable to download webpage') + ) { + return { + rejectionClass: 'YOUTUBE_EXTRACTION_FAILURE', + rejectionSummary: 'YouTube extraction failed', + }; + } + + return { + rejectionClass: 'DOWNLOAD_FAILED', + rejectionSummary: 'Download failed', + }; + } + async retry(id: number): Promise { const track = await this.get(id); @@ -118,11 +299,10 @@ export class TrackService { return; } - const rejectedUrls = this.parseRejectedYoutubeUrls(track); - - if (track.youtubeUrl && !rejectedUrls.includes(track.youtubeUrl)) { - rejectedUrls.push(track.youtubeUrl); - } + const rejectedCandidateFields = this.rejectCurrentYoutubeCandidate(track, { + rejectionClass: 'MANUAL_RETRY', + rejectionSummary: 'Rejected by manual retry', + }); await this.update(id, { ...track, @@ -131,7 +311,7 @@ export class TrackService { selectedYoutubeAuthor: null, selectedYoutubeScore: null, selectedYoutubeReason: null, - rejectedYoutubeUrls: this.stringifyRejectedYoutubeUrls(rejectedUrls), + ...rejectedCandidateFields, downloadAttemptCount: 0, error: null, status: TrackStatusEnum.New, @@ -293,13 +473,17 @@ export class TrackService { error = toSafeErrorMessage(err); } - const rejectedUrls = this.parseRejectedYoutubeUrls(track); const nextAttemptCount = (track.downloadAttemptCount || 0) + 1; const maxAttempts = this.getMaxDownloadAttempts(); - - if (error && track.youtubeUrl && !rejectedUrls.includes(track.youtubeUrl)) { - rejectedUrls.push(track.youtubeUrl); - } + const rejectedCandidateFields = error + ? this.rejectCurrentYoutubeCandidate(track, this.classifyRejection(error)) + : { + rejectedYoutubeUrls: this.stringifyRejectedYoutubeUrls( + this.parseRejectedYoutubeUrls(track), + ), + rejectedYoutubeCandidatesJson: + track.rejectedYoutubeCandidatesJson || JSON.stringify([]), + }; if (error && nextAttemptCount < maxAttempts) { this.logger.warn( @@ -314,7 +498,7 @@ export class TrackService { selectedYoutubeAuthor: null, selectedYoutubeScore: null, selectedYoutubeReason: null, - rejectedYoutubeUrls: this.stringifyRejectedYoutubeUrls(rejectedUrls), + ...rejectedCandidateFields, downloadAttemptCount: nextAttemptCount, error: null, status: TrackStatusEnum.New, @@ -327,7 +511,7 @@ export class TrackService { const updatedTrack = { ...track, status: error ? TrackStatusEnum.Error : TrackStatusEnum.Completed, - rejectedYoutubeUrls: this.stringifyRejectedYoutubeUrls(rejectedUrls), + ...rejectedCandidateFields, downloadAttemptCount: error ? nextAttemptCount : 0, ...(error ? { error } : { error: null }), };