Persist rejected YouTube candidate truth

docs/spooty-operations-snapshot.md

@@ -9,16 +9,16 @@ The MVP reports truth Spooty already owns without a schema migration:
 - selected YouTube candidate title, author, score, and reason from `TrackEntity`
 - raw failure evidence from `TrackEntity.error`
 - download retry count from `TrackEntity.downloadAttemptCount`
-- rejected YouTube candidates from `TrackEntity.rejectedYoutubeUrls`
+- rejected YouTube candidate explanations from `TrackEntity.rejectedYoutubeCandidatesJson`
+- legacy rejected YouTube candidate URLs from `TrackEntity.rejectedYoutubeUrls`
 - playlist and single-song counts from `PlaylistEntity.isTrack`
 - queue depth and active jobs from BullMQ queues `track-search-processor` and `track-download-processor`
 - Spotify connection presence from the stored Spotify user token
 
-The endpoint intentionally does not infer candidate scoring history. Selected candidate metadata is persisted when Spooty chooses a candidate. Rejected candidates are still persisted only as URL strings.
+The endpoint intentionally does not infer candidate scoring history. Selected candidate metadata is persisted when Spooty chooses a candidate. Rejected candidate metadata is persisted when a selected candidate is rejected by manual retry or download failure. Older rows and URL-only cases fall back to rejected candidate objects containing only `url`.
 
 Future persistence work is required for:
 
 - full searched candidate history
-- structured rejected candidate metadata beyond URL
 - structured error class fields stored at write time instead of classified from the persisted raw error string
 - operation attempt history across search and download phases

src/backend/src/operations/operations.service.ts

@@ -10,6 +10,7 @@ import {
   ErrorClass,
   FailureTruth,
   OperationsSnapshot,
+  RejectedCandidateTruth,
   SpootyOperation,
   TrackTruth,
 } from './operations.types';
@@ -204,7 +205,15 @@ export class OperationsService {
     };
   }
 
-  private parseRejectedCandidates(track: TrackEntity): string[] {
+  private parseRejectedCandidates(
+    track: TrackEntity,
+  ): RejectedCandidateTruth[] {
+    const structured = this.parseRejectedCandidateJson(track);
+
+    if (structured.length) {
+      return structured;
+    }
+
     if (!track.rejectedYoutubeUrls) {
       return [];
     }
@@ -212,7 +221,52 @@ export class OperationsService {
     try {
       const parsed = JSON.parse(track.rejectedYoutubeUrls);
       return Array.isArray(parsed)
-        ? parsed.filter((item) => typeof item === 'string')
+        ? parsed
+            .filter((item) => typeof item === 'string')
+            .map((url) => ({ url }))
+        : [];
+    } catch {
+      return [];
+    }
+  }
+
+  private parseRejectedCandidateJson(
+    track: TrackEntity,
+  ): RejectedCandidateTruth[] {
+    if (!track.rejectedYoutubeCandidatesJson) {
+      return [];
+    }
+
+    try {
+      const parsed = JSON.parse(track.rejectedYoutubeCandidatesJson);
+      return Array.isArray(parsed)
+        ? parsed
+            .filter(
+              (item) =>
+                !!item &&
+                typeof item === 'object' &&
+                typeof item.url === 'string',
+            )
+            .map((item) => ({
+              url: item.url,
+              ...(typeof item.title === 'string' ? { title: item.title } : {}),
+              ...(typeof item.author === 'string'
+                ? { author: item.author }
+                : {}),
+              ...(typeof item.score === 'number' ? { score: item.score } : {}),
+              ...(typeof item.reason === 'string'
+                ? { reason: item.reason }
+                : {}),
+              ...(typeof item.rejectionClass === 'string'
+                ? { rejectionClass: item.rejectionClass }
+                : {}),
+              ...(typeof item.rejectionSummary === 'string'
+                ? { rejectionSummary: item.rejectionSummary }
+                : {}),
+              ...(typeof item.rejectedAt === 'string'
+                ? { rejectedAt: item.rejectedAt }
+                : {}),
+            }))
         : [];
     } catch {
       return [];

src/backend/src/operations/operations.types.ts

@@ -16,6 +16,27 @@ export type ErrorClass =
   | 'FILE_WRITE_FAILED'
   | 'UNKNOWN_DOWNLOAD_ERROR';
 
+export type RejectionClass =
+  | 'DOWNLOAD_FAILED'
+  | 'YOUTUBE_VIDEO_UNAVAILABLE'
+  | 'YOUTUBE_AGE_GATED'
+  | 'YOUTUBE_NO_FORMATS'
+  | 'YOUTUBE_PRIVATE_VIDEO'
+  | 'YOUTUBE_EXTRACTION_FAILURE'
+  | 'UNKNOWN_DOWNLOAD_ERROR'
+  | 'MANUAL_RETRY';
+
+export interface RejectedCandidateTruth {
+  url: string;
+  title?: string;
+  author?: string;
+  score?: number;
+  reason?: string;
+  rejectionClass?: RejectionClass;
+  rejectionSummary?: string;
+  rejectedAt?: string;
+}
+
 export interface TrackTruth {
   id: number;
   artist: string;
@@ -35,7 +56,7 @@ export interface TrackTruth {
   errorSummary?: string;
   errorDetail?: string;
   rejectedCandidateCount: number;
-  rejectedCandidates: string[];
+  rejectedCandidates: RejectedCandidateTruth[];
 }
 
 export interface FailureTruth extends TrackTruth {}

src/backend/src/track/track.entity.ts

@@ -42,6 +42,9 @@ export class TrackEntity {
   @Column({ nullable: true, type: 'text' })
   rejectedYoutubeUrls?: string;
 
+  @Column({ nullable: true, type: 'text' })
+  rejectedYoutubeCandidatesJson?: string;
+
   @Column({ default: 0 })
   downloadAttemptCount?: number;
 

src/backend/src/track/track.service.ts

@@ -24,6 +24,32 @@ type ClientTrack = Omit<Partial<TrackEntity>, 'rejectedYoutubeUrls'> & {
   rejectedYoutubeUrls?: string[];
 };
 
+type RejectionClass =
+  | 'DOWNLOAD_FAILED'
+  | 'YOUTUBE_VIDEO_UNAVAILABLE'
+  | 'YOUTUBE_AGE_GATED'
+  | 'YOUTUBE_NO_FORMATS'
+  | 'YOUTUBE_PRIVATE_VIDEO'
+  | 'YOUTUBE_EXTRACTION_FAILURE'
+  | 'UNKNOWN_DOWNLOAD_ERROR'
+  | 'MANUAL_RETRY';
+
+interface RejectedYoutubeCandidate {
+  url: string;
+  title?: string;
+  author?: string;
+  score?: number;
+  reason?: string;
+  rejectionClass?: RejectionClass;
+  rejectionSummary?: string;
+  rejectedAt: string;
+}
+
+interface RejectionReason {
+  rejectionClass: RejectionClass;
+  rejectionSummary: string;
+}
+
 @WebSocketGateway()
 @Injectable()
 export class TrackService {
@@ -110,6 +136,161 @@ export class TrackService {
     return JSON.stringify([...new Set(urls)].slice(0, 20));
   }
 
+  private parseRejectedYoutubeCandidates(
+    track: TrackEntity,
+  ): RejectedYoutubeCandidate[] {
+    if (!track.rejectedYoutubeCandidatesJson) {
+      return [];
+    }
+
+    try {
+      const parsed = JSON.parse(track.rejectedYoutubeCandidatesJson);
+      return Array.isArray(parsed)
+        ? parsed.filter(
+            (item): item is RejectedYoutubeCandidate =>
+              !!item &&
+              typeof item === 'object' &&
+              typeof item.url === 'string' &&
+              typeof item.rejectedAt === 'string',
+          )
+        : [];
+    } catch {
+      return [];
+    }
+  }
+
+  private stringifyRejectedYoutubeCandidates(
+    candidates: RejectedYoutubeCandidate[],
+  ): string {
+    const byUrl = new Map<string, RejectedYoutubeCandidate>();
+
+    for (const candidate of candidates) {
+      if (candidate.url) {
+        byUrl.set(candidate.url, candidate);
+      }
+    }
+
+    return JSON.stringify([...byUrl.values()].slice(-20));
+  }
+
+  private buildRejectedYoutubeCandidate(
+    track: TrackEntity,
+    reason: RejectionReason,
+  ): RejectedYoutubeCandidate | null {
+    if (!track.youtubeUrl) {
+      return null;
+    }
+
+    return {
+      url: track.youtubeUrl,
+      ...(track.selectedYoutubeTitle
+        ? { title: track.selectedYoutubeTitle }
+        : {}),
+      ...(track.selectedYoutubeAuthor
+        ? { author: track.selectedYoutubeAuthor }
+        : {}),
+      ...(typeof track.selectedYoutubeScore === 'number'
+        ? { score: track.selectedYoutubeScore }
+        : {}),
+      ...(track.selectedYoutubeReason
+        ? { reason: track.selectedYoutubeReason }
+        : {}),
+      ...reason,
+      rejectedAt: new Date().toISOString(),
+    };
+  }
+
+  private rejectCurrentYoutubeCandidate(
+    track: TrackEntity,
+    reason: RejectionReason,
+  ): {
+    rejectedYoutubeUrls: string;
+    rejectedYoutubeCandidatesJson: string;
+  } {
+    const rejectedUrls = this.parseRejectedYoutubeUrls(track);
+    const rejectedCandidates = this.parseRejectedYoutubeCandidates(track);
+    const rejectedCandidate = this.buildRejectedYoutubeCandidate(track, reason);
+
+    if (rejectedCandidate) {
+      if (!rejectedUrls.includes(rejectedCandidate.url)) {
+        rejectedUrls.push(rejectedCandidate.url);
+      }
+
+      rejectedCandidates.push(rejectedCandidate);
+    }
+
+    return {
+      rejectedYoutubeUrls: this.stringifyRejectedYoutubeUrls(rejectedUrls),
+      rejectedYoutubeCandidatesJson:
+        this.stringifyRejectedYoutubeCandidates(rejectedCandidates),
+    };
+  }
+
+  private classifyRejection(error?: string): RejectionReason {
+    if (!error) {
+      return {
+        rejectionClass: 'UNKNOWN_DOWNLOAD_ERROR',
+        rejectionSummary: 'Unknown download error',
+      };
+    }
+
+    const normalized = error.toLowerCase();
+
+    if (normalized.includes('sign in to confirm your age')) {
+      return {
+        rejectionClass: 'YOUTUBE_AGE_GATED',
+        rejectionSummary: 'YouTube video is age-gated',
+      };
+    }
+
+    if (normalized.includes('private video')) {
+      return {
+        rejectionClass: 'YOUTUBE_PRIVATE_VIDEO',
+        rejectionSummary: 'YouTube video is private',
+      };
+    }
+
+    if (
+      normalized.includes('only images are available') ||
+      normalized.includes('requested format is not available') ||
+      normalized.includes('no downloadable audio/video formats')
+    ) {
+      return {
+        rejectionClass: 'YOUTUBE_NO_FORMATS',
+        rejectionSummary: 'No downloadable YouTube audio format',
+      };
+    }
+
+    if (
+      normalized.includes('video unavailable') ||
+      normalized.includes('this video is not available') ||
+      normalized.includes('selected video unavailable')
+    ) {
+      return {
+        rejectionClass: 'YOUTUBE_VIDEO_UNAVAILABLE',
+        rejectionSummary: 'YouTube video is unavailable',
+      };
+    }
+
+    if (
+      normalized.includes('failed to parse yt-dlp search output') ||
+      normalized.includes('yt-dlp exited with code') ||
+      normalized.includes('failed to start yt-dlp') ||
+      normalized.includes('yt-dlp exceeded maximum runtime') ||
+      normalized.includes('unable to download webpage')
+    ) {
+      return {
+        rejectionClass: 'YOUTUBE_EXTRACTION_FAILURE',
+        rejectionSummary: 'YouTube extraction failed',
+      };
+    }
+
+    return {
+      rejectionClass: 'DOWNLOAD_FAILED',
+      rejectionSummary: 'Download failed',
+    };
+  }
+
   async retry(id: number): Promise<void> {
     const track = await this.get(id);
 
@@ -118,11 +299,10 @@ export class TrackService {
       return;
     }
 
-    const rejectedUrls = this.parseRejectedYoutubeUrls(track);
-
-    if (track.youtubeUrl && !rejectedUrls.includes(track.youtubeUrl)) {
-      rejectedUrls.push(track.youtubeUrl);
-    }
+    const rejectedCandidateFields = this.rejectCurrentYoutubeCandidate(track, {
+      rejectionClass: 'MANUAL_RETRY',
+      rejectionSummary: 'Rejected by manual retry',
+    });
 
     await this.update(id, {
       ...track,
@@ -131,7 +311,7 @@ export class TrackService {
       selectedYoutubeAuthor: null,
       selectedYoutubeScore: null,
       selectedYoutubeReason: null,
-      rejectedYoutubeUrls: this.stringifyRejectedYoutubeUrls(rejectedUrls),
+      ...rejectedCandidateFields,
       downloadAttemptCount: 0,
       error: null,
       status: TrackStatusEnum.New,
@@ -293,13 +473,17 @@ export class TrackService {
       error = toSafeErrorMessage(err);
     }
 
-    const rejectedUrls = this.parseRejectedYoutubeUrls(track);
     const nextAttemptCount = (track.downloadAttemptCount || 0) + 1;
     const maxAttempts = this.getMaxDownloadAttempts();
-
-    if (error && track.youtubeUrl && !rejectedUrls.includes(track.youtubeUrl)) {
-      rejectedUrls.push(track.youtubeUrl);
-    }
+    const rejectedCandidateFields = error
+      ? this.rejectCurrentYoutubeCandidate(track, this.classifyRejection(error))
+      : {
+          rejectedYoutubeUrls: this.stringifyRejectedYoutubeUrls(
+            this.parseRejectedYoutubeUrls(track),
+          ),
+          rejectedYoutubeCandidatesJson:
+            track.rejectedYoutubeCandidatesJson || JSON.stringify([]),
+        };
 
     if (error && nextAttemptCount < maxAttempts) {
       this.logger.warn(
@@ -314,7 +498,7 @@ export class TrackService {
         selectedYoutubeAuthor: null,
         selectedYoutubeScore: null,
         selectedYoutubeReason: null,
-        rejectedYoutubeUrls: this.stringifyRejectedYoutubeUrls(rejectedUrls),
+        ...rejectedCandidateFields,
         downloadAttemptCount: nextAttemptCount,
         error: null,
         status: TrackStatusEnum.New,
@@ -327,7 +511,7 @@ export class TrackService {
     const updatedTrack = {
       ...track,
       status: error ? TrackStatusEnum.Error : TrackStatusEnum.Completed,
-      rejectedYoutubeUrls: this.stringifyRejectedYoutubeUrls(rejectedUrls),
+      ...rejectedCandidateFields,
       downloadAttemptCount: error ? nextAttemptCount : 0,
       ...(error ? { error } : { error: null }),
     };