Persist rejected YouTube candidate truth

docs/spooty-operations-snapshot.md

@@ -6,19 +6,19 @@ The MVP reports truth Spooty already owns without a schema migration:
 
 - track state from `TrackEntity.status`
 - selected YouTube URL from `TrackEntity.youtubeUrl`
+- selected YouTube candidate title, author, score, and reason from `TrackEntity`
 - raw failure evidence from `TrackEntity.error`
 - download retry count from `TrackEntity.downloadAttemptCount`
-- rejected YouTube candidates from `TrackEntity.rejectedYoutubeUrls`
+- rejected YouTube candidate explanations from `TrackEntity.rejectedYoutubeCandidatesJson`
+- legacy rejected YouTube candidate URLs from `TrackEntity.rejectedYoutubeUrls`
 - playlist and single-song counts from `PlaylistEntity.isTrack`
 - queue depth and active jobs from BullMQ queues `track-search-processor` and `track-download-processor`
 - Spotify connection presence from the stored Spotify user token
 
-The endpoint intentionally does not infer candidate scoring history. Today, selected candidate title, author, score, and reason are logged during search but are not persisted. Rejected candidates are persisted only as URL strings.
+The endpoint intentionally does not infer candidate scoring history. Selected candidate metadata is persisted when Spooty chooses a candidate. Rejected candidate metadata is persisted when a selected candidate is rejected by manual retry or download failure. Older rows and URL-only cases fall back to rejected candidate objects containing only `url`.
 
 Future persistence work is required for:
 
-- selected candidate title, author, score, and scoring reason
 - full searched candidate history
-- structured rejected candidate metadata beyond URL
 - structured error class fields stored at write time instead of classified from the persisted raw error string
 - operation attempt history across search and download phases

src/backend/src/operations/operations.service.ts

@@ -10,6 +10,7 @@ import {
   ErrorClass,
   FailureTruth,
   OperationsSnapshot,
+  RejectedCandidateTruth,
   SpootyOperation,
   TrackTruth,
 } from './operations.types';
@@ -176,6 +177,25 @@ export class OperationsService {
       status: TrackStatusEnum[track.status] || String(track.status),
       ...(track.spotifyUrl ? { spotifyUrl: track.spotifyUrl } : {}),
       ...(track.youtubeUrl ? { youtubeUrl: track.youtubeUrl } : {}),
+      ...(track.youtubeUrl
+        ? {
+            selectedCandidate: {
+              url: track.youtubeUrl,
+              ...(track.selectedYoutubeTitle
+                ? { title: track.selectedYoutubeTitle }
+                : {}),
+              ...(track.selectedYoutubeAuthor
+                ? { author: track.selectedYoutubeAuthor }
+                : {}),
+              ...(typeof track.selectedYoutubeScore === 'number'
+                ? { score: track.selectedYoutubeScore }
+                : {}),
+              ...(track.selectedYoutubeReason
+                ? { reason: track.selectedYoutubeReason }
+                : {}),
+            },
+          }
+        : {}),
       ...(typeof track.downloadAttemptCount === 'number'
         ? { downloadAttemptCount: track.downloadAttemptCount }
         : {}),
@@ -185,7 +205,15 @@ export class OperationsService {
     };
   }
 
-  private parseRejectedCandidates(track: TrackEntity): string[] {
+  private parseRejectedCandidates(
+    track: TrackEntity,
+  ): RejectedCandidateTruth[] {
+    const structured = this.parseRejectedCandidateJson(track);
+
+    if (structured.length) {
+      return structured;
+    }
+
     if (!track.rejectedYoutubeUrls) {
       return [];
     }
@@ -193,7 +221,52 @@ export class OperationsService {
     try {
       const parsed = JSON.parse(track.rejectedYoutubeUrls);
       return Array.isArray(parsed)
-        ? parsed.filter((item) => typeof item === 'string')
+        ? parsed
+            .filter((item) => typeof item === 'string')
+            .map((url) => ({ url }))
+        : [];
+    } catch {
+      return [];
+    }
+  }
+
+  private parseRejectedCandidateJson(
+    track: TrackEntity,
+  ): RejectedCandidateTruth[] {
+    if (!track.rejectedYoutubeCandidatesJson) {
+      return [];
+    }
+
+    try {
+      const parsed = JSON.parse(track.rejectedYoutubeCandidatesJson);
+      return Array.isArray(parsed)
+        ? parsed
+            .filter(
+              (item) =>
+                !!item &&
+                typeof item === 'object' &&
+                typeof item.url === 'string',
+            )
+            .map((item) => ({
+              url: item.url,
+              ...(typeof item.title === 'string' ? { title: item.title } : {}),
+              ...(typeof item.author === 'string'
+                ? { author: item.author }
+                : {}),
+              ...(typeof item.score === 'number' ? { score: item.score } : {}),
+              ...(typeof item.reason === 'string'
+                ? { reason: item.reason }
+                : {}),
+              ...(typeof item.rejectionClass === 'string'
+                ? { rejectionClass: item.rejectionClass }
+                : {}),
+              ...(typeof item.rejectionSummary === 'string'
+                ? { rejectionSummary: item.rejectionSummary }
+                : {}),
+              ...(typeof item.rejectedAt === 'string'
+                ? { rejectedAt: item.rejectedAt }
+                : {}),
+            }))
         : [];
     } catch {
       return [];

src/backend/src/operations/operations.types.ts

@@ -16,6 +16,27 @@ export type ErrorClass =
   | 'FILE_WRITE_FAILED'
   | 'UNKNOWN_DOWNLOAD_ERROR';
 
+export type RejectionClass =
+  | 'DOWNLOAD_FAILED'
+  | 'YOUTUBE_VIDEO_UNAVAILABLE'
+  | 'YOUTUBE_AGE_GATED'
+  | 'YOUTUBE_NO_FORMATS'
+  | 'YOUTUBE_PRIVATE_VIDEO'
+  | 'YOUTUBE_EXTRACTION_FAILURE'
+  | 'UNKNOWN_DOWNLOAD_ERROR'
+  | 'MANUAL_RETRY';
+
+export interface RejectedCandidateTruth {
+  url: string;
+  title?: string;
+  author?: string;
+  score?: number;
+  reason?: string;
+  rejectionClass?: RejectionClass;
+  rejectionSummary?: string;
+  rejectedAt?: string;
+}
+
 export interface TrackTruth {
   id: number;
   artist: string;
@@ -23,12 +44,19 @@ export interface TrackTruth {
   status: string;
   spotifyUrl?: string;
   youtubeUrl?: string;
+  selectedCandidate?: {
+    url: string;
+    title?: string;
+    author?: string;
+    score?: number;
+    reason?: string;
+  };
   downloadAttemptCount?: number;
   errorClass?: ErrorClass;
   errorSummary?: string;
   errorDetail?: string;
   rejectedCandidateCount: number;
-  rejectedCandidates: string[];
+  rejectedCandidates: RejectedCandidateTruth[];
 }
 
 export interface FailureTruth extends TrackTruth {}

src/backend/src/track/track.entity.ts

@@ -27,9 +27,24 @@ export class TrackEntity {
   @Column({ nullable: true })
   youtubeUrl?: string;
 
+  @Column({ nullable: true })
+  selectedYoutubeTitle?: string;
+
+  @Column({ nullable: true })
+  selectedYoutubeAuthor?: string;
+
+  @Column({ nullable: true })
+  selectedYoutubeScore?: number;
+
+  @Column({ nullable: true, type: 'text' })
+  selectedYoutubeReason?: string;
+
   @Column({ nullable: true, type: 'text' })
   rejectedYoutubeUrls?: string;
 
+  @Column({ nullable: true, type: 'text' })
+  rejectedYoutubeCandidatesJson?: string;
+
   @Column({ default: 0 })
   downloadAttemptCount?: number;
 

src/backend/src/track/track.service.ts

@@ -24,6 +24,32 @@ type ClientTrack = Omit<Partial<TrackEntity>, 'rejectedYoutubeUrls'> & {
   rejectedYoutubeUrls?: string[];
 };
 
+type RejectionClass =
+  | 'DOWNLOAD_FAILED'
+  | 'YOUTUBE_VIDEO_UNAVAILABLE'
+  | 'YOUTUBE_AGE_GATED'
+  | 'YOUTUBE_NO_FORMATS'
+  | 'YOUTUBE_PRIVATE_VIDEO'
+  | 'YOUTUBE_EXTRACTION_FAILURE'
+  | 'UNKNOWN_DOWNLOAD_ERROR'
+  | 'MANUAL_RETRY';
+
+interface RejectedYoutubeCandidate {
+  url: string;
+  title?: string;
+  author?: string;
+  score?: number;
+  reason?: string;
+  rejectionClass?: RejectionClass;
+  rejectionSummary?: string;
+  rejectedAt: string;
+}
+
+interface RejectionReason {
+  rejectionClass: RejectionClass;
+  rejectionSummary: string;
+}
+
 @WebSocketGateway()
 @Injectable()
 export class TrackService {
@@ -98,7 +124,9 @@ export class TrackService {
 
     try {
       const parsed = JSON.parse(track.rejectedYoutubeUrls);
-      return Array.isArray(parsed) ? parsed.filter((item) => typeof item === 'string') : [];
+      return Array.isArray(parsed)
+        ? parsed.filter((item) => typeof item === 'string')
+        : [];
     } catch {
       return [];
     }
@@ -108,6 +136,161 @@ export class TrackService {
     return JSON.stringify([...new Set(urls)].slice(0, 20));
   }
 
+  private parseRejectedYoutubeCandidates(
+    track: TrackEntity,
+  ): RejectedYoutubeCandidate[] {
+    if (!track.rejectedYoutubeCandidatesJson) {
+      return [];
+    }
+
+    try {
+      const parsed = JSON.parse(track.rejectedYoutubeCandidatesJson);
+      return Array.isArray(parsed)
+        ? parsed.filter(
+            (item): item is RejectedYoutubeCandidate =>
+              !!item &&
+              typeof item === 'object' &&
+              typeof item.url === 'string' &&
+              typeof item.rejectedAt === 'string',
+          )
+        : [];
+    } catch {
+      return [];
+    }
+  }
+
+  private stringifyRejectedYoutubeCandidates(
+    candidates: RejectedYoutubeCandidate[],
+  ): string {
+    const byUrl = new Map<string, RejectedYoutubeCandidate>();
+
+    for (const candidate of candidates) {
+      if (candidate.url) {
+        byUrl.set(candidate.url, candidate);
+      }
+    }
+
+    return JSON.stringify([...byUrl.values()].slice(-20));
+  }
+
+  private buildRejectedYoutubeCandidate(
+    track: TrackEntity,
+    reason: RejectionReason,
+  ): RejectedYoutubeCandidate | null {
+    if (!track.youtubeUrl) {
+      return null;
+    }
+
+    return {
+      url: track.youtubeUrl,
+      ...(track.selectedYoutubeTitle
+        ? { title: track.selectedYoutubeTitle }
+        : {}),
+      ...(track.selectedYoutubeAuthor
+        ? { author: track.selectedYoutubeAuthor }
+        : {}),
+      ...(typeof track.selectedYoutubeScore === 'number'
+        ? { score: track.selectedYoutubeScore }
+        : {}),
+      ...(track.selectedYoutubeReason
+        ? { reason: track.selectedYoutubeReason }
+        : {}),
+      ...reason,
+      rejectedAt: new Date().toISOString(),
+    };
+  }
+
+  private rejectCurrentYoutubeCandidate(
+    track: TrackEntity,
+    reason: RejectionReason,
+  ): {
+    rejectedYoutubeUrls: string;
+    rejectedYoutubeCandidatesJson: string;
+  } {
+    const rejectedUrls = this.parseRejectedYoutubeUrls(track);
+    const rejectedCandidates = this.parseRejectedYoutubeCandidates(track);
+    const rejectedCandidate = this.buildRejectedYoutubeCandidate(track, reason);
+
+    if (rejectedCandidate) {
+      if (!rejectedUrls.includes(rejectedCandidate.url)) {
+        rejectedUrls.push(rejectedCandidate.url);
+      }
+
+      rejectedCandidates.push(rejectedCandidate);
+    }
+
+    return {
+      rejectedYoutubeUrls: this.stringifyRejectedYoutubeUrls(rejectedUrls),
+      rejectedYoutubeCandidatesJson:
+        this.stringifyRejectedYoutubeCandidates(rejectedCandidates),
+    };
+  }
+
+  private classifyRejection(error?: string): RejectionReason {
+    if (!error) {
+      return {
+        rejectionClass: 'UNKNOWN_DOWNLOAD_ERROR',
+        rejectionSummary: 'Unknown download error',
+      };
+    }
+
+    const normalized = error.toLowerCase();
+
+    if (normalized.includes('sign in to confirm your age')) {
+      return {
+        rejectionClass: 'YOUTUBE_AGE_GATED',
+        rejectionSummary: 'YouTube video is age-gated',
+      };
+    }
+
+    if (normalized.includes('private video')) {
+      return {
+        rejectionClass: 'YOUTUBE_PRIVATE_VIDEO',
+        rejectionSummary: 'YouTube video is private',
+      };
+    }
+
+    if (
+      normalized.includes('only images are available') ||
+      normalized.includes('requested format is not available') ||
+      normalized.includes('no downloadable audio/video formats')
+    ) {
+      return {
+        rejectionClass: 'YOUTUBE_NO_FORMATS',
+        rejectionSummary: 'No downloadable YouTube audio format',
+      };
+    }
+
+    if (
+      normalized.includes('video unavailable') ||
+      normalized.includes('this video is not available') ||
+      normalized.includes('selected video unavailable')
+    ) {
+      return {
+        rejectionClass: 'YOUTUBE_VIDEO_UNAVAILABLE',
+        rejectionSummary: 'YouTube video is unavailable',
+      };
+    }
+
+    if (
+      normalized.includes('failed to parse yt-dlp search output') ||
+      normalized.includes('yt-dlp exited with code') ||
+      normalized.includes('failed to start yt-dlp') ||
+      normalized.includes('yt-dlp exceeded maximum runtime') ||
+      normalized.includes('unable to download webpage')
+    ) {
+      return {
+        rejectionClass: 'YOUTUBE_EXTRACTION_FAILURE',
+        rejectionSummary: 'YouTube extraction failed',
+      };
+    }
+
+    return {
+      rejectionClass: 'DOWNLOAD_FAILED',
+      rejectionSummary: 'Download failed',
+    };
+  }
+
   async retry(id: number): Promise<void> {
     const track = await this.get(id);
 
@@ -116,16 +299,19 @@ export class TrackService {
       return;
     }
 
-    const rejectedUrls = this.parseRejectedYoutubeUrls(track);
+    const rejectedCandidateFields = this.rejectCurrentYoutubeCandidate(track, {
-
+      rejectionClass: 'MANUAL_RETRY',
-    if (track.youtubeUrl && !rejectedUrls.includes(track.youtubeUrl)) {
+      rejectionSummary: 'Rejected by manual retry',
-      rejectedUrls.push(track.youtubeUrl);
+    });
-    }
 
     await this.update(id, {
       ...track,
       youtubeUrl: null,
-      rejectedYoutubeUrls: this.stringifyRejectedYoutubeUrls(rejectedUrls),
+      selectedYoutubeTitle: null,
+      selectedYoutubeAuthor: null,
+      selectedYoutubeScore: null,
+      selectedYoutubeReason: null,
+      ...rejectedCandidateFields,
       downloadAttemptCount: 0,
       error: null,
       status: TrackStatusEnum.New,
@@ -161,6 +347,10 @@ export class TrackService {
       updatedTrack = {
         ...track,
         youtubeUrl: youtubeMatch.url,
+        selectedYoutubeTitle: youtubeMatch.title,
+        selectedYoutubeAuthor: youtubeMatch.author,
+        selectedYoutubeScore: youtubeMatch.score,
+        selectedYoutubeReason: youtubeMatch.reason,
         status: TrackStatusEnum.Queued,
       };
       this.logger.debug(
@@ -193,7 +383,9 @@ export class TrackService {
     const existingJob = await this.trackSearchQueue.getJob(jobId);
 
     if (existingJob) {
-      this.logger.warn(`Removing existing search job for track ${id} before enqueue`);
+      this.logger.warn(
+        `Removing existing search job for track ${id} before enqueue`,
+      );
       await existingJob.remove();
     }
 
@@ -212,7 +404,9 @@ export class TrackService {
     const existingJob = await this.trackDownloadQueue.getJob(jobId);
 
     if (existingJob) {
-      this.logger.warn(`Removing existing download job for track ${id} before enqueue`);
+      this.logger.warn(
+        `Removing existing download job for track ${id} before enqueue`,
+      );
       await existingJob.remove();
     }
 
@@ -279,13 +473,17 @@ export class TrackService {
       error = toSafeErrorMessage(err);
     }
 
-    const rejectedUrls = this.parseRejectedYoutubeUrls(track);
     const nextAttemptCount = (track.downloadAttemptCount || 0) + 1;
     const maxAttempts = this.getMaxDownloadAttempts();
-
+    const rejectedCandidateFields = error
-    if (error && track.youtubeUrl && !rejectedUrls.includes(track.youtubeUrl)) {
+      ? this.rejectCurrentYoutubeCandidate(track, this.classifyRejection(error))
-      rejectedUrls.push(track.youtubeUrl);
+      : {
-    }
+          rejectedYoutubeUrls: this.stringifyRejectedYoutubeUrls(
+            this.parseRejectedYoutubeUrls(track),
+          ),
+          rejectedYoutubeCandidatesJson:
+            track.rejectedYoutubeCandidatesJson || JSON.stringify([]),
+        };
 
     if (error && nextAttemptCount < maxAttempts) {
       this.logger.warn(
@@ -296,7 +494,11 @@ export class TrackService {
       await this.update(track.id, {
         ...track,
         youtubeUrl: null,
-        rejectedYoutubeUrls: this.stringifyRejectedYoutubeUrls(rejectedUrls),
+        selectedYoutubeTitle: null,
+        selectedYoutubeAuthor: null,
+        selectedYoutubeScore: null,
+        selectedYoutubeReason: null,
+        ...rejectedCandidateFields,
         downloadAttemptCount: nextAttemptCount,
         error: null,
         status: TrackStatusEnum.New,
@@ -309,7 +511,7 @@ export class TrackService {
     const updatedTrack = {
       ...track,
       status: error ? TrackStatusEnum.Error : TrackStatusEnum.Completed,
-      rejectedYoutubeUrls: this.stringifyRejectedYoutubeUrls(rejectedUrls),
+      ...rejectedCandidateFields,
       downloadAttemptCount: error ? nextAttemptCount : 0,
       ...(error ? { error } : { error: null }),
     };
@@ -320,7 +522,8 @@ export class TrackService {
     const safeArtist = track.artist || 'unknown_artist';
     const safeName = track.name || 'unknown_track';
     const fileName = `${safeArtist} - ${safeName}`;
-    const format = this.configService.get<string>(EnvironmentEnum.FORMAT) || 'mp3';
+    const format =
+      this.configService.get<string>(EnvironmentEnum.FORMAT) || 'mp3';
     return `${this.utilsService.stripFileIllegalChars(fileName)}.${format}`;
   }