Use direct yt-dlp CLI with deterministic fallback handling

src/backend/src/playlist/playlist.service.ts

@@ -51,9 +51,12 @@ export class PlaylistService {
   async create(playlist: PlaylistEntity): Promise<void> {
     // Detect if URL is for a single track or a playlist and route accordingly
     const isTrack = this.spotifyService.isTrackUrl(playlist.spotifyUrl);
+    const isArtist = this.spotifyService.isArtistUrl(playlist.spotifyUrl);
 
     if (isTrack) {
       await this.createSingleTrack(playlist);
+    } else if (isArtist) {
+      await this.createArtistLibrary(playlist);
     } else {
       await this.createPlaylist(playlist);
     }
@@ -105,6 +108,107 @@ export class PlaylistService {
     }
   }
 
+
+  private async createArtistLibrary(playlist: PlaylistEntity): Promise<void> {
+    let detail: { tracks: any[]; name: string; image: string };
+    let playlist2Save: PlaylistEntity;
+
+    try {
+      detail = await this.spotifyService.getArtistLibrary(playlist.spotifyUrl);
+      this.logger.debug(
+        `Artist library retrieved with ${detail.tracks?.length || 0} tracks`,
+      );
+
+      playlist2Save = {
+        ...playlist,
+        name: detail.name,
+        coverUrl: detail.image,
+        active: false,
+      };
+      this.createPlaylistFolderStructure(playlist2Save.name);
+    } catch (err) {
+      this.logger.error(`Error getting artist library details: ${err}`);
+      playlist2Save = {
+        ...playlist,
+        name: 'Failed artist library import',
+        error: toSafeErrorMessage(err),
+        active: false,
+      };
+    }
+
+    const savedPlaylist = await this.save(playlist2Save);
+
+    if (detail?.tracks && detail.tracks.length > 0) {
+      await this.createTracksForSavedPlaylist(detail.tracks, savedPlaylist);
+    } else {
+      this.logger.warn(`No tracks found for artist library ${savedPlaylist.name}`);
+    }
+  }
+
+  private async createTracksForSavedPlaylist(
+    tracks: any[],
+    savedPlaylist: PlaylistEntity,
+  ): Promise<void> {
+    this.logger.debug(
+      `Starting to process ${tracks.length} tracks for ${savedPlaylist.name}`,
+    );
+
+    let processedCount = 0;
+    let skippedCount = 0;
+    let errorCount = 0;
+
+    for (const track of tracks) {
+      try {
+        if (!track.artist || !track.name) {
+          this.logger.warn(
+            `Skipping track ${processedCount + skippedCount + 1}: Missing artist or name information`,
+          );
+          skippedCount++;
+          continue;
+        }
+
+        if (track.unavailable === true) {
+          this.logger.warn(
+            `Skipping unavailable track ${processedCount + skippedCount + 1}: ${track.artist} - ${track.name}`,
+          );
+          skippedCount++;
+          continue;
+        }
+
+        await this.trackService.create(
+          {
+            artist: track.artist,
+            name: track.name,
+            spotifyUrl: track.previewUrl || null,
+            coverUrl: track.coverUrl || savedPlaylist.coverUrl,
+            durationMs: track.durationMs,
+          },
+          savedPlaylist,
+        );
+
+        processedCount++;
+
+        if (processedCount % 100 === 0) {
+          this.logger.debug(
+            `Processed ${processedCount} tracks so far for ${savedPlaylist.name}`,
+          );
+        }
+      } catch (error) {
+        this.logger.error(
+          `Error creating track "${
+            track?.artist || 'Unknown'
+          } - ${track?.name || 'Unknown'}": ${error.message}`,
+        );
+        errorCount++;
+      }
+    }
+
+    this.logger.debug(
+      `Finished processing ${savedPlaylist.name}: ` +
+        `${processedCount} tracks processed, ${skippedCount} skipped, ${errorCount} errors`,
+    );
+  }
+
   private async createPlaylist(playlist: PlaylistEntity): Promise<void> {
     let detail: { tracks: any; name: any; image: any };
     let playlist2Save: PlaylistEntity;
@@ -162,6 +266,7 @@ export class PlaylistService {
               name: track.name,
               spotifyUrl: track.previewUrl || null,
               coverUrl: track.coverUrl || savedPlaylist.coverUrl, // Use track's album art, fallback to playlist cover
+              durationMs: track.durationMs,
             },
             savedPlaylist,
           );

src/backend/src/shared/spotify-api.service.ts

@@ -60,6 +60,30 @@ export class SpotifyApiService {
     }
   }
 
+  isArtistUrl(url: string): boolean {
+    try {
+      const urlObj = new URL(url);
+      return urlObj.pathname.includes('/artist/');
+    } catch {
+      return false;
+    }
+  }
+
+  private getArtistId(url: string): string {
+    try {
+      const urlObj = new URL(url);
+      const pathParts = urlObj.pathname.split('/');
+      const artistIndex = pathParts.findIndex((part) => part === 'artist');
+      if (artistIndex >= 0 && pathParts.length > artistIndex + 1) {
+        return pathParts[artistIndex + 1].split('?')[0];
+      }
+      throw new Error('Invalid Spotify artist URL');
+    } catch (error) {
+      this.logger.error(`Failed to extract artist ID: ${error.message}`);
+      throw error;
+    }
+  }
+
   private getClientId(): string {
     const clientId = process.env.SPOTIFY_CLIENT_ID;
     if (!clientId) {
@@ -412,4 +436,196 @@ export class SpotifyApiService {
       throw error;
     }
   }
+
+  async getArtistLibrary(
+    spotifyUrl: string,
+  ): Promise<{ name: string; image: string; tracks: any[] }> {
+    try {
+      this.logger.debug(`Getting artist library for ${spotifyUrl}`);
+      const artistId = this.getArtistId(spotifyUrl);
+      const accessToken = await this.getUserAccessToken();
+
+      const artistResponse = await fetch(
+        `https://api.spotify.com/v1/artists/${artistId}`,
+        {
+          headers: {
+            Authorization: `Bearer ${accessToken}`,
+          },
+        },
+      );
+
+      if (!artistResponse.ok) {
+        const errorText = await artistResponse.text();
+        throw new Error(`Failed to fetch artist metadata: ${artistResponse.status} ${errorText}`);
+      }
+
+      const artist = await artistResponse.json();
+      const albums = await this.getAllArtistAlbums(artistId, accessToken);
+      const tracks = await this.getTracksForAlbums(albums, accessToken, artist.name);
+
+      return {
+        name: `Artist Library: ${artist.name}`,
+        image: artist.images?.[0]?.url || '',
+        tracks,
+      };
+    } catch (error) {
+      this.logger.error(`Failed to get artist library: ${error.message}`);
+      throw error;
+    }
+  }
+
+  private async getAllArtistAlbums(artistId: string, accessToken: string): Promise<any[]> {
+    const includeGroups = process.env.SPOTIFY_ARTIST_INCLUDE_GROUPS || 'album,single';
+    const albumByKey = new Map<string, any>();
+    const artistAlbumPageLimit = 20;
+    let offset = 0;
+    let hasMoreAlbums = true;
+
+    while (hasMoreAlbums) {
+      this.logger.debug(
+        `Fetching artist albums from Spotify API with offset ${offset}`,
+      );
+
+      const albumUrl = new URL(
+        `https://api.spotify.com/v1/artists/${artistId}/albums`,
+      );
+      albumUrl.searchParams.set('include_groups', includeGroups);
+      albumUrl.searchParams.set('offset', String(offset));
+
+      this.logger.debug(`Spotify artist albums URL: ${albumUrl.toString()}`);
+
+      const response = await fetch(albumUrl.toString(), {
+        headers: {
+          Authorization: `Bearer ${accessToken}`,
+        },
+      });
+
+      if (!response.ok) {
+        const errorText = await response.text();
+        throw new Error(`Failed to fetch artist albums: ${response.status} ${errorText}`);
+      }
+
+      const data = await response.json();
+
+      for (const album of data.items || []) {
+        const key = this.normalizeLibraryKey(`${album.name}|${album.release_date}|${album.total_tracks}`);
+        if (!albumByKey.has(key)) {
+          albumByKey.set(key, album);
+        }
+      }
+
+      if (!data.next) {
+        hasMoreAlbums = false;
+      } else {
+        offset += artistAlbumPageLimit;
+      }
+    }
+
+    const albums = [...albumByKey.values()];
+    this.logger.debug(`Retrieved ${albums.length} deduplicated artist albums/singles`);
+    return albums;
+  }
+
+  private async getTracksForAlbums(
+    albums: any[],
+    accessToken: string,
+    artistName: string,
+  ): Promise<any[]> {
+    const trackByKey = new Map<string, any>();
+    const maxTracks = Number(process.env.SPOTIFY_ARTIST_LIBRARY_MAX_TRACKS || 1000);
+
+    for (const album of albums) {
+      let offset = 0;
+      let hasMoreTracks = true;
+
+      while (hasMoreTracks) {
+        const response = await fetch(
+          `https://api.spotify.com/v1/albums/${album.id}/tracks?limit=50&offset=${offset}`,
+          {
+            headers: {
+              Authorization: `Bearer ${accessToken}`,
+            },
+          },
+        );
+
+        if (!response.ok) {
+          const errorText = await response.text();
+          throw new Error(`Failed to fetch album tracks: ${response.status} ${errorText}`);
+        }
+
+        const data = await response.json();
+
+        for (const item of data.items || []) {
+          if (!item?.name || !item?.artists?.length) {
+            continue;
+          }
+
+          const fullTrack = await this.getTrackById(item.id, accessToken);
+          const isrc = fullTrack?.external_ids?.isrc;
+          const durationMs = fullTrack?.duration_ms || item.duration_ms;
+          const artist = item.artists.map((a) => a.name).join(', ');
+          const key = isrc
+            ? `isrc:${isrc}`
+            : this.normalizeLibraryKey(`${artist}|${item.name}|${Math.round((durationMs || 0) / 2000)}`);
+
+          if (!trackByKey.has(key)) {
+            trackByKey.set(key, {
+              id: item.id,
+              name: item.name,
+              artist,
+              previewUrl: item.external_urls?.spotify || null,
+              coverUrl: album.images?.[0]?.url || null,
+              durationMs,
+              albumName: album.name,
+              primaryArtist: artistName,
+            });
+          }
+
+          if (trackByKey.size >= maxTracks) {
+            this.logger.warn(`Artist library max track limit reached: ${maxTracks}`);
+            return [...trackByKey.values()];
+          }
+        }
+
+        if (!data.next) {
+          hasMoreTracks = false;
+        } else {
+          offset += 50;
+        }
+      }
+    }
+
+    const tracks = [...trackByKey.values()];
+    this.logger.debug(`Retrieved ${tracks.length} deduplicated artist library tracks`);
+    return tracks;
+  }
+
+  private async getTrackById(trackId: string, accessToken: string): Promise<any> {
+    const response = await fetch(
+      `https://api.spotify.com/v1/tracks/${trackId}`,
+      {
+        headers: {
+          Authorization: `Bearer ${accessToken}`,
+        },
+      },
+    );
+
+    if (!response.ok) {
+      return null;
+    }
+
+    return response.json();
+  }
+
+  private normalizeLibraryKey(value: string): string {
+    return value
+      .toLowerCase()
+      .normalize('NFD')
+      .replace(/[\u0300-\u036f]/g, '')
+      .replace(/&/g, ' and ')
+      .replace(/[^a-z0-9]+/g, ' ')
+      .replace(/\s+/g, ' ')
+      .trim();
+  }
+
 }

src/backend/src/shared/spotify.service.ts

@@ -16,6 +16,10 @@ export class SpotifyService {
     return this.spotifyApiService.isTrackUrl(url);
   }
 
+  isArtistUrl(url: string): boolean {
+    return this.spotifyApiService.isArtistUrl(url);
+  }
+
   async getTrackDetail(
     spotifyUrl: string,
   ): Promise<{ name: string; artist: string; image: string }> {
@@ -61,6 +65,13 @@ export class SpotifyService {
     }
   }
 
+  async getArtistLibrary(
+    spotifyUrl: string,
+  ): Promise<{ name: string; tracks: any[]; image: string }> {
+    this.logger.debug(`Get artist library ${spotifyUrl} on Spotify`);
+    return this.spotifyApiService.getArtistLibrary(spotifyUrl);
+  }
+
   async getPlaylistTracks(spotifyUrl: string): Promise<any[]> {
     this.logger.debug(`Get playlist ${spotifyUrl} on Spotify`);
     try {

src/backend/src/shared/youtube.service.ts

@@ -3,9 +3,9 @@ import { TrackEntity } from '../track/track.entity';
 import { EnvironmentEnum } from '../environmentEnum';
 import { TrackService } from '../track/track.service';
 import { ConfigService } from '@nestjs/config';
-import { YtDlp } from 'ytdlp-nodejs';
 import * as yts from 'yt-search';
 import * as fs from 'fs';
+import { spawn } from 'child_process';
 const NodeID3 = require('node-id3');
 
 const HEADERS = {
@@ -73,13 +73,24 @@ export class YoutubeService {
     artist: string,
     name: string,
     durationMs?: number,
+    excludedUrls: string[] = [],
   ): Promise<YoutubeMatch> {
     const query = `${artist} - ${name}`;
     this.logger.debug(`Searching ${query} on YT`);
 
     const result = await yts(query);
+    const excludedVideoIds = new Set(
+      excludedUrls
+        .map((url) => this.getYoutubeVideoId(url))
+        .filter((id): id is string => !!id),
+    );
+
     const candidates = (result.videos || [])
       .filter((video: any) => !!video?.url && !!video?.title)
+      .filter((video: any) => {
+        const videoId = this.getYoutubeVideoId(String(video.url));
+        return !videoId || !excludedVideoIds.has(videoId);
+      })
       .slice(0, 15)
       .map((video: any) => this.scoreCandidate(video, artist, name, durationMs))
       .filter((match: CandidateScore) => {
@@ -106,7 +117,10 @@ export class YoutubeService {
     const accepted = candidates.find((candidate) => !candidate.rejected);
 
     if (!accepted) {
-      throw new Error(`No acceptable YouTube result found for: ${query}`);
+      throw new Error(
+        `No acceptable YouTube result found for: ${query}` +
+          (excludedUrls.length ? ` after excluding ${excludedUrls.length} failed candidate(s)` : ''),
+      );
     }
 
     this.logger.debug(
@@ -244,6 +258,20 @@ export class YoutubeService {
     return undefined;
   }
 
+  private getYoutubeVideoId(url: string): string | null {
+    try {
+      const parsed = new URL(url);
+
+      if (parsed.hostname === 'youtu.be') {
+        return parsed.pathname.replace(/^\//, '') || null;
+      }
+
+      return parsed.searchParams.get('v');
+    } catch {
+      return null;
+    }
+  }
+
   assertValidYoutubeUrl(url: string): void {
     let parsed: URL;
 
@@ -262,25 +290,93 @@ export class YoutubeService {
     }
   }
 
-  private getCookiesOptions(): {
-    cookiesFromBrowser?: string;
-    cookies?: string;
-  } {
-    const cookiesBrowser = this.configService.get<string>(
-      EnvironmentEnum.YT_COOKIES,
-    );
-    if (cookiesBrowser) {
-      this.logger.debug(`Using cookies from browser: ${cookiesBrowser}`);
-      return { cookiesFromBrowser: cookiesBrowser };
-    }
+  private getCookiesFile(): string | null {
     const cookiesFile = this.configService.get<string>(
       EnvironmentEnum.YT_COOKIES_FILE,
     );
+
     if (cookiesFile && fs.existsSync(cookiesFile)) {
       this.logger.debug(`Using cookies file: ${cookiesFile}`);
-      return { cookies: cookiesFile };
+      return cookiesFile;
     }
-    return {};
+
+    return null;
+  }
+
+  private classifyYtDlpError(output: string): string {
+    const text = output || '';
+
+    if (text.includes('Sign in to confirm your age')) {
+      return 'YouTube age-gated: cookies required or invalid';
+    }
+
+    if (text.includes('Only images are available')) {
+      return 'YouTube unavailable: no downloadable audio/video formats';
+    }
+
+    if (text.includes('Requested format is not available')) {
+      return 'YouTube unavailable: requested audio format is not available';
+    }
+
+    if (text.includes('Private video')) {
+      return 'YouTube unavailable: private video';
+    }
+
+    if (text.includes('Video unavailable')) {
+      return 'YouTube unavailable';
+    }
+
+    if (text.includes('Unable to download webpage')) {
+      return 'YouTube unavailable: unable to download webpage';
+    }
+
+    return text.trim().slice(0, 1200) || 'Unknown yt-dlp error';
+  }
+
+  private prepareWritableCookiesFile(cookiesFile: string | null): string | null {
+    if (!cookiesFile) {
+      return null;
+    }
+
+    const tempCookiesFile = `/tmp/jarri-spooty-youtube-${process.pid}.cookies.txt`;
+    fs.copyFileSync(cookiesFile, tempCookiesFile);
+    return tempCookiesFile;
+  }
+
+  private runYtDlp(args: string[]): Promise<void> {
+    return new Promise((resolvePromise, rejectPromise) => {
+      const child = spawn('yt-dlp', args, {
+        stdio: ['ignore', 'pipe', 'pipe'],
+      });
+
+      let stdout = '';
+      let stderr = '';
+
+      child.stdout.on('data', (chunk) => {
+        stdout += chunk.toString();
+      });
+
+      child.stderr.on('data', (chunk) => {
+        stderr += chunk.toString();
+      });
+
+      child.on('error', (error) => {
+        rejectPromise(new Error(`Failed to start yt-dlp: ${error.message}`));
+      });
+
+      child.on('close', (code) => {
+        if (code === 0) {
+          resolvePromise();
+          return;
+        }
+
+        rejectPromise(
+          new Error(
+            `yt-dlp exited with code ${code}: ${this.classifyYtDlpError(stderr || stdout)}`,
+          ),
+        );
+      });
+    });
   }
 
   async downloadAndFormat(track: TrackEntity, output: string): Promise<void> {
@@ -294,18 +390,32 @@ export class YoutubeService {
 
     this.assertValidYoutubeUrl(track.youtubeUrl);
 
-    const ytdlp = new YtDlp();
-    await ytdlp.downloadAudio(
-      track.youtubeUrl,
-      this.configService.get<'m4a'>(EnvironmentEnum.FORMAT),
-      {
-        output,
-        ...this.getCookiesOptions(),
-        headers: HEADERS,
-        jsRuntime: 'node',
-        audioQuality: this.configService.get<string>('QUALITY'),
-      },
-    );
+    const format = this.configService.get<string>(EnvironmentEnum.FORMAT) || 'mp3';
+    const quality = this.configService.get<string>('QUALITY') || '0';
+    const cookiesFile = this.prepareWritableCookiesFile(this.getCookiesFile());
+
+    const args = [
+      '--no-playlist',
+      '--no-cache-dir',
+      '--no-cookies-from-browser',
+      '--extract-audio',
+      '--audio-format',
+      format,
+      '--audio-quality',
+      quality,
+      '--add-header',
+      `User-Agent:${HEADERS['User-Agent']}`,
+      '-o',
+      output,
+    ];
+
+    if (cookiesFile) {
+      args.push('--cookies', cookiesFile);
+    }
+
+    args.push(track.youtubeUrl);
+
+    await this.runYtDlp(args);
 
     this.logger.debug(
       `Downloaded ${track.artist} - ${track.name} to ${output}`,

src/backend/src/track/track.entity.ts

@@ -27,6 +27,12 @@ export class TrackEntity {
   @Column({ nullable: true })
   youtubeUrl?: string;
 
+  @Column({ nullable: true, type: 'text' })
+  rejectedYoutubeUrls?: string;
+
+  @Column({ default: 0 })
+  downloadAttemptCount?: number;
+
   @Column({ default: TrackStatusEnum.New })
   status?: TrackStatusEnum;
 

src/backend/src/track/track.service.ts

@@ -70,6 +70,23 @@ export class TrackService {
     this.io.emit(WsTrackOperation.Update, track);
   }
 
+  private parseRejectedYoutubeUrls(track: TrackEntity): string[] {
+    if (!track.rejectedYoutubeUrls) {
+      return [];
+    }
+
+    try {
+      const parsed = JSON.parse(track.rejectedYoutubeUrls);
+      return Array.isArray(parsed) ? parsed.filter((item) => typeof item === 'string') : [];
+    } catch {
+      return [];
+    }
+  }
+
+  private stringifyRejectedYoutubeUrls(urls: string[]): string {
+    return JSON.stringify([...new Set(urls)].slice(0, 20));
+  }
+
   async retry(id: number): Promise<void> {
     const track = await this.get(id);
 
@@ -78,12 +95,22 @@ export class TrackService {
       return;
     }
 
-    await this.enqueueSearch(id);
+    const rejectedUrls = this.parseRejectedYoutubeUrls(track);
+
+    if (track.youtubeUrl && !rejectedUrls.includes(track.youtubeUrl)) {
+      rejectedUrls.push(track.youtubeUrl);
+    }
+
     await this.update(id, {
       ...track,
+      youtubeUrl: null,
+      rejectedYoutubeUrls: this.stringifyRejectedYoutubeUrls(rejectedUrls),
+      downloadAttemptCount: 0,
       error: null,
       status: TrackStatusEnum.New,
     });
+
+    await this.enqueueSearch(id);
   }
 
   async findOnYoutube(track: TrackEntity): Promise<void> {
@@ -102,9 +129,13 @@ export class TrackService {
     });
     let updatedTrack: TrackEntity;
     try {
+      const rejectedUrls = this.parseRejectedYoutubeUrls(track);
+
       const youtubeMatch = await this.youtubeService.findBestYoutubeMatch(
         track.artist,
         track.name,
+        track.durationMs,
+        rejectedUrls,
       );
       updatedTrack = {
         ...track,
@@ -141,8 +172,8 @@ export class TrackService {
     const existingJob = await this.trackSearchQueue.getJob(jobId);
 
     if (existingJob) {
-      this.logger.warn(`Search job already exists for track ${id}`);
-      return;
+      this.logger.warn(`Removing existing search job for track ${id} before enqueue`);
+      await existingJob.remove();
     }
 
     await this.trackSearchQueue.add('search-track', { id }, { jobId });
@@ -160,13 +191,23 @@ export class TrackService {
     const existingJob = await this.trackDownloadQueue.getJob(jobId);
 
     if (existingJob) {
-      this.logger.warn(`Download job already exists for track ${id}`);
-      return;
+      this.logger.warn(`Removing existing download job for track ${id} before enqueue`);
+      await existingJob.remove();
     }
 
     await this.trackDownloadQueue.add('download-track', { id }, { jobId });
   }
 
+  private getMaxDownloadAttempts(): number {
+    const parsed = Number(process.env.YT_DOWNLOAD_FALLBACK_ATTEMPTS || 3);
+
+    if (!Number.isFinite(parsed) || parsed < 1) {
+      return 3;
+    }
+
+    return Math.max(1, Math.min(10, Math.floor(parsed)));
+  }
+
   async downloadFromYoutube(track: TrackEntity): Promise<void> {
     const dbTrack = await this.get(track.id);
     if (!dbTrack) {
@@ -217,10 +258,39 @@ export class TrackService {
       error = toSafeErrorMessage(err);
     }
 
+    const rejectedUrls = this.parseRejectedYoutubeUrls(track);
+    const nextAttemptCount = (track.downloadAttemptCount || 0) + 1;
+    const maxAttempts = this.getMaxDownloadAttempts();
+
+    if (error && track.youtubeUrl && !rejectedUrls.includes(track.youtubeUrl)) {
+      rejectedUrls.push(track.youtubeUrl);
+    }
+
+    if (error && nextAttemptCount < maxAttempts) {
+      this.logger.warn(
+        `Download failed for track ${track.id}; rejecting current YouTube candidate and retrying search ` +
+          `(${nextAttemptCount}/${maxAttempts})`,
+      );
+
+      await this.update(track.id, {
+        ...track,
+        youtubeUrl: null,
+        rejectedYoutubeUrls: this.stringifyRejectedYoutubeUrls(rejectedUrls),
+        downloadAttemptCount: nextAttemptCount,
+        error: null,
+        status: TrackStatusEnum.New,
+      });
+
+      await this.enqueueSearch(track.id);
+      return;
+    }
+
     const updatedTrack = {
       ...track,
       status: error ? TrackStatusEnum.Error : TrackStatusEnum.Completed,
-      ...(error ? { error } : {}),
+      rejectedYoutubeUrls: this.stringifyRejectedYoutubeUrls(rejectedUrls),
+      downloadAttemptCount: error ? nextAttemptCount : 0,
+      ...(error ? { error } : { error: null }),
     };
     await this.update(track.id, updatedTrack);
   }