feat(youtube): use youtube cookies to bypass limitation & add timeout between each downloads

src/backend/.env.default

@@ -10,3 +10,6 @@ REDIS_RUN=false
 # Credentials for Spotify API
 SPOTIFY_CLIENT_ID=your_client_id
 SPOTIFY_CLIENT_SECRET=your_client_secret
+
+YT_COOKIES="AEC=xxx; APISID=xxx; APISID=xxx; HSID=xxx; HSID=xxx; LOGIN_INFO=xxx; NID=xxx; PREF=xxx; SAPISID=xxx; SAPISID=Nxxx; SEARCH_SAMESITE=xxx; SID=xxx; SID=xxx; SIDCC=xxx; SIDCC=xxx; SSID=xxx; SSID=xxx; STRP=xxx; VISITOR_PRIVACY_METADATA=xxx; YSC=xxx"
+YT_DOWNLOADS_PER_MINUTE=3

src/backend/.env.docker

@@ -10,3 +10,6 @@ REDIS_RUN=true
 # Credentials for Spotify API
 SPOTIFY_CLIENT_ID=your_client_id
 SPOTIFY_CLIENT_SECRET=your_client_secret
+
+YT_COOKIES=
+YT_DOWNLOADS_PER_MINUTE=3

src/backend/package.json

@@ -84,4 +84,4 @@
     "coverageDirectory": "../coverage",
     "testEnvironment": "node"
   }
-}
+}

src/backend/src/app.module.ts

@@ -14,7 +14,7 @@ import { BullModule } from '@nestjs/bullmq';
 
 @Module({
   imports: [
-    ConfigModule.forRoot(),
+    ConfigModule.forRoot({ isGlobal: true }),
     ScheduleModule.forRoot(),
     TypeOrmModule.forRootAsync({
       imports: [ConfigModule],

src/backend/src/shared/youtube.service.ts

@@ -10,21 +10,39 @@ import * as ytdl from '@distube/ytdl-core';
 import { Readable } from 'stream';
 const NodeID3 = require('node-id3');
 
-const YT_SETTINGS: ytdl.downloadOptions = {
-  quality: 'highestaudio',
-  filter: 'audioonly',
-};
-
 enum StreamStates {
   Finish = 'finish',
   Error = 'error',
 }
 
+function parseCookies(
+  cookieString?: string,
+): { name: string; value: string }[] | undefined {
+  if (!cookieString) return undefined;
+  return cookieString
+    .split(';')
+    .map((c) => {
+      const [name, ...rest] = c.split('=');
+      return { name: name.trim(), value: rest.join('=').trim() };
+    })
+    .filter((c) => c.name && c.value);
+}
+
 @Injectable()
 export class YoutubeService {
   private readonly logger = new Logger(TrackService.name);
+  private readonly ytCookies: string | undefined;
+  private readonly ytAgent: any | undefined;
 
-  constructor(private readonly configService: ConfigService) {}
+  constructor(private readonly configService: ConfigService) {
+    this.ytCookies = this.configService.get<string>('YT_COOKIES');
+    if (this.ytCookies) {
+      const cookiesArr = parseCookies(this.ytCookies);
+      if (cookiesArr && cookiesArr.length > 0) {
+        this.ytAgent = ytdl.createAgent(cookiesArr);
+      }
+    }
+  }
 
   async findOnYoutubeOne(artist: string, name: string): Promise<string> {
     this.logger.debug(`Searching ${artist} - ${name} on YT`);
@@ -86,7 +104,23 @@ export class YoutubeService {
     youtubeUrl: string,
     reject: (reason: any) => void,
   ): Readable {
-    return ytdl(youtubeUrl, YT_SETTINGS).on(StreamStates.Error, (err) =>
+    if (!youtubeUrl) {
+      this.logger.error('youtubeUrl is null or undefined');
+      reject('youtubeUrl is null or undefined');
+      return null;
+    }
+    const options: ytdl.downloadOptions = {
+      quality: 'highestaudio',
+      filter: 'audioonly',
+      agent: this.ytAgent,
+      requestOptions: {
+        headers: {
+          'User-Agent':
+            'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
+        },
+      },
+    };
+    return ytdl(youtubeUrl, options).on(StreamStates.Error, (err) =>
       reject(err),
     );
   }

src/backend/src/track/track-download.processor.ts

@@ -9,7 +9,10 @@ export class TrackDownloadProcessor extends WorkerHost {
     super();
   }
 
-  async process(job: Job<TrackEntity, void, string>): Promise<void> {
+  async process(job: Job<TrackEntity, void>): Promise<void> {
+    const maxPerMinute = Number(process.env.YT_DOWNLOADS_PER_MINUTE || 3);
+    const sleepMs = Math.floor(60000 / maxPerMinute);
+    await new Promise((res) => setTimeout(res, sleepMs));
     await this.trackService.downloadFromYoutube(job.data);
   }
 }

src/backend/src/track/track.module.ts

@@ -6,12 +6,8 @@ import { TrackController } from './track.controller';
 import { ConfigModule } from '@nestjs/config';
 import { SharedModule } from '../shared/shared.module';
 import { BullModule } from '@nestjs/bullmq';
-import {
-  TrackDownloadProcessor,
-} from './track-download.processor';
-import {
-  TrackSearchProcessor,
-} from './track-search.processor';
+import { TrackDownloadProcessor } from './track-download.processor';
+import { TrackSearchProcessor } from './track-search.processor';
 
 @Module({
   imports: [

src/backend/src/track/track.service.ts

@@ -110,6 +110,17 @@ export class TrackService {
     if (!(await this.get(track.id))) {
       return;
     }
+    if (
+      !track.name ||
+      !track.artist ||
+      !track.playlist ||
+      !track.playlist.coverUrl
+    ) {
+      this.logger.error(
+        `Track or playlist field is null or undefined: name=${track.name}, artist=${track.artist}, playlist=${track.playlist ? 'ok' : 'null'}, coverUrl=${track.playlist?.coverUrl}`,
+      );
+      return;
+    }
     await this.update(track.id, {
       ...track,
       status: TrackStatusEnum.Downloading,
@@ -137,13 +148,16 @@ export class TrackService {
   }
 
   getTrackFileName(track: TrackEntity): string {
-    const fileName = `${track.artist} - ${track.name.replace('/', '')}`;
+    const safeArtist = track.artist || 'unknown_artist';
+    const safeName = (track.name || 'unknown_track').replace('/', '');
+    const fileName = `${safeArtist} - ${safeName}`;
     return `${this.utilsService.stripFileIllegalChars(fileName)}.${this.configService.get<string>(EnvironmentEnum.FORMAT)}`;
   }
 
   getFolderName(track: TrackEntity, playlist: PlaylistEntity): string {
+    const safePlaylistName = playlist?.name || 'unknown_playlist';
     return resolve(
-      this.utilsService.getPlaylistFolderPath(playlist.name),
+      this.utilsService.getPlaylistFolderPath(safePlaylistName),
       this.getTrackFileName(track),
     );
   }