feat(youtube): use youtube cookies to bypass limitation & add timeout between each downloads

README.md

@@ -112,6 +112,17 @@ Some behaviour and settings of Spooty can be configured using environment variab
  RUN_REDIS            | false                                       | Whenever Redis server should be started from backend (recommended for Docker environment)                                                                        |
  SPOTIFY_CLIENT_ID    | your_client_id                              | Client ID of your Spotify application (required)                                                                                                                  |
  SPOTIFY_CLIENT_SECRET| your_client_secret                          | Client Secret of your Spotify application (required)                                                                                                              |
+ YT_DOWNLOADS_PER_MINUTE | 3                                           | Set the maximum number of YouTube downloads started per minute                                                                                                  |
+ YT_COOKIES           |                                             | Allows you to pass your YouTube cookies to bypass some download restrictions. See [below](#how-to-get-your-youtube-cookies) for instructions.                   |
+
+How to get your YouTube cookies (using browser dev tools):
+1. Go to https://www.youtube.com and log in if needed.
+2. Open the browser developer tools (F12 or right click > Inspect).
+3. Go to the "Application" tab (in Chrome) or "Storage" (in Firefox).
+4. In the left menu, find "Cookies" and select https://www.youtube.com.
+5. Copy all the cookies (name=value) and join them with a semicolon and a space, like:
+   VISITOR_INFO1_LIVE=xxxx; YSC=xxxx; SID=xxxx; ...
+6. Paste this string into the YT_COOKIES environment variable (in your .env or Docker config).
 
 # ⚖️ License
 [MIT](https://choosealicense.com/licenses/mit/)

package-lock.json

@@ -383,44 +383,6 @@
         "url": "https://github.com/chalk/chalk?sponsor=1"
       }
     },
-    "node_modules/@angular-devkit/schematics-cli/node_modules/chokidar": {
-      "version": "3.6.0",
-      "dev": true,
-      "license": "MIT",
-      "optional": true,
-      "peer": true,
-      "dependencies": {
-        "anymatch": "~3.1.2",
-        "braces": "~3.0.2",
-        "glob-parent": "~5.1.2",
-        "is-binary-path": "~2.1.0",
-        "is-glob": "~4.0.1",
-        "normalize-path": "~3.0.0",
-        "readdirp": "~3.6.0"
-      },
-      "engines": {
-        "node": ">= 8.10.0"
-      },
-      "funding": {
-        "url": "https://paulmillr.com/funding/"
-      },
-      "optionalDependencies": {
-        "fsevents": "~2.3.2"
-      }
-    },
-    "node_modules/@angular-devkit/schematics-cli/node_modules/glob-parent": {
-      "version": "5.1.2",
-      "dev": true,
-      "license": "ISC",
-      "optional": true,
-      "peer": true,
-      "dependencies": {
-        "is-glob": "^4.0.1"
-      },
-      "engines": {
-        "node": ">= 6"
-      }
-    },
     "node_modules/@angular-devkit/schematics-cli/node_modules/inquirer": {
       "version": "9.2.15",
       "dev": true,
@@ -481,32 +443,6 @@
         "url": "https://github.com/sponsors/jonschlinkert"
       }
     },
-    "node_modules/@angular-devkit/schematics-cli/node_modules/readdirp": {
-      "version": "3.6.0",
-      "dev": true,
-      "license": "MIT",
-      "optional": true,
-      "peer": true,
-      "dependencies": {
-        "picomatch": "^2.2.1"
-      },
-      "engines": {
-        "node": ">=8.10.0"
-      }
-    },
-    "node_modules/@angular-devkit/schematics-cli/node_modules/readdirp/node_modules/picomatch": {
-      "version": "2.3.1",
-      "dev": true,
-      "license": "MIT",
-      "optional": true,
-      "peer": true,
-      "engines": {
-        "node": ">=8.6"
-      },
-      "funding": {
-        "url": "https://github.com/sponsors/jonschlinkert"
-      }
-    },
     "node_modules/@angular-devkit/schematics-cli/node_modules/run-async": {
       "version": "3.0.0",
       "dev": true,
@@ -4074,44 +4010,6 @@
         }
       }
     },
-    "node_modules/@nestjs/schematics/node_modules/chokidar": {
-      "version": "3.6.0",
-      "dev": true,
-      "license": "MIT",
-      "optional": true,
-      "peer": true,
-      "dependencies": {
-        "anymatch": "~3.1.2",
-        "braces": "~3.0.2",
-        "glob-parent": "~5.1.2",
-        "is-binary-path": "~2.1.0",
-        "is-glob": "~4.0.1",
-        "normalize-path": "~3.0.0",
-        "readdirp": "~3.6.0"
-      },
-      "engines": {
-        "node": ">= 8.10.0"
-      },
-      "funding": {
-        "url": "https://paulmillr.com/funding/"
-      },
-      "optionalDependencies": {
-        "fsevents": "~2.3.2"
-      }
-    },
-    "node_modules/@nestjs/schematics/node_modules/glob-parent": {
-      "version": "5.1.2",
-      "dev": true,
-      "license": "ISC",
-      "optional": true,
-      "peer": true,
-      "dependencies": {
-        "is-glob": "^4.0.1"
-      },
-      "engines": {
-        "node": ">= 6"
-      }
-    },
     "node_modules/@nestjs/schematics/node_modules/magic-string": {
       "version": "0.30.8",
       "dev": true,
@@ -4134,32 +4032,6 @@
         "url": "https://github.com/sponsors/jonschlinkert"
       }
     },
-    "node_modules/@nestjs/schematics/node_modules/readdirp": {
-      "version": "3.6.0",
-      "dev": true,
-      "license": "MIT",
-      "optional": true,
-      "peer": true,
-      "dependencies": {
-        "picomatch": "^2.2.1"
-      },
-      "engines": {
-        "node": ">=8.10.0"
-      }
-    },
-    "node_modules/@nestjs/schematics/node_modules/readdirp/node_modules/picomatch": {
-      "version": "2.3.1",
-      "dev": true,
-      "license": "MIT",
-      "optional": true,
-      "peer": true,
-      "engines": {
-        "node": ">=8.6"
-      },
-      "funding": {
-        "url": "https://github.com/sponsors/jonschlinkert"
-      }
-    },
     "node_modules/@nestjs/serve-static": {
       "version": "4.0.2",
       "license": "MIT",

package.json

@@ -31,4 +31,4 @@
     "release-it-docker-plugin": "^2.0.0",
     "rimraf": "^6.0.1"
   }
-}
+}

src/backend/.env.default

@@ -10,3 +10,6 @@ REDIS_RUN=false
 # Credentials for Spotify API
 SPOTIFY_CLIENT_ID=your_client_id
 SPOTIFY_CLIENT_SECRET=your_client_secret
+
+YT_COOKIES="AEC=xxx; APISID=xxx; APISID=xxx; HSID=xxx; HSID=xxx; LOGIN_INFO=xxx; NID=xxx; PREF=xxx; SAPISID=xxx; SAPISID=Nxxx; SEARCH_SAMESITE=xxx; SID=xxx; SID=xxx; SIDCC=xxx; SIDCC=xxx; SSID=xxx; SSID=xxx; STRP=xxx; VISITOR_PRIVACY_METADATA=xxx; YSC=xxx"
+YT_DOWNLOADS_PER_MINUTE=3

src/backend/.env.docker

@@ -10,3 +10,6 @@ REDIS_RUN=true
 # Credentials for Spotify API
 SPOTIFY_CLIENT_ID=your_client_id
 SPOTIFY_CLIENT_SECRET=your_client_secret
+
+YT_COOKIES=
+YT_DOWNLOADS_PER_MINUTE=3

src/backend/package.json

@@ -84,4 +84,4 @@
     "coverageDirectory": "../coverage",
     "testEnvironment": "node"
   }
-}
+}

src/backend/src/app.module.ts

@@ -14,7 +14,7 @@ import { BullModule } from '@nestjs/bullmq';
 
 @Module({
   imports: [
-    ConfigModule.forRoot(),
+    ConfigModule.forRoot({ isGlobal: true }),
     ScheduleModule.forRoot(),
     TypeOrmModule.forRootAsync({
       imports: [ConfigModule],

src/backend/src/shared/youtube.service.ts

@@ -10,21 +10,39 @@ import * as ytdl from '@distube/ytdl-core';
 import { Readable } from 'stream';
 const NodeID3 = require('node-id3');
 
-const YT_SETTINGS: ytdl.downloadOptions = {
-  quality: 'highestaudio',
-  filter: 'audioonly',
-};
-
 enum StreamStates {
   Finish = 'finish',
   Error = 'error',
 }
 
+function parseCookies(
+  cookieString?: string,
+): { name: string; value: string }[] | undefined {
+  if (!cookieString) return undefined;
+  return cookieString
+    .split(';')
+    .map((c) => {
+      const [name, ...rest] = c.split('=');
+      return { name: name.trim(), value: rest.join('=').trim() };
+    })
+    .filter((c) => c.name && c.value);
+}
+
 @Injectable()
 export class YoutubeService {
   private readonly logger = new Logger(TrackService.name);
+  private readonly ytCookies: string | undefined;
+  private readonly ytAgent: any | undefined;
 
-  constructor(private readonly configService: ConfigService) {}
+  constructor(private readonly configService: ConfigService) {
+    this.ytCookies = this.configService.get<string>('YT_COOKIES');
+    if (this.ytCookies) {
+      const cookiesArr = parseCookies(this.ytCookies);
+      if (cookiesArr && cookiesArr.length > 0) {
+        this.ytAgent = ytdl.createAgent(cookiesArr);
+      }
+    }
+  }
 
   async findOnYoutubeOne(artist: string, name: string): Promise<string> {
     this.logger.debug(`Searching ${artist} - ${name} on YT`);
@@ -86,7 +104,23 @@ export class YoutubeService {
     youtubeUrl: string,
     reject: (reason: any) => void,
   ): Readable {
-    return ytdl(youtubeUrl, YT_SETTINGS).on(StreamStates.Error, (err) =>
+    if (!youtubeUrl) {
+      this.logger.error('youtubeUrl is null or undefined');
+      reject('youtubeUrl is null or undefined');
+      return null;
+    }
+    const options: ytdl.downloadOptions = {
+      quality: 'highestaudio',
+      filter: 'audioonly',
+      agent: this.ytAgent,
+      requestOptions: {
+        headers: {
+          'User-Agent':
+            'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
+        },
+      },
+    };
+    return ytdl(youtubeUrl, options).on(StreamStates.Error, (err) =>
       reject(err),
     );
   }

src/backend/src/track/track-download.processor.ts

@@ -9,7 +9,10 @@ export class TrackDownloadProcessor extends WorkerHost {
     super();
   }
 
-  async process(job: Job<TrackEntity, void, string>): Promise<void> {
+  async process(job: Job<TrackEntity, void>): Promise<void> {
+    const maxPerMinute = Number(process.env.YT_DOWNLOADS_PER_MINUTE || 3);
+    const sleepMs = Math.floor(60000 / maxPerMinute);
+    await new Promise((res) => setTimeout(res, sleepMs));
     await this.trackService.downloadFromYoutube(job.data);
   }
 }

src/backend/src/track/track.module.ts

@@ -6,12 +6,8 @@ import { TrackController } from './track.controller';
 import { ConfigModule } from '@nestjs/config';
 import { SharedModule } from '../shared/shared.module';
 import { BullModule } from '@nestjs/bullmq';
-import {
+import { TrackDownloadProcessor } from './track-download.processor';
-  TrackDownloadProcessor,
+import { TrackSearchProcessor } from './track-search.processor';
-} from './track-download.processor';
-import {
-  TrackSearchProcessor,
-} from './track-search.processor';
 
 @Module({
   imports: [

src/backend/src/track/track.service.ts

@@ -110,6 +110,17 @@ export class TrackService {
     if (!(await this.get(track.id))) {
       return;
     }
+    if (
+      !track.name ||
+      !track.artist ||
+      !track.playlist ||
+      !track.playlist.coverUrl
+    ) {
+      this.logger.error(
+        `Track or playlist field is null or undefined: name=${track.name}, artist=${track.artist}, playlist=${track.playlist ? 'ok' : 'null'}, coverUrl=${track.playlist?.coverUrl}`,
+      );
+      return;
+    }
     await this.update(track.id, {
       ...track,
       status: TrackStatusEnum.Downloading,
@@ -137,13 +148,16 @@ export class TrackService {
   }
 
   getTrackFileName(track: TrackEntity): string {
-    const fileName = `${track.artist} - ${track.name.replace('/', '')}`;
+    const safeArtist = track.artist || 'unknown_artist';
+    const safeName = (track.name || 'unknown_track').replace('/', '');
+    const fileName = `${safeArtist} - ${safeName}`;
     return `${this.utilsService.stripFileIllegalChars(fileName)}.${this.configService.get<string>(EnvironmentEnum.FORMAT)}`;
   }
 
   getFolderName(track: TrackEntity, playlist: PlaylistEntity): string {
+    const safePlaylistName = playlist?.name || 'unknown_playlist';
     return resolve(
-      this.utilsService.getPlaylistFolderPath(playlist.name),
+      this.utilsService.getPlaylistFolderPath(safePlaylistName),
       this.getTrackFileName(track),
     );
   }

src/frontend/package.json

@@ -48,4 +48,4 @@
     "karma-jasmine-html-reporter": "~2.1.0",
     "typescript": "~5.6.3"
   }
-}
+}