From 4c051782fd8808a808fe49fe1509ec0b0db9d05e Mon Sep 17 00:00:00 2001 From: AlanCarrer Date: Sat, 13 Sep 2025 13:18:14 +0200 Subject: [PATCH] feat(youtube): use youtube cookies to bypass limitation & add timeout between each downloads --- README.md | 11 ++ package-lock.json | 128 ------------------ package.json | 2 +- src/backend/.env.default | 3 + src/backend/.env.docker | 3 + src/backend/package.json | 2 +- src/backend/src/app.module.ts | 2 +- src/backend/src/shared/youtube.service.ts | 48 ++++++- .../src/track/track-download.processor.ts | 5 +- src/backend/src/track/track.module.ts | 8 +- src/backend/src/track/track.service.ts | 18 ++- src/frontend/package.json | 2 +- 12 files changed, 84 insertions(+), 148 deletions(-) diff --git a/README.md b/README.md index f148fea..5f29d0f 100644 --- a/README.md +++ b/README.md @@ -112,6 +112,17 @@ Some behaviour and settings of Spooty can be configured using environment variab RUN_REDIS | false | Whenever Redis server should be started from backend (recommended for Docker environment) | SPOTIFY_CLIENT_ID | your_client_id | Client ID of your Spotify application (required) | SPOTIFY_CLIENT_SECRET| your_client_secret | Client Secret of your Spotify application (required) | + YT_DOWNLOADS_PER_MINUTE | 3 | Set the maximum number of YouTube downloads started per minute | + YT_COOKIES | | Allows you to pass your YouTube cookies to bypass some download restrictions. See [below](#how-to-get-your-youtube-cookies) for instructions. | + +How to get your YouTube cookies (using browser dev tools): +1. Go to https://www.youtube.com and log in if needed. +2. Open the browser developer tools (F12 or right click > Inspect). +3. Go to the "Application" tab (in Chrome) or "Storage" (in Firefox). +4. In the left menu, find "Cookies" and select https://www.youtube.com. +5. Copy all the cookies (name=value) and join them with a semicolon and a space, like: + VISITOR_INFO1_LIVE=xxxx; YSC=xxxx; SID=xxxx; ... +6. Paste this string into the YT_COOKIES environment variable (in your .env or Docker config). # ⚖️ License [MIT](https://choosealicense.com/licenses/mit/) diff --git a/package-lock.json b/package-lock.json index 0f4707d..0f7d39b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -383,44 +383,6 @@ "url": "https://github.com/chalk/chalk?sponsor=1" } }, - "node_modules/@angular-devkit/schematics-cli/node_modules/chokidar": { - "version": "3.6.0", - "dev": true, - "license": "MIT", - "optional": true, - "peer": true, - "dependencies": { - "anymatch": "~3.1.2", - "braces": "~3.0.2", - "glob-parent": "~5.1.2", - "is-binary-path": "~2.1.0", - "is-glob": "~4.0.1", - "normalize-path": "~3.0.0", - "readdirp": "~3.6.0" - }, - "engines": { - "node": ">= 8.10.0" - }, - "funding": { - "url": "https://paulmillr.com/funding/" - }, - "optionalDependencies": { - "fsevents": "~2.3.2" - } - }, - "node_modules/@angular-devkit/schematics-cli/node_modules/glob-parent": { - "version": "5.1.2", - "dev": true, - "license": "ISC", - "optional": true, - "peer": true, - "dependencies": { - "is-glob": "^4.0.1" - }, - "engines": { - "node": ">= 6" - } - }, "node_modules/@angular-devkit/schematics-cli/node_modules/inquirer": { "version": "9.2.15", "dev": true, @@ -481,32 +443,6 @@ "url": "https://github.com/sponsors/jonschlinkert" } }, - "node_modules/@angular-devkit/schematics-cli/node_modules/readdirp": { - "version": "3.6.0", - "dev": true, - "license": "MIT", - "optional": true, - "peer": true, - "dependencies": { - "picomatch": "^2.2.1" - }, - "engines": { - "node": ">=8.10.0" - } - }, - "node_modules/@angular-devkit/schematics-cli/node_modules/readdirp/node_modules/picomatch": { - "version": "2.3.1", - "dev": true, - "license": "MIT", - "optional": true, - "peer": true, - "engines": { - "node": ">=8.6" - }, - "funding": { - "url": "https://github.com/sponsors/jonschlinkert" - } - }, "node_modules/@angular-devkit/schematics-cli/node_modules/run-async": { "version": "3.0.0", "dev": true, @@ -4074,44 +4010,6 @@ } } }, - "node_modules/@nestjs/schematics/node_modules/chokidar": { - "version": "3.6.0", - "dev": true, - "license": "MIT", - "optional": true, - "peer": true, - "dependencies": { - "anymatch": "~3.1.2", - "braces": "~3.0.2", - "glob-parent": "~5.1.2", - "is-binary-path": "~2.1.0", - "is-glob": "~4.0.1", - "normalize-path": "~3.0.0", - "readdirp": "~3.6.0" - }, - "engines": { - "node": ">= 8.10.0" - }, - "funding": { - "url": "https://paulmillr.com/funding/" - }, - "optionalDependencies": { - "fsevents": "~2.3.2" - } - }, - "node_modules/@nestjs/schematics/node_modules/glob-parent": { - "version": "5.1.2", - "dev": true, - "license": "ISC", - "optional": true, - "peer": true, - "dependencies": { - "is-glob": "^4.0.1" - }, - "engines": { - "node": ">= 6" - } - }, "node_modules/@nestjs/schematics/node_modules/magic-string": { "version": "0.30.8", "dev": true, @@ -4134,32 +4032,6 @@ "url": "https://github.com/sponsors/jonschlinkert" } }, - "node_modules/@nestjs/schematics/node_modules/readdirp": { - "version": "3.6.0", - "dev": true, - "license": "MIT", - "optional": true, - "peer": true, - "dependencies": { - "picomatch": "^2.2.1" - }, - "engines": { - "node": ">=8.10.0" - } - }, - "node_modules/@nestjs/schematics/node_modules/readdirp/node_modules/picomatch": { - "version": "2.3.1", - "dev": true, - "license": "MIT", - "optional": true, - "peer": true, - "engines": { - "node": ">=8.6" - }, - "funding": { - "url": "https://github.com/sponsors/jonschlinkert" - } - }, "node_modules/@nestjs/serve-static": { "version": "4.0.2", "license": "MIT", diff --git a/package.json b/package.json index 4f11de9..e25b720 100644 --- a/package.json +++ b/package.json @@ -31,4 +31,4 @@ "release-it-docker-plugin": "^2.0.0", "rimraf": "^6.0.1" } -} +} \ No newline at end of file diff --git a/src/backend/.env.default b/src/backend/.env.default index 9694f7a..f2adc5a 100644 --- a/src/backend/.env.default +++ b/src/backend/.env.default @@ -10,3 +10,6 @@ REDIS_RUN=false # Credentials for Spotify API SPOTIFY_CLIENT_ID=your_client_id SPOTIFY_CLIENT_SECRET=your_client_secret + +YT_COOKIES="AEC=xxx; APISID=xxx; APISID=xxx; HSID=xxx; HSID=xxx; LOGIN_INFO=xxx; NID=xxx; PREF=xxx; SAPISID=xxx; SAPISID=Nxxx; SEARCH_SAMESITE=xxx; SID=xxx; SID=xxx; SIDCC=xxx; SIDCC=xxx; SSID=xxx; SSID=xxx; STRP=xxx; VISITOR_PRIVACY_METADATA=xxx; YSC=xxx" +YT_DOWNLOADS_PER_MINUTE=3 \ No newline at end of file diff --git a/src/backend/.env.docker b/src/backend/.env.docker index 20ab21a..74f06ae 100644 --- a/src/backend/.env.docker +++ b/src/backend/.env.docker @@ -10,3 +10,6 @@ REDIS_RUN=true # Credentials for Spotify API SPOTIFY_CLIENT_ID=your_client_id SPOTIFY_CLIENT_SECRET=your_client_secret + +YT_COOKIES= +YT_DOWNLOADS_PER_MINUTE=3 \ No newline at end of file diff --git a/src/backend/package.json b/src/backend/package.json index e9c71fc..9fea5ff 100644 --- a/src/backend/package.json +++ b/src/backend/package.json @@ -84,4 +84,4 @@ "coverageDirectory": "../coverage", "testEnvironment": "node" } -} +} \ No newline at end of file diff --git a/src/backend/src/app.module.ts b/src/backend/src/app.module.ts index aa617d9..166e489 100644 --- a/src/backend/src/app.module.ts +++ b/src/backend/src/app.module.ts @@ -14,7 +14,7 @@ import { BullModule } from '@nestjs/bullmq'; @Module({ imports: [ - ConfigModule.forRoot(), + ConfigModule.forRoot({ isGlobal: true }), ScheduleModule.forRoot(), TypeOrmModule.forRootAsync({ imports: [ConfigModule], diff --git a/src/backend/src/shared/youtube.service.ts b/src/backend/src/shared/youtube.service.ts index 71e26bf..66bfbb2 100644 --- a/src/backend/src/shared/youtube.service.ts +++ b/src/backend/src/shared/youtube.service.ts @@ -10,21 +10,39 @@ import * as ytdl from '@distube/ytdl-core'; import { Readable } from 'stream'; const NodeID3 = require('node-id3'); -const YT_SETTINGS: ytdl.downloadOptions = { - quality: 'highestaudio', - filter: 'audioonly', -}; - enum StreamStates { Finish = 'finish', Error = 'error', } +function parseCookies( + cookieString?: string, +): { name: string; value: string }[] | undefined { + if (!cookieString) return undefined; + return cookieString + .split(';') + .map((c) => { + const [name, ...rest] = c.split('='); + return { name: name.trim(), value: rest.join('=').trim() }; + }) + .filter((c) => c.name && c.value); +} + @Injectable() export class YoutubeService { private readonly logger = new Logger(TrackService.name); + private readonly ytCookies: string | undefined; + private readonly ytAgent: any | undefined; - constructor(private readonly configService: ConfigService) {} + constructor(private readonly configService: ConfigService) { + this.ytCookies = this.configService.get('YT_COOKIES'); + if (this.ytCookies) { + const cookiesArr = parseCookies(this.ytCookies); + if (cookiesArr && cookiesArr.length > 0) { + this.ytAgent = ytdl.createAgent(cookiesArr); + } + } + } async findOnYoutubeOne(artist: string, name: string): Promise { this.logger.debug(`Searching ${artist} - ${name} on YT`); @@ -86,7 +104,23 @@ export class YoutubeService { youtubeUrl: string, reject: (reason: any) => void, ): Readable { - return ytdl(youtubeUrl, YT_SETTINGS).on(StreamStates.Error, (err) => + if (!youtubeUrl) { + this.logger.error('youtubeUrl is null or undefined'); + reject('youtubeUrl is null or undefined'); + return null; + } + const options: ytdl.downloadOptions = { + quality: 'highestaudio', + filter: 'audioonly', + agent: this.ytAgent, + requestOptions: { + headers: { + 'User-Agent': + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36', + }, + }, + }; + return ytdl(youtubeUrl, options).on(StreamStates.Error, (err) => reject(err), ); } diff --git a/src/backend/src/track/track-download.processor.ts b/src/backend/src/track/track-download.processor.ts index f7c0916..8cdf7b1 100644 --- a/src/backend/src/track/track-download.processor.ts +++ b/src/backend/src/track/track-download.processor.ts @@ -9,7 +9,10 @@ export class TrackDownloadProcessor extends WorkerHost { super(); } - async process(job: Job): Promise { + async process(job: Job): Promise { + const maxPerMinute = Number(process.env.YT_DOWNLOADS_PER_MINUTE || 3); + const sleepMs = Math.floor(60000 / maxPerMinute); + await new Promise((res) => setTimeout(res, sleepMs)); await this.trackService.downloadFromYoutube(job.data); } } diff --git a/src/backend/src/track/track.module.ts b/src/backend/src/track/track.module.ts index 170996f..2384bd1 100644 --- a/src/backend/src/track/track.module.ts +++ b/src/backend/src/track/track.module.ts @@ -6,12 +6,8 @@ import { TrackController } from './track.controller'; import { ConfigModule } from '@nestjs/config'; import { SharedModule } from '../shared/shared.module'; import { BullModule } from '@nestjs/bullmq'; -import { - TrackDownloadProcessor, -} from './track-download.processor'; -import { - TrackSearchProcessor, -} from './track-search.processor'; +import { TrackDownloadProcessor } from './track-download.processor'; +import { TrackSearchProcessor } from './track-search.processor'; @Module({ imports: [ diff --git a/src/backend/src/track/track.service.ts b/src/backend/src/track/track.service.ts index 5d0eb2d..0834257 100644 --- a/src/backend/src/track/track.service.ts +++ b/src/backend/src/track/track.service.ts @@ -110,6 +110,17 @@ export class TrackService { if (!(await this.get(track.id))) { return; } + if ( + !track.name || + !track.artist || + !track.playlist || + !track.playlist.coverUrl + ) { + this.logger.error( + `Track or playlist field is null or undefined: name=${track.name}, artist=${track.artist}, playlist=${track.playlist ? 'ok' : 'null'}, coverUrl=${track.playlist?.coverUrl}`, + ); + return; + } await this.update(track.id, { ...track, status: TrackStatusEnum.Downloading, @@ -137,13 +148,16 @@ export class TrackService { } getTrackFileName(track: TrackEntity): string { - const fileName = `${track.artist} - ${track.name.replace('/', '')}`; + const safeArtist = track.artist || 'unknown_artist'; + const safeName = (track.name || 'unknown_track').replace('/', ''); + const fileName = `${safeArtist} - ${safeName}`; return `${this.utilsService.stripFileIllegalChars(fileName)}.${this.configService.get(EnvironmentEnum.FORMAT)}`; } getFolderName(track: TrackEntity, playlist: PlaylistEntity): string { + const safePlaylistName = playlist?.name || 'unknown_playlist'; return resolve( - this.utilsService.getPlaylistFolderPath(playlist.name), + this.utilsService.getPlaylistFolderPath(safePlaylistName), this.getTrackFileName(track), ); } diff --git a/src/frontend/package.json b/src/frontend/package.json index 04fe7bd..fa9d38e 100644 --- a/src/frontend/package.json +++ b/src/frontend/package.json @@ -48,4 +48,4 @@ "karma-jasmine-html-reporter": "~2.1.0", "typescript": "~5.6.3" } -} +} \ No newline at end of file