Harden YouTube search and download handling
This commit is contained in:
@@ -13,6 +13,14 @@ const HEADERS = {
|
|||||||
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
|
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
|
||||||
};
|
};
|
||||||
|
|
||||||
|
const ALLOWED_YOUTUBE_HOSTS = new Set([
|
||||||
|
'youtube.com',
|
||||||
|
'www.youtube.com',
|
||||||
|
'm.youtube.com',
|
||||||
|
'music.youtube.com',
|
||||||
|
'youtu.be',
|
||||||
|
]);
|
||||||
|
|
||||||
@Injectable()
|
@Injectable()
|
||||||
export class YoutubeService {
|
export class YoutubeService {
|
||||||
private readonly logger = new Logger(TrackService.name);
|
private readonly logger = new Logger(TrackService.name);
|
||||||
@@ -20,10 +28,37 @@ export class YoutubeService {
|
|||||||
constructor(private readonly configService: ConfigService) {}
|
constructor(private readonly configService: ConfigService) {}
|
||||||
|
|
||||||
async findOnYoutubeOne(artist: string, name: string): Promise<string> {
|
async findOnYoutubeOne(artist: string, name: string): Promise<string> {
|
||||||
this.logger.debug(`Searching ${artist} - ${name} on YT`);
|
const query = `${artist} - ${name}`;
|
||||||
const url = (await yts(`${artist} - ${name}`)).videos[0].url;
|
this.logger.debug(`Searching ${query} on YT`);
|
||||||
this.logger.debug(`Found ${artist} - ${name} on ${url}`);
|
|
||||||
return url;
|
const result = await yts(query);
|
||||||
|
const firstVideo = result.videos?.[0];
|
||||||
|
|
||||||
|
if (!firstVideo?.url) {
|
||||||
|
throw new Error(`No YouTube result found for: ${query}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
this.assertValidYoutubeUrl(firstVideo.url);
|
||||||
|
this.logger.debug(`Found ${query} on ${firstVideo.url}`);
|
||||||
|
return firstVideo.url;
|
||||||
|
}
|
||||||
|
|
||||||
|
assertValidYoutubeUrl(url: string): void {
|
||||||
|
let parsed: URL;
|
||||||
|
|
||||||
|
try {
|
||||||
|
parsed = new URL(url);
|
||||||
|
} catch {
|
||||||
|
throw new Error('Invalid YouTube URL');
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!['http:', 'https:'].includes(parsed.protocol)) {
|
||||||
|
throw new Error('Rejected non-HTTP YouTube URL');
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!ALLOWED_YOUTUBE_HOSTS.has(parsed.hostname.toLowerCase())) {
|
||||||
|
throw new Error(`Rejected non-YouTube URL: ${parsed.hostname}`);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private getCookiesOptions(): {
|
private getCookiesOptions(): {
|
||||||
@@ -51,10 +86,13 @@ export class YoutubeService {
|
|||||||
this.logger.debug(
|
this.logger.debug(
|
||||||
`Downloading ${track.artist} - ${track.name} (${track.youtubeUrl}) from YT`,
|
`Downloading ${track.artist} - ${track.name} (${track.youtubeUrl}) from YT`,
|
||||||
);
|
);
|
||||||
|
|
||||||
if (!track.youtubeUrl) {
|
if (!track.youtubeUrl) {
|
||||||
this.logger.error('youtubeUrl is null or undefined');
|
throw new Error('youtubeUrl is null or undefined');
|
||||||
throw Error('youtubeUrl is null or undefined');
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
this.assertValidYoutubeUrl(track.youtubeUrl);
|
||||||
|
|
||||||
const ytdlp = new YtDlp();
|
const ytdlp = new YtDlp();
|
||||||
await ytdlp.downloadAudio(
|
await ytdlp.downloadAudio(
|
||||||
track.youtubeUrl,
|
track.youtubeUrl,
|
||||||
@@ -67,6 +105,7 @@ export class YoutubeService {
|
|||||||
audioQuality: this.configService.get<string>('QUALITY'),
|
audioQuality: this.configService.get<string>('QUALITY'),
|
||||||
},
|
},
|
||||||
);
|
);
|
||||||
|
|
||||||
this.logger.debug(
|
this.logger.debug(
|
||||||
`Downloaded ${track.artist} - ${track.name} to ${output}`,
|
`Downloaded ${track.artist} - ${track.name} to ${output}`,
|
||||||
);
|
);
|
||||||
@@ -78,8 +117,15 @@ export class YoutubeService {
|
|||||||
title: string,
|
title: string,
|
||||||
artist: string,
|
artist: string,
|
||||||
): Promise<void> {
|
): Promise<void> {
|
||||||
if (coverUrl) {
|
if (!coverUrl) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
const res = await fetch(coverUrl);
|
const res = await fetch(coverUrl);
|
||||||
|
if (!res.ok) {
|
||||||
|
throw new Error(`Failed to fetch cover art: ${res.status}`);
|
||||||
|
}
|
||||||
|
|
||||||
const arrayBuf = await res.arrayBuffer();
|
const arrayBuf = await res.arrayBuffer();
|
||||||
const imageBuffer = Buffer.from(arrayBuf);
|
const imageBuffer = Buffer.from(arrayBuf);
|
||||||
|
|
||||||
@@ -98,4 +144,3 @@ export class YoutubeService {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|||||||
@@ -3,6 +3,16 @@ import { Job } from 'bullmq';
|
|||||||
import { TrackService } from './track.service';
|
import { TrackService } from './track.service';
|
||||||
import { TrackEntity } from './track.entity';
|
import { TrackEntity } from './track.entity';
|
||||||
|
|
||||||
|
function getDownloadsPerMinute(): number {
|
||||||
|
const parsed = Number(process.env.YT_DOWNLOADS_PER_MINUTE || 3);
|
||||||
|
|
||||||
|
if (!Number.isFinite(parsed) || parsed <= 0) {
|
||||||
|
return 3;
|
||||||
|
}
|
||||||
|
|
||||||
|
return Math.max(1, Math.min(30, Math.floor(parsed)));
|
||||||
|
}
|
||||||
|
|
||||||
@Processor('track-download-processor')
|
@Processor('track-download-processor')
|
||||||
export class TrackDownloadProcessor extends WorkerHost {
|
export class TrackDownloadProcessor extends WorkerHost {
|
||||||
constructor(private readonly trackService: TrackService) {
|
constructor(private readonly trackService: TrackService) {
|
||||||
@@ -10,7 +20,7 @@ export class TrackDownloadProcessor extends WorkerHost {
|
|||||||
}
|
}
|
||||||
|
|
||||||
async process(job: Job<TrackEntity, void>): Promise<void> {
|
async process(job: Job<TrackEntity, void>): Promise<void> {
|
||||||
const maxPerMinute = Number(process.env.YT_DOWNLOADS_PER_MINUTE || 3);
|
const maxPerMinute = getDownloadsPerMinute();
|
||||||
const sleepMs = Math.floor(60000 / maxPerMinute);
|
const sleepMs = Math.floor(60000 / maxPerMinute);
|
||||||
await new Promise((res) => setTimeout(res, sleepMs));
|
await new Promise((res) => setTimeout(res, sleepMs));
|
||||||
await this.trackService.downloadFromYoutube(job.data);
|
await this.trackService.downloadFromYoutube(job.data);
|
||||||
|
|||||||
@@ -99,17 +99,24 @@ export class TrackService {
|
|||||||
error: String(err),
|
error: String(err),
|
||||||
status: TrackStatusEnum.Error,
|
status: TrackStatusEnum.Error,
|
||||||
};
|
};
|
||||||
|
await this.update(track.id, updatedTrack);
|
||||||
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
await this.update(track.id, updatedTrack);
|
||||||
await this.trackDownloadQueue.add('', updatedTrack, {
|
await this.trackDownloadQueue.add('', updatedTrack, {
|
||||||
jobId: `id-${updatedTrack.id}`,
|
jobId: `id-${updatedTrack.id}`,
|
||||||
});
|
});
|
||||||
await this.update(track.id, updatedTrack);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
async downloadFromYoutube(track: TrackEntity): Promise<void> {
|
async downloadFromYoutube(track: TrackEntity): Promise<void> {
|
||||||
if (!(await this.get(track.id))) {
|
const dbTrack = await this.get(track.id);
|
||||||
|
if (!dbTrack) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
track = dbTrack;
|
||||||
|
|
||||||
if (!track.name || !track.artist || !track.playlist) {
|
if (!track.name || !track.artist || !track.playlist) {
|
||||||
this.logger.error(
|
this.logger.error(
|
||||||
`Track or playlist field is null or undefined: name=${track.name}, artist=${track.artist}, playlist=${track.playlist ? 'ok' : 'null'}`,
|
`Track or playlist field is null or undefined: name=${track.name}, artist=${track.artist}, playlist=${track.playlist ? 'ok' : 'null'}`,
|
||||||
|
|||||||
Reference in New Issue
Block a user