Add Workspace single-track intake and hardened YouTube matching
This commit is contained in:
@@ -49,7 +49,7 @@ import { ArchiveModule } from './archive/archive.module';
|
|||||||
ThrottlerModule.forRoot([
|
ThrottlerModule.forRoot([
|
||||||
{
|
{
|
||||||
ttl: 60000,
|
ttl: 60000,
|
||||||
limit: 60,
|
limit: Number(process.env.API_THROTTLE_LIMIT || 600),
|
||||||
},
|
},
|
||||||
]),
|
]),
|
||||||
BullModule.forRootAsync({
|
BullModule.forRootAsync({
|
||||||
|
|||||||
@@ -0,0 +1,11 @@
|
|||||||
|
import { IsString, MaxLength } from 'class-validator';
|
||||||
|
|
||||||
|
export class SearchTrackDto {
|
||||||
|
@IsString()
|
||||||
|
@MaxLength(512)
|
||||||
|
artist: string;
|
||||||
|
|
||||||
|
@IsString()
|
||||||
|
@MaxLength(512)
|
||||||
|
title: string;
|
||||||
|
}
|
||||||
@@ -12,6 +12,7 @@ import { PlaylistService } from './playlist.service';
|
|||||||
import { PlaylistEntity } from './playlist.entity';
|
import { PlaylistEntity } from './playlist.entity';
|
||||||
import { CreatePlaylistDto } from './dto/create-playlist.dto';
|
import { CreatePlaylistDto } from './dto/create-playlist.dto';
|
||||||
import { UpdatePlaylistDto } from './dto/update-playlist.dto';
|
import { UpdatePlaylistDto } from './dto/update-playlist.dto';
|
||||||
|
import { SearchTrackDto } from './dto/search-track.dto';
|
||||||
|
|
||||||
@Controller('playlist')
|
@Controller('playlist')
|
||||||
export class PlaylistController {
|
export class PlaylistController {
|
||||||
@@ -27,6 +28,13 @@ export class PlaylistController {
|
|||||||
await this.service.create(playlist as PlaylistEntity);
|
await this.service.create(playlist as PlaylistEntity);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Post('search-track')
|
||||||
|
async searchTrack(
|
||||||
|
@Body() body: SearchTrackDto,
|
||||||
|
): Promise<{ spotifyUrl: string; name: string; artist: string }> {
|
||||||
|
return this.service.createFromSearch(body.artist, body.title);
|
||||||
|
}
|
||||||
|
|
||||||
@Put(':id')
|
@Put(':id')
|
||||||
update(
|
update(
|
||||||
@Param('id', ParseIntPipe) id: number,
|
@Param('id', ParseIntPipe) id: number,
|
||||||
|
|||||||
@@ -48,6 +48,27 @@ export class PlaylistService {
|
|||||||
this.io.emit(WsPlaylistOperation.Delete, { id });
|
this.io.emit(WsPlaylistOperation.Delete, { id });
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
async createFromSearch(
|
||||||
|
artist: string,
|
||||||
|
title: string,
|
||||||
|
): Promise<{ spotifyUrl: string; name: string; artist: string }> {
|
||||||
|
const track = await this.spotifyService.searchTrack(artist, title);
|
||||||
|
|
||||||
|
await this.create({
|
||||||
|
spotifyUrl: track.spotifyUrl,
|
||||||
|
name: track.name,
|
||||||
|
active: false,
|
||||||
|
isTrack: true,
|
||||||
|
} as PlaylistEntity);
|
||||||
|
|
||||||
|
return {
|
||||||
|
spotifyUrl: track.spotifyUrl,
|
||||||
|
name: track.name,
|
||||||
|
artist: track.artist,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
async create(playlist: PlaylistEntity): Promise<void> {
|
async create(playlist: PlaylistEntity): Promise<void> {
|
||||||
// Detect if URL is for a single track or a playlist and route accordingly
|
// Detect if URL is for a single track or a playlist and route accordingly
|
||||||
const isTrack = this.spotifyService.isTrackUrl(playlist.spotifyUrl);
|
const isTrack = this.spotifyService.isTrackUrl(playlist.spotifyUrl);
|
||||||
@@ -63,7 +84,7 @@ export class PlaylistService {
|
|||||||
}
|
}
|
||||||
|
|
||||||
private async createSingleTrack(playlist: PlaylistEntity): Promise<void> {
|
private async createSingleTrack(playlist: PlaylistEntity): Promise<void> {
|
||||||
let trackDetail: { name: string; artist: string; image: string };
|
let trackDetail: { name: string; artist: string; image: string; durationMs?: number };
|
||||||
let playlist2Save: PlaylistEntity;
|
let playlist2Save: PlaylistEntity;
|
||||||
try {
|
try {
|
||||||
trackDetail = await this.spotifyService.getTrackDetail(
|
trackDetail = await this.spotifyService.getTrackDetail(
|
||||||
@@ -97,6 +118,7 @@ export class PlaylistService {
|
|||||||
name: trackDetail.name,
|
name: trackDetail.name,
|
||||||
spotifyUrl: playlist.spotifyUrl,
|
spotifyUrl: playlist.spotifyUrl,
|
||||||
coverUrl: trackDetail.image,
|
coverUrl: trackDetail.image,
|
||||||
|
durationMs: trackDetail.durationMs,
|
||||||
},
|
},
|
||||||
savedPlaylist,
|
savedPlaylist,
|
||||||
);
|
);
|
||||||
|
|||||||
@@ -12,27 +12,57 @@ export class SpotifyAuthController {
|
|||||||
}
|
}
|
||||||
|
|
||||||
@Get('login')
|
@Get('login')
|
||||||
login(@Res() res: Response): void {
|
login(
|
||||||
res.redirect(this.spotifyApiService.getAuthorizationUrl());
|
@Query('returnTo') returnTo: string | undefined,
|
||||||
|
@Res() res: Response,
|
||||||
|
): void {
|
||||||
|
const state =
|
||||||
|
returnTo && /^https?:\/\/localhost:\d+\/?$/.test(returnTo)
|
||||||
|
? Buffer.from(JSON.stringify({ returnTo }), 'utf8').toString('base64url')
|
||||||
|
: undefined;
|
||||||
|
|
||||||
|
res.redirect(this.spotifyApiService.getAuthorizationUrl(state));
|
||||||
}
|
}
|
||||||
|
|
||||||
@Get('callback')
|
@Get('callback')
|
||||||
async callback(
|
async callback(
|
||||||
@Query('code') code: string | undefined,
|
@Query('code') code: string | undefined,
|
||||||
@Query('error') error: string | undefined,
|
@Query('error') error: string | undefined,
|
||||||
|
@Query('state') state: string | undefined,
|
||||||
@Res() res: Response,
|
@Res() res: Response,
|
||||||
): Promise<void> {
|
): Promise<void> {
|
||||||
|
const returnTo = this.parseReturnTo(state);
|
||||||
|
|
||||||
if (error) {
|
if (error) {
|
||||||
res.redirect(`/?spotify_error=${encodeURIComponent(error)}`);
|
res.redirect(`${returnTo}?spotify_error=${encodeURIComponent(error)}`);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (!code) {
|
if (!code) {
|
||||||
res.redirect('/?spotify_error=missing_code');
|
res.redirect(`${returnTo}?spotify_error=missing_code`);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
await this.spotifyApiService.exchangeAuthorizationCode(code);
|
await this.spotifyApiService.exchangeAuthorizationCode(code);
|
||||||
res.redirect('/?spotify=connected');
|
res.redirect(`${returnTo}?spotify=connected`);
|
||||||
|
}
|
||||||
|
|
||||||
|
private parseReturnTo(state: string | undefined): string {
|
||||||
|
if (!state) {
|
||||||
|
return '/';
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
const parsed = JSON.parse(Buffer.from(state, 'base64url').toString('utf8'));
|
||||||
|
const returnTo = parsed?.returnTo;
|
||||||
|
|
||||||
|
if (typeof returnTo === 'string' && /^https?:\/\/localhost:\d+\/?$/.test(returnTo)) {
|
||||||
|
return returnTo;
|
||||||
|
}
|
||||||
|
} catch {
|
||||||
|
return '/';
|
||||||
|
}
|
||||||
|
|
||||||
|
return '/';
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -151,7 +151,7 @@ export class SpotifyApiService {
|
|||||||
return !!this.readUserToken();
|
return !!this.readUserToken();
|
||||||
}
|
}
|
||||||
|
|
||||||
getAuthorizationUrl(): string {
|
getAuthorizationUrl(state?: string): string {
|
||||||
const url = new URL('https://accounts.spotify.com/authorize');
|
const url = new URL('https://accounts.spotify.com/authorize');
|
||||||
|
|
||||||
url.searchParams.set('response_type', 'code');
|
url.searchParams.set('response_type', 'code');
|
||||||
@@ -167,6 +167,10 @@ export class SpotifyApiService {
|
|||||||
url.searchParams.set('redirect_uri', this.getRedirectUri());
|
url.searchParams.set('redirect_uri', this.getRedirectUri());
|
||||||
url.searchParams.set('show_dialog', 'true');
|
url.searchParams.set('show_dialog', 'true');
|
||||||
|
|
||||||
|
if (state) {
|
||||||
|
url.searchParams.set('state', state);
|
||||||
|
}
|
||||||
|
|
||||||
return url.toString();
|
return url.toString();
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -336,6 +340,85 @@ export class SpotifyApiService {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
async searchTrack(
|
||||||
|
artist: string,
|
||||||
|
title: string,
|
||||||
|
): Promise<{ spotifyUrl: string; name: string; artist: string; image: string; durationMs?: number }> {
|
||||||
|
const accessToken = await this.getClientCredentialsAccessToken();
|
||||||
|
const plainQuery = `${artist} ${title}`.trim();
|
||||||
|
const structuredQuery = `track:"${title}" artist:"${artist}"`;
|
||||||
|
|
||||||
|
const url = new URL('https://api.spotify.com/v1/search');
|
||||||
|
url.searchParams.set('q', structuredQuery || plainQuery);
|
||||||
|
url.searchParams.set('type', 'track');
|
||||||
|
url.searchParams.set('limit', '10');
|
||||||
|
|
||||||
|
const response = await fetch(url.toString(), {
|
||||||
|
headers: { Authorization: `Bearer ${accessToken}` },
|
||||||
|
});
|
||||||
|
|
||||||
|
if (!response.ok) {
|
||||||
|
const errorText = await response.text();
|
||||||
|
throw new Error(`Spotify track search failed: ${response.status} ${errorText}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
const data = await response.json();
|
||||||
|
const tracks = data?.tracks?.items || [];
|
||||||
|
|
||||||
|
if (!tracks.length) {
|
||||||
|
throw new Error(`No Spotify track found for ${artist} - ${title}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
const normalize = (value: string) =>
|
||||||
|
(value || '')
|
||||||
|
.toLowerCase()
|
||||||
|
.normalize('NFD')
|
||||||
|
.replace(/[\u0300-\u036f]/g, '')
|
||||||
|
.replace(/[^a-z0-9]+/g, ' ')
|
||||||
|
.replace(/\s+/g, ' ')
|
||||||
|
.trim();
|
||||||
|
|
||||||
|
const wantedArtist = normalize(artist);
|
||||||
|
const wantedTitle = normalize(title);
|
||||||
|
|
||||||
|
const scoreTrack = (track: any): number => {
|
||||||
|
const trackName = normalize(track.name);
|
||||||
|
const artistNames = (track.artists || []).map((item: any) => normalize(item.name));
|
||||||
|
const allArtists = artistNames.join(' ');
|
||||||
|
|
||||||
|
let score = 0;
|
||||||
|
if (trackName === wantedTitle) score += 70;
|
||||||
|
else if (trackName.includes(wantedTitle) || wantedTitle.includes(trackName)) score += 40;
|
||||||
|
|
||||||
|
if (artistNames.some((name: string) => name === wantedArtist)) score += 50;
|
||||||
|
else if (allArtists.includes(wantedArtist) || wantedArtist.includes(allArtists)) score += 25;
|
||||||
|
|
||||||
|
if (track.explicit !== undefined) score += 1;
|
||||||
|
if (track.popularity) score += Math.min(20, Math.floor(track.popularity / 5));
|
||||||
|
|
||||||
|
return score;
|
||||||
|
};
|
||||||
|
|
||||||
|
const ranked = [...tracks]
|
||||||
|
.map((track: any) => ({ track, score: scoreTrack(track) }))
|
||||||
|
.sort((a, b) => b.score - a.score);
|
||||||
|
|
||||||
|
const best = ranked[0]?.track || tracks[0];
|
||||||
|
|
||||||
|
this.logger.debug(
|
||||||
|
`Spotify search "${plainQuery}" selected "${best?.artists?.map((item: any) => item.name).join(', ')} - ${best?.name}" score=${ranked[0]?.score ?? 0}`,
|
||||||
|
);
|
||||||
|
|
||||||
|
return {
|
||||||
|
spotifyUrl: best.external_urls.spotify,
|
||||||
|
name: best.name,
|
||||||
|
artist: (best.artists || []).map((item: any) => item.name).join(', '),
|
||||||
|
image: best.album?.images?.[0]?.url || '',
|
||||||
|
durationMs: best.duration_ms,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
async getAllPlaylistTracks(spotifyUrl: string): Promise<any[]> {
|
async getAllPlaylistTracks(spotifyUrl: string): Promise<any[]> {
|
||||||
try {
|
try {
|
||||||
this.logger.debug(`Getting all tracks for playlist ${spotifyUrl}`);
|
this.logger.debug(`Getting all tracks for playlist ${spotifyUrl}`);
|
||||||
|
|||||||
@@ -72,6 +72,14 @@ export class SpotifyService {
|
|||||||
return this.spotifyApiService.getArtistLibrary(spotifyUrl);
|
return this.spotifyApiService.getArtistLibrary(spotifyUrl);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
async searchTrack(
|
||||||
|
artist: string,
|
||||||
|
title: string,
|
||||||
|
): Promise<{ spotifyUrl: string; name: string; artist: string; image: string; durationMs?: number }> {
|
||||||
|
return this.spotifyApiService.searchTrack(artist, title);
|
||||||
|
}
|
||||||
|
|
||||||
async getPlaylistTracks(spotifyUrl: string): Promise<any[]> {
|
async getPlaylistTracks(spotifyUrl: string): Promise<any[]> {
|
||||||
this.logger.debug(`Get playlist ${spotifyUrl} on Spotify`);
|
this.logger.debug(`Get playlist ${spotifyUrl} on Spotify`);
|
||||||
try {
|
try {
|
||||||
|
|||||||
@@ -38,6 +38,19 @@ const BAD_MATCH_TERMS = [
|
|||||||
'reaction',
|
'reaction',
|
||||||
'tutorial',
|
'tutorial',
|
||||||
'instrumental',
|
'instrumental',
|
||||||
|
'reaction',
|
||||||
|
'review',
|
||||||
|
'analysis',
|
||||||
|
'breakdown',
|
||||||
|
'explained',
|
||||||
|
'secret message',
|
||||||
|
'meaning',
|
||||||
|
'interview',
|
||||||
|
'podcast',
|
||||||
|
'news',
|
||||||
|
'documentary',
|
||||||
|
'fifa world cup',
|
||||||
|
'song style',
|
||||||
];
|
];
|
||||||
|
|
||||||
const OFFICIALISH_TERMS = [
|
const OFFICIALISH_TERMS = [
|
||||||
@@ -123,11 +136,12 @@ export class YoutubeService {
|
|||||||
.join(' | '),
|
.join(' | '),
|
||||||
);
|
);
|
||||||
|
|
||||||
const accepted = candidates.find((candidate) => !candidate.rejected);
|
const minimumScore = Number(process.env.YT_MIN_CANDIDATE_SCORE || 60);
|
||||||
|
const accepted = candidates.find((candidate) => !candidate.rejected && candidate.score >= minimumScore);
|
||||||
|
|
||||||
if (!accepted) {
|
if (!accepted) {
|
||||||
throw new Error(
|
throw new Error(
|
||||||
`No acceptable YouTube result found for: ${query}` +
|
`No acceptable YouTube result found for: ${query} with minimum score ${minimumScore}` +
|
||||||
(excludedUrls.length ? ` after excluding ${excludedUrls.length} failed candidate(s)` : ''),
|
(excludedUrls.length ? ` after excluding ${excludedUrls.length} failed candidate(s)` : ''),
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
@@ -143,11 +157,12 @@ export class YoutubeService {
|
|||||||
const searchTarget = `ytsearch15:${query}`;
|
const searchTarget = `ytsearch15:${query}`;
|
||||||
const args = [
|
const args = [
|
||||||
'--dump-single-json',
|
'--dump-single-json',
|
||||||
'--flat-playlist',
|
|
||||||
'--skip-download',
|
'--skip-download',
|
||||||
'--no-playlist',
|
'--no-playlist',
|
||||||
'--no-cache-dir',
|
'--no-cache-dir',
|
||||||
'--no-cookies-from-browser',
|
'--no-cookies-from-browser',
|
||||||
|
'--extractor-args',
|
||||||
|
'youtube:player_client=android_vr',
|
||||||
'--add-header',
|
'--add-header',
|
||||||
`User-Agent:${HEADERS['User-Agent']}`,
|
`User-Agent:${HEADERS['User-Agent']}`,
|
||||||
searchTarget,
|
searchTarget,
|
||||||
@@ -198,6 +213,7 @@ export class YoutubeService {
|
|||||||
const author = String(video.author?.name || video.author || '');
|
const author = String(video.author?.name || video.author || '');
|
||||||
const url = String(video.url || '');
|
const url = String(video.url || '');
|
||||||
const seconds = this.getVideoSeconds(video);
|
const seconds = this.getVideoSeconds(video);
|
||||||
|
this.logger.debug(`YT candidate raw duration title="${title}" duration=${video.duration} seconds=${video.seconds} parsed=${seconds}`);
|
||||||
|
|
||||||
const normalizedTitle = this.normalize(title);
|
const normalizedTitle = this.normalize(title);
|
||||||
const normalizedAuthor = this.normalize(author);
|
const normalizedAuthor = this.normalize(author);
|
||||||
@@ -213,14 +229,21 @@ export class YoutubeService {
|
|||||||
reasons.push('title=45');
|
reasons.push('title=45');
|
||||||
}
|
}
|
||||||
|
|
||||||
|
let hasArtistSignal = false;
|
||||||
for (const artistPart of this.artistParts(normalizedArtist)) {
|
for (const artistPart of this.artistParts(normalizedArtist)) {
|
||||||
if (artistPart.length >= 3 && (normalizedTitle.includes(artistPart) || normalizedAuthor.includes(artistPart))) {
|
if (artistPart.length >= 3 && (normalizedTitle.includes(artistPart) || normalizedAuthor.includes(artistPart))) {
|
||||||
score += 20;
|
score += 20;
|
||||||
reasons.push(`artist=${artistPart}:20`);
|
reasons.push(`artist=${artistPart}:20`);
|
||||||
|
hasArtistSignal = true;
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (!hasArtistSignal) {
|
||||||
|
score -= 35;
|
||||||
|
reasons.push('artist=missing:-35');
|
||||||
|
}
|
||||||
|
|
||||||
const officialish = OFFICIALISH_TERMS.find((term) =>
|
const officialish = OFFICIALISH_TERMS.find((term) =>
|
||||||
`${normalizedTitle} ${normalizedAuthor}`.includes(this.normalize(term)),
|
`${normalizedTitle} ${normalizedAuthor}`.includes(this.normalize(term)),
|
||||||
);
|
);
|
||||||
@@ -233,8 +256,9 @@ export class YoutubeService {
|
|||||||
normalizedTitle.includes(this.normalize(term)),
|
normalizedTitle.includes(this.normalize(term)),
|
||||||
);
|
);
|
||||||
if (badTerm) {
|
if (badTerm) {
|
||||||
score -= 25;
|
score -= 80;
|
||||||
reasons.push(`bad=${badTerm}:-25`);
|
rejected = true;
|
||||||
|
reasons.push(`bad=${badTerm}:-80 reject`);
|
||||||
}
|
}
|
||||||
|
|
||||||
if (durationMs && seconds) {
|
if (durationMs && seconds) {
|
||||||
@@ -495,6 +519,8 @@ export class YoutubeService {
|
|||||||
'--no-playlist',
|
'--no-playlist',
|
||||||
'--no-cache-dir',
|
'--no-cache-dir',
|
||||||
'--no-cookies-from-browser',
|
'--no-cookies-from-browser',
|
||||||
|
'-f',
|
||||||
|
'bestaudio/best',
|
||||||
'--extract-audio',
|
'--extract-audio',
|
||||||
'--audio-format',
|
'--audio-format',
|
||||||
format,
|
format,
|
||||||
|
|||||||
@@ -320,7 +320,8 @@ export class TrackService {
|
|||||||
const safeArtist = track.artist || 'unknown_artist';
|
const safeArtist = track.artist || 'unknown_artist';
|
||||||
const safeName = track.name || 'unknown_track';
|
const safeName = track.name || 'unknown_track';
|
||||||
const fileName = `${safeArtist} - ${safeName}`;
|
const fileName = `${safeArtist} - ${safeName}`;
|
||||||
return `${this.utilsService.stripFileIllegalChars(fileName)}.${this.configService.get<string>(EnvironmentEnum.FORMAT)}`;
|
const format = this.configService.get<string>(EnvironmentEnum.FORMAT) || 'mp3';
|
||||||
|
return `${this.utilsService.stripFileIllegalChars(fileName)}.${format}`;
|
||||||
}
|
}
|
||||||
|
|
||||||
getFolderName(track: TrackEntity, playlist: PlaylistEntity): string {
|
getFolderName(track: TrackEntity, playlist: PlaylistEntity): string {
|
||||||
|
|||||||
Reference in New Issue
Block a user