mapping albums and artists from track requests

This commit is contained in:
andy 2022-03-03 22:32:29 +00:00
parent 12842c240b
commit 6292d6c3f0
4 changed files with 54 additions and 199 deletions

View File

@ -24,26 +24,14 @@ namespace Selector.CLI
simulOption.AddAlias("-s"); simulOption.AddAlias("-s");
AddOption(simulOption); AddOption(simulOption);
var limitOption = new Option<int?>("--limit", "limit number of objects to poll"); var limitOption = new Option<int?>("--limit", getDefaultValue: () => 200, "limit number of objects to poll");
limitOption.AddAlias("-l"); limitOption.AddAlias("-l");
AddOption(limitOption); AddOption(limitOption);
var artists = new Option("--artist", "map scrobble artists to spotify"); Handler = CommandHandler.Create(async (int delay, int simultaneous, int? limit, CancellationToken token) => await Execute(delay, simultaneous, limit, token));
artists.AddAlias("-ar");
AddOption(artists);
var albums = new Option("--album", "map scrobble albums to spotify");
albums.AddAlias("-al");
AddOption(albums);
var tracks = new Option("--track", "map scrobble tracks to spotify");
tracks.AddAlias("-tr");
AddOption(tracks);
Handler = CommandHandler.Create(async (int delay, int simultaneous, int? limit, bool artist, bool album, bool track, CancellationToken token) => await Execute(delay, simultaneous, limit, artist, album, track, token));
} }
public static async Task<int> Execute(int delay, int simultaneous, int? limit, bool artists, bool albums, bool tracks, CancellationToken token) public static async Task<int> Execute(int delay, int simultaneous, int? limit, CancellationToken token)
{ {
try try
{ {
@ -58,10 +46,7 @@ namespace Selector.CLI
{ {
InterRequestDelay = new TimeSpan(0, 0, 0, 0, delay), InterRequestDelay = new TimeSpan(0, 0, 0, 0, delay),
SimultaneousConnections = simultaneous, SimultaneousConnections = simultaneous,
Limit = limit, Limit = limit
Artists = artists,
Albums = albums,
Tracks = tracks
}, },
new ScrobbleRepository(db), new ScrobbleRepository(db),
new ScrobbleMappingRepository(db), new ScrobbleMappingRepository(db),

View File

@ -80,7 +80,9 @@ namespace Selector.CLI.Extensions
using var db = new ApplicationDbContext(context.DatabaseConfig.Options, NullLogger<ApplicationDbContext>.Instance); using var db = new ApplicationDbContext(context.DatabaseConfig.Options, NullLogger<ApplicationDbContext>.Instance);
refreshToken = db.Users.FirstOrDefault(u => u.UserName == "sarsoo")?.SpotifyRefreshToken; var user = db.Users.FirstOrDefault(u => u.UserName == "sarsoo");
refreshToken = user?.SpotifyRefreshToken;
} }
var configFactory = new RefreshTokenFactory(context.Config.ClientId, context.Config.ClientSecret, refreshToken); var configFactory = new RefreshTokenFactory(context.Config.ClientId, context.Config.ClientSecret, refreshToken);

View File

@ -33,7 +33,7 @@
}, },
"Selector.CLI.Scrobble.Map": { "Selector.CLI.Scrobble.Map": {
"commandName": "Project", "commandName": "Project",
"commandLineArgs": "scrobble map --artist -l 50 --album --track", "commandLineArgs": "scrobble map",
"environmentVariables": { "environmentVariables": {
"DOTNET_ENVIRONMENT": "Development" "DOTNET_ENVIRONMENT": "Development"
}, },

View File

@ -4,6 +4,7 @@ using Selector.Model;
using Selector.Operations; using Selector.Operations;
using SpotifyAPI.Web; using SpotifyAPI.Web;
using System; using System;
using System.Collections.Generic;
using System.Linq; using System.Linq;
using System.Threading; using System.Threading;
using System.Threading.Tasks; using System.Threading.Tasks;
@ -16,9 +17,6 @@ namespace Selector
public TimeSpan Timeout { get; set; } = new TimeSpan(0, 20, 0); public TimeSpan Timeout { get; set; } = new TimeSpan(0, 20, 0);
public int SimultaneousConnections { get; set; } = 3; public int SimultaneousConnections { get; set; } = 3;
public int? Limit { get; set; } = null; public int? Limit { get; set; } = null;
public bool Tracks { get; set; } = false;
public bool Albums { get; set; } = false;
public bool Artists { get; set; } = false;
} }
public class ScrobbleMapper public class ScrobbleMapper
@ -44,163 +42,11 @@ namespace Selector
public async Task Execute(CancellationToken token) public async Task Execute(CancellationToken token)
{ {
if (config.Artists) await MapTracks(token);
{
await MapArtists(token);
}
if (config.Albums)
{
await MapAlbums(token);
}
if (config.Tracks)
{
await MapTracks(token);
}
await mappingRepo.Save(); await mappingRepo.Save();
} }
private async Task MapArtists(CancellationToken token)
{
logger.LogInformation("Mapping scrobble artists");
var currentArtists = mappingRepo.GetArtists();
var scrobbleArtists = scrobbleRepo.GetAll()
.GroupBy(x => x.ArtistName)
.Select(x => (x.Key, x.Count()))
.OrderByDescending(x => x.Item2)
.Select(x => x.Key);
var artistsToPull = scrobbleArtists
.ExceptBy(currentArtists.Select(a => a.LastfmArtistName), a => a);
if (config.Limit is not null)
{
artistsToPull = artistsToPull.Take(config.Limit.Value);
}
var requests = artistsToPull.Select(a => new ScrobbleArtistMapping(
searchClient,
loggerFactory.CreateLogger<ScrobbleArtistMapping>() ?? NullLogger<ScrobbleArtistMapping>.Instance,
a)
).ToArray();
logger.LogInformation("Found {} artists to map, starting", requests.Length);
var batchRequest = new BatchingOperation<ScrobbleArtistMapping>(
config.InterRequestDelay,
config.Timeout,
config.SimultaneousConnections,
requests
);
await batchRequest.TriggerRequests(token);
logger.LogInformation("Finished mapping artists");
var newArtists = batchRequest.DoneRequests
.Select(a => a.Result)
.Cast<FullArtist>()
.Where(a => a is not null);
var newMappings = newArtists.Select(a => new ArtistLastfmSpotifyMapping()
{
LastfmArtistName = a.Name,
SpotifyUri = a.Uri
});
var existingUris = currentArtists.Select(a => a.SpotifyUri).ToArray();
foreach (var candidateMapping in newMappings)
{
if (existingUris.Contains(candidateMapping.SpotifyUri))
{
var duplicates = currentArtists.Where(a => a.LastfmArtistName.Equals(candidateMapping.LastfmArtistName, StringComparison.OrdinalIgnoreCase));
logger.LogWarning("Found duplicate Spotify uri ({}), [{}], {}",
candidateMapping.SpotifyUri,
candidateMapping.LastfmArtistName,
string.Join(", ", duplicates.Select(d => d.LastfmArtistName))
);
}
else
{
mappingRepo.Add(candidateMapping);
}
}
}
private async Task MapAlbums(CancellationToken token)
{
logger.LogInformation("Mapping scrobble albums");
var currentAlbums = mappingRepo.GetAlbums();
var scrobbleAlbums = scrobbleRepo.GetAll()
.GroupBy(x => (x.ArtistName, x.AlbumName))
.Select(x => (x.Key, x.Count()))
.OrderByDescending(x => x.Item2)
.Select(x => x.Key);
var albumsToPull = scrobbleAlbums
.ExceptBy(currentAlbums.Select(a => (a.LastfmArtistName, a.LastfmAlbumName)), a => a);
if (config.Limit is not null)
{
albumsToPull = albumsToPull.Take(config.Limit.Value);
}
var requests = albumsToPull.Select(a => new ScrobbleAlbumMapping(
searchClient,
loggerFactory.CreateLogger<ScrobbleAlbumMapping>() ?? NullLogger<ScrobbleAlbumMapping>.Instance,
a.AlbumName, a.ArtistName)
).ToArray();
logger.LogInformation("Found {} albums to map, starting", requests.Length);
var batchRequest = new BatchingOperation<ScrobbleAlbumMapping>(
config.InterRequestDelay,
config.Timeout,
config.SimultaneousConnections,
requests
);
await batchRequest.TriggerRequests(token);
logger.LogInformation("Finished mapping albums");
var newArtists = batchRequest.DoneRequests
.Select(a => a.Result)
.Cast<SimpleAlbum>()
.Where(a => a is not null);
var newMappings = newArtists.Select(a => new AlbumLastfmSpotifyMapping()
{
LastfmAlbumName = a.Name,
LastfmArtistName = a.Artists.FirstOrDefault()?.Name,
SpotifyUri = a.Uri
});
var existingUris = currentAlbums.Select(a => a.SpotifyUri).ToArray();
foreach(var candidateMapping in newMappings)
{
if(existingUris.Contains(candidateMapping.SpotifyUri))
{
var duplicates = currentAlbums.Where(a => a.LastfmArtistName.Equals(candidateMapping.LastfmArtistName, StringComparison.OrdinalIgnoreCase)
&& a.LastfmAlbumName.Equals(candidateMapping.LastfmAlbumName, StringComparison.OrdinalIgnoreCase));
logger.LogWarning("Found duplicate Spotify uri ({}), [{}, {}] {}",
candidateMapping.SpotifyUri,
candidateMapping.LastfmAlbumName,
candidateMapping.LastfmArtistName,
string.Join(", ", duplicates.Select(d => $"{d.LastfmAlbumName} {d.LastfmArtistName}"))
);
}
else
{
mappingRepo.Add(candidateMapping);
}
}
}
private async Task MapTracks(CancellationToken token) private async Task MapTracks(CancellationToken token)
{ {
logger.LogInformation("Mapping scrobble tracks"); logger.LogInformation("Mapping scrobble tracks");
@ -228,48 +74,70 @@ namespace Selector
logger.LogInformation("Found {} tracks to map, starting", requests.Length); logger.LogInformation("Found {} tracks to map, starting", requests.Length);
var batchRequest = new BatchingOperation<ScrobbleTrackMapping>( var batchRequest = GetOperation(requests);
config.InterRequestDelay,
config.Timeout,
config.SimultaneousConnections,
requests
);
await batchRequest.TriggerRequests(token); await batchRequest.TriggerRequests(token);
logger.LogInformation("Finished mapping tracks"); logger.LogInformation("Finished mapping tracks");
var newArtists = batchRequest.DoneRequests var newTracks = batchRequest.DoneRequests
.Select(a => a.Result) .Select(a => a.Result)
.Cast<FullTrack>() .Cast<FullTrack>()
.Where(a => a is not null); .Where(a => a is not null);
var newMappings = newArtists.Select(a => new TrackLastfmSpotifyMapping()
{
LastfmTrackName = a.Name,
LastfmArtistName = a.Artists.FirstOrDefault()?.Name,
SpotifyUri = a.Uri
});
var existingUris = currentTracks.Select(a => a.SpotifyUri).ToArray(); var existingTrackUris = currentTracks.Select(a => a.SpotifyUri).ToArray();
var existingAlbumUris = mappingRepo.GetAlbums().Select(a => a.SpotifyUri).ToArray();
var existingArtistUris = mappingRepo.GetArtists().Select(a => a.SpotifyUri).ToArray();
foreach (var candidateMapping in newMappings) foreach (var track in newTracks)
{ {
if (existingUris.Contains(candidateMapping.SpotifyUri)) if (existingTrackUris.Contains(track.Uri))
{ {
var duplicates = currentTracks.Where(a => a.LastfmArtistName.Equals(candidateMapping.LastfmArtistName, StringComparison.OrdinalIgnoreCase) var artistName = track.Artists.FirstOrDefault()?.Name;
&& a.LastfmTrackName.Equals(candidateMapping.LastfmTrackName, StringComparison.OrdinalIgnoreCase)); var duplicates = currentTracks.Where(a => a.LastfmArtistName.Equals(artistName, StringComparison.OrdinalIgnoreCase)
&& a.LastfmTrackName.Equals(track.Name, StringComparison.OrdinalIgnoreCase));
logger.LogWarning("Found duplicate Spotify uri ({}), [{}, {}] {}", logger.LogWarning("Found duplicate Spotify uri ({}), [{}, {}] {}",
candidateMapping.SpotifyUri, track.Uri,
candidateMapping.LastfmTrackName, track.Name,
candidateMapping.LastfmArtistName, artistName,
string.Join(", ", duplicates.Select(d => $"{d.LastfmTrackName} {d.LastfmArtistName}")) string.Join(", ", duplicates.Select(d => $"{d.LastfmTrackName} {d.LastfmArtistName}"))
); );
} }
else else
{ {
mappingRepo.Add(candidateMapping); mappingRepo.Add(new TrackLastfmSpotifyMapping()
{
LastfmTrackName = track.Name,
LastfmArtistName = track.Artists.FirstOrDefault()?.Name,
SpotifyUri = track.Uri
});
}
if(!existingAlbumUris.Contains(track.Album.Uri))
{
mappingRepo.Add(new AlbumLastfmSpotifyMapping()
{
LastfmAlbumName = track.Album.Name,
LastfmArtistName = track.Album.Artists.FirstOrDefault()?.Name,
SpotifyUri = track.Album.Uri
});
}
foreach(var artist in track.Artists.UnionBy(track.Album.Artists, a => a.Name))
{
if (!existingArtistUris.Contains(artist.Uri))
{
mappingRepo.Add(new ArtistLastfmSpotifyMapping()
{
LastfmArtistName = artist.Name,
SpotifyUri = artist.Uri
});
}
} }
} }
} }
private BatchingOperation<T> GetOperation<T>(IEnumerable<T> requests) where T: IOperation
=> new (config.InterRequestDelay, config.Timeout, config.SimultaneousConnections, requests);
} }
} }