From 6292d6c3f0b2c85390f60983ca1cb8485ef49380 Mon Sep 17 00:00:00 2001 From: andy Date: Thu, 3 Mar 2022 22:32:29 +0000 Subject: [PATCH] mapping albums and artists from track requests --- Selector.CLI/Command/Scrobble/ScrobbleMap.cs | 23 +- .../Extensions/CommandContextExtensions.cs | 4 +- Selector.CLI/Properties/launchSettings.json | 2 +- Selector.CLI/ScrobbleMapper.cs | 224 ++++-------------- 4 files changed, 54 insertions(+), 199 deletions(-) diff --git a/Selector.CLI/Command/Scrobble/ScrobbleMap.cs b/Selector.CLI/Command/Scrobble/ScrobbleMap.cs index b232d58..bc2bd25 100644 --- a/Selector.CLI/Command/Scrobble/ScrobbleMap.cs +++ b/Selector.CLI/Command/Scrobble/ScrobbleMap.cs @@ -24,26 +24,14 @@ namespace Selector.CLI simulOption.AddAlias("-s"); AddOption(simulOption); - var limitOption = new Option("--limit", "limit number of objects to poll"); + var limitOption = new Option("--limit", getDefaultValue: () => 200, "limit number of objects to poll"); limitOption.AddAlias("-l"); AddOption(limitOption); - var artists = new Option("--artist", "map scrobble artists to spotify"); - artists.AddAlias("-ar"); - AddOption(artists); - - var albums = new Option("--album", "map scrobble albums to spotify"); - albums.AddAlias("-al"); - AddOption(albums); - - var tracks = new Option("--track", "map scrobble tracks to spotify"); - tracks.AddAlias("-tr"); - AddOption(tracks); - - Handler = CommandHandler.Create(async (int delay, int simultaneous, int? limit, bool artist, bool album, bool track, CancellationToken token) => await Execute(delay, simultaneous, limit, artist, album, track, token)); + Handler = CommandHandler.Create(async (int delay, int simultaneous, int? limit, CancellationToken token) => await Execute(delay, simultaneous, limit, token)); } - public static async Task Execute(int delay, int simultaneous, int? limit, bool artists, bool albums, bool tracks, CancellationToken token) + public static async Task Execute(int delay, int simultaneous, int? limit, CancellationToken token) { try { @@ -58,10 +46,7 @@ namespace Selector.CLI { InterRequestDelay = new TimeSpan(0, 0, 0, 0, delay), SimultaneousConnections = simultaneous, - Limit = limit, - Artists = artists, - Albums = albums, - Tracks = tracks + Limit = limit }, new ScrobbleRepository(db), new ScrobbleMappingRepository(db), diff --git a/Selector.CLI/Extensions/CommandContextExtensions.cs b/Selector.CLI/Extensions/CommandContextExtensions.cs index 0ffe2d5..cf279c1 100644 --- a/Selector.CLI/Extensions/CommandContextExtensions.cs +++ b/Selector.CLI/Extensions/CommandContextExtensions.cs @@ -80,7 +80,9 @@ namespace Selector.CLI.Extensions using var db = new ApplicationDbContext(context.DatabaseConfig.Options, NullLogger.Instance); - refreshToken = db.Users.FirstOrDefault(u => u.UserName == "sarsoo")?.SpotifyRefreshToken; + var user = db.Users.FirstOrDefault(u => u.UserName == "sarsoo"); + + refreshToken = user?.SpotifyRefreshToken; } var configFactory = new RefreshTokenFactory(context.Config.ClientId, context.Config.ClientSecret, refreshToken); diff --git a/Selector.CLI/Properties/launchSettings.json b/Selector.CLI/Properties/launchSettings.json index fc2ac50..1ddf5a9 100644 --- a/Selector.CLI/Properties/launchSettings.json +++ b/Selector.CLI/Properties/launchSettings.json @@ -33,7 +33,7 @@ }, "Selector.CLI.Scrobble.Map": { "commandName": "Project", - "commandLineArgs": "scrobble map --artist -l 50 --album --track", + "commandLineArgs": "scrobble map", "environmentVariables": { "DOTNET_ENVIRONMENT": "Development" }, diff --git a/Selector.CLI/ScrobbleMapper.cs b/Selector.CLI/ScrobbleMapper.cs index 84ce73e..4214d04 100644 --- a/Selector.CLI/ScrobbleMapper.cs +++ b/Selector.CLI/ScrobbleMapper.cs @@ -4,6 +4,7 @@ using Selector.Model; using Selector.Operations; using SpotifyAPI.Web; using System; +using System.Collections.Generic; using System.Linq; using System.Threading; using System.Threading.Tasks; @@ -16,9 +17,6 @@ namespace Selector public TimeSpan Timeout { get; set; } = new TimeSpan(0, 20, 0); public int SimultaneousConnections { get; set; } = 3; public int? Limit { get; set; } = null; - public bool Tracks { get; set; } = false; - public bool Albums { get; set; } = false; - public bool Artists { get; set; } = false; } public class ScrobbleMapper @@ -44,163 +42,11 @@ namespace Selector public async Task Execute(CancellationToken token) { - if (config.Artists) - { - await MapArtists(token); - } - - if (config.Albums) - { - await MapAlbums(token); - } - - if (config.Tracks) - { - await MapTracks(token); - } + await MapTracks(token); await mappingRepo.Save(); } - private async Task MapArtists(CancellationToken token) - { - logger.LogInformation("Mapping scrobble artists"); - - var currentArtists = mappingRepo.GetArtists(); - var scrobbleArtists = scrobbleRepo.GetAll() - .GroupBy(x => x.ArtistName) - .Select(x => (x.Key, x.Count())) - .OrderByDescending(x => x.Item2) - .Select(x => x.Key); - - var artistsToPull = scrobbleArtists - .ExceptBy(currentArtists.Select(a => a.LastfmArtistName), a => a); - - if (config.Limit is not null) - { - artistsToPull = artistsToPull.Take(config.Limit.Value); - } - - var requests = artistsToPull.Select(a => new ScrobbleArtistMapping( - searchClient, - loggerFactory.CreateLogger() ?? NullLogger.Instance, - a) - ).ToArray(); - - logger.LogInformation("Found {} artists to map, starting", requests.Length); - - var batchRequest = new BatchingOperation( - config.InterRequestDelay, - config.Timeout, - config.SimultaneousConnections, - requests - ); - - await batchRequest.TriggerRequests(token); - - logger.LogInformation("Finished mapping artists"); - - var newArtists = batchRequest.DoneRequests - .Select(a => a.Result) - .Cast() - .Where(a => a is not null); - var newMappings = newArtists.Select(a => new ArtistLastfmSpotifyMapping() - { - LastfmArtistName = a.Name, - SpotifyUri = a.Uri - }); - - var existingUris = currentArtists.Select(a => a.SpotifyUri).ToArray(); - - foreach (var candidateMapping in newMappings) - { - if (existingUris.Contains(candidateMapping.SpotifyUri)) - { - var duplicates = currentArtists.Where(a => a.LastfmArtistName.Equals(candidateMapping.LastfmArtistName, StringComparison.OrdinalIgnoreCase)); - logger.LogWarning("Found duplicate Spotify uri ({}), [{}], {}", - candidateMapping.SpotifyUri, - candidateMapping.LastfmArtistName, - string.Join(", ", duplicates.Select(d => d.LastfmArtistName)) - ); - } - else - { - mappingRepo.Add(candidateMapping); - } - } - } - - private async Task MapAlbums(CancellationToken token) - { - logger.LogInformation("Mapping scrobble albums"); - - var currentAlbums = mappingRepo.GetAlbums(); - var scrobbleAlbums = scrobbleRepo.GetAll() - .GroupBy(x => (x.ArtistName, x.AlbumName)) - .Select(x => (x.Key, x.Count())) - .OrderByDescending(x => x.Item2) - .Select(x => x.Key); - - var albumsToPull = scrobbleAlbums - .ExceptBy(currentAlbums.Select(a => (a.LastfmArtistName, a.LastfmAlbumName)), a => a); - - if (config.Limit is not null) - { - albumsToPull = albumsToPull.Take(config.Limit.Value); - } - - var requests = albumsToPull.Select(a => new ScrobbleAlbumMapping( - searchClient, - loggerFactory.CreateLogger() ?? NullLogger.Instance, - a.AlbumName, a.ArtistName) - ).ToArray(); - - logger.LogInformation("Found {} albums to map, starting", requests.Length); - - var batchRequest = new BatchingOperation( - config.InterRequestDelay, - config.Timeout, - config.SimultaneousConnections, - requests - ); - - await batchRequest.TriggerRequests(token); - - logger.LogInformation("Finished mapping albums"); - - var newArtists = batchRequest.DoneRequests - .Select(a => a.Result) - .Cast() - .Where(a => a is not null); - var newMappings = newArtists.Select(a => new AlbumLastfmSpotifyMapping() - { - LastfmAlbumName = a.Name, - LastfmArtistName = a.Artists.FirstOrDefault()?.Name, - SpotifyUri = a.Uri - }); - - var existingUris = currentAlbums.Select(a => a.SpotifyUri).ToArray(); - - foreach(var candidateMapping in newMappings) - { - if(existingUris.Contains(candidateMapping.SpotifyUri)) - { - var duplicates = currentAlbums.Where(a => a.LastfmArtistName.Equals(candidateMapping.LastfmArtistName, StringComparison.OrdinalIgnoreCase) - && a.LastfmAlbumName.Equals(candidateMapping.LastfmAlbumName, StringComparison.OrdinalIgnoreCase)); - logger.LogWarning("Found duplicate Spotify uri ({}), [{}, {}] {}", - candidateMapping.SpotifyUri, - candidateMapping.LastfmAlbumName, - candidateMapping.LastfmArtistName, - string.Join(", ", duplicates.Select(d => $"{d.LastfmAlbumName} {d.LastfmArtistName}")) - ); - } - else - { - mappingRepo.Add(candidateMapping); - } - } - } - private async Task MapTracks(CancellationToken token) { logger.LogInformation("Mapping scrobble tracks"); @@ -228,48 +74,70 @@ namespace Selector logger.LogInformation("Found {} tracks to map, starting", requests.Length); - var batchRequest = new BatchingOperation( - config.InterRequestDelay, - config.Timeout, - config.SimultaneousConnections, - requests - ); + var batchRequest = GetOperation(requests); await batchRequest.TriggerRequests(token); logger.LogInformation("Finished mapping tracks"); - var newArtists = batchRequest.DoneRequests + var newTracks = batchRequest.DoneRequests .Select(a => a.Result) .Cast() .Where(a => a is not null); - var newMappings = newArtists.Select(a => new TrackLastfmSpotifyMapping() - { - LastfmTrackName = a.Name, - LastfmArtistName = a.Artists.FirstOrDefault()?.Name, - SpotifyUri = a.Uri - }); - var existingUris = currentTracks.Select(a => a.SpotifyUri).ToArray(); + var existingTrackUris = currentTracks.Select(a => a.SpotifyUri).ToArray(); + var existingAlbumUris = mappingRepo.GetAlbums().Select(a => a.SpotifyUri).ToArray(); + var existingArtistUris = mappingRepo.GetArtists().Select(a => a.SpotifyUri).ToArray(); - foreach (var candidateMapping in newMappings) + foreach (var track in newTracks) { - if (existingUris.Contains(candidateMapping.SpotifyUri)) + if (existingTrackUris.Contains(track.Uri)) { - var duplicates = currentTracks.Where(a => a.LastfmArtistName.Equals(candidateMapping.LastfmArtistName, StringComparison.OrdinalIgnoreCase) - && a.LastfmTrackName.Equals(candidateMapping.LastfmTrackName, StringComparison.OrdinalIgnoreCase)); + var artistName = track.Artists.FirstOrDefault()?.Name; + var duplicates = currentTracks.Where(a => a.LastfmArtistName.Equals(artistName, StringComparison.OrdinalIgnoreCase) + && a.LastfmTrackName.Equals(track.Name, StringComparison.OrdinalIgnoreCase)); logger.LogWarning("Found duplicate Spotify uri ({}), [{}, {}] {}", - candidateMapping.SpotifyUri, - candidateMapping.LastfmTrackName, - candidateMapping.LastfmArtistName, + track.Uri, + track.Name, + artistName, string.Join(", ", duplicates.Select(d => $"{d.LastfmTrackName} {d.LastfmArtistName}")) ); } else { - mappingRepo.Add(candidateMapping); + mappingRepo.Add(new TrackLastfmSpotifyMapping() + { + LastfmTrackName = track.Name, + LastfmArtistName = track.Artists.FirstOrDefault()?.Name, + SpotifyUri = track.Uri + }); + } + + if(!existingAlbumUris.Contains(track.Album.Uri)) + { + mappingRepo.Add(new AlbumLastfmSpotifyMapping() + { + LastfmAlbumName = track.Album.Name, + LastfmArtistName = track.Album.Artists.FirstOrDefault()?.Name, + SpotifyUri = track.Album.Uri + }); + } + + foreach(var artist in track.Artists.UnionBy(track.Album.Artists, a => a.Name)) + { + if (!existingArtistUris.Contains(artist.Uri)) + { + mappingRepo.Add(new ArtistLastfmSpotifyMapping() + { + LastfmArtistName = artist.Name, + SpotifyUri = artist.Uri + }); + } } } } + + private BatchingOperation GetOperation(IEnumerable requests) where T: IOperation + => new (config.InterRequestDelay, config.Timeout, config.SimultaneousConnections, requests); } }