From fbbd42f97a2aaed75ab977ea6599dd0c1b95e4ef Mon Sep 17 00:00:00 2001 From: andy Date: Thu, 24 Feb 2022 00:27:34 +0000 Subject: [PATCH] working scrobble mapping command added scrobble mapping repo --- Selector.CLI/Command/CommandContext.cs | 2 + Selector.CLI/Command/Scrobble/Scrobble.cs | 3 + Selector.CLI/Command/Scrobble/ScrobbleMap.cs | 81 ++++++++++ Selector.CLI/Command/Scrobble/ScrobbleSave.cs | 25 ++- .../Extensions/CommandContextExtensions.cs | 31 ++++ Selector.CLI/Extensions/ServiceExtensions.cs | 2 + Selector.CLI/Options.cs | 4 + Selector.CLI/ScrobbleMapper.cs | 108 +++++++++++++ Selector.CLI/ScrobbleSaver.cs | 135 ++++++---------- .../Scrobble/IScrobbleMappingRepository.cs | 29 ++++ .../Scrobble/IScrobbleRepository.cs | 21 +++ .../{ => Scrobble}/LastfmSpotifyMapping.cs | 0 .../Scrobble/ScrobbleMappingRepository.cs | 146 ++++++++++++++++++ Selector.Model/Scrobble/ScrobbleRepository.cs | 122 +++++++++++++++ Selector.Model/{ => Scrobble}/UserScrobble.cs | 0 Selector/Operations/BatchingOperation.cs | 74 +++++++++ Selector/Operations/IOperation.cs | 15 ++ Selector/Scrobble/ScrobbleAlbumMapping.cs | 39 +++++ Selector/Scrobble/ScrobbleArtistMapping.cs | 38 +++++ Selector/Scrobble/ScrobbleMapping.cs | 80 ++++++++++ Selector/Scrobble/ScrobbleRequest.cs | 14 +- Selector/Scrobble/ScrobbleTrackMapping.cs | 41 +++++ 22 files changed, 902 insertions(+), 108 deletions(-) create mode 100644 Selector.CLI/Command/Scrobble/ScrobbleMap.cs create mode 100644 Selector.CLI/ScrobbleMapper.cs create mode 100644 Selector.Model/Scrobble/IScrobbleMappingRepository.cs create mode 100644 Selector.Model/Scrobble/IScrobbleRepository.cs rename Selector.Model/{ => Scrobble}/LastfmSpotifyMapping.cs (100%) create mode 100644 Selector.Model/Scrobble/ScrobbleMappingRepository.cs create mode 100644 Selector.Model/Scrobble/ScrobbleRepository.cs rename Selector.Model/{ => Scrobble}/UserScrobble.cs (100%) create mode 100644 Selector/Operations/BatchingOperation.cs create mode 100644 Selector/Operations/IOperation.cs create mode 100644 Selector/Scrobble/ScrobbleAlbumMapping.cs create mode 100644 Selector/Scrobble/ScrobbleArtistMapping.cs create mode 100644 Selector/Scrobble/ScrobbleMapping.cs create mode 100644 Selector/Scrobble/ScrobbleTrackMapping.cs diff --git a/Selector.CLI/Command/CommandContext.cs b/Selector.CLI/Command/CommandContext.cs index f139972..96d38fd 100644 --- a/Selector.CLI/Command/CommandContext.cs +++ b/Selector.CLI/Command/CommandContext.cs @@ -2,6 +2,7 @@ using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Logging; using Selector.Model; +using SpotifyAPI.Web; namespace Selector.CLI { @@ -9,6 +10,7 @@ namespace Selector.CLI { public RootOptions Config { get; set; } public ILoggerFactory Logger { get; set; } + public ISpotifyClient Spotify{ get; set; } public DbContextOptionsBuilder DatabaseConfig { get; set; } public LastfmClient LastFmClient { get; set; } diff --git a/Selector.CLI/Command/Scrobble/Scrobble.cs b/Selector.CLI/Command/Scrobble/Scrobble.cs index 47a6d2c..2671871 100644 --- a/Selector.CLI/Command/Scrobble/Scrobble.cs +++ b/Selector.CLI/Command/Scrobble/Scrobble.cs @@ -11,6 +11,9 @@ namespace Selector.CLI var clearCommand = new ScrobbleClearCommand("clear", "clear user scrobbles"); AddCommand(clearCommand); + + var mapCommand = new ScrobbleMapCommand("map", "map last.fm data to spotify uris"); + AddCommand(mapCommand); } } } diff --git a/Selector.CLI/Command/Scrobble/ScrobbleMap.cs b/Selector.CLI/Command/Scrobble/ScrobbleMap.cs new file mode 100644 index 0000000..b232d58 --- /dev/null +++ b/Selector.CLI/Command/Scrobble/ScrobbleMap.cs @@ -0,0 +1,81 @@ +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Logging; +using Selector.CLI.Extensions; +using Selector.Model; +using Selector.Model.Extensions; +using System; +using System.CommandLine; +using System.CommandLine.Invocation; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; + +namespace Selector.CLI +{ + public class ScrobbleMapCommand : Command + { + public ScrobbleMapCommand(string name, string description = null) : base(name, description) + { + var delayOption = new Option("--delay", getDefaultValue: () => 100, "milliseconds to delay"); + delayOption.AddAlias("-d"); + AddOption(delayOption); + + var simulOption = new Option("--simultaneous", getDefaultValue: () => 3, "simultaneous connections when pulling"); + simulOption.AddAlias("-s"); + AddOption(simulOption); + + var limitOption = new Option("--limit", "limit number of objects to poll"); + limitOption.AddAlias("-l"); + AddOption(limitOption); + + var artists = new Option("--artist", "map scrobble artists to spotify"); + artists.AddAlias("-ar"); + AddOption(artists); + + var albums = new Option("--album", "map scrobble albums to spotify"); + albums.AddAlias("-al"); + AddOption(albums); + + var tracks = new Option("--track", "map scrobble tracks to spotify"); + tracks.AddAlias("-tr"); + AddOption(tracks); + + Handler = CommandHandler.Create(async (int delay, int simultaneous, int? limit, bool artist, bool album, bool track, CancellationToken token) => await Execute(delay, simultaneous, limit, artist, album, track, token)); + } + + public static async Task Execute(int delay, int simultaneous, int? limit, bool artists, bool albums, bool tracks, CancellationToken token) + { + try + { + var context = new CommandContext().WithLogger().WithDb().WithSpotify(); + var logger = context.Logger.CreateLogger("Scrobble"); + + using var db = new ApplicationDbContext(context.DatabaseConfig.Options, context.Logger.CreateLogger()); + + await new ScrobbleMapper( + context.Spotify.Search, + new () + { + InterRequestDelay = new TimeSpan(0, 0, 0, 0, delay), + SimultaneousConnections = simultaneous, + Limit = limit, + Artists = artists, + Albums = albums, + Tracks = tracks + }, + new ScrobbleRepository(db), + new ScrobbleMappingRepository(db), + context.Logger.CreateLogger(), + context.Logger) + .Execute(token); + } + catch (Exception ex) + { + Console.WriteLine(ex); + return 1; + } + + return 0; + } + } +} diff --git a/Selector.CLI/Command/Scrobble/ScrobbleSave.cs b/Selector.CLI/Command/Scrobble/ScrobbleSave.cs index 5c85666..cb838cd 100644 --- a/Selector.CLI/Command/Scrobble/ScrobbleSave.cs +++ b/Selector.CLI/Command/Scrobble/ScrobbleSave.cs @@ -57,21 +57,32 @@ namespace Selector.CLI var logger = context.Logger.CreateLogger("Scrobble"); using var db = new ApplicationDbContext(context.DatabaseConfig.Options, context.Logger.CreateLogger()); + var repo = new ScrobbleRepository(db); - logger.LogInformation("Running from {0} to {1}", from, to); + logger.LogInformation("Running from {} to {}", from, to); - logger.LogInformation("Searching for {0}", username); + logger.LogInformation("Searching for {}", username); var user = db.Users.AsNoTracking().FirstOrDefault(u => u.UserName == username); if (user is not null) { if (user.LastFmConnected()) { - logger.LogInformation("Last.fm username found ({0}), starting...", user.LastFmUsername); + logger.LogInformation("Last.fm username found ({}), starting...", user.LastFmUsername); + + if(from.Kind != DateTimeKind.Utc) + { + from = from.ToUniversalTime(); + } + + if (to.Kind != DateTimeKind.Utc) + { + to = to.ToUniversalTime(); + } await new ScrobbleSaver( context.LastFmClient.User, - new ScrobbleSaverConfig() + new () { User = user, InterRequestDelay = new TimeSpan(0, 0, 0, 0, delay), @@ -82,19 +93,19 @@ namespace Selector.CLI DontRemove = noRemove, SimultaneousConnections = simul }, - db, + repo, context.Logger.CreateLogger(), context.Logger) .Execute(token); } else { - logger.LogError("{0} doesn't have a Last.fm username", username); + logger.LogError("{} doesn't have a Last.fm username", username); } } else { - logger.LogError("{0} not found", username); + logger.LogError("{} not found", username); } } diff --git a/Selector.CLI/Extensions/CommandContextExtensions.cs b/Selector.CLI/Extensions/CommandContextExtensions.cs index a23dbaf..0ffe2d5 100644 --- a/Selector.CLI/Extensions/CommandContextExtensions.cs +++ b/Selector.CLI/Extensions/CommandContextExtensions.cs @@ -2,7 +2,10 @@ using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; using Selector.Model; +using SpotifyAPI.Web; +using System.Linq; namespace Selector.CLI.Extensions { @@ -58,5 +61,33 @@ namespace Selector.CLI.Extensions return context; } + + public static CommandContext WithSpotify(this CommandContext context) + { + if (context.Config is null) + { + context.WithConfig(); + } + + var refreshToken = context.Config.RefreshToken; + + if(string.IsNullOrWhiteSpace(refreshToken)) + { + if (context.DatabaseConfig is null) + { + context.WithDb(); + } + + using var db = new ApplicationDbContext(context.DatabaseConfig.Options, NullLogger.Instance); + + refreshToken = db.Users.FirstOrDefault(u => u.UserName == "sarsoo")?.SpotifyRefreshToken; + } + + var configFactory = new RefreshTokenFactory(context.Config.ClientId, context.Config.ClientSecret, refreshToken); + + context.Spotify = new SpotifyClient(configFactory.GetConfig().Result); + + return context; + } } } diff --git a/Selector.CLI/Extensions/ServiceExtensions.cs b/Selector.CLI/Extensions/ServiceExtensions.cs index 63e0d2d..3fbf4cf 100644 --- a/Selector.CLI/Extensions/ServiceExtensions.cs +++ b/Selector.CLI/Extensions/ServiceExtensions.cs @@ -45,6 +45,8 @@ namespace Selector.CLI.Extensions options.UseNpgsql(config.DatabaseOptions.ConnectionString) ); + services.AddTransient(); + services.AddHostedService(); } diff --git a/Selector.CLI/Options.cs b/Selector.CLI/Options.cs index 4cb4507..c73f596 100644 --- a/Selector.CLI/Options.cs +++ b/Selector.CLI/Options.cs @@ -36,6 +36,10 @@ namespace Selector.CLI /// Spotify app secret /// public string ClientSecret { get; set; } + /// + /// Service account refresh token for tool spotify usage + /// + public string RefreshToken { get; set; } public string LastfmClient { get; set; } public string LastfmSecret { get; set; } public WatcherOptions WatcherOptions { get; set; } = new(); diff --git a/Selector.CLI/ScrobbleMapper.cs b/Selector.CLI/ScrobbleMapper.cs new file mode 100644 index 0000000..4654d71 --- /dev/null +++ b/Selector.CLI/ScrobbleMapper.cs @@ -0,0 +1,108 @@ +using IF.Lastfm.Core.Api; +using IF.Lastfm.Core.Api.Helpers; +using IF.Lastfm.Core.Objects; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Selector.Model; +using Selector.Operations; +using SpotifyAPI.Web; +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using System.Text; +using System.Threading; +using System.Threading.Tasks; + +namespace Selector +{ + public class ScrobbleMapperConfig + { + public TimeSpan InterRequestDelay { get; set; } + public TimeSpan Timeout { get; set; } = new TimeSpan(0, 20, 0); + public int SimultaneousConnections { get; set; } = 3; + public int? Limit { get; set; } = null; + public bool Tracks { get; set; } = false; + public bool Albums { get; set; } = false; + public bool Artists { get; set; } = false; + } + + public class ScrobbleMapper + { + private readonly ILogger logger; + private readonly ILoggerFactory loggerFactory; + private readonly ISearchClient searchClient; + + private readonly ScrobbleMapperConfig config; + + private readonly IScrobbleRepository scrobbleRepo; + private readonly IScrobbleMappingRepository mappingRepo; + + public ScrobbleMapper(ISearchClient _searchClient, ScrobbleMapperConfig _config, IScrobbleRepository _scrobbleRepository, IScrobbleMappingRepository _scrobbleMappingRepository, ILogger _logger, ILoggerFactory _loggerFactory = null) + { + searchClient = _searchClient; + config = _config; + scrobbleRepo = _scrobbleRepository; + mappingRepo = _scrobbleMappingRepository; + logger = _logger; + loggerFactory = _loggerFactory; + } + + public async Task Execute(CancellationToken token) + { + if (config.Artists) + { + logger.LogInformation("Mapping scrobble artists"); + + var currentArtists = mappingRepo.GetArtists(); + var scrobbleArtists = scrobbleRepo.GetAll() + .GroupBy(x => x.ArtistName) + .Select(x => (x.Key, x.Count())) + .OrderByDescending(x => x.Item2) + .Select(x => x.Key); + + if(config.Limit is not null) + { + scrobbleArtists = scrobbleArtists.Take(config.Limit.Value); + } + + var artistsToPull = scrobbleArtists + .ExceptBy(currentArtists.Select(a => a.LastfmArtistName), a => a); + + + var requests = artistsToPull.Select(a => new ScrobbleArtistMapping( + searchClient, + loggerFactory.CreateLogger() ?? NullLogger.Instance, + a) + ).ToList(); + + logger.LogInformation("Found {} artists to map, starting", requests.Count); + + var batchRequest = new BatchingOperation( + config.InterRequestDelay, + config.Timeout, + config.SimultaneousConnections, + requests + ); + + await batchRequest.TriggerRequests(token); + + logger.LogInformation("Finished mapping artists"); + + var newArtists = batchRequest.DoneRequests + .Where(a => a is not null) + .Select(a => (FullArtist) a.Result) + .Where(a => a is not null); + var newMappings = newArtists.Select(a => new ArtistLastfmSpotifyMapping() { + LastfmArtistName = a.Name, + SpotifyUri = a.Uri + }); + + mappingRepo.AddRange(newMappings); + } + + await mappingRepo.Save(); + } + } +} diff --git a/Selector.CLI/ScrobbleSaver.cs b/Selector.CLI/ScrobbleSaver.cs index a14a665..16dacde 100644 --- a/Selector.CLI/ScrobbleSaver.cs +++ b/Selector.CLI/ScrobbleSaver.cs @@ -1,9 +1,9 @@ using IF.Lastfm.Core.Api; -using IF.Lastfm.Core.Api.Helpers; using IF.Lastfm.Core.Objects; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging.Abstractions; using Selector.Model; +using Selector.Operations; using System; using System.Collections.Concurrent; using System.Collections.Generic; @@ -36,27 +36,23 @@ namespace Selector private readonly IUserApi userClient; private readonly ScrobbleSaverConfig config; - private CancellationToken _token; - private Task aggregateNetworkTask; + private Task batchTask; + private BatchingOperation batchOperation; - private readonly ApplicationDbContext db; + private readonly IScrobbleRepository scrobbleRepo; - private ConcurrentQueue waitingRequests = new(); - private ConcurrentQueue runRequests = new(); - - public ScrobbleSaver(IUserApi _userClient, ScrobbleSaverConfig _config, ApplicationDbContext _db, ILogger _logger, ILoggerFactory _loggerFactory = null) + public ScrobbleSaver(IUserApi _userClient, ScrobbleSaverConfig _config, IScrobbleRepository _scrobbleRepository, ILogger _logger, ILoggerFactory _loggerFactory = null) { userClient = _userClient; config = _config; - db = _db; + scrobbleRepo = _scrobbleRepository; logger = _logger; loggerFactory = _loggerFactory; } public async Task Execute(CancellationToken token) { - logger.LogInformation("Saving scrobbles for {0}/{1}", config.User.UserName, config.User.LastFmUsername); - _token = token; + logger.LogInformation("Saving scrobbles for {}/{}", config.User.UserName, config.User.LastFmUsername); var page1 = new ScrobbleRequest(userClient, loggerFactory?.CreateLogger() ?? NullLogger.Instance, @@ -66,21 +62,37 @@ namespace Selector config.From, config.To); await page1.Execute(); - runRequests.Enqueue(page1); if (page1.Succeeded) { if (page1.TotalPages > 1) { - TriggerNetworkRequests(page1.TotalPages, token); + batchOperation = new BatchingOperation( + config.InterRequestDelay, + config.Timeout, + config.SimultaneousConnections, + GetRequestsFromPageNumbers(2, page1.TotalPages) + ); + + batchTask = batchOperation.TriggerRequests(token); } + logger.LogDebug("Pulling currently stored scrobbles"); - var currentScrobbles = GetDbScrobbles(); + var currentScrobbles = scrobbleRepo.GetAll(userId: config.User.Id, from: config.From, to: config.To); - await aggregateNetworkTask; - var scrobbles = runRequests.SelectMany(r => r.Scrobbles); + if(batchTask is not null) + { + await batchTask; + } + + var scrobbles = page1.Scrobbles; + + if(batchOperation is not null) + { + scrobbles.AddRange(batchOperation.DoneRequests.SelectMany(r => r.Scrobbles)); + } IdentifyDuplicates(scrobbles); @@ -97,7 +109,7 @@ namespace Selector return nativeScrobble; }); - logger.LogInformation("Completed database scrobble pulling for {0}, pulled {1:n0}", config.User.UserName, nativeScrobbles.Count()); + logger.LogInformation("Completed database scrobble pulling for {}, pulled {:n0}", config.User.UserName, nativeScrobbles.Count()); logger.LogDebug("Identifying difference sets"); var time = Stopwatch.StartNew(); @@ -105,99 +117,39 @@ namespace Selector (var toAdd, var toRemove) = ScrobbleMatcher.IdentifyDiffs(currentScrobbles, nativeScrobbles); time.Stop(); - logger.LogTrace("Finished diffing: {0:n}ms", time.ElapsedMilliseconds); + logger.LogTrace("Finished diffing: {:n}ms", time.ElapsedMilliseconds); var timeDbOps = Stopwatch.StartNew(); if(!config.DontAdd) { - await db.Scrobble.AddRangeAsync(toAdd.Cast()); + scrobbleRepo.AddRange(toAdd.Cast()); } else { - logger.LogInformation("Skipping adding of {0} scrobbles", toAdd.Count()); + logger.LogInformation("Skipping adding of {} scrobbles", toAdd.Count()); } if (!config.DontRemove) { - db.Scrobble.RemoveRange(toRemove.Cast()); + scrobbleRepo.RemoveRange(toRemove.Cast()); } else { - logger.LogInformation("Skipping removal of {0} scrobbles", toRemove.Count()); + logger.LogInformation("Skipping removal of {} scrobbles", toRemove.Count()); } - await db.SaveChangesAsync(); + await scrobbleRepo.Save(); timeDbOps.Stop(); - logger.LogTrace("DB ops: {0:n}ms", timeDbOps.ElapsedMilliseconds); + logger.LogTrace("DB ops: {:n}ms", timeDbOps.ElapsedMilliseconds); - logger.LogInformation("Completed scrobble pulling for {0}, +{1:n0}, -{2:n0}", config.User.UserName, toAdd.Count(), toRemove.Count()); + logger.LogInformation("Completed scrobble pulling for {}, +{:n0}, -{:n0}", config.User.UserName, toAdd.Count(), toRemove.Count()); } else { - logger.LogError("Failed to pull first scrobble page for {0}/{1}", config.User.UserName, config.User.LastFmUsername); + logger.LogError("Failed to pull first scrobble page for {}/{}", config.User.UserName, config.User.LastFmUsername); } } - private async void HandleSuccessfulRequest(object o, EventArgs e) - { - await Task.Delay(config.InterRequestDelay, _token); - TransitionRequest(); - } - - private void TransitionRequest() - { - if (waitingRequests.TryDequeue(out var request)) - { - request.Success += HandleSuccessfulRequest; - _ = request.Execute(); - runRequests.Enqueue(request); - } - } - - private void TriggerNetworkRequests(int totalPages, CancellationToken token) - { - foreach (var req in GetRequestsFromPageNumbers(2, totalPages)) - { - waitingRequests.Enqueue(req); - } - - foreach (var _ in Enumerable.Range(1, config.SimultaneousConnections)) - { - TransitionRequest(); - } - - var timeoutTask = Task.Delay(config.Timeout, token); - var allTasks = waitingRequests.Union(runRequests).Select(r => r.Task).ToList(); - - aggregateNetworkTask = Task.WhenAny(timeoutTask, Task.WhenAll(allTasks)); - - aggregateNetworkTask.ContinueWith(t => - { - if (timeoutTask.IsCompleted) - { - throw new TimeoutException($"Timed-out pulling scrobbles, took {config.Timeout}"); - } - }); - } - - private IEnumerable GetDbScrobbles() - { - var currentScrobbles = db.Scrobble.AsEnumerable() - .Where(s => s.UserId == config.User.Id); - - if (config.From is not null) - { - currentScrobbles = currentScrobbles.Where(s => s.Timestamp > config.From); - } - - if (config.To is not null) - { - currentScrobbles = currentScrobbles.Where(s => s.Timestamp < config.To); - } - - return currentScrobbles; - } - private IEnumerable GetRequestsFromPageNumbers(int start, int totalPages) => Enumerable.Range(start, totalPages - 1) .Select(n => new ScrobbleRequest( @@ -207,7 +159,8 @@ namespace Selector n, config.PageSize, config.From, - config.To)); + config.To, + config.Retries)); private void IdentifyDuplicates(IEnumerable tracks) { @@ -223,18 +176,16 @@ namespace Selector foreach(var scrobble in dupe) { - dupeString.Append("("); + dupeString.Append('('); dupeString.Append(scrobble.Name); dupeString.Append(", "); dupeString.Append(scrobble.AlbumName); dupeString.Append(", "); dupeString.Append(scrobble.ArtistName); - dupeString.Append(")"); - - dupeString.Append(" "); + dupeString.Append(") "); } - logger.LogInformation("Duplicate at {0}: {1}", dupe.Key, dupeString.ToString()); + logger.LogInformation("Duplicate at {}: {}", dupe.Key, dupeString.ToString()); } } diff --git a/Selector.Model/Scrobble/IScrobbleMappingRepository.cs b/Selector.Model/Scrobble/IScrobbleMappingRepository.cs new file mode 100644 index 0000000..7881adc --- /dev/null +++ b/Selector.Model/Scrobble/IScrobbleMappingRepository.cs @@ -0,0 +1,29 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Selector.Model +{ + public interface IScrobbleMappingRepository + { + void Add(TrackLastfmSpotifyMapping item); + void Add(AlbumLastfmSpotifyMapping item); + void Add(ArtistLastfmSpotifyMapping item); + void AddRange(IEnumerable item); + void AddRange(IEnumerable item); + void AddRange(IEnumerable item); + IEnumerable GetTracks(string include = null, string trackName = null, string albumName = null, string artistName = null); + IEnumerable GetAlbums(string include = null, string albumName = null, string artistName = null); + IEnumerable GetArtists(string include = null, string artistName = null); + + public void Remove(TrackLastfmSpotifyMapping mapping); + public void Remove(AlbumLastfmSpotifyMapping mapping); + public void Remove(ArtistLastfmSpotifyMapping mapping); + public void RemoveRange(IEnumerable mappings); + public void RemoveRange(IEnumerable mappings); + public void RemoveRange(IEnumerable mappings); + Task Save(); + } +} diff --git a/Selector.Model/Scrobble/IScrobbleRepository.cs b/Selector.Model/Scrobble/IScrobbleRepository.cs new file mode 100644 index 0000000..d9cbfab --- /dev/null +++ b/Selector.Model/Scrobble/IScrobbleRepository.cs @@ -0,0 +1,21 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Selector.Model +{ + public interface IScrobbleRepository + { + void Add(UserScrobble item); + void AddRange(IEnumerable item); + IEnumerable GetAll(string include = null, string userId = null, string username = null, string trackName = null, string albumName = null, string artistName = null, DateTime? from = null, DateTime? to = null); + UserScrobble Find(int key, string include = null); + void Remove(int key); + public void Remove(UserScrobble scrobble); + public void RemoveRange(IEnumerable scrobbles); + void Update(UserScrobble item); + Task Save(); + } +} diff --git a/Selector.Model/LastfmSpotifyMapping.cs b/Selector.Model/Scrobble/LastfmSpotifyMapping.cs similarity index 100% rename from Selector.Model/LastfmSpotifyMapping.cs rename to Selector.Model/Scrobble/LastfmSpotifyMapping.cs diff --git a/Selector.Model/Scrobble/ScrobbleMappingRepository.cs b/Selector.Model/Scrobble/ScrobbleMappingRepository.cs new file mode 100644 index 0000000..f252b4d --- /dev/null +++ b/Selector.Model/Scrobble/ScrobbleMappingRepository.cs @@ -0,0 +1,146 @@ +using Microsoft.Extensions.Logging; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore; + +namespace Selector.Model +{ + public class ScrobbleMappingRepository : IScrobbleMappingRepository + { + private readonly ApplicationDbContext db; + + public ScrobbleMappingRepository(ApplicationDbContext context) + { + db = context; + } + + public void Add(TrackLastfmSpotifyMapping item) + { + db.TrackMapping.Add(item); + } + + public void Add(AlbumLastfmSpotifyMapping item) + { + db.AlbumMapping.Add(item); + } + + public void Add(ArtistLastfmSpotifyMapping item) + { + db.ArtistMapping.Add(item); + } + + public void AddRange(IEnumerable item) + { + db.TrackMapping.AddRange(item); + } + + public void AddRange(IEnumerable item) + { + db.AlbumMapping.AddRange(item); + } + + public void AddRange(IEnumerable item) + { + db.ArtistMapping.AddRange(item); + } + + public IEnumerable GetAlbums(string include = null, string albumName = null, string artistName = null) + { + var mappings = db.AlbumMapping.AsQueryable(); + + if (!string.IsNullOrWhiteSpace(include)) + { + mappings = mappings.Include(include); + } + + if (!string.IsNullOrWhiteSpace(albumName)) + { + mappings = mappings.Where(s => s.LastfmAlbumName == albumName); + } + + if (!string.IsNullOrWhiteSpace(artistName)) + { + mappings = mappings.Where(s => s.LastfmArtistName == artistName); + } + + return mappings.AsEnumerable(); + } + + public IEnumerable GetArtists(string include = null, string artistName = null) + { + var mappings = db.ArtistMapping.AsQueryable(); + + if (!string.IsNullOrWhiteSpace(include)) + { + mappings = mappings.Include(include); + } + + if (!string.IsNullOrWhiteSpace(artistName)) + { + mappings = mappings.Where(s => s.LastfmArtistName == artistName); + } + + return mappings.AsEnumerable(); + } + + public IEnumerable GetTracks(string include = null, string trackName = null, string albumName = null, string artistName = null) + { + var mappings = db.TrackMapping.AsQueryable(); + + if (!string.IsNullOrWhiteSpace(include)) + { + mappings = mappings.Include(include); + } + + if (!string.IsNullOrWhiteSpace(trackName)) + { + mappings = mappings.Where(s => s.LastfmTrackName == trackName); + } + + if (!string.IsNullOrWhiteSpace(artistName)) + { + mappings = mappings.Where(s => s.LastfmArtistName == artistName); + } + + return mappings.AsEnumerable(); + } + + public void Remove(TrackLastfmSpotifyMapping mapping) + { + db.TrackMapping.Remove(mapping); + } + + public void Remove(AlbumLastfmSpotifyMapping mapping) + { + db.AlbumMapping.Remove(mapping); + } + + public void Remove(ArtistLastfmSpotifyMapping mapping) + { + db.ArtistMapping.Remove(mapping); + } + + public void RemoveRange(IEnumerable mappings) + { + db.TrackMapping.RemoveRange(mappings); + } + + public void RemoveRange(IEnumerable mappings) + { + db.AlbumMapping.RemoveRange(mappings); + } + + public void RemoveRange(IEnumerable mappings) + { + db.ArtistMapping.RemoveRange(mappings); + } + + public Task Save() + { + return db.SaveChangesAsync(); + } + } +} diff --git a/Selector.Model/Scrobble/ScrobbleRepository.cs b/Selector.Model/Scrobble/ScrobbleRepository.cs new file mode 100644 index 0000000..7663efb --- /dev/null +++ b/Selector.Model/Scrobble/ScrobbleRepository.cs @@ -0,0 +1,122 @@ +using Microsoft.Extensions.Logging; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore; + +namespace Selector.Model +{ + public class ScrobbleRepository : IScrobbleRepository + { + private readonly ApplicationDbContext db; + + public ScrobbleRepository(ApplicationDbContext context) + { + db = context; + } + + public void Add(UserScrobble item) + { + db.Scrobble.Add(item); + } + + public void AddRange(IEnumerable item) + { + db.Scrobble.AddRange(item); + } + + public UserScrobble Find(int key, string include = null) + { + var scrobbles = db.Scrobble.Where(s => s.Id == key); + + if (!string.IsNullOrWhiteSpace(include)) + { + scrobbles = scrobbles.Include(include); + } + + return scrobbles.FirstOrDefault(); + } + + public IEnumerable GetAll(string include = null, string userId = null, string username = null, string trackName = null, string albumName = null, string artistName = null, DateTime? from = null, DateTime? to = null) + { + var scrobbles = db.Scrobble.AsQueryable(); + + if (!string.IsNullOrWhiteSpace(include)) + { + scrobbles = scrobbles.Include(include); + } + + if (!string.IsNullOrWhiteSpace(userId)) + { + scrobbles = scrobbles.Where(s => s.UserId == userId); + } + + if (!string.IsNullOrWhiteSpace(username)) + { + var user = db.Users.AsNoTracking().Where(u => u.UserName.Equals(username, StringComparison.InvariantCultureIgnoreCase)).FirstOrDefault(); + if (user is not null) + { + scrobbles = scrobbles.Where(s => s.UserId == user.Id); + } + else + { + scrobbles = Enumerable.Empty().AsQueryable(); + } + } + + if (!string.IsNullOrWhiteSpace(trackName)) + { + scrobbles = scrobbles.Where(s => s.TrackName == trackName); + } + + if (!string.IsNullOrWhiteSpace(albumName)) + { + scrobbles = scrobbles.Where(s => s.AlbumName == albumName); + } + + if (!string.IsNullOrWhiteSpace(artistName)) + { + scrobbles = scrobbles.Where(s => s.ArtistName == artistName); + } + + if (from is not null) + { + scrobbles = scrobbles.Where(u => u.Timestamp >= from.Value); + } + + if (to is not null) + { + scrobbles = scrobbles.Where(u => u.Timestamp < to.Value); + } + + return scrobbles.AsEnumerable(); + } + + public void Remove(int key) + { + Remove(Find(key)); + } + + public void Remove(UserScrobble scrobble) + { + db.Scrobble.Remove(scrobble); + } + + public void RemoveRange(IEnumerable scrobbles) + { + db.Scrobble.RemoveRange(scrobbles); + } + + public void Update(UserScrobble item) + { + db.Scrobble.Update(item); + } + + public Task Save() + { + return db.SaveChangesAsync(); + } + } +} diff --git a/Selector.Model/UserScrobble.cs b/Selector.Model/Scrobble/UserScrobble.cs similarity index 100% rename from Selector.Model/UserScrobble.cs rename to Selector.Model/Scrobble/UserScrobble.cs diff --git a/Selector/Operations/BatchingOperation.cs b/Selector/Operations/BatchingOperation.cs new file mode 100644 index 0000000..da56147 --- /dev/null +++ b/Selector/Operations/BatchingOperation.cs @@ -0,0 +1,74 @@ +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading; +using System.Threading.Tasks; + +namespace Selector.Operations +{ + public class BatchingOperation where T : IOperation + { + protected ILogger> logger; + protected CancellationToken _token; + protected Task aggregateNetworkTask; + + public ConcurrentQueue WaitingRequests { get; private set; } = new(); + public ConcurrentQueue DoneRequests { get; private set; } = new(); + + private TimeSpan interRequestDelay; + private TimeSpan timeout; + private int simultaneousRequests; + + public BatchingOperation(TimeSpan _interRequestDelay, TimeSpan _timeout, int _simultaneous, IEnumerable requests, ILogger> _logger = null) + { + interRequestDelay = _interRequestDelay; + timeout = _timeout; + simultaneousRequests = _simultaneous; + logger = _logger ?? NullLogger>.Instance; + + foreach(var request in requests) + { + WaitingRequests.Enqueue(request); + } + } + + public bool TimedOut { get; private set; } = false; + + private async void HandleSuccessfulRequest(object o, EventArgs e) + { + await Task.Delay(interRequestDelay, _token); + TransitionRequest(); + } + + private void TransitionRequest() + { + if (WaitingRequests.TryDequeue(out var request)) + { + request.Success += HandleSuccessfulRequest; + _ = request.Execute(); + DoneRequests.Enqueue(request); + + logger.LogInformation("Executing request {} of {}", DoneRequests.Count, WaitingRequests.Count + DoneRequests.Count); + } + } + + public async Task TriggerRequests(CancellationToken token) + { + foreach (var _ in Enumerable.Range(1, simultaneousRequests)) + { + TransitionRequest(); + } + + var timeoutTask = Task.Delay(timeout, token); + var allTasks = WaitingRequests.Union(DoneRequests).Select(r => r.Task).ToList(); + + var firstToFinish = await Task.WhenAny(timeoutTask, Task.WhenAll(allTasks)); + + TimedOut = firstToFinish == timeoutTask; + } + } +} diff --git a/Selector/Operations/IOperation.cs b/Selector/Operations/IOperation.cs new file mode 100644 index 0000000..3662f08 --- /dev/null +++ b/Selector/Operations/IOperation.cs @@ -0,0 +1,15 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Selector.Operations +{ + public interface IOperation + { + event EventHandler Success; + Task Execute(); + Task Task { get; } + } +} diff --git a/Selector/Scrobble/ScrobbleAlbumMapping.cs b/Selector/Scrobble/ScrobbleAlbumMapping.cs new file mode 100644 index 0000000..7baf390 --- /dev/null +++ b/Selector/Scrobble/ScrobbleAlbumMapping.cs @@ -0,0 +1,39 @@ +using Microsoft.Extensions.Logging; +using SpotifyAPI.Web; +using System; +using System.Linq; +using System.Threading.Tasks; + +namespace Selector +{ + public class ScrobbleAlbumMapping : ScrobbleMapping + { + public string AlbumName { get; set; } + public string ArtistName { get; set; } + + public ScrobbleAlbumMapping(ISearchClient _searchClient, ILogger _logger, string albumName, string artistName) : base(_searchClient, _logger) + { + AlbumName = albumName; + ArtistName = artistName; + } + + private SimpleAlbum result; + public override object Result => result; + + public override string Query => $"{AlbumName} {ArtistName}"; + + public override SearchRequest.Types QueryType => SearchRequest.Types.Album; + + public override void HandleResponse(Task response) + { + var topResult = response.Result.Albums.Items.FirstOrDefault(); + + if (topResult is not null + && topResult.Name.Equals(AlbumName, StringComparison.InvariantCultureIgnoreCase) + && topResult.Artists.First().Name.Equals(ArtistName, StringComparison.InvariantCultureIgnoreCase)) + { + result = topResult; + } + } + } +} diff --git a/Selector/Scrobble/ScrobbleArtistMapping.cs b/Selector/Scrobble/ScrobbleArtistMapping.cs new file mode 100644 index 0000000..259358f --- /dev/null +++ b/Selector/Scrobble/ScrobbleArtistMapping.cs @@ -0,0 +1,38 @@ +using Microsoft.Extensions.Logging; +using SpotifyAPI.Web; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Selector +{ + public class ScrobbleArtistMapping : ScrobbleMapping + { + public string ArtistName { get; set; } + + public ScrobbleArtistMapping(ISearchClient _searchClient, ILogger _logger, string artistName) : base(_searchClient, _logger) + { + ArtistName = artistName; + } + + private FullArtist result; + public override object Result => result; + + public override string Query => ArtistName; + + public override SearchRequest.Types QueryType => SearchRequest.Types.Artist; + + public override void HandleResponse(Task response) + { + var topResult = response.Result.Artists.Items.FirstOrDefault(); + + if (topResult is not null + && topResult.Name.Equals(ArtistName, StringComparison.InvariantCultureIgnoreCase)) + { + result = topResult; + } + } + } +} diff --git a/Selector/Scrobble/ScrobbleMapping.cs b/Selector/Scrobble/ScrobbleMapping.cs new file mode 100644 index 0000000..39b203d --- /dev/null +++ b/Selector/Scrobble/ScrobbleMapping.cs @@ -0,0 +1,80 @@ +using Microsoft.Extensions.Logging; +using Selector.Operations; +using SpotifyAPI.Web; +using System; +using System.Threading.Tasks; + +namespace Selector +{ + public enum LastfmObject{ + Track, Album, Artist + } + + public abstract class ScrobbleMapping : IOperation + { + private readonly ILogger logger; + private readonly ISearchClient searchClient; + + public event EventHandler Success; + + public int MaxAttempts { get; private set; } = 5; + public int Attempts { get; private set; } + + private Task currentTask { get; set; } + public bool Succeeded { get; private set; } = false; + + public abstract object Result { get; } + public abstract string Query { get; } + public abstract SearchRequest.Types QueryType { get; } + + private TaskCompletionSource AggregateTaskSource { get; set; } = new(); + public Task Task => AggregateTaskSource.Task; + + public ScrobbleMapping(ISearchClient _searchClient, ILogger _logger) + { + logger = _logger; + searchClient = _searchClient; + } + + public Task Execute() + { + logger.LogInformation("Mapping Last.fm {} ({}) to Spotify", Query, QueryType); + + currentTask = searchClient.Item(new (QueryType, Query)); + currentTask.ContinueWith(async t => + { + if (t.IsCompletedSuccessfully) + { + HandleResponse(t); + OnSuccess(); + AggregateTaskSource.SetResult(); + } + else + { + if(t.Exception.InnerException is APITooManyRequestsException ex) + { + logger.LogError("Spotify search request too many requests, waiting for {}", ex.RetryAfter); + await Task.Delay(ex.RetryAfter); + Execute(); + } + else + { + logger.LogError("Spotify search request task faulted, {}", t.Exception); + AggregateTaskSource.SetException(t.Exception); + } + } + }); + + Attempts++; + return Task.CompletedTask; + } + + public abstract void HandleResponse(Task response); + + protected virtual void OnSuccess() + { + // Raise the event in a thread-safe manner using the ?. operator. + Success?.Invoke(this, new EventArgs()); + } + } +} diff --git a/Selector/Scrobble/ScrobbleRequest.cs b/Selector/Scrobble/ScrobbleRequest.cs index f8c2d3f..11bd503 100644 --- a/Selector/Scrobble/ScrobbleRequest.cs +++ b/Selector/Scrobble/ScrobbleRequest.cs @@ -2,6 +2,7 @@ using IF.Lastfm.Core.Api.Helpers; using IF.Lastfm.Core.Objects; using Microsoft.Extensions.Logging; +using Selector.Operations; using System; using System.Collections.Generic; using System.Linq; @@ -9,14 +10,14 @@ using System.Threading.Tasks; namespace Selector { - public class ScrobbleRequest + public class ScrobbleRequest : IOperation { private readonly ILogger logger; private readonly IUserApi userClient; public event EventHandler Success; - public int MaxAttempts { get; private set; } = 5; + public int MaxAttempts { get; private set; } public int Attempts { get; private set; } public List Scrobbles { get; private set; } public int TotalPages { get; private set; } @@ -32,7 +33,7 @@ namespace Selector private TaskCompletionSource AggregateTaskSource { get; set; } = new(); public Task Task => AggregateTaskSource.Task; - public ScrobbleRequest(IUserApi _userClient, ILogger _logger, string _username, int _pageNumber, int _pageSize, DateTime? _from, DateTime? _to) + public ScrobbleRequest(IUserApi _userClient, ILogger _logger, string _username, int _pageNumber, int _pageSize, DateTime? _from, DateTime? _to, int maxRetries = 5) { userClient = _userClient; logger = _logger; @@ -42,12 +43,8 @@ namespace Selector pageSize = _pageSize; from = _from; to = _to; - } - protected virtual void RaiseSampleEvent() - { - // Raise the event in a thread-safe manner using the ?. operator. - Success?.Invoke(this, new EventArgs()); + MaxAttempts = maxRetries; } public Task Execute() @@ -95,7 +92,6 @@ namespace Selector protected virtual void OnSuccess() { - // Raise the event in a thread-safe manner using the ?. operator. Success?.Invoke(this, new EventArgs()); } } diff --git a/Selector/Scrobble/ScrobbleTrackMapping.cs b/Selector/Scrobble/ScrobbleTrackMapping.cs new file mode 100644 index 0000000..5b95e56 --- /dev/null +++ b/Selector/Scrobble/ScrobbleTrackMapping.cs @@ -0,0 +1,41 @@ +using Microsoft.Extensions.Logging; +using SpotifyAPI.Web; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Selector +{ + public class ScrobbleTrackMapping : ScrobbleMapping + { + public string TrackName { get; set; } + public string ArtistName { get; set; } + + public ScrobbleTrackMapping(ISearchClient _searchClient, ILogger _logger, string trackName, string artistName) : base(_searchClient, _logger) + { + TrackName = trackName; + ArtistName = artistName; + } + + private FullTrack result; + public override object Result => result; + + public override string Query => $"{TrackName} {ArtistName}"; + + public override SearchRequest.Types QueryType => SearchRequest.Types.Track; + + public override void HandleResponse(Task response) + { + var topResult = response.Result.Tracks.Items.FirstOrDefault(); + + if(topResult is not null + && topResult.Name.Equals(TrackName, StringComparison.InvariantCultureIgnoreCase) + && topResult.Artists.First().Name.Equals(ArtistName, StringComparison.InvariantCultureIgnoreCase)) + { + result = topResult; + } + } + } +}