sarsoo/Selector
1
0
Fork 0
Code Issues 21 Pull Requests Actions Packages Projects Releases Wiki Activity

more efficient linq, adding album and track last fm mapping

Browse Source
This commit is contained in:
andy 2022-02-24 21:57:26 +00:00
parent 369595f80c
commit 56271ea32e
5 changed files with 196 additions and 76 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
Selector.CLI/Command/Scrobble/ScrobbleSave.cs
View File

@ -46,10 +46,10 @@ namespace Selector.CLI
dontRemove.AddAlias("-nr");
AddOption(dontRemove);
Handler = CommandHandler.Create(async (DateTime from, DateTime to, int page, int delay, int simul, string username, bool noAdd, bool noRemove, CancellationToken token) => await Execute(from, to, page, delay, simul, username, noAdd, noRemove, token));
Handler = CommandHandler.Create(async (DateTime from, DateTime to, int page, int delay, int simultaneous, string username, bool noAdd, bool noRemove, CancellationToken token) => await Execute(from, to, page, delay, simultaneous, username, noAdd, noRemove, token));
}
public static async Task<int> Execute(DateTime from, DateTime to, int page, int delay, int simul, string username, bool noAdd, bool noRemove, CancellationToken token)
public static async Task<int> Execute(DateTime from, DateTime to, int page, int delay, int simultaneous, string username, bool noAdd, bool noRemove, CancellationToken token)
{
try
{
@ -91,7 +91,7 @@ namespace Selector.CLI
PageSize = page,
DontAdd = noAdd,
DontRemove = noRemove,
SimultaneousConnections = simul
SimultaneousConnections = simultaneous
},
repo,
context.Logger.CreateLogger<ScrobbleSaver>(),

218
Selector.CLI/ScrobbleMapper.cs
View File

@ -1,17 +1,10 @@
using IF.Lastfm.Core.Api;
using IF.Lastfm.Core.Api.Helpers;
using IF.Lastfm.Core.Objects;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Logging.Abstractions;
using Selector.Model;
using Selector.Operations;
using SpotifyAPI.Web;
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Diagnostics;
using System.Linq;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
@ -53,56 +46,175 @@ namespace Selector
{
if (config.Artists)
{
logger.LogInformation("Mapping scrobble artists");
await MapArtists(token);
}
var currentArtists = mappingRepo.GetArtists();
var scrobbleArtists = scrobbleRepo.GetAll()
.GroupBy(x => x.ArtistName)
.Select(x => (x.Key, x.Count()))
.OrderByDescending(x => x.Item2)
.Select(x => x.Key);
if (config.Albums)
{
await MapAlbums(token);
}
if(config.Limit is not null)
{
scrobbleArtists = scrobbleArtists.Take(config.Limit.Value);
}
var artistsToPull = scrobbleArtists
.ExceptBy(currentArtists.Select(a => a.LastfmArtistName), a => a);
var requests = artistsToPull.Select(a => new ScrobbleArtistMapping(
searchClient,
loggerFactory.CreateLogger<ScrobbleArtistMapping>() ?? NullLogger<ScrobbleArtistMapping>.Instance,
a)
).ToList();
logger.LogInformation("Found {} artists to map, starting", requests.Count);
var batchRequest = new BatchingOperation<ScrobbleArtistMapping>(
config.InterRequestDelay,
config.Timeout,
config.SimultaneousConnections,
requests
);
await batchRequest.TriggerRequests(token);
logger.LogInformation("Finished mapping artists");
var newArtists = batchRequest.DoneRequests
.Where(a => a is not null)
.Select(a => (FullArtist) a.Result)
.Where(a => a is not null);
var newMappings = newArtists.Select(a => new ArtistLastfmSpotifyMapping() {
LastfmArtistName = a.Name,
SpotifyUri = a.Uri
});
mappingRepo.AddRange(newMappings);
if (config.Tracks)
{
await MapTracks(token);
}
await mappingRepo.Save();
}
private async Task MapArtists(CancellationToken token)
{
logger.LogInformation("Mapping scrobble artists");
var currentArtists = mappingRepo.GetArtists();
var scrobbleArtists = scrobbleRepo.GetAll()
.GroupBy(x => x.ArtistName)
.Select(x => (x.Key, x.Count()))
.OrderByDescending(x => x.Item2)
.Select(x => x.Key);
var artistsToPull = scrobbleArtists
.ExceptBy(currentArtists.Select(a => a.LastfmArtistName), a => a);
if (config.Limit is not null)
{
artistsToPull = artistsToPull.Take(config.Limit.Value);
}
var requests = artistsToPull.Select(a => new ScrobbleArtistMapping(
searchClient,
loggerFactory.CreateLogger<ScrobbleArtistMapping>() ?? NullLogger<ScrobbleArtistMapping>.Instance,
a)
).ToArray();
logger.LogInformation("Found {} artists to map, starting", requests.Length);
var batchRequest = new BatchingOperation<ScrobbleArtistMapping>(
config.InterRequestDelay,
config.Timeout,
config.SimultaneousConnections,
requests
);
await batchRequest.TriggerRequests(token);
logger.LogInformation("Finished mapping artists");
var newArtists = batchRequest.DoneRequests
.Select(a => a.Result)
.Cast<FullArtist>()
.Where(a => a is not null);
var newMappings = newArtists.Select(a => new ArtistLastfmSpotifyMapping()
{
LastfmArtistName = a.Name,
SpotifyUri = a.Uri
});
mappingRepo.AddRange(newMappings);
}
private async Task MapAlbums(CancellationToken token)
{
logger.LogInformation("Mapping scrobble albums");
var currentAlbums = mappingRepo.GetAlbums();
var scrobbleAlbums = scrobbleRepo.GetAll()
.GroupBy(x => (x.ArtistName, x.AlbumName))
.Select(x => (x.Key, x.Count()))
.OrderByDescending(x => x.Item2)
.Select(x => x.Key);
var albumsToPull = scrobbleAlbums
.ExceptBy(currentAlbums.Select(a => (a.LastfmArtistName, a.LastfmAlbumName)), a => a);
if (config.Limit is not null)
{
albumsToPull = albumsToPull.Take(config.Limit.Value);
}
var requests = albumsToPull.Select(a => new ScrobbleAlbumMapping(
searchClient,
loggerFactory.CreateLogger<ScrobbleAlbumMapping>() ?? NullLogger<ScrobbleAlbumMapping>.Instance,
a.AlbumName, a.ArtistName)
).ToArray();
logger.LogInformation("Found {} albums to map, starting", requests.Length);
var batchRequest = new BatchingOperation<ScrobbleAlbumMapping>(
config.InterRequestDelay,
config.Timeout,
config.SimultaneousConnections,
requests
);
await batchRequest.TriggerRequests(token);
logger.LogInformation("Finished mapping albums");
var newArtists = batchRequest.DoneRequests
.Select(a => a.Result)
.Cast<SimpleAlbum>()
.Where(a => a is not null);
var newMappings = newArtists.Select(a => new AlbumLastfmSpotifyMapping()
{
LastfmAlbumName = a.Name,
LastfmArtistName = a.Artists.FirstOrDefault()?.Name,
SpotifyUri = a.Uri
});
mappingRepo.AddRange(newMappings);
}
private async Task MapTracks(CancellationToken token)
{
logger.LogInformation("Mapping scrobble tracks");
var currentTracks = mappingRepo.GetTracks();
var scrobbleTracks = scrobbleRepo.GetAll()
.GroupBy(x => (x.ArtistName, x.TrackName))
.Select(x => (x.Key, x.Count()))
.OrderByDescending(x => x.Item2)
.Select(x => x.Key);
if (config.Limit is not null)
{
scrobbleTracks = scrobbleTracks.Take(config.Limit.Value);
}
var tracksToPull = scrobbleTracks
.ExceptBy(currentTracks.Select(a => (a.LastfmArtistName, a.LastfmTrackName)), a => a);
var requests = tracksToPull.Select(a => new ScrobbleTrackMapping(
searchClient,
loggerFactory.CreateLogger<ScrobbleTrackMapping>() ?? NullLogger<ScrobbleTrackMapping>.Instance,
a.TrackName, a.ArtistName)
).ToArray();
logger.LogInformation("Found {} tracks to map, starting", requests.Length);
var batchRequest = new BatchingOperation<ScrobbleTrackMapping>(
config.InterRequestDelay,
config.Timeout,
config.SimultaneousConnections,
requests
);
await batchRequest.TriggerRequests(token);
logger.LogInformation("Finished mapping tracks");
var newArtists = batchRequest.DoneRequests
.Select(a => a.Result)
.Cast<FullTrack>()
.Where(a => a is not null);
var newMappings = newArtists.Select(a => new TrackLastfmSpotifyMapping()
{
LastfmTrackName = a.Name,
LastfmArtistName = a.Artists.FirstOrDefault()?.Name,
SpotifyUri = a.Uri
});
mappingRepo.AddRange(newMappings);
}
}
}

42
Selector.CLI/ScrobbleSaver.cs
View File

@ -87,29 +87,28 @@ namespace Selector
await batchTask;
}
var scrobbles = page1.Scrobbles;
IEnumerable<LastTrack> scrobbles;
if(batchOperation is not null)
{
scrobbles.AddRange(batchOperation.DoneRequests.SelectMany(r => r.Scrobbles));
scrobbles = page1.Scrobbles.Union(batchOperation.DoneRequests.SelectMany(r => r.Scrobbles));
}
else
{
scrobbles = page1.Scrobbles;
}
IdentifyDuplicates(scrobbles);
logger.LogDebug("Ordering and filtering pulled scrobbles");
RemoveNowPlaying(scrobbles.ToList());
scrobbles = RemoveNowPlaying(scrobbles);
var nativeScrobbles = scrobbles
.DistinctBy(s => new { s.TimePlayed?.UtcDateTime, s.Name, s.ArtistName })
.Select(s =>
{
var nativeScrobble = (UserScrobble)s;
nativeScrobble.UserId = config.User.Id;
return nativeScrobble;
});
.Select(s => (UserScrobble) s)
.ToArray();
logger.LogInformation("Completed database scrobble pulling for {}, pulled {:n0}", config.User.UserName, nativeScrobbles.Count());
logger.LogInformation("Completed database scrobble pulling for {}, pulled {:n0}", config.User.UserName, nativeScrobbles.Length);
logger.LogDebug("Identifying difference sets");
var time = Stopwatch.StartNew();
@ -123,7 +122,12 @@ namespace Selector
if(!config.DontAdd)
{
scrobbleRepo.AddRange(toAdd.Cast<UserScrobble>());
foreach(var add in toAdd)
{
var scrobble = (UserScrobble) add;
scrobble.UserId = config.User.Id;
scrobbleRepo.Add(scrobble);
}
}
else
{
@ -131,7 +135,12 @@ namespace Selector
}
if (!config.DontRemove)
{
scrobbleRepo.RemoveRange(toRemove.Cast<UserScrobble>());
foreach (var remove in toRemove)
{
var scrobble = (UserScrobble) remove;
scrobble.UserId = config.User.Id;
scrobbleRepo.Remove(scrobble);
}
}
else
{
@ -189,19 +198,18 @@ namespace Selector
}
}
private bool RemoveNowPlaying(List<LastTrack> scrobbles)
private IEnumerable<LastTrack> RemoveNowPlaying(IEnumerable<LastTrack> scrobbles)
{
var newestScrobble = scrobbles.FirstOrDefault();
if (newestScrobble is not null)
{
if (newestScrobble.IsNowPlaying is bool playing && playing)
{
scrobbles.Remove(newestScrobble);
return true;
scrobbles = scrobbles.Skip(1);
}
}
return false;
return scrobbles;
}
}
}

2
Selector/Operations/BatchingOperation.cs
View File

@ -64,7 +64,7 @@ namespace Selector.Operations
}
var timeoutTask = Task.Delay(timeout, token);
var allTasks = WaitingRequests.Union(DoneRequests).Select(r => r.Task).ToList();
var allTasks = WaitingRequests.Union(DoneRequests).Select(r => r.Task);
var firstToFinish = await Task.WhenAny(timeoutTask, Task.WhenAll(allTasks));

4
Selector/Scrobble/ScrobbleRequest.cs
View File

@ -19,7 +19,7 @@ namespace Selector
public int MaxAttempts { get; private set; }
public int Attempts { get; private set; }
public List<LastTrack> Scrobbles { get; private set; }
public IEnumerable<LastTrack> Scrobbles { get; private set; }
public int TotalPages { get; private set; }
private Task<PageResponse<LastTrack>> currentTask { get; set; }
public bool Succeeded { get; private set; } = false;
@ -60,7 +60,7 @@ namespace Selector
if (Succeeded)
{
Scrobbles = result.Content.ToList();
Scrobbles = result.Content.ToArray();
TotalPages = result.TotalPages;
OnSuccess();
AggregateTaskSource.SetResult();