2022-02-16 23:38:45 +00:00
|
|
|
|
using IF.Lastfm.Core.Api;
|
|
|
|
|
using IF.Lastfm.Core.Api.Helpers;
|
|
|
|
|
using IF.Lastfm.Core.Objects;
|
|
|
|
|
using Microsoft.Extensions.Logging;
|
2022-02-20 21:22:32 +00:00
|
|
|
|
using Microsoft.Extensions.Logging.Abstractions;
|
2022-02-16 23:38:45 +00:00
|
|
|
|
using Selector.Model;
|
|
|
|
|
using System;
|
2022-02-20 21:22:32 +00:00
|
|
|
|
using System.Collections.Concurrent;
|
2022-02-16 23:38:45 +00:00
|
|
|
|
using System.Collections.Generic;
|
2022-02-18 00:08:42 +00:00
|
|
|
|
using System.Diagnostics;
|
2022-02-16 23:38:45 +00:00
|
|
|
|
using System.Linq;
|
2022-02-18 19:47:11 +00:00
|
|
|
|
using System.Text;
|
2022-02-16 23:38:45 +00:00
|
|
|
|
using System.Threading;
|
|
|
|
|
using System.Threading.Tasks;
|
|
|
|
|
|
|
|
|
|
namespace Selector
|
|
|
|
|
{
|
|
|
|
|
public class ScrobbleSaverConfig
|
|
|
|
|
{
|
|
|
|
|
public ApplicationUser User { get; set; }
|
|
|
|
|
public TimeSpan InterRequestDelay { get; set; }
|
2022-02-20 21:22:32 +00:00
|
|
|
|
public TimeSpan Timeout { get; set; } = new TimeSpan(0, 20, 0);
|
2022-02-16 23:38:45 +00:00
|
|
|
|
public DateTime? From { get; set; }
|
|
|
|
|
public DateTime? To { get; set; }
|
|
|
|
|
public int PageSize { get; set; } = 100;
|
2022-02-18 00:08:42 +00:00
|
|
|
|
public int Retries { get; set; } = 5;
|
2022-02-20 21:22:32 +00:00
|
|
|
|
public int SimultaneousConnections { get; set; } = 3;
|
2022-02-18 00:08:42 +00:00
|
|
|
|
public bool DontAdd { get; set; } = false;
|
|
|
|
|
public bool DontRemove { get; set; } = false;
|
2022-02-16 23:38:45 +00:00
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
public class ScrobbleSaver
|
|
|
|
|
{
|
|
|
|
|
private readonly ILogger<ScrobbleSaver> logger;
|
2022-02-20 21:22:32 +00:00
|
|
|
|
private readonly ILoggerFactory loggerFactory;
|
2022-02-16 23:38:45 +00:00
|
|
|
|
|
|
|
|
|
private readonly IUserApi userClient;
|
|
|
|
|
private readonly ScrobbleSaverConfig config;
|
2022-02-20 21:22:32 +00:00
|
|
|
|
private CancellationToken _token;
|
|
|
|
|
private Task aggregateNetworkTask;
|
|
|
|
|
|
2022-02-18 00:08:42 +00:00
|
|
|
|
private readonly ApplicationDbContext db;
|
2022-02-16 23:38:45 +00:00
|
|
|
|
|
2022-02-20 21:22:32 +00:00
|
|
|
|
private ConcurrentQueue<ScrobbleRequest> waitingRequests = new();
|
|
|
|
|
private ConcurrentQueue<ScrobbleRequest> runRequests = new();
|
|
|
|
|
|
|
|
|
|
public ScrobbleSaver(IUserApi _userClient, ScrobbleSaverConfig _config, ApplicationDbContext _db, ILogger<ScrobbleSaver> _logger, ILoggerFactory _loggerFactory = null)
|
2022-02-16 23:38:45 +00:00
|
|
|
|
{
|
|
|
|
|
userClient = _userClient;
|
|
|
|
|
config = _config;
|
2022-02-18 00:08:42 +00:00
|
|
|
|
db = _db;
|
2022-02-16 23:38:45 +00:00
|
|
|
|
logger = _logger;
|
2022-02-20 21:22:32 +00:00
|
|
|
|
loggerFactory = _loggerFactory;
|
2022-02-16 23:38:45 +00:00
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
public async Task Execute(CancellationToken token)
|
|
|
|
|
{
|
2022-02-18 00:08:42 +00:00
|
|
|
|
logger.LogInformation("Saving scrobbles for {0}/{1}", config.User.UserName, config.User.LastFmUsername);
|
2022-02-20 21:22:32 +00:00
|
|
|
|
_token = token;
|
2022-02-16 23:38:45 +00:00
|
|
|
|
|
2022-02-20 21:22:32 +00:00
|
|
|
|
var page1 = new ScrobbleRequest(userClient,
|
|
|
|
|
loggerFactory?.CreateLogger<ScrobbleRequest>() ?? NullLogger<ScrobbleRequest>.Instance,
|
|
|
|
|
config.User.UserName,
|
|
|
|
|
1,
|
|
|
|
|
config.PageSize,
|
|
|
|
|
config.From, config.To);
|
2022-02-16 23:38:45 +00:00
|
|
|
|
|
2022-02-20 21:22:32 +00:00
|
|
|
|
await page1.Execute();
|
|
|
|
|
runRequests.Enqueue(page1);
|
2022-02-16 23:38:45 +00:00
|
|
|
|
|
2022-02-20 21:22:32 +00:00
|
|
|
|
if (page1.Succeeded)
|
|
|
|
|
{
|
2022-02-18 00:08:42 +00:00
|
|
|
|
if (page1.TotalPages > 1)
|
2022-02-16 23:38:45 +00:00
|
|
|
|
{
|
2022-02-20 21:22:32 +00:00
|
|
|
|
TriggerNetworkRequests(page1.TotalPages, token);
|
2022-02-16 23:38:45 +00:00
|
|
|
|
}
|
|
|
|
|
|
2022-02-20 21:22:32 +00:00
|
|
|
|
logger.LogDebug("Pulling currently stored scrobbles");
|
|
|
|
|
|
2022-02-20 21:54:01 +00:00
|
|
|
|
var currentScrobbles = GetDbScrobbles();
|
2022-02-20 21:22:32 +00:00
|
|
|
|
|
|
|
|
|
await aggregateNetworkTask;
|
|
|
|
|
var scrobbles = runRequests.SelectMany(r => r.Scrobbles);
|
|
|
|
|
|
2022-02-18 19:47:11 +00:00
|
|
|
|
IdentifyDuplicates(scrobbles);
|
|
|
|
|
|
2022-02-18 00:08:42 +00:00
|
|
|
|
logger.LogDebug("Ordering and filtering pulled scrobbles");
|
|
|
|
|
|
2022-02-20 21:22:32 +00:00
|
|
|
|
RemoveNowPlaying(scrobbles.ToList());
|
|
|
|
|
|
2022-02-16 23:38:45 +00:00
|
|
|
|
var nativeScrobbles = scrobbles
|
2022-02-20 21:54:01 +00:00
|
|
|
|
.DistinctBy(s => new { s.TimePlayed?.UtcDateTime, s.Name, s.ArtistName })
|
2022-02-16 23:38:45 +00:00
|
|
|
|
.Select(s =>
|
|
|
|
|
{
|
2022-02-20 21:22:32 +00:00
|
|
|
|
var nativeScrobble = (UserScrobble)s;
|
2022-02-16 23:38:45 +00:00
|
|
|
|
nativeScrobble.UserId = config.User.Id;
|
|
|
|
|
return nativeScrobble;
|
|
|
|
|
});
|
2022-02-18 00:08:42 +00:00
|
|
|
|
|
2022-02-20 21:22:32 +00:00
|
|
|
|
logger.LogInformation("Completed database scrobble pulling for {0}, pulled {1:n0}", config.User.UserName, nativeScrobbles.Count());
|
2022-02-18 00:08:42 +00:00
|
|
|
|
|
|
|
|
|
logger.LogDebug("Identifying difference sets");
|
|
|
|
|
var time = Stopwatch.StartNew();
|
2022-02-16 23:38:45 +00:00
|
|
|
|
|
2022-02-20 21:54:01 +00:00
|
|
|
|
(var toAdd, var toRemove) = ScrobbleMatcher.IdentifyDiffs(currentScrobbles, nativeScrobbles);
|
2022-02-18 00:08:42 +00:00
|
|
|
|
|
|
|
|
|
time.Stop();
|
|
|
|
|
logger.LogTrace("Finished diffing: {0:n}ms", time.ElapsedMilliseconds);
|
|
|
|
|
|
|
|
|
|
var timeDbOps = Stopwatch.StartNew();
|
|
|
|
|
|
|
|
|
|
if(!config.DontAdd)
|
|
|
|
|
{
|
2022-02-20 21:22:32 +00:00
|
|
|
|
await db.Scrobble.AddRangeAsync(toAdd.Cast<UserScrobble>());
|
2022-02-18 00:08:42 +00:00
|
|
|
|
}
|
|
|
|
|
else
|
|
|
|
|
{
|
2022-02-20 21:22:32 +00:00
|
|
|
|
logger.LogInformation("Skipping adding of {0} scrobbles", toAdd.Count());
|
2022-02-18 00:08:42 +00:00
|
|
|
|
}
|
|
|
|
|
if (!config.DontRemove)
|
|
|
|
|
{
|
2022-02-20 21:22:32 +00:00
|
|
|
|
db.Scrobble.RemoveRange(toRemove.Cast<UserScrobble>());
|
2022-02-18 00:08:42 +00:00
|
|
|
|
}
|
|
|
|
|
else
|
|
|
|
|
{
|
2022-02-20 21:22:32 +00:00
|
|
|
|
logger.LogInformation("Skipping removal of {0} scrobbles", toRemove.Count());
|
2022-02-18 00:08:42 +00:00
|
|
|
|
}
|
2022-02-16 23:38:45 +00:00
|
|
|
|
await db.SaveChangesAsync();
|
2022-02-18 00:08:42 +00:00
|
|
|
|
|
|
|
|
|
timeDbOps.Stop();
|
|
|
|
|
logger.LogTrace("DB ops: {0:n}ms", timeDbOps.ElapsedMilliseconds);
|
|
|
|
|
|
2022-02-20 21:22:32 +00:00
|
|
|
|
logger.LogInformation("Completed scrobble pulling for {0}, +{1:n0}, -{2:n0}", config.User.UserName, toAdd.Count(), toRemove.Count());
|
2022-02-16 23:38:45 +00:00
|
|
|
|
}
|
|
|
|
|
else
|
|
|
|
|
{
|
|
|
|
|
logger.LogError("Failed to pull first scrobble page for {0}/{1}", config.User.UserName, config.User.LastFmUsername);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2022-02-20 21:22:32 +00:00
|
|
|
|
private async void HandleSuccessfulRequest(object o, EventArgs e)
|
2022-02-16 23:38:45 +00:00
|
|
|
|
{
|
2022-02-20 21:22:32 +00:00
|
|
|
|
await Task.Delay(config.InterRequestDelay, _token);
|
|
|
|
|
TransitionRequest();
|
|
|
|
|
}
|
2022-02-16 23:38:45 +00:00
|
|
|
|
|
2022-02-20 21:22:32 +00:00
|
|
|
|
private void TransitionRequest()
|
|
|
|
|
{
|
|
|
|
|
if (waitingRequests.TryDequeue(out var request))
|
|
|
|
|
{
|
|
|
|
|
request.Success += HandleSuccessfulRequest;
|
|
|
|
|
_ = request.Execute();
|
|
|
|
|
runRequests.Enqueue(request);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private void TriggerNetworkRequests(int totalPages, CancellationToken token)
|
|
|
|
|
{
|
|
|
|
|
foreach (var req in GetRequestsFromPageNumbers(2, totalPages))
|
2022-02-16 23:38:45 +00:00
|
|
|
|
{
|
2022-02-20 21:22:32 +00:00
|
|
|
|
waitingRequests.Enqueue(req);
|
2022-02-16 23:38:45 +00:00
|
|
|
|
}
|
|
|
|
|
|
2022-02-20 21:22:32 +00:00
|
|
|
|
foreach (var _ in Enumerable.Range(1, config.SimultaneousConnections))
|
|
|
|
|
{
|
|
|
|
|
TransitionRequest();
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
var timeoutTask = Task.Delay(config.Timeout, token);
|
|
|
|
|
var allTasks = waitingRequests.Union(runRequests).Select(r => r.Task).ToList();
|
|
|
|
|
|
|
|
|
|
aggregateNetworkTask = Task.WhenAny(timeoutTask, Task.WhenAll(allTasks));
|
|
|
|
|
|
|
|
|
|
aggregateNetworkTask.ContinueWith(t =>
|
|
|
|
|
{
|
|
|
|
|
if (timeoutTask.IsCompleted)
|
|
|
|
|
{
|
|
|
|
|
throw new TimeoutException($"Timed-out pulling scrobbles, took {config.Timeout}");
|
|
|
|
|
}
|
|
|
|
|
});
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private IEnumerable<UserScrobble> GetDbScrobbles()
|
|
|
|
|
{
|
|
|
|
|
var currentScrobbles = db.Scrobble.AsEnumerable()
|
|
|
|
|
.Where(s => s.UserId == config.User.Id);
|
|
|
|
|
|
|
|
|
|
if (config.From is not null)
|
|
|
|
|
{
|
|
|
|
|
currentScrobbles = currentScrobbles.Where(s => s.Timestamp > config.From);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (config.To is not null)
|
|
|
|
|
{
|
|
|
|
|
currentScrobbles = currentScrobbles.Where(s => s.Timestamp < config.To);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return currentScrobbles;
|
2022-02-16 23:38:45 +00:00
|
|
|
|
}
|
2022-02-18 19:47:11 +00:00
|
|
|
|
|
2022-02-20 21:22:32 +00:00
|
|
|
|
private IEnumerable<ScrobbleRequest> GetRequestsFromPageNumbers(int start, int totalPages)
|
|
|
|
|
=> Enumerable.Range(start, totalPages - 1)
|
|
|
|
|
.Select(n => new ScrobbleRequest(
|
|
|
|
|
userClient,
|
|
|
|
|
loggerFactory.CreateLogger<ScrobbleRequest>() ?? NullLogger<ScrobbleRequest>.Instance,
|
|
|
|
|
config.User.UserName,
|
|
|
|
|
n,
|
|
|
|
|
config.PageSize,
|
|
|
|
|
config.From,
|
|
|
|
|
config.To));
|
|
|
|
|
|
2022-02-18 19:47:11 +00:00
|
|
|
|
private void IdentifyDuplicates(IEnumerable<LastTrack> tracks)
|
|
|
|
|
{
|
|
|
|
|
logger.LogDebug("Identifying duplicates");
|
|
|
|
|
|
|
|
|
|
var duplicates = tracks
|
|
|
|
|
.GroupBy(t => t.TimePlayed?.UtcDateTime)
|
|
|
|
|
.Where(g => g.Count() > 1);
|
|
|
|
|
|
|
|
|
|
foreach(var dupe in duplicates)
|
|
|
|
|
{
|
|
|
|
|
var dupeString = new StringBuilder();
|
|
|
|
|
|
|
|
|
|
foreach(var scrobble in dupe)
|
|
|
|
|
{
|
|
|
|
|
dupeString.Append("(");
|
|
|
|
|
dupeString.Append(scrobble.Name);
|
|
|
|
|
dupeString.Append(", ");
|
|
|
|
|
dupeString.Append(scrobble.AlbumName);
|
|
|
|
|
dupeString.Append(", ");
|
|
|
|
|
dupeString.Append(scrobble.ArtistName);
|
|
|
|
|
dupeString.Append(")");
|
|
|
|
|
|
|
|
|
|
dupeString.Append(" ");
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
logger.LogInformation("Duplicate at {0}: {1}", dupe.Key, dupeString.ToString());
|
|
|
|
|
}
|
|
|
|
|
}
|
2022-02-20 21:22:32 +00:00
|
|
|
|
|
|
|
|
|
private bool RemoveNowPlaying(List<LastTrack> scrobbles)
|
|
|
|
|
{
|
|
|
|
|
var newestScrobble = scrobbles.FirstOrDefault();
|
|
|
|
|
if (newestScrobble is not null)
|
|
|
|
|
{
|
|
|
|
|
if (newestScrobble.IsNowPlaying is bool playing && playing)
|
|
|
|
|
{
|
|
|
|
|
scrobbles.Remove(newestScrobble);
|
|
|
|
|
return true;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return false;
|
|
|
|
|
}
|
2022-02-16 23:38:45 +00:00
|
|
|
|
}
|
|
|
|
|
}
|