adding scrobble saving to ui, adding full scrobble save

This commit is contained in:
andy 2022-02-26 23:42:29 +00:00
parent 64b7105514
commit a2f874ec27
6 changed files with 181 additions and 103 deletions

View File

@ -55,23 +55,48 @@ namespace Selector.CLI.Extensions
tp.MaxConcurrency = 5;
});
var scrobbleKey = new JobKey("scrobble-watcher", "scrobble");
if (config.JobOptions.Scrobble.Enabled)
{
Console.WriteLine("> Adding Scrobble Jobs...");
options.AddJob<ScrobbleWatcherJob>(j => j
.WithDescription("Watch recent scrobbles and mirror to database")
.WithIdentity(scrobbleKey)
);
var scrobbleKey = new JobKey("scrobble-watcher-agile", "scrobble");
options.AddTrigger(t => t
.WithIdentity("scrobble-watcher-trigger")
.ForJob(scrobbleKey)
.StartNow()
.WithSimpleSchedule(x => x.WithInterval(config.JobOptions.Scrobble.InterJobDelay).RepeatForever())
.WithDescription("Periodic trigger for scrobble watcher")
);
options.AddJob<ScrobbleWatcherJob>(j => j
.WithDescription("Watch recent scrobbles and mirror to database")
.WithIdentity(scrobbleKey)
.UsingJobData("IsFull", false)
);
options.AddTrigger(t => t
.WithIdentity("scrobble-watcher-agile-trigger")
.ForJob(scrobbleKey)
.StartNow()
.WithSimpleSchedule(x => x.WithInterval(config.JobOptions.Scrobble.InterJobDelay).RepeatForever())
.WithDescription("Periodic trigger for scrobble watcher")
);
var fullScrobbleKey = new JobKey("scrobble-watcher-full", "scrobble");
options.AddJob<ScrobbleWatcherJob>(j => j
.WithDescription("Check all scrobbles and mirror to database")
.WithIdentity(fullScrobbleKey)
.UsingJobData("IsFull", true)
);
options.AddTrigger(t => t
.WithIdentity("scrobble-watcher-full-trigger")
.ForJob(fullScrobbleKey)
.WithCronSchedule(config.JobOptions.Scrobble.FullScrobbleCron)
.WithDescription("Periodic trigger for scrobble watcher")
);
}
else
{
Console.WriteLine("> Skipping Scrobble Jobs...");
}
});
services.AddQuartzHostedService(options =>{
services.AddQuartzHostedService(options => {
options.WaitForJobsToComplete = true;
});

View File

@ -23,6 +23,10 @@ namespace Selector.CLI.Jobs
private readonly ApplicationDbContext db;
private readonly ScrobbleWatcherJobOptions options;
private static object databaseLock = new();
public bool IsFull { get; set; }
public ScrobbleWatcherJob(
IUserApi _userApi,
IScrobbleRepository _scrobbleRepo,
@ -42,47 +46,50 @@ namespace Selector.CLI.Jobs
public async Task Execute(IJobExecutionContext context)
{
logger.LogInformation("Starting scrobble watching job");
var users = db.Users
.AsEnumerable()
.Where(u => u.ScrobbleSavingEnabled())
.ToArray();
foreach (var user in users)
try
{
logger.LogInformation("Saving scrobbles for {}/{}", user.UserName, user.LastFmUsername);
logger.LogInformation("Starting scrobble watching job");
if (options.From is not null && options.From.Value.Kind != DateTimeKind.Utc)
var users = db.Users
.AsEnumerable()
.Where(u => u.ScrobbleSavingEnabled())
.ToArray();
foreach (var user in users)
{
options.From = options.From.Value.ToUniversalTime();
}
logger.LogInformation("Saving scrobbles for {}/{}", user.UserName, user.LastFmUsername);
if (options.To is not null && options.To.Value.Kind != DateTimeKind.Utc)
{
options.To = options.To.Value.ToUniversalTime();
}
var saver = new ScrobbleSaver(
userApi,
new()
DateTime? from = null;
if (options.From is not null && !IsFull)
{
User = user,
InterRequestDelay = options.InterRequestDelay,
From = options.From,
To = options.To,
PageSize = options.PageSize,
DontAdd = false,
DontRemove = false,
SimultaneousConnections = options.Simultaneous
},
scrobbleRepo,
loggerFactory.CreateLogger<ScrobbleSaver>(),
loggerFactory);
from = options.From.Value.ToUniversalTime();
}
await saver.Execute(context.CancellationToken);
var saver = new ScrobbleSaver(
userApi,
new()
{
User = user,
InterRequestDelay = options.InterRequestDelay,
From = from,
PageSize = options.PageSize,
DontAdd = false,
DontRemove = false,
SimultaneousConnections = options.Simultaneous
},
scrobbleRepo,
loggerFactory.CreateLogger<ScrobbleSaver>(),
loggerFactory,
databaseLock);
logger.LogInformation("Finished scrobbles for {}/{}", user.UserName, user.LastFmUsername);
await saver.Execute(context.CancellationToken);
logger.LogInformation("Finished scrobbles for {}/{}", user.UserName, user.LastFmUsername);
}
}
catch (Exception ex)
{
logger.LogError(ex, "Error occured while saving scrobbles");
}
}
}

View File

@ -124,10 +124,10 @@ namespace Selector.CLI
public const string Key = "Scrobble";
public bool Enabled { get; set; } = true;
public string FullScrobbleCron { get; set; } = "0 2 * * * ?";
public TimeSpan InterJobDelay { get; set; } = TimeSpan.FromMinutes(5);
public TimeSpan InterRequestDelay { get; set; } = TimeSpan.FromMilliseconds(100);
public DateTime? From { get; set; } = DateTime.UtcNow.AddDays(-14);
public DateTime? To { get; set; }
public int PageSize { get; set; } = 200;
public int Simultaneous { get; set; } = 3;
}

View File

@ -36,18 +36,21 @@ namespace Selector
private readonly IUserApi userClient;
private readonly ScrobbleSaverConfig config;
private Task batchTask;
private BatchingOperation<ScrobbleRequest> batchOperation;
private readonly IScrobbleRepository scrobbleRepo;
public ScrobbleSaver(IUserApi _userClient, ScrobbleSaverConfig _config, IScrobbleRepository _scrobbleRepository, ILogger<ScrobbleSaver> _logger, ILoggerFactory _loggerFactory = null)
private readonly object dbLock;
public ScrobbleSaver(IUserApi _userClient, ScrobbleSaverConfig _config, IScrobbleRepository _scrobbleRepository, ILogger<ScrobbleSaver> _logger, ILoggerFactory _loggerFactory = null, object _dbLock = null)
{
userClient = _userClient;
config = _config;
scrobbleRepo = _scrobbleRepository;
logger = _logger;
loggerFactory = _loggerFactory;
dbLock = _dbLock ?? new();
}
public async Task Execute(CancellationToken token)
@ -74,17 +77,7 @@ namespace Selector
GetRequestsFromPageNumbers(2, page1.TotalPages)
);
batchTask = batchOperation.TriggerRequests(token);
}
logger.LogDebug("Pulling currently stored scrobbles");
var currentScrobbles = scrobbleRepo.GetAll(userId: config.User.Id, from: config.From, to: config.To);
if(batchTask is not null)
{
await batchTask;
await batchOperation.TriggerRequests(token);
}
IEnumerable<LastTrack> scrobbles;
@ -108,50 +101,57 @@ namespace Selector
.Select(s => (UserScrobble) s)
.ToArray();
logger.LogInformation("Completed database scrobble pulling for {}, pulled {:n0}", config.User.UserName, nativeScrobbles.Length);
logger.LogInformation("Completed network scrobble pulling for {}, pulled {:n0}", config.User.UserName, nativeScrobbles.Length);
logger.LogDebug("Identifying difference sets");
var time = Stopwatch.StartNew();
logger.LogDebug("Pulling currently stored scrobbles");
(var toAdd, var toRemove) = ScrobbleMatcher.IdentifyDiffs(currentScrobbles, nativeScrobbles);
time.Stop();
logger.LogTrace("Finished diffing: {:n}ms", time.ElapsedMilliseconds);
var timeDbOps = Stopwatch.StartNew();
if(!config.DontAdd)
lock (dbLock)
{
foreach(var add in toAdd)
var currentScrobbles = scrobbleRepo.GetAll(userId: config.User.Id, from: config.From, to: config.To);
logger.LogDebug("Identifying difference sets");
var time = Stopwatch.StartNew();
(var toAdd, var toRemove) = ScrobbleMatcher.IdentifyDiffs(currentScrobbles, nativeScrobbles);
time.Stop();
logger.LogTrace("Finished diffing: {:n}ms", time.ElapsedMilliseconds);
var timeDbOps = Stopwatch.StartNew();
if (!config.DontAdd)
{
var scrobble = (UserScrobble) add;
scrobble.UserId = config.User.Id;
scrobbleRepo.Add(scrobble);
foreach (var add in toAdd)
{
var scrobble = (UserScrobble)add;
scrobble.UserId = config.User.Id;
scrobbleRepo.Add(scrobble);
}
}
}
else
{
logger.LogInformation("Skipping adding of {} scrobbles", toAdd.Count());
}
if (!config.DontRemove)
{
foreach (var remove in toRemove)
else
{
var scrobble = (UserScrobble) remove;
scrobble.UserId = config.User.Id;
scrobbleRepo.Remove(scrobble);
logger.LogInformation("Skipping adding of {} scrobbles", toAdd.Count());
}
}
else
{
logger.LogInformation("Skipping removal of {} scrobbles", toRemove.Count());
}
await scrobbleRepo.Save();
if (!config.DontRemove)
{
foreach (var remove in toRemove)
{
var scrobble = (UserScrobble)remove;
scrobble.UserId = config.User.Id;
scrobbleRepo.Remove(scrobble);
}
}
else
{
logger.LogInformation("Skipping removal of {} scrobbles", toRemove.Count());
}
_ = scrobbleRepo.Save().Result;
timeDbOps.Stop();
logger.LogTrace("DB ops: {:n}ms", timeDbOps.ElapsedMilliseconds);
timeDbOps.Stop();
logger.LogTrace("DB ops: {:n}ms", timeDbOps.ElapsedMilliseconds);
logger.LogInformation("Completed scrobble pulling for {}, +{:n0}, -{:n0}", config.User.UserName, toAdd.Count(), toRemove.Count());
logger.LogInformation("Completed scrobble pulling for {}, +{:n0}, -{:n0}", config.User.UserName, toAdd.Count(), toRemove.Count());
}
}
else
{

View File

@ -16,7 +16,12 @@
<input asp-for="Input.Username" class="form-control" />
<span asp-validation-for="Input.Username" class="text-danger"></span>
</div>
<button id="change-username-button" type="submit" asp-page-handler="ChangeUsername" class="btn btn-primary form-element">Change Username</button>
<div class="form-group form-element">
<label asp-for="Input.ScrobbleSaving"></label>
<input asp-for="Input.ScrobbleSaving" />
<span asp-validation-for="Input.ScrobbleSaving" class="text-danger"></span>
</div>
<button id="save-button" type="submit" asp-page-handler="Save" class="btn btn-primary form-element">Save</button>
</form>
</div>
</div>

View File

@ -13,6 +13,7 @@ using Microsoft.AspNetCore.WebUtilities;
using Selector.Model;
using Selector.Events;
using Microsoft.Extensions.Logging;
namespace Selector.Web.Areas.Identity.Pages.Account.Manage
{
@ -21,12 +22,17 @@ namespace Selector.Web.Areas.Identity.Pages.Account.Manage
private readonly UserManager<ApplicationUser> _userManager;
private readonly UserEventBus UserEvent;
private readonly ILogger<LastFmModel> logger;
public LastFmModel(
UserManager<ApplicationUser> userManager,
UserEventBus userEvent)
UserEventBus userEvent,
ILogger<LastFmModel> _logger)
{
_userManager = userManager;
UserEvent = userEvent;
logger = _logger;
}
[TempData]
@ -39,6 +45,9 @@ namespace Selector.Web.Areas.Identity.Pages.Account.Manage
{
[Display(Name = "Username")]
public string Username { get; set; }
[Display(Name = "Scrobble Saving")]
public bool ScrobbleSaving { get; set; }
}
private Task LoadAsync(ApplicationUser user)
@ -46,6 +55,7 @@ namespace Selector.Web.Areas.Identity.Pages.Account.Manage
Input = new InputModel
{
Username = user.LastFmUsername,
ScrobbleSaving = user.SaveScrobbles
};
return Task.CompletedTask;
@ -63,7 +73,7 @@ namespace Selector.Web.Areas.Identity.Pages.Account.Manage
return Page();
}
public async Task<IActionResult> OnPostChangeUsernameAsync()
public async Task<IActionResult> OnPostSaveAsync()
{
var user = await _userManager.GetUserAsync(User);
if (user == null)
@ -77,23 +87,54 @@ namespace Selector.Web.Areas.Identity.Pages.Account.Manage
return Page();
}
var changed = false;
if (Input.Username != user.LastFmUsername)
{
var oldUsername = user.LastFmUsername;
user.LastFmUsername = Input.Username?.Trim();
await _userManager.UpdateAsync(user);
changed = true;
UserEvent.OnLastfmCredChange(this, new LastfmChange {
UserId = user.Id,
PreviousUsername = oldUsername,
NewUsername = user.LastFmUsername
});
logger.LogInformation("Changing username from {} to {}", oldUsername, user.LastFmUsername);
StatusMessage = "Username changed";
return RedirectToPage();
}
StatusMessage = "Username unchanged";
if (Input.ScrobbleSaving != user.SaveScrobbles)
{
user.SaveScrobbles = Input.ScrobbleSaving;
logger.LogInformation("Changing scrobble saving from {} to {}", !Input.ScrobbleSaving, Input.ScrobbleSaving);
if (changed)
{
StatusMessage += ", scrobble saving updated";
}
else
{
StatusMessage = "Scrobble saving updated";
changed = true;
}
}
if (changed)
{
logger.LogInformation("Saving Last.fm settings for {}", user.LastFmUsername);
await _userManager.UpdateAsync(user);
}
else
{
StatusMessage = "Settings unchanged";
}
return RedirectToPage();
}
}