adding scrobbles and lastfm/spotify mappings

This commit is contained in:
andy 2022-02-16 23:38:45 +00:00
parent bdd63b5ffa
commit ca5b2cf0f0
20 changed files with 1160 additions and 13 deletions

View File

@ -1,4 +1,5 @@
using System.Collections.Generic;
using System;
using System.Collections.Generic;
using Microsoft.Extensions.Configuration;
namespace Selector.CLI
@ -10,6 +11,7 @@ namespace Selector.CLI
config.GetSection(FormatKeys( new[] { RootOptions.Key, WatcherOptions.Key})).Bind(options.WatcherOptions);
config.GetSection(FormatKeys( new[] { RootOptions.Key, DatabaseOptions.Key})).Bind(options.DatabaseOptions);
config.GetSection(FormatKeys( new[] { RootOptions.Key, RedisOptions.Key})).Bind(options.RedisOptions);
config.GetSection(FormatKeys( new[] { RootOptions.Key, ScrobbleMonitorOptions.Key})).Bind(options.ScrobbleOptions);
}
public static RootOptions ConfigureOptions(IConfiguration config)
@ -37,6 +39,7 @@ namespace Selector.CLI
public string LastfmClient { get; set; }
public string LastfmSecret { get; set; }
public WatcherOptions WatcherOptions { get; set; } = new();
public ScrobbleMonitorOptions ScrobbleOptions { get; set; } = new();
public DatabaseOptions DatabaseOptions { get; set; } = new();
public RedisOptions RedisOptions { get; set; } = new();
public EqualityChecker Equality { get; set; } = EqualityChecker.Uri;
@ -85,4 +88,12 @@ namespace Selector.CLI
public bool Enabled { get; set; } = false;
public string ConnectionString { get; set; }
}
public class ScrobbleMonitorOptions
{
public const string Key = "Scrobble";
public bool Enabled { get; set; } = true;
public TimeSpan InterRequestDelay { get; set; } = new(0, 0, 0, 1, 0);
}
}

View File

@ -13,6 +13,7 @@ using Selector.Cache;
using Selector.Cache.Extensions;
using Selector.Events;
using Selector.Model.Services;
using Selector.CLI.Services;
namespace Selector.CLI
{
@ -150,6 +151,13 @@ namespace Selector.CLI
services.AddHostedService<DbWatcherService>();
}
}
if (config.ScrobbleOptions.Enabled)
{
Console.WriteLine("> Adding Scrobble Monitor Service");
services.AddHostedService<ScrobbleMonitor>();
}
}
public static void ConfigureDefaultNlog(HostBuilderContext context, ILoggingBuilder builder)

View File

@ -0,0 +1,123 @@
using IF.Lastfm.Core.Api;
using IF.Lastfm.Core.Api.Helpers;
using IF.Lastfm.Core.Objects;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;
using Selector.Model;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
namespace Selector
{
public class ScrobbleSaverConfig
{
public ApplicationUser User { get; set; }
public TimeSpan InterRequestDelay { get; set; }
public DateTime? From { get; set; }
public DateTime? To { get; set; }
public int PageSize { get; set; } = 100;
}
public class ScrobbleSaver
{
private readonly ILogger<ScrobbleSaver> logger;
private readonly IUserApi userClient;
private readonly ScrobbleSaverConfig config;
private readonly IServiceScopeFactory serviceScopeFactory;
public ScrobbleSaver(IUserApi _userClient, ScrobbleSaverConfig _config, IServiceScopeFactory _serviceScopeFactory, ILogger<ScrobbleSaver> _logger)
{
userClient = _userClient;
config = _config;
serviceScopeFactory = _serviceScopeFactory;
logger = _logger;
}
public async Task Execute(CancellationToken token)
{
logger.LogInformation("Saving all scrobbles for {0}/{1}", config.User.UserName, config.User.LastFmUsername);
using var scope = serviceScopeFactory.CreateScope();
using var db = scope.ServiceProvider.GetRequiredService<ApplicationDbContext>();
var page1 = await userClient.GetRecentScrobbles(config.User.LastFmUsername, count: config.PageSize, from: config.From, to: config.To);
if(page1.Success)
{
var scrobbles = page1.Content.ToList();
if(page1.TotalPages > 1)
{
var tasks = await GetScrobblesFromPageNumbers(Enumerable.Range(2, page1.TotalPages - 1), token);
var taskResults = await Task.WhenAll(tasks);
foreach (var result in taskResults)
{
if (result.Success)
{
scrobbles.AddRange(result.Content);
}
else
{
logger.LogInformation("Failed to get a subset of scrobbles for {0}/{1}", config.User.UserName, config.User.LastFmUsername);
}
}
}
var nativeScrobbles = scrobbles
.DistinctBy(s => s.TimePlayed)
.OrderBy(s => s.TimePlayed)
.Select(s =>
{
var nativeScrobble = (UserScrobble) s;
nativeScrobble.UserId = config.User.Id;
return nativeScrobble;
});
var currentScrobbles = db.Scrobble
.AsEnumerable()
.OrderBy(s => s.Timestamp)
.Where(s => s.UserId == config.User.Id);
if (config.From is not null)
{
currentScrobbles = currentScrobbles.Where(s => s.Timestamp > config.From);
}
if (config.To is not null)
{
currentScrobbles = currentScrobbles.Where(s => s.Timestamp < config.To);
}
(var toAdd, var toRemove) = ScrobbleMatcher.IdentifyDiffsContains(currentScrobbles, nativeScrobbles);
await db.Scrobble.AddRangeAsync(toAdd.Cast<UserScrobble>());
db.Scrobble.RemoveRange(toRemove.Cast<UserScrobble>());
await db.SaveChangesAsync();
}
else
{
logger.LogError("Failed to pull first scrobble page for {0}/{1}", config.User.UserName, config.User.LastFmUsername);
}
}
private async Task<List<Task<PageResponse<LastTrack>>>> GetScrobblesFromPageNumbers(IEnumerable<int> pageNumbers, CancellationToken token)
{
var tasks = new List<Task<PageResponse<LastTrack>>>();
foreach (var pageNumber in pageNumbers)
{
logger.LogInformation("Pulling page {2} for {0}/{1}", config.User.UserName, config.User.LastFmUsername, pageNumber);
tasks.Add(userClient.GetRecentScrobbles(config.User.LastFmUsername, pagenumber: pageNumber, count: config.PageSize, from: config.From, to: config.To));
await Task.Delay(config.InterRequestDelay, token);
}
return tasks;
}
}
}

View File

@ -10,6 +10,7 @@ using Microsoft.Extensions.Logging;
using Selector.Cache;
using Selector.Model;
using Selector.Model.Extensions;
using Selector.Events;
using System.Collections.Concurrent;
@ -131,7 +132,7 @@ namespace Selector.CLI
if (UserEventFirerFactory is not null) consumers.Add(await UserEventFirerFactory.Get());
if (!string.IsNullOrWhiteSpace(dbWatcher.User.LastFmUsername))
if (dbWatcher.User.LastFmConnected())
{
consumers.Add(await PlayCounterFactory.Get(creds: new() { Username = dbWatcher.User.LastFmUsername }));
}

View File

@ -0,0 +1,67 @@
using IF.Lastfm.Core.Api;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using Selector.Model;
using Selector.Model.Extensions;
using System;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
namespace Selector.CLI.Services
{
public class ScrobbleMonitor : IHostedService
{
private readonly ILogger<ScrobbleMonitor> logger;
private readonly ILoggerFactory loggerFactory;
private readonly ScrobbleMonitorOptions config;
private readonly IUserApi userApi;
private readonly IServiceScopeFactory serviceScopeFactory;
private Task task;
public ScrobbleMonitor(ILogger<ScrobbleMonitor> _logger, IOptions<ScrobbleMonitorOptions> _options, IUserApi _userApi, IServiceScopeFactory _serviceScopeFactory, ILoggerFactory _loggerFactory)
{
logger = _logger;
userApi = _userApi;
config = _options.Value;
serviceScopeFactory = _serviceScopeFactory;
loggerFactory = _loggerFactory;
}
public async Task StartAsync(CancellationToken cancellationToken)
{
logger.LogInformation("Starting scrobble monitor");
using var scope = serviceScopeFactory.CreateScope();
using var db = scope.ServiceProvider.GetRequiredService<ApplicationDbContext>();
await RunScrobbleSavers(db, cancellationToken);
}
public async Task RunScrobbleSavers(ApplicationDbContext db, CancellationToken token)
{
foreach (var user in db.Users
.AsNoTracking()
.AsEnumerable()
.Where(u => u.ScrobbleSavingEnabled()))
{
logger.LogInformation("Starting scrobble saver for {0}/{1}", user.UserName, user.LastFmUsername);
await new ScrobbleSaver(userApi, new ScrobbleSaverConfig()
{
User = user,
InterRequestDelay = config.InterRequestDelay,
From = DateTime.UtcNow.AddDays(-3)
}, serviceScopeFactory, loggerFactory.CreateLogger<ScrobbleSaver>()).Execute(token);
}
}
public Task StopAsync(CancellationToken cancellationToken)
{
return Task.CompletedTask;
}
}
}

View File

@ -4,7 +4,8 @@
"ClientSecret": "",
"Equality": "uri",
"Watcher": {
"localenabled": false,
"enabled": false,
"localenabled": false,
"Instances": [
{
"type": "player",
@ -18,7 +19,7 @@
"enabled": true
},
"Redis": {
"enabled": true
"enabled": false
}
},
"Logging": {

View File

@ -19,6 +19,10 @@ namespace Selector.Model
private readonly ILogger<ApplicationDbContext> Logger;
public DbSet<Watcher> Watcher { get; set; }
public DbSet<UserScrobble> Scrobble { get; set; }
public DbSet<TrackLastfmSpotifyMapping> TrackMapping { get; set; }
public DbSet<AlbumLastfmSpotifyMapping> AlbumMapping { get; set; }
public DbSet<ArtistLastfmSpotifyMapping> ArtistMapping { get; set; }
public ApplicationDbContext(
DbContextOptions<ApplicationDbContext> options,
@ -37,11 +41,56 @@ namespace Selector.Model
{
base.OnModelCreating(modelBuilder);
modelBuilder.HasCollation("case_insensitive", locale: "en-u-ks-primary", provider: "icu", deterministic: false);
modelBuilder.Entity<ApplicationUser>()
.Property(u => u.SpotifyIsLinked)
.IsRequired();
modelBuilder.Entity<ApplicationUser>()
.Property(u => u.LastFmUsername)
.UseCollation("case_insensitive");
modelBuilder.Entity<Watcher>()
.HasOne(w => w.User)
.WithMany(u => u.Watchers)
.HasForeignKey(w => w.UserId);
modelBuilder.Entity<UserScrobble>().HasKey(s => new { s.UserId, s.Timestamp });
modelBuilder.Entity<UserScrobble>()
.HasOne(w => w.User)
.WithMany(u => u.Scrobbles)
.HasForeignKey(w => w.UserId);
modelBuilder.Entity<UserScrobble>()
.Property(s => s.TrackName)
.UseCollation("case_insensitive");
modelBuilder.Entity<UserScrobble>()
.Property(s => s.AlbumName)
.UseCollation("case_insensitive");
modelBuilder.Entity<UserScrobble>()
.Property(s => s.ArtistName)
.UseCollation("case_insensitive");
modelBuilder.Entity<TrackLastfmSpotifyMapping>().HasKey(s => s.Id);
modelBuilder.Entity<TrackLastfmSpotifyMapping>()
.Property(s => s.LastfmTrackName)
.UseCollation("case_insensitive");
modelBuilder.Entity<TrackLastfmSpotifyMapping>()
.Property(s => s.LastfmArtistName)
.UseCollation("case_insensitive");
modelBuilder.Entity<AlbumLastfmSpotifyMapping>().HasKey(s => s.Id);
modelBuilder.Entity<AlbumLastfmSpotifyMapping>()
.Property(s => s.LastfmAlbumName)
.UseCollation("case_insensitive");
modelBuilder.Entity<AlbumLastfmSpotifyMapping>()
.Property(s => s.LastfmArtistName)
.UseCollation("case_insensitive");
modelBuilder.Entity<ArtistLastfmSpotifyMapping>().HasKey(s => s.Id);
modelBuilder.Entity<ArtistLastfmSpotifyMapping>()
.Property(s => s.LastfmArtistName)
.UseCollation("case_insensitive");
SeedData.Seed(modelBuilder);
}

View File

@ -8,7 +8,6 @@ namespace Selector.Model
{
public class ApplicationUser : IdentityUser
{
[Required]
public bool SpotifyIsLinked { get; set; }
public DateTime SpotifyLastRefresh { get; set; }
public int SpotifyTokenExpiry { get; set; }
@ -16,8 +15,10 @@ namespace Selector.Model
public string SpotifyRefreshToken { get; set; }
public string LastFmUsername { get; set; }
public bool SaveScrobbles { get; set; }
public List<Watcher> Watchers { get; set; }
public List<UserScrobble> Scrobbles { get; set; }
}
public class ApplicationUserDTO

View File

@ -0,0 +1,13 @@

namespace Selector.Model.Extensions
{
public static class ScrobbleExtensions
{
/// <summary>
/// Helper to get the <see cref="Scrobble.AlbumArtistName"/> if possible, if not fall back to artist name
/// </summary>
/// <param name="scrobble"></param>
/// <returns></returns>
public static string AlbumArtist(this Scrobble scrobble) => scrobble.AlbumArtistName ?? scrobble.ArtistName;
}
}

View File

@ -0,0 +1,17 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace Selector.Model.Extensions
{
public static class UserExtensions
{
public static bool LastFmConnected(this ApplicationUser user)
=> !string.IsNullOrEmpty(user.LastFmUsername);
public static bool ScrobbleSavingEnabled(this ApplicationUser user)
=> user.LastFmConnected() && user.SaveScrobbles;
}
}

View File

@ -0,0 +1,26 @@
namespace Selector.Model
{
public abstract class LastfmSpotifyMapping
{
public int Id { get; set; }
public string SpotifyUri { get; set; }
}
public class TrackLastfmSpotifyMapping: LastfmSpotifyMapping
{
public string LastfmTrackName { get; set; }
public string LastfmArtistName { get; set; }
}
public class AlbumLastfmSpotifyMapping : LastfmSpotifyMapping
{
public string LastfmAlbumName { get; set; }
public string LastfmArtistName { get; set; }
}
public class ArtistLastfmSpotifyMapping : LastfmSpotifyMapping
{
public string LastfmArtistName { get; set; }
}
}

View File

@ -0,0 +1,455 @@
// <auto-generated />
using System;
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Infrastructure;
using Microsoft.EntityFrameworkCore.Migrations;
using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata;
using Selector.Model;
#nullable disable
namespace Selector.Model.Migrations
{
[DbContext(typeof(ApplicationDbContext))]
[Migration("20220216214339_scrobble_and_lastfm_spotify_mappings")]
partial class scrobble_and_lastfm_spotify_mappings
{
protected override void BuildTargetModel(ModelBuilder modelBuilder)
{
#pragma warning disable 612, 618
modelBuilder
.HasAnnotation("Npgsql:CollationDefinition:case_insensitive", "en-u-ks-primary,en-u-ks-primary,icu,False")
.HasAnnotation("ProductVersion", "6.0.2")
.HasAnnotation("Relational:MaxIdentifierLength", 63);
NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder);
modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityRole", b =>
{
b.Property<string>("Id")
.HasColumnType("text");
b.Property<string>("ConcurrencyStamp")
.IsConcurrencyToken()
.HasColumnType("text");
b.Property<string>("Name")
.HasMaxLength(256)
.HasColumnType("character varying(256)");
b.Property<string>("NormalizedName")
.HasMaxLength(256)
.HasColumnType("character varying(256)");
b.HasKey("Id");
b.HasIndex("NormalizedName")
.IsUnique()
.HasDatabaseName("RoleNameIndex");
b.ToTable("AspNetRoles", (string)null);
b.HasData(
new
{
Id = "00c64c0a-3387-4933-9575-83443fa9092b",
ConcurrencyStamp = "b91c880d-e280-4a17-a528-d34fdc35f291",
Name = "Admin",
NormalizedName = "ADMIN"
});
});
modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityRoleClaim<string>", b =>
{
b.Property<int>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("integer");
NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property<int>("Id"));
b.Property<string>("ClaimType")
.HasColumnType("text");
b.Property<string>("ClaimValue")
.HasColumnType("text");
b.Property<string>("RoleId")
.IsRequired()
.HasColumnType("text");
b.HasKey("Id");
b.HasIndex("RoleId");
b.ToTable("AspNetRoleClaims", (string)null);
});
modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityUserClaim<string>", b =>
{
b.Property<int>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("integer");
NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property<int>("Id"));
b.Property<string>("ClaimType")
.HasColumnType("text");
b.Property<string>("ClaimValue")
.HasColumnType("text");
b.Property<string>("UserId")
.IsRequired()
.HasColumnType("text");
b.HasKey("Id");
b.HasIndex("UserId");
b.ToTable("AspNetUserClaims", (string)null);
});
modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityUserLogin<string>", b =>
{
b.Property<string>("LoginProvider")
.HasColumnType("text");
b.Property<string>("ProviderKey")
.HasColumnType("text");
b.Property<string>("ProviderDisplayName")
.HasColumnType("text");
b.Property<string>("UserId")
.IsRequired()
.HasColumnType("text");
b.HasKey("LoginProvider", "ProviderKey");
b.HasIndex("UserId");
b.ToTable("AspNetUserLogins", (string)null);
});
modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityUserRole<string>", b =>
{
b.Property<string>("UserId")
.HasColumnType("text");
b.Property<string>("RoleId")
.HasColumnType("text");
b.HasKey("UserId", "RoleId");
b.HasIndex("RoleId");
b.ToTable("AspNetUserRoles", (string)null);
});
modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityUserToken<string>", b =>
{
b.Property<string>("UserId")
.HasColumnType("text");
b.Property<string>("LoginProvider")
.HasColumnType("text");
b.Property<string>("Name")
.HasColumnType("text");
b.Property<string>("Value")
.HasColumnType("text");
b.HasKey("UserId", "LoginProvider", "Name");
b.ToTable("AspNetUserTokens", (string)null);
});
modelBuilder.Entity("Selector.Model.AlbumLastfmSpotifyMapping", b =>
{
b.Property<int>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("integer");
NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property<int>("Id"));
b.Property<string>("LastfmAlbumName")
.HasColumnType("text")
.UseCollation("case_insensitive");
b.Property<string>("LastfmArtistName")
.HasColumnType("text")
.UseCollation("case_insensitive");
b.Property<string>("SpotifyUri")
.HasColumnType("text");
b.HasKey("Id");
b.ToTable("AlbumMapping");
});
modelBuilder.Entity("Selector.Model.ApplicationUser", b =>
{
b.Property<string>("Id")
.HasColumnType("text");
b.Property<int>("AccessFailedCount")
.HasColumnType("integer");
b.Property<string>("ConcurrencyStamp")
.IsConcurrencyToken()
.HasColumnType("text");
b.Property<string>("Email")
.HasMaxLength(256)
.HasColumnType("character varying(256)");
b.Property<bool>("EmailConfirmed")
.HasColumnType("boolean");
b.Property<string>("LastFmUsername")
.HasColumnType("text")
.UseCollation("case_insensitive");
b.Property<bool>("LockoutEnabled")
.HasColumnType("boolean");
b.Property<DateTimeOffset?>("LockoutEnd")
.HasColumnType("timestamp with time zone");
b.Property<string>("NormalizedEmail")
.HasMaxLength(256)
.HasColumnType("character varying(256)");
b.Property<string>("NormalizedUserName")
.HasMaxLength(256)
.HasColumnType("character varying(256)");
b.Property<string>("PasswordHash")
.HasColumnType("text");
b.Property<string>("PhoneNumber")
.HasColumnType("text");
b.Property<bool>("PhoneNumberConfirmed")
.HasColumnType("boolean");
b.Property<bool>("SaveScrobbles")
.HasColumnType("boolean");
b.Property<string>("SecurityStamp")
.HasColumnType("text");
b.Property<string>("SpotifyAccessToken")
.HasColumnType("text");
b.Property<bool>("SpotifyIsLinked")
.HasColumnType("boolean");
b.Property<DateTime>("SpotifyLastRefresh")
.HasColumnType("timestamp with time zone");
b.Property<string>("SpotifyRefreshToken")
.HasColumnType("text");
b.Property<int>("SpotifyTokenExpiry")
.HasColumnType("integer");
b.Property<bool>("TwoFactorEnabled")
.HasColumnType("boolean");
b.Property<string>("UserName")
.HasMaxLength(256)
.HasColumnType("character varying(256)");
b.HasKey("Id");
b.HasIndex("NormalizedEmail")
.HasDatabaseName("EmailIndex");
b.HasIndex("NormalizedUserName")
.IsUnique()
.HasDatabaseName("UserNameIndex");
b.ToTable("AspNetUsers", (string)null);
});
modelBuilder.Entity("Selector.Model.ArtistLastfmSpotifyMapping", b =>
{
b.Property<int>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("integer");
NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property<int>("Id"));
b.Property<string>("LastfmArtistName")
.HasColumnType("text")
.UseCollation("case_insensitive");
b.Property<string>("SpotifyUri")
.HasColumnType("text");
b.HasKey("Id");
b.ToTable("ArtistMapping");
});
modelBuilder.Entity("Selector.Model.TrackLastfmSpotifyMapping", b =>
{
b.Property<int>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("integer");
NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property<int>("Id"));
b.Property<string>("LastfmArtistName")
.HasColumnType("text")
.UseCollation("case_insensitive");
b.Property<string>("LastfmTrackName")
.HasColumnType("text")
.UseCollation("case_insensitive");
b.Property<string>("SpotifyUri")
.HasColumnType("text");
b.HasKey("Id");
b.ToTable("TrackMapping");
});
modelBuilder.Entity("Selector.Model.UserScrobble", b =>
{
b.Property<string>("UserId")
.HasColumnType("text");
b.Property<DateTime>("Timestamp")
.HasColumnType("timestamp with time zone");
b.Property<string>("AlbumArtistName")
.HasColumnType("text");
b.Property<string>("AlbumName")
.HasColumnType("text")
.UseCollation("case_insensitive");
b.Property<string>("ArtistName")
.HasColumnType("text")
.UseCollation("case_insensitive");
b.Property<string>("TrackName")
.HasColumnType("text")
.UseCollation("case_insensitive");
b.HasKey("UserId", "Timestamp");
b.ToTable("Scrobble");
});
modelBuilder.Entity("Selector.Model.Watcher", b =>
{
b.Property<int>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("integer");
NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property<int>("Id"));
b.Property<int>("Type")
.HasColumnType("integer");
b.Property<string>("UserId")
.IsRequired()
.HasColumnType("text");
b.HasKey("Id");
b.HasIndex("UserId");
b.ToTable("Watcher");
});
modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityRoleClaim<string>", b =>
{
b.HasOne("Microsoft.AspNetCore.Identity.IdentityRole", null)
.WithMany()
.HasForeignKey("RoleId")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
});
modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityUserClaim<string>", b =>
{
b.HasOne("Selector.Model.ApplicationUser", null)
.WithMany()
.HasForeignKey("UserId")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
});
modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityUserLogin<string>", b =>
{
b.HasOne("Selector.Model.ApplicationUser", null)
.WithMany()
.HasForeignKey("UserId")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
});
modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityUserRole<string>", b =>
{
b.HasOne("Microsoft.AspNetCore.Identity.IdentityRole", null)
.WithMany()
.HasForeignKey("RoleId")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
b.HasOne("Selector.Model.ApplicationUser", null)
.WithMany()
.HasForeignKey("UserId")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
});
modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityUserToken<string>", b =>
{
b.HasOne("Selector.Model.ApplicationUser", null)
.WithMany()
.HasForeignKey("UserId")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
});
modelBuilder.Entity("Selector.Model.UserScrobble", b =>
{
b.HasOne("Selector.Model.ApplicationUser", "User")
.WithMany("Scrobbles")
.HasForeignKey("UserId")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
b.Navigation("User");
});
modelBuilder.Entity("Selector.Model.Watcher", b =>
{
b.HasOne("Selector.Model.ApplicationUser", "User")
.WithMany("Watchers")
.HasForeignKey("UserId")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
b.Navigation("User");
});
modelBuilder.Entity("Selector.Model.ApplicationUser", b =>
{
b.Navigation("Scrobbles");
b.Navigation("Watchers");
});
#pragma warning restore 612, 618
}
}
}

View File

@ -0,0 +1,146 @@
using System;
using Microsoft.EntityFrameworkCore.Migrations;
using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata;
#nullable disable
namespace Selector.Model.Migrations
{
public partial class scrobble_and_lastfm_spotify_mappings : Migration
{
protected override void Up(MigrationBuilder migrationBuilder)
{
migrationBuilder.AlterDatabase()
.Annotation("Npgsql:CollationDefinition:case_insensitive", "en-u-ks-primary,en-u-ks-primary,icu,False");
migrationBuilder.AlterColumn<string>(
name: "LastFmUsername",
table: "AspNetUsers",
type: "text",
nullable: true,
collation: "case_insensitive",
oldClrType: typeof(string),
oldType: "text",
oldNullable: true);
migrationBuilder.AddColumn<bool>(
name: "SaveScrobbles",
table: "AspNetUsers",
type: "boolean",
nullable: false,
defaultValue: false);
migrationBuilder.CreateTable(
name: "AlbumMapping",
columns: table => new
{
Id = table.Column<int>(type: "integer", nullable: false)
.Annotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn),
LastfmAlbumName = table.Column<string>(type: "text", nullable: true, collation: "case_insensitive"),
LastfmArtistName = table.Column<string>(type: "text", nullable: true, collation: "case_insensitive"),
SpotifyUri = table.Column<string>(type: "text", nullable: true)
},
constraints: table =>
{
table.PrimaryKey("PK_AlbumMapping", x => x.Id);
});
migrationBuilder.CreateTable(
name: "ArtistMapping",
columns: table => new
{
Id = table.Column<int>(type: "integer", nullable: false)
.Annotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn),
LastfmArtistName = table.Column<string>(type: "text", nullable: true, collation: "case_insensitive"),
SpotifyUri = table.Column<string>(type: "text", nullable: true)
},
constraints: table =>
{
table.PrimaryKey("PK_ArtistMapping", x => x.Id);
});
migrationBuilder.CreateTable(
name: "Scrobble",
columns: table => new
{
Timestamp = table.Column<DateTime>(type: "timestamp with time zone", nullable: false),
UserId = table.Column<string>(type: "text", nullable: false),
TrackName = table.Column<string>(type: "text", nullable: true, collation: "case_insensitive"),
AlbumName = table.Column<string>(type: "text", nullable: true, collation: "case_insensitive"),
AlbumArtistName = table.Column<string>(type: "text", nullable: true),
ArtistName = table.Column<string>(type: "text", nullable: true, collation: "case_insensitive")
},
constraints: table =>
{
table.PrimaryKey("PK_Scrobble", x => new { x.UserId, x.Timestamp });
table.ForeignKey(
name: "FK_Scrobble_AspNetUsers_UserId",
column: x => x.UserId,
principalTable: "AspNetUsers",
principalColumn: "Id",
onDelete: ReferentialAction.Cascade);
});
migrationBuilder.CreateTable(
name: "TrackMapping",
columns: table => new
{
Id = table.Column<int>(type: "integer", nullable: false)
.Annotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn),
LastfmTrackName = table.Column<string>(type: "text", nullable: true, collation: "case_insensitive"),
LastfmArtistName = table.Column<string>(type: "text", nullable: true, collation: "case_insensitive"),
SpotifyUri = table.Column<string>(type: "text", nullable: true)
},
constraints: table =>
{
table.PrimaryKey("PK_TrackMapping", x => x.Id);
});
migrationBuilder.UpdateData(
table: "AspNetRoles",
keyColumn: "Id",
keyValue: "00c64c0a-3387-4933-9575-83443fa9092b",
column: "ConcurrencyStamp",
value: "b91c880d-e280-4a17-a528-d34fdc35f291");
}
protected override void Down(MigrationBuilder migrationBuilder)
{
migrationBuilder.DropTable(
name: "AlbumMapping");
migrationBuilder.DropTable(
name: "ArtistMapping");
migrationBuilder.DropTable(
name: "Scrobble");
migrationBuilder.DropTable(
name: "TrackMapping");
migrationBuilder.DropColumn(
name: "SaveScrobbles",
table: "AspNetUsers");
migrationBuilder.AlterDatabase()
.OldAnnotation("Npgsql:CollationDefinition:case_insensitive", "en-u-ks-primary,en-u-ks-primary,icu,False");
migrationBuilder.AlterColumn<string>(
name: "LastFmUsername",
table: "AspNetUsers",
type: "text",
nullable: true,
oldClrType: typeof(string),
oldType: "text",
oldNullable: true,
oldCollation: "case_insensitive");
migrationBuilder.UpdateData(
table: "AspNetRoles",
keyColumn: "Id",
keyValue: "00c64c0a-3387-4933-9575-83443fa9092b",
column: "ConcurrencyStamp",
value: "0801d9f2-0f90-4ca7-bb85-eaa36046fc86");
}
}
}

View File

@ -17,7 +17,8 @@ namespace Selector.Model.Migrations
{
#pragma warning disable 612, 618
modelBuilder
.HasAnnotation("ProductVersion", "6.0.0")
.HasAnnotation("Npgsql:CollationDefinition:case_insensitive", "en-u-ks-primary,en-u-ks-primary,icu,False")
.HasAnnotation("ProductVersion", "6.0.2")
.HasAnnotation("Relational:MaxIdentifierLength", 63);
NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder);
@ -51,7 +52,7 @@ namespace Selector.Model.Migrations
new
{
Id = "00c64c0a-3387-4933-9575-83443fa9092b",
ConcurrencyStamp = "0801d9f2-0f90-4ca7-bb85-eaa36046fc86",
ConcurrencyStamp = "b91c880d-e280-4a17-a528-d34fdc35f291",
Name = "Admin",
NormalizedName = "ADMIN"
});
@ -163,6 +164,30 @@ namespace Selector.Model.Migrations
b.ToTable("AspNetUserTokens", (string)null);
});
modelBuilder.Entity("Selector.Model.AlbumLastfmSpotifyMapping", b =>
{
b.Property<int>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("integer");
NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property<int>("Id"));
b.Property<string>("LastfmAlbumName")
.HasColumnType("text")
.UseCollation("case_insensitive");
b.Property<string>("LastfmArtistName")
.HasColumnType("text")
.UseCollation("case_insensitive");
b.Property<string>("SpotifyUri")
.HasColumnType("text");
b.HasKey("Id");
b.ToTable("AlbumMapping");
});
modelBuilder.Entity("Selector.Model.ApplicationUser", b =>
{
b.Property<string>("Id")
@ -183,7 +208,8 @@ namespace Selector.Model.Migrations
.HasColumnType("boolean");
b.Property<string>("LastFmUsername")
.HasColumnType("text");
.HasColumnType("text")
.UseCollation("case_insensitive");
b.Property<bool>("LockoutEnabled")
.HasColumnType("boolean");
@ -208,6 +234,9 @@ namespace Selector.Model.Migrations
b.Property<bool>("PhoneNumberConfirmed")
.HasColumnType("boolean");
b.Property<bool>("SaveScrobbles")
.HasColumnType("boolean");
b.Property<string>("SecurityStamp")
.HasColumnType("text");
@ -245,6 +274,78 @@ namespace Selector.Model.Migrations
b.ToTable("AspNetUsers", (string)null);
});
modelBuilder.Entity("Selector.Model.ArtistLastfmSpotifyMapping", b =>
{
b.Property<int>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("integer");
NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property<int>("Id"));
b.Property<string>("LastfmArtistName")
.HasColumnType("text")
.UseCollation("case_insensitive");
b.Property<string>("SpotifyUri")
.HasColumnType("text");
b.HasKey("Id");
b.ToTable("ArtistMapping");
});
modelBuilder.Entity("Selector.Model.TrackLastfmSpotifyMapping", b =>
{
b.Property<int>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("integer");
NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property<int>("Id"));
b.Property<string>("LastfmArtistName")
.HasColumnType("text")
.UseCollation("case_insensitive");
b.Property<string>("LastfmTrackName")
.HasColumnType("text")
.UseCollation("case_insensitive");
b.Property<string>("SpotifyUri")
.HasColumnType("text");
b.HasKey("Id");
b.ToTable("TrackMapping");
});
modelBuilder.Entity("Selector.Model.UserScrobble", b =>
{
b.Property<string>("UserId")
.HasColumnType("text");
b.Property<DateTime>("Timestamp")
.HasColumnType("timestamp with time zone");
b.Property<string>("AlbumArtistName")
.HasColumnType("text");
b.Property<string>("AlbumName")
.HasColumnType("text")
.UseCollation("case_insensitive");
b.Property<string>("ArtistName")
.HasColumnType("text")
.UseCollation("case_insensitive");
b.Property<string>("TrackName")
.HasColumnType("text")
.UseCollation("case_insensitive");
b.HasKey("UserId", "Timestamp");
b.ToTable("Scrobble");
});
modelBuilder.Entity("Selector.Model.Watcher", b =>
{
b.Property<int>("Id")
@ -318,6 +419,17 @@ namespace Selector.Model.Migrations
.IsRequired();
});
modelBuilder.Entity("Selector.Model.UserScrobble", b =>
{
b.HasOne("Selector.Model.ApplicationUser", "User")
.WithMany("Scrobbles")
.HasForeignKey("UserId")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
b.Navigation("User");
});
modelBuilder.Entity("Selector.Model.Watcher", b =>
{
b.HasOne("Selector.Model.ApplicationUser", "User")
@ -331,6 +443,8 @@ namespace Selector.Model.Migrations
modelBuilder.Entity("Selector.Model.ApplicationUser", b =>
{
b.Navigation("Scrobbles");
b.Navigation("Watchers");
});
#pragma warning restore 612, 618

View File

@ -1,7 +1,9 @@
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using System;
using System.Threading;
using System.Threading.Tasks;
@ -9,13 +11,13 @@ namespace Selector.Model.Services
{
public class MigratorService : IHostedService
{
private readonly ApplicationDbContext context;
private readonly IServiceScopeFactory scopeProvider;
private readonly DatabaseOptions options;
private readonly ILogger<MigratorService> logger;
public MigratorService(ApplicationDbContext _context, IOptions<DatabaseOptions> _options, ILogger<MigratorService> _logger)
public MigratorService(IServiceScopeFactory _scopeProvider, IOptions<DatabaseOptions> _options, ILogger<MigratorService> _logger)
{
context = _context;
scopeProvider = _scopeProvider;
options = _options.Value;
logger = _logger;
}
@ -24,8 +26,10 @@ namespace Selector.Model.Services
{
if(options.Migrate)
{
using var scope = scopeProvider.CreateScope();
logger.LogInformation("Applying migrations");
context.Database.Migrate();
scope.ServiceProvider.GetRequiredService<ApplicationDbContext>().Database.Migrate();
}
return Task.CompletedTask;

View File

@ -0,0 +1,20 @@
using System;
namespace Selector.Model
{
public class UserScrobble: Scrobble
{
public string UserId { get; set; }
public ApplicationUser User { get; set; }
public static explicit operator UserScrobble(IF.Lastfm.Core.Objects.LastTrack track) => new()
{
Timestamp = track.TimePlayed?.UtcDateTime ?? DateTime.MinValue,
TrackName = track.Name,
AlbumName = track.AlbumName,
ArtistName = track.ArtistName,
};
}
}

View File

@ -11,6 +11,7 @@ using StackExchange.Redis;
using Selector.Cache;
using Selector.Model;
using Selector.Model.Extensions;
namespace Selector.Web.Hubs
{
@ -88,7 +89,7 @@ namespace Selector.Web.Hubs
.SingleOrDefault()
?? throw new SqlNullValueException("No user returned");
if(!string.IsNullOrWhiteSpace(user.LastFmUsername))
if(user.LastFmConnected())
{
var playCount = await PlayCountPuller.Get(user.LastFmUsername, track, artist, album, albumArtist);

View File

@ -0,0 +1,25 @@
using System;
namespace Selector
{
public class Scrobble
{
public DateTime Timestamp { get; set; }
public string TrackName { get; set; }
public string AlbumName { get; set; }
/// <summary>
/// Not populated by default from the service, where not the same as <see cref="ArtistName"/> these have been manually entered
/// </summary>
public string AlbumArtistName { get; set; }
public string ArtistName { get; set; }
public static explicit operator Scrobble(IF.Lastfm.Core.Objects.LastTrack track) => new()
{
Timestamp = track.TimePlayed?.UtcDateTime ?? DateTime.MinValue,
TrackName = track.Name,
AlbumName = track.AlbumName,
ArtistName = track.ArtistName,
};
}
}

View File

@ -0,0 +1,65 @@
using IF.Lastfm.Core.Objects;
using System.Collections.Generic;
using System.Diagnostics.CodeAnalysis;
using System.Linq;
namespace Selector
{
public static class ScrobbleMatcher
{
public static bool MatchTime(Scrobble nativeScrobble, LastTrack serviceScrobble)
=> serviceScrobble.TimePlayed.Equals(nativeScrobble);
public static bool MatchTime(Scrobble nativeScrobble, Scrobble serviceScrobble)
=> serviceScrobble.Timestamp.Equals(nativeScrobble);
public static (IEnumerable<Scrobble>, IEnumerable<Scrobble>) IdentifyDiffs(IEnumerable<Scrobble> existing, IEnumerable<Scrobble> toApply)
{
existing = existing.OrderBy(s => s.Timestamp);
toApply = toApply.OrderBy(s => s.Timestamp);
var toApplyIter = toApply.GetEnumerator();
var toAdd = new List<Scrobble>();
var toRemove = new List<Scrobble>();
if(existing.Any())
{
foreach (var currentExisting in existing)
{
while (toApplyIter.Current.Timestamp < currentExisting.Timestamp)
{
toAdd.Add(toApplyIter.Current);
if (!toApplyIter.MoveNext()) break;
}
if (!MatchTime(currentExisting, toApplyIter.Current))
{
toRemove.Add(currentExisting);
}
}
}
else
{
toAdd.AddRange(toApply);
}
return (toAdd, toRemove);
}
public static (IEnumerable<Scrobble>, IEnumerable<Scrobble>) IdentifyDiffsContains(IEnumerable<Scrobble> existing, IEnumerable<Scrobble> toApply)
{
var toAdd = toApply.Where(s => !existing.Contains(s, new ScrobbleComp()));
var toRemove = existing.Where(s => !toApply.Contains(s, new ScrobbleComp()));
return (toAdd, toRemove);
}
public class ScrobbleComp : IEqualityComparer<Scrobble>
{
public bool Equals(Scrobble x, Scrobble y) => x.Timestamp == y.Timestamp;
public int GetHashCode([DisallowNull] Scrobble obj) => obj.Timestamp.GetHashCode();
}
}
}