Merge pull request #50 from Sarsoo/spotifyhistory

Spotify History storing from CLI and inclusion in the front end

Selector.CLI/Command/SpotHistoryCommand.cs

@@ -0,0 +1,90 @@
+using Microsoft.EntityFrameworkCore;
+using Microsoft.Extensions.Logging;
+using Selector.CLI.Extensions;
+using Selector.Data;
+using Selector.Model;
+using System;
+using System.IO;
+using System.CommandLine;
+using System.CommandLine.Invocation;
+using System.Linq;
+using System.Collections.Generic;
+
+namespace Selector.CLI
+{
+    public class SpotifyHistoryCommand : Command
+    {
+        public SpotifyHistoryCommand(string name, string description = null) : base(name, description)
+        {
+            var connectionString = new Option<string>("--connection", "database to migrate");
+            connectionString.AddAlias("-c");
+            AddOption(connectionString);
+
+            var pathString = new Option<string>("--path", "path to find data");
+            pathString.AddAlias("-i");
+            AddOption(pathString);
+
+            var username = new Option<string>("--username", "user to pulls scrobbles for");
+            username.AddAlias("-u");
+            AddOption(username);
+
+            Handler = CommandHandler.Create((string connectionString, string path, string username) => Execute(connectionString, path, username));
+        }
+
+        public static int Execute(string connectionString, string path, string username)
+        {
+            var streams = new List<FileStream>();
+
+            try
+            {
+                var context = new CommandContext().WithLogger().WithDb(connectionString).WithLastfmApi();
+                var logger = context.Logger.CreateLogger("Scrobble");
+
+                using var db = new ApplicationDbContext(context.DatabaseConfig.Options, context.Logger.CreateLogger<ApplicationDbContext>());
+
+                var historyPersister = new HistoryPersister(db, new DataJsonContext(), new()
+                {
+                    Username = username
+                }, context.Logger.CreateLogger<HistoryPersister>());
+
+                logger.LogInformation("Preparing to parse from {} for {}", path, username);
+
+                var directoryContents = Directory.EnumerateFiles(path);
+                var endSongs = directoryContents.Where(f => f.Contains("endsong_")).ToArray();
+
+                foreach(var file in endSongs)
+                {
+                    streams.Add(File.OpenRead(file));
+                }
+                           
+                Console.WriteLine("Parse {0} historical data files? (y/n) ", endSongs.Length);
+                var input = Console.ReadLine();
+
+                if (input.Trim().Equals("y", StringComparison.OrdinalIgnoreCase))
+                {
+                    logger.LogInformation("Parsing files");
+
+                    historyPersister.Process(streams).Wait();
+                }
+                else
+                {
+                    logger.LogInformation("Exiting");
+                }
+            }
+            catch (Exception ex)
+            {
+                Console.WriteLine(ex);
+                return 1;
+            }
+            finally
+            {
+                foreach(var stream in streams)
+                {
+                    stream.Dispose();
+                }
+            }
+
+            return 0;
+        }
+    }
+}

Selector.CLI/Extensions/ServiceExtensions.cs

@@ -118,7 +118,12 @@ namespace Selector.CLI.Extensions
                     options.UseNpgsql(config.DatabaseOptions.ConnectionString)
                 );
 
-                services.AddTransient<IScrobbleRepository, ScrobbleRepository>();
+                services.AddTransient<IScrobbleRepository, ScrobbleRepository>()
+                        .AddTransient<ISpotifyListenRepository, SpotifyListenRepository>();
+
+                services.AddTransient<IListenRepository, MetaListenRepository>();
+                //services.AddTransient<IListenRepository, SpotifyListenRepository>();
+
                 services.AddTransient<IScrobbleMappingRepository, ScrobbleMappingRepository>();
 
                 services.AddHostedService<MigratorService>();

Selector.CLI/Extensions/SpotifyExtensions.cs

@@ -0,0 +1,39 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading.Tasks;
+using Microsoft.Extensions.Logging;
+using SpotifyAPI.Web;
+
+namespace Selector.CLI.Extensions
+{
+    public static class SpotifyExtensions
+    {
+        public static async Task<(FullPlaylist, IEnumerable<PlaylistTrack<IPlayableItem>>)> GetPopulated(this ISpotifyClient client, string playlistId, ILogger logger = null)
+        {
+            try
+            {
+                var playlist = await client.Playlists.Get(playlistId);
+                var items = await client.Paginate(playlist.Tracks).ToListAsync();
+
+                return (playlist, items);
+            }
+            catch (APIUnauthorizedException e)
+            {
+                logger?.LogDebug("Unauthorised error: [{message}] (should be refreshed and retried?)", e.Message);
+                throw e;
+            }
+            catch (APITooManyRequestsException e)
+            {
+                logger?.LogDebug("Too many requests error: [{message}]", e.Message);
+                throw e;
+            }
+            catch (APIException e)
+            {
+                logger?.LogDebug("API error: [{message}]", e.Message);
+                throw e;
+            }
+        }
+    }
+}
+

Selector.CLI/Program.cs

@@ -9,6 +9,7 @@ namespace Selector.CLI
             var cmd = new HostRootCommand();
             cmd.AddCommand(new ScrobbleCommand("scrobble", "Manipulate scrobbles"));
             cmd.AddCommand(new MigrateCommand("migrate", "Migrate database"));
+            cmd.AddCommand(new SpotifyHistoryCommand("history", "Insert Spotify history"));
 
             cmd.Invoke(args);
         }

Selector.CLI/ScrobbleSaver.cs

@@ -112,7 +112,7 @@ namespace Selector
                     logger.LogDebug("Identifying difference sets");
                     var time = Stopwatch.StartNew();
 
-                    (var toAdd, var toRemove) = ScrobbleMatcher.IdentifyDiffs(currentScrobbles, nativeScrobbles);
+                    (var toAdd, var toRemove) = ListenMatcher.IdentifyDiffs(currentScrobbles, nativeScrobbles);
 
                     time.Stop();
                     logger.LogTrace("Finished diffing: {:n}ms", time.ElapsedMilliseconds);

Selector.CLI/Selector.CLI.csproj

@@ -31,6 +31,7 @@
     <ProjectReference Include="..\Selector\Selector.csproj" />
     <ProjectReference Include="..\Selector.Model\Selector.Model.csproj" />
     <ProjectReference Include="..\Selector.Cache\Selector.Cache.csproj" />
+    <ProjectReference Include="..\Selector.Data\Selector.Data.csproj" />
     <ProjectReference Include="..\Selector.Event\Selector.Event.csproj" />
   </ItemGroup>
 

Selector.Data/DataJsonContext.cs

@@ -0,0 +1,14 @@
+using System;
+using System.Text.Json;
+using System.Text.Json.Serialization;
+using System.Text.Json.Serialization.Metadata;
+
+namespace Selector.Data;
+
+[JsonSerializable(typeof(EndSong))]
+[JsonSerializable(typeof(EndSong[]))]
+public partial class DataJsonContext : JsonSerializerContext
+{
+
+}
+

Selector.Data/EndSong.cs

@@ -0,0 +1,27 @@
+namespace Selector.Data;
+
+public record struct EndSong
+{
+    public string conn_country { get; set; }
+    public string episode_name { get; set; }
+    public string episode_show_name { get; set; }
+    public bool? incognito_mode { get; set; }
+    public string ip_addr_decrypted { get; set; }
+    public string master_metadata_album_album_name { get; set; }
+    public string master_metadata_album_artist_name { get; set; }
+    public string master_metadata_track_name { get; set; }
+    public int ms_played { get; set; }
+    public bool? offline { get; set; }
+    public long? offline_timestamp { get; set; }
+    public string platform { get; set; }
+    public string reason_end { get; set; }
+    public string reason_start { get; set; }
+    public bool shuffle { get; set; }
+    public bool? skipped { get; set; }
+    public string spotify_episode_uri { get; set; }
+    public string spotify_track_uri { get; set; }
+    public string ts { get; set; }
+    public string user_agent_decrypted { get; set; }
+    public string username { get; set; }
+}
+

Selector.Data/HistoryPersister.cs

@@ -0,0 +1,102 @@
+using System.Text.Json;
+using Microsoft.Extensions.Logging;
+using Selector.Model;
+
+namespace Selector.Data;
+
+public class HistoryPersisterConfig
+{
+    public string Username { get; set; }
+    public bool InitialClear { get; set; } = true;
+    public bool Apply50PercentRule { get; set; } = false;
+}
+
+public class HistoryPersister
+{
+    private HistoryPersisterConfig Config { get; set; }
+    private ApplicationDbContext Db { get; set; }
+    private DataJsonContext Json { get; set; }
+
+    private ILogger<HistoryPersister> Logger { get; set; }
+
+    public HistoryPersister(ApplicationDbContext db, DataJsonContext json, HistoryPersisterConfig config, ILogger<HistoryPersister> logger = null)
+    {
+        Config = config;
+        Db = db;
+        Json = json;
+        Logger = logger;
+    }
+
+    public void Process(string input)
+    {
+        var parsed = JsonSerializer.Deserialize(input, Json.EndSongArray);
+        Process(parsed).Wait();
+    }
+
+    public async Task Process(Stream input)
+    {
+        var parsed = await JsonSerializer.DeserializeAsync(input, Json.EndSongArray);
+        await Process(parsed);
+    }
+
+    public async Task Process(IEnumerable<Stream> input)
+    {
+        var songs = Enumerable.Empty<EndSong>();
+
+        foreach(var singleInput in input)
+        {
+            var parsed = await JsonSerializer.DeserializeAsync(singleInput, Json.EndSongArray);
+            songs = songs.Concat(parsed);
+
+            Logger?.LogDebug("Parsed {.2f} items for {}", parsed.Length, Config.Username);
+        }
+
+        await Process(songs);
+    }
+
+    public async Task Process(IEnumerable<EndSong> input)
+    {
+        if (Config.InitialClear)
+        {
+            var latestTime = input.OrderBy(x => x.ts).Last().ts;
+            var time = DateTime.Parse(latestTime).ToUniversalTime();
+            Db.SpotifyListen.RemoveRange(Db.SpotifyListen.Where(x => x.User.UserName == Config.Username && x.Timestamp <= time));
+        }
+
+        var user = Db.Users.Single(u => u.UserName == Config.Username);
+
+        var counter = 0;
+
+        var filtered = input.Where(x => x.ms_played > 30000)
+            .DistinctBy(x => (x.offline_timestamp, x.ts, x.spotify_track_uri))
+            .ToArray();
+
+        Logger.LogInformation("{.2f} items after filtering", filtered.Length);
+
+        foreach (var item in filtered)
+        {
+            if(!string.IsNullOrWhiteSpace(item.master_metadata_track_name))
+            {
+                Db.SpotifyListen.Add(new()
+                {
+                    TrackName = item.master_metadata_track_name,
+                    AlbumName = item.master_metadata_album_album_name,
+                    ArtistName = item.master_metadata_album_artist_name,
+
+                    Timestamp = DateTime.Parse(item.ts).ToUniversalTime(),
+                    PlayedDuration = item.ms_played,
+
+                    TrackUri = item.spotify_track_uri,
+                    UserId = user.Id
+                });
+
+                counter++;
+            }
+        }
+
+        Logger?.LogInformation("Added {} historical items for {}", counter, user.UserName);
+
+        await Db.SaveChangesAsync();
+    }
+}
+

Selector.Data/Selector.Data.csproj

@@ -0,0 +1,16 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>net6.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\Selector.Model\Selector.Model.csproj" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <PackageReference Include="Microsoft.EntityFrameworkCore" Version="6.0.9" />
+  </ItemGroup>
+
+</Project>

Selector.Model/ApplicationDbContext.cs

@@ -24,6 +24,8 @@ namespace Selector.Model
         public DbSet<AlbumLastfmSpotifyMapping> AlbumMapping { get; set; }
         public DbSet<ArtistLastfmSpotifyMapping> ArtistMapping { get; set; }
 
+        public DbSet<SpotifyListen> SpotifyListen { get; set; }
+
         public ApplicationDbContext(
             DbContextOptions<ApplicationDbContext> options, 
             ILogger<ApplicationDbContext> logger
@@ -90,6 +92,17 @@ namespace Selector.Model
                 .Property(s => s.LastfmArtistName)
                 .UseCollation("case_insensitive");
 
+            modelBuilder.Entity<SpotifyListen>().HasKey(s => s.Id);
+            modelBuilder.Entity<SpotifyListen>()
+                .Property(s => s.TrackName)
+                .UseCollation("case_insensitive");
+            modelBuilder.Entity<SpotifyListen>()
+                .Property(s => s.AlbumName)
+                .UseCollation("case_insensitive");
+            modelBuilder.Entity<SpotifyListen>()
+                .Property(s => s.ArtistName)
+                .UseCollation("case_insensitive");
+
             SeedData.Seed(modelBuilder);
         }
 

Selector.Model/Listen/IListenRepository.cs

@@ -0,0 +1,12 @@
+using System;
+using System.Collections.Generic;
+
+namespace Selector.Model
+{
+    public interface IListenRepository
+    {
+        IEnumerable<IListen> GetAll(string include = null, string userId = null, string username = null, string trackName = null, string albumName = null, string artistName = null, DateTime? from = null, DateTime? to = null);
+        int Count(string userId = null, string username = null, string trackName = null, string albumName = null, string artistName = null, DateTime? from = null, DateTime? to = null);
+    }
+}
+

Selector.Model/Listen/ISpotifyListenRepository.cs

@@ -0,0 +1,21 @@
+using System;
+using System.Collections.Generic;
+using System.Threading.Tasks;
+
+namespace Selector.Model
+{
+    public interface ISpotifyListenRepository: IListenRepository
+    {
+        void Add(SpotifyListen item);
+        void AddRange(IEnumerable<SpotifyListen> item);
+        //IEnumerable<SpotifyListen> GetAll(string include = null, string userId = null, string username = null, string trackName = null, string albumName = null, string artistName = null, DateTime? from = null, DateTime? to = null);
+        SpotifyListen Find(DateTime key, string include = null);
+        void Remove(DateTime key);
+        public void Remove(SpotifyListen scrobble);
+        public void RemoveRange(IEnumerable<SpotifyListen> scrobbles);
+        void Update(SpotifyListen item);
+        Task<int> Save();
+        //int Count(string userId = null, string username = null, string trackName = null, string albumName = null, string artistName = null, DateTime? from = null, DateTime? to = null);
+    }
+}
+

Selector.Model/Listen/IUserListen.cs

@@ -0,0 +1,10 @@
+using System;
+
+namespace Selector.Model;
+
+public interface IUserListen: IListen
+{
+    string UserId { get; set; }
+    ApplicationUser User { get; set; }
+}
+

Selector.Model/Listen/MetaListenRepository.cs

@@ -0,0 +1,71 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+
+namespace Selector.Model;
+
+public class MetaListenRepository: IListenRepository
+{
+    private readonly IScrobbleRepository scrobbleRepository;
+    private readonly ISpotifyListenRepository spotifyRepository;
+
+    public MetaListenRepository(IScrobbleRepository scrobbleRepository, ISpotifyListenRepository listenRepository)
+    {
+        this.scrobbleRepository = scrobbleRepository;
+        spotifyRepository = listenRepository;
+    }
+
+    public int Count(
+        string userId = null,
+        string username = null,
+        string trackName = null,
+        string albumName = null,
+        string artistName = null,
+        DateTime? from = null,
+        DateTime? to = null) => GetAll(userId: userId,
+            username: username,
+            trackName: trackName,
+            albumName: albumName,
+            artistName: artistName,
+            from: from,
+            to:to).Count();
+
+    public IEnumerable<IListen> GetAll(
+        string includes = null,
+        string userId = null,
+        string username = null,
+        string trackName = null,
+        string albumName = null,
+        string artistName = null,
+        DateTime? from = null,
+        DateTime? to = null)
+    {
+        var scrobbles = scrobbleRepository.GetAll(
+            include: includes,
+            userId: userId,
+            username: username,
+            trackName: trackName,
+            albumName: albumName,
+            artistName: artistName,
+            from: from,
+            to: to)
+            .OrderBy(x => x.Timestamp)
+            .ToArray();
+
+        var spotListens = spotifyRepository.GetAll(
+            include: includes,
+            userId: userId,
+            username: username,
+            trackName: trackName,
+            albumName: albumName,
+            artistName: artistName,
+            from: from,
+            to: scrobbles.FirstOrDefault()?.Timestamp)
+            .OrderBy(x => x.Timestamp)
+            .ToArray();
+        
+
+        return spotListens.Concat(scrobbles);
+    }
+}
+

Selector.Model/Listen/SpotifyListen.cs

@@ -0,0 +1,16 @@
+using System;
+
+namespace Selector.Model;
+
+public class SpotifyListen: Listen, IUserListen
+{
+    public int Id { get; set; }
+
+    public int? PlayedDuration { get; set; }
+
+    public string TrackUri { get; set; }
+
+    public string UserId { get; set; }
+    public ApplicationUser User { get; set; }
+}
+

Selector.Model/Listen/SpotifyListenRepository.cs

@@ -0,0 +1,129 @@
+using Microsoft.Extensions.Logging;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using Microsoft.EntityFrameworkCore;
+
+namespace Selector.Model
+{
+    public class SpotifyListenRepository : ISpotifyListenRepository
+    {
+        private readonly ApplicationDbContext db;
+
+        public SpotifyListenRepository(ApplicationDbContext context)
+        {
+            db = context;
+        }
+
+        public void Add(SpotifyListen item)
+        {
+            db.SpotifyListen.Add(item);
+        }
+
+        public void AddRange(IEnumerable<SpotifyListen> item)
+        {
+            db.SpotifyListen.AddRange(item);
+        }
+
+        public SpotifyListen Find(DateTime key, string include = null)
+        {
+            var listens = db.SpotifyListen.Where(s => s.Timestamp == key);
+            
+            if (!string.IsNullOrWhiteSpace(include))
+            {
+                listens = listens.Include(include);
+            }
+                
+            return listens.FirstOrDefault();
+        }
+
+        private IQueryable<SpotifyListen> GetAllQueryable(string include = null, string userId = null, string username = null, string trackName = null, string albumName = null, string artistName = null, DateTime? from = null, DateTime? to = null)
+        {
+            var listens = db.SpotifyListen.AsQueryable();
+
+            if (!string.IsNullOrWhiteSpace(include))
+            {
+                listens = listens.Include(include);
+            }
+
+            if (!string.IsNullOrWhiteSpace(userId))
+            {
+                listens = listens.Where(s => s.UserId == userId);
+            }
+
+            if (!string.IsNullOrWhiteSpace(username))
+            {
+                var normalUsername = username.ToUpperInvariant();
+                var user = db.Users.AsNoTracking().Where(u => u.NormalizedUserName == normalUsername).FirstOrDefault();
+                if (user is not null)
+                {
+                    listens = listens.Where(s => s.UserId == user.Id);
+                }
+                else
+                {
+                    listens = Enumerable.Empty<SpotifyListen>().AsQueryable();
+                }
+            }
+
+            if (!string.IsNullOrWhiteSpace(trackName))
+            {
+                listens = listens.Where(s => s.TrackName == trackName);
+            }
+
+            if (!string.IsNullOrWhiteSpace(albumName))
+            {
+                listens = listens.Where(s => s.AlbumName == albumName);
+            }
+
+            if (!string.IsNullOrWhiteSpace(artistName))
+            {
+                listens = listens.Where(s => s.ArtistName == artistName);
+            }
+
+            if (from is not null)
+            {
+                listens = listens.Where(u => u.Timestamp >= from.Value);
+            }
+
+            if (to is not null)
+            {
+                listens = listens.Where(u => u.Timestamp < to.Value);
+            }
+
+            return listens;
+        }
+
+        public IEnumerable<IListen> GetAll(string include = null, string userId = null, string username = null, string trackName = null, string albumName = null, string artistName = null, DateTime? from = null, DateTime? to = null)
+            => GetAllQueryable(include: include, userId: userId, username: username, trackName: trackName, albumName: albumName, artistName: artistName, from: from, to: to).AsEnumerable();
+
+        public void Remove(DateTime key)
+        {
+            Remove(Find(key));
+        }
+
+        public void Remove(SpotifyListen scrobble)
+        {
+            db.SpotifyListen.Remove(scrobble);
+        }
+
+        public void RemoveRange(IEnumerable<SpotifyListen> scrobbles)
+        {
+            db.SpotifyListen.RemoveRange(scrobbles);
+        }
+
+        public void Update(SpotifyListen item)
+        {
+            db.SpotifyListen.Update(item);
+        }
+
+        public Task<int> Save()
+        {
+            return db.SaveChangesAsync();
+        }
+
+        public int Count(string userId = null, string username = null, string trackName = null, string albumName = null, string artistName = null, DateTime? from = null, DateTime? to = null)
+            => GetAllQueryable(userId: userId, username: username, trackName: trackName, albumName: albumName, artistName: artistName, from: from, to: to).Count();
+    }
+}

Selector.Model/Migrations/20221007214100_SpotifyHistory.Designer.cs

@@ -0,0 +1,491 @@
+// <auto-generated />
+using System;
+using Microsoft.EntityFrameworkCore;
+using Microsoft.EntityFrameworkCore.Infrastructure;
+using Microsoft.EntityFrameworkCore.Migrations;
+using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
+using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata;
+using Selector.Model;
+
+#nullable disable
+
+namespace Selector.Model.Migrations
+{
+    [DbContext(typeof(ApplicationDbContext))]
+    [Migration("20221007214100_SpotifyHistory")]
+    partial class SpotifyHistory
+    {
+        protected override void BuildTargetModel(ModelBuilder modelBuilder)
+        {
+#pragma warning disable 612, 618
+            modelBuilder
+                .HasAnnotation("Npgsql:CollationDefinition:case_insensitive", "en-u-ks-primary,en-u-ks-primary,icu,False")
+                .HasAnnotation("ProductVersion", "6.0.9")
+                .HasAnnotation("Relational:MaxIdentifierLength", 63);
+
+            NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder);
+
+            modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityRole", b =>
+                {
+                    b.Property<string>("Id")
+                        .HasColumnType("text");
+
+                    b.Property<string>("ConcurrencyStamp")
+                        .IsConcurrencyToken()
+                        .HasColumnType("text");
+
+                    b.Property<string>("Name")
+                        .HasMaxLength(256)
+                        .HasColumnType("character varying(256)");
+
+                    b.Property<string>("NormalizedName")
+                        .HasMaxLength(256)
+                        .HasColumnType("character varying(256)");
+
+                    b.HasKey("Id");
+
+                    b.HasIndex("NormalizedName")
+                        .IsUnique()
+                        .HasDatabaseName("RoleNameIndex");
+
+                    b.ToTable("AspNetRoles", (string)null);
+
+                    b.HasData(
+                        new
+                        {
+                            Id = "00c64c0a-3387-4933-9575-83443fa9092b",
+                            ConcurrencyStamp = "4b4a37c7-cc65-485a-ac0e-d88ef6dede78",
+                            Name = "Admin",
+                            NormalizedName = "ADMIN"
+                        });
+                });
+
+            modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityRoleClaim<string>", b =>
+                {
+                    b.Property<int>("Id")
+                        .ValueGeneratedOnAdd()
+                        .HasColumnType("integer");
+
+                    NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property<int>("Id"));
+
+                    b.Property<string>("ClaimType")
+                        .HasColumnType("text");
+
+                    b.Property<string>("ClaimValue")
+                        .HasColumnType("text");
+
+                    b.Property<string>("RoleId")
+                        .IsRequired()
+                        .HasColumnType("text");
+
+                    b.HasKey("Id");
+
+                    b.HasIndex("RoleId");
+
+                    b.ToTable("AspNetRoleClaims", (string)null);
+                });
+
+            modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityUserClaim<string>", b =>
+                {
+                    b.Property<int>("Id")
+                        .ValueGeneratedOnAdd()
+                        .HasColumnType("integer");
+
+                    NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property<int>("Id"));
+
+                    b.Property<string>("ClaimType")
+                        .HasColumnType("text");
+
+                    b.Property<string>("ClaimValue")
+                        .HasColumnType("text");
+
+                    b.Property<string>("UserId")
+                        .IsRequired()
+                        .HasColumnType("text");
+
+                    b.HasKey("Id");
+
+                    b.HasIndex("UserId");
+
+                    b.ToTable("AspNetUserClaims", (string)null);
+                });
+
+            modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityUserLogin<string>", b =>
+                {
+                    b.Property<string>("LoginProvider")
+                        .HasColumnType("text");
+
+                    b.Property<string>("ProviderKey")
+                        .HasColumnType("text");
+
+                    b.Property<string>("ProviderDisplayName")
+                        .HasColumnType("text");
+
+                    b.Property<string>("UserId")
+                        .IsRequired()
+                        .HasColumnType("text");
+
+                    b.HasKey("LoginProvider", "ProviderKey");
+
+                    b.HasIndex("UserId");
+
+                    b.ToTable("AspNetUserLogins", (string)null);
+                });
+
+            modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityUserRole<string>", b =>
+                {
+                    b.Property<string>("UserId")
+                        .HasColumnType("text");
+
+                    b.Property<string>("RoleId")
+                        .HasColumnType("text");
+
+                    b.HasKey("UserId", "RoleId");
+
+                    b.HasIndex("RoleId");
+
+                    b.ToTable("AspNetUserRoles", (string)null);
+                });
+
+            modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityUserToken<string>", b =>
+                {
+                    b.Property<string>("UserId")
+                        .HasColumnType("text");
+
+                    b.Property<string>("LoginProvider")
+                        .HasColumnType("text");
+
+                    b.Property<string>("Name")
+                        .HasColumnType("text");
+
+                    b.Property<string>("Value")
+                        .HasColumnType("text");
+
+                    b.HasKey("UserId", "LoginProvider", "Name");
+
+                    b.ToTable("AspNetUserTokens", (string)null);
+                });
+
+            modelBuilder.Entity("Selector.Model.AlbumLastfmSpotifyMapping", b =>
+                {
+                    b.Property<string>("SpotifyUri")
+                        .HasColumnType("text");
+
+                    b.Property<string>("LastfmAlbumName")
+                        .HasColumnType("text")
+                        .UseCollation("case_insensitive");
+
+                    b.Property<string>("LastfmArtistName")
+                        .HasColumnType("text")
+                        .UseCollation("case_insensitive");
+
+                    b.HasKey("SpotifyUri");
+
+                    b.ToTable("AlbumMapping");
+                });
+
+            modelBuilder.Entity("Selector.Model.ApplicationUser", b =>
+                {
+                    b.Property<string>("Id")
+                        .HasColumnType("text");
+
+                    b.Property<int>("AccessFailedCount")
+                        .HasColumnType("integer");
+
+                    b.Property<string>("ConcurrencyStamp")
+                        .IsConcurrencyToken()
+                        .HasColumnType("text");
+
+                    b.Property<string>("Email")
+                        .HasMaxLength(256)
+                        .HasColumnType("character varying(256)");
+
+                    b.Property<bool>("EmailConfirmed")
+                        .HasColumnType("boolean");
+
+                    b.Property<string>("LastFmUsername")
+                        .HasColumnType("text")
+                        .UseCollation("case_insensitive");
+
+                    b.Property<bool>("LockoutEnabled")
+                        .HasColumnType("boolean");
+
+                    b.Property<DateTimeOffset?>("LockoutEnd")
+                        .HasColumnType("timestamp with time zone");
+
+                    b.Property<string>("NormalizedEmail")
+                        .HasMaxLength(256)
+                        .HasColumnType("character varying(256)");
+
+                    b.Property<string>("NormalizedUserName")
+                        .HasMaxLength(256)
+                        .HasColumnType("character varying(256)");
+
+                    b.Property<string>("PasswordHash")
+                        .HasColumnType("text");
+
+                    b.Property<string>("PhoneNumber")
+                        .HasColumnType("text");
+
+                    b.Property<bool>("PhoneNumberConfirmed")
+                        .HasColumnType("boolean");
+
+                    b.Property<bool>("SaveScrobbles")
+                        .HasColumnType("boolean");
+
+                    b.Property<string>("SecurityStamp")
+                        .HasColumnType("text");
+
+                    b.Property<string>("SpotifyAccessToken")
+                        .HasColumnType("text");
+
+                    b.Property<bool>("SpotifyIsLinked")
+                        .HasColumnType("boolean");
+
+                    b.Property<DateTime>("SpotifyLastRefresh")
+                        .HasColumnType("timestamp with time zone");
+
+                    b.Property<string>("SpotifyRefreshToken")
+                        .HasColumnType("text");
+
+                    b.Property<int>("SpotifyTokenExpiry")
+                        .HasColumnType("integer");
+
+                    b.Property<bool>("TwoFactorEnabled")
+                        .HasColumnType("boolean");
+
+                    b.Property<string>("UserName")
+                        .HasMaxLength(256)
+                        .HasColumnType("character varying(256)");
+
+                    b.HasKey("Id");
+
+                    b.HasIndex("NormalizedEmail")
+                        .HasDatabaseName("EmailIndex");
+
+                    b.HasIndex("NormalizedUserName")
+                        .IsUnique()
+                        .HasDatabaseName("UserNameIndex");
+
+                    b.ToTable("AspNetUsers", (string)null);
+                });
+
+            modelBuilder.Entity("Selector.Model.ArtistLastfmSpotifyMapping", b =>
+                {
+                    b.Property<string>("SpotifyUri")
+                        .HasColumnType("text");
+
+                    b.Property<string>("LastfmArtistName")
+                        .HasColumnType("text")
+                        .UseCollation("case_insensitive");
+
+                    b.HasKey("SpotifyUri");
+
+                    b.ToTable("ArtistMapping");
+                });
+
+            modelBuilder.Entity("Selector.Model.SpotifyListen", b =>
+                {
+                    b.Property<int>("Id")
+                        .ValueGeneratedOnAdd()
+                        .HasColumnType("integer");
+
+                    NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property<int>("Id"));
+
+                    b.Property<string>("AlbumName")
+                        .HasColumnType("text")
+                        .UseCollation("case_insensitive");
+
+                    b.Property<string>("ArtistName")
+                        .HasColumnType("text")
+                        .UseCollation("case_insensitive");
+
+                    b.Property<int?>("PlayedDuration")
+                        .HasColumnType("integer");
+
+                    b.Property<DateTime>("Timestamp")
+                        .HasColumnType("timestamp with time zone");
+
+                    b.Property<string>("TrackName")
+                        .HasColumnType("text")
+                        .UseCollation("case_insensitive");
+
+                    b.Property<string>("TrackUri")
+                        .HasColumnType("text");
+
+                    b.Property<string>("UserId")
+                        .HasColumnType("text");
+
+                    b.HasKey("Id");
+
+                    b.HasIndex("UserId");
+
+                    b.ToTable("SpotifyListen");
+                });
+
+            modelBuilder.Entity("Selector.Model.TrackLastfmSpotifyMapping", b =>
+                {
+                    b.Property<string>("SpotifyUri")
+                        .HasColumnType("text");
+
+                    b.Property<string>("LastfmArtistName")
+                        .HasColumnType("text")
+                        .UseCollation("case_insensitive");
+
+                    b.Property<string>("LastfmTrackName")
+                        .HasColumnType("text")
+                        .UseCollation("case_insensitive");
+
+                    b.HasKey("SpotifyUri");
+
+                    b.ToTable("TrackMapping");
+                });
+
+            modelBuilder.Entity("Selector.Model.UserScrobble", b =>
+                {
+                    b.Property<int>("Id")
+                        .ValueGeneratedOnAdd()
+                        .HasColumnType("integer");
+
+                    NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property<int>("Id"));
+
+                    b.Property<string>("AlbumArtistName")
+                        .HasColumnType("text");
+
+                    b.Property<string>("AlbumName")
+                        .HasColumnType("text")
+                        .UseCollation("case_insensitive");
+
+                    b.Property<string>("ArtistName")
+                        .HasColumnType("text")
+                        .UseCollation("case_insensitive");
+
+                    b.Property<DateTime>("Timestamp")
+                        .HasColumnType("timestamp with time zone");
+
+                    b.Property<string>("TrackName")
+                        .HasColumnType("text")
+                        .UseCollation("case_insensitive");
+
+                    b.Property<string>("UserId")
+                        .HasColumnType("text");
+
+                    b.HasKey("Id");
+
+                    b.HasIndex("UserId");
+
+                    b.ToTable("Scrobble");
+                });
+
+            modelBuilder.Entity("Selector.Model.Watcher", b =>
+                {
+                    b.Property<int>("Id")
+                        .ValueGeneratedOnAdd()
+                        .HasColumnType("integer");
+
+                    NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property<int>("Id"));
+
+                    b.Property<int>("Type")
+                        .HasColumnType("integer");
+
+                    b.Property<string>("UserId")
+                        .IsRequired()
+                        .HasColumnType("text");
+
+                    b.HasKey("Id");
+
+                    b.HasIndex("UserId");
+
+                    b.ToTable("Watcher");
+                });
+
+            modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityRoleClaim<string>", b =>
+                {
+                    b.HasOne("Microsoft.AspNetCore.Identity.IdentityRole", null)
+                        .WithMany()
+                        .HasForeignKey("RoleId")
+                        .OnDelete(DeleteBehavior.Cascade)
+                        .IsRequired();
+                });
+
+            modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityUserClaim<string>", b =>
+                {
+                    b.HasOne("Selector.Model.ApplicationUser", null)
+                        .WithMany()
+                        .HasForeignKey("UserId")
+                        .OnDelete(DeleteBehavior.Cascade)
+                        .IsRequired();
+                });
+
+            modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityUserLogin<string>", b =>
+                {
+                    b.HasOne("Selector.Model.ApplicationUser", null)
+                        .WithMany()
+                        .HasForeignKey("UserId")
+                        .OnDelete(DeleteBehavior.Cascade)
+                        .IsRequired();
+                });
+
+            modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityUserRole<string>", b =>
+                {
+                    b.HasOne("Microsoft.AspNetCore.Identity.IdentityRole", null)
+                        .WithMany()
+                        .HasForeignKey("RoleId")
+                        .OnDelete(DeleteBehavior.Cascade)
+                        .IsRequired();
+
+                    b.HasOne("Selector.Model.ApplicationUser", null)
+                        .WithMany()
+                        .HasForeignKey("UserId")
+                        .OnDelete(DeleteBehavior.Cascade)
+                        .IsRequired();
+                });
+
+            modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityUserToken<string>", b =>
+                {
+                    b.HasOne("Selector.Model.ApplicationUser", null)
+                        .WithMany()
+                        .HasForeignKey("UserId")
+                        .OnDelete(DeleteBehavior.Cascade)
+                        .IsRequired();
+                });
+
+            modelBuilder.Entity("Selector.Model.SpotifyListen", b =>
+                {
+                    b.HasOne("Selector.Model.ApplicationUser", "User")
+                        .WithMany()
+                        .HasForeignKey("UserId");
+
+                    b.Navigation("User");
+                });
+
+            modelBuilder.Entity("Selector.Model.UserScrobble", b =>
+                {
+                    b.HasOne("Selector.Model.ApplicationUser", "User")
+                        .WithMany("Scrobbles")
+                        .HasForeignKey("UserId");
+
+                    b.Navigation("User");
+                });
+
+            modelBuilder.Entity("Selector.Model.Watcher", b =>
+                {
+                    b.HasOne("Selector.Model.ApplicationUser", "User")
+                        .WithMany("Watchers")
+                        .HasForeignKey("UserId")
+                        .OnDelete(DeleteBehavior.Cascade)
+                        .IsRequired();
+
+                    b.Navigation("User");
+                });
+
+            modelBuilder.Entity("Selector.Model.ApplicationUser", b =>
+                {
+                    b.Navigation("Scrobbles");
+
+                    b.Navigation("Watchers");
+                });
+#pragma warning restore 612, 618
+        }
+    }
+}

Selector.Model/Migrations/20221007214100_SpotifyHistory.cs

@@ -0,0 +1,63 @@
+using System;
+using Microsoft.EntityFrameworkCore.Migrations;
+using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata;
+
+#nullable disable
+
+namespace Selector.Model.Migrations
+{
+    public partial class SpotifyHistory : Migration
+    {
+        protected override void Up(MigrationBuilder migrationBuilder)
+        {
+            migrationBuilder.CreateTable(
+                name: "SpotifyListen",
+                columns: table => new
+                {
+                    Id = table.Column<int>(type: "integer", nullable: false)
+                        .Annotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn),
+                    PlayedDuration = table.Column<int>(type: "integer", nullable: true),
+                    TrackUri = table.Column<string>(type: "text", nullable: true),
+                    UserId = table.Column<string>(type: "text", nullable: true),
+                    TrackName = table.Column<string>(type: "text", nullable: true, collation: "case_insensitive"),
+                    AlbumName = table.Column<string>(type: "text", nullable: true, collation: "case_insensitive"),
+                    ArtistName = table.Column<string>(type: "text", nullable: true, collation: "case_insensitive"),
+                    Timestamp = table.Column<DateTime>(type: "timestamp with time zone", nullable: false)
+                },
+                constraints: table =>
+                {
+                    table.PrimaryKey("PK_SpotifyListen", x => x.Id);
+                    table.ForeignKey(
+                        name: "FK_SpotifyListen_AspNetUsers_UserId",
+                        column: x => x.UserId,
+                        principalTable: "AspNetUsers",
+                        principalColumn: "Id");
+                });
+
+            migrationBuilder.UpdateData(
+                table: "AspNetRoles",
+                keyColumn: "Id",
+                keyValue: "00c64c0a-3387-4933-9575-83443fa9092b",
+                column: "ConcurrencyStamp",
+                value: "4b4a37c7-cc65-485a-ac0e-d88ef6dede78");
+
+            migrationBuilder.CreateIndex(
+                name: "IX_SpotifyListen_UserId",
+                table: "SpotifyListen",
+                column: "UserId");
+        }
+
+        protected override void Down(MigrationBuilder migrationBuilder)
+        {
+            migrationBuilder.DropTable(
+                name: "SpotifyListen");
+
+            migrationBuilder.UpdateData(
+                table: "AspNetRoles",
+                keyColumn: "Id",
+                keyValue: "00c64c0a-3387-4933-9575-83443fa9092b",
+                column: "ConcurrencyStamp",
+                value: "ec454f56-2b26-4bd8-be8e-a7fd34981ac2");
+        }
+    }
+}

Selector.Model/Migrations/ApplicationDbContextModelSnapshot.cs

@@ -18,7 +18,7 @@ namespace Selector.Model.Migrations
 #pragma warning disable 612, 618
             modelBuilder
                 .HasAnnotation("Npgsql:CollationDefinition:case_insensitive", "en-u-ks-primary,en-u-ks-primary,icu,False")
-                .HasAnnotation("ProductVersion", "6.0.2")
+                .HasAnnotation("ProductVersion", "6.0.9")
                 .HasAnnotation("Relational:MaxIdentifierLength", 63);
 
             NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder);
@@ -52,7 +52,7 @@ namespace Selector.Model.Migrations
                         new
                         {
                             Id = "00c64c0a-3387-4933-9575-83443fa9092b",
-                            ConcurrencyStamp = "ec454f56-2b26-4bd8-be8e-a7fd34981ac2",
+                            ConcurrencyStamp = "4b4a37c7-cc65-485a-ac0e-d88ef6dede78",
                             Name = "Admin",
                             NormalizedName = "ADMIN"
                         });
@@ -282,6 +282,45 @@ namespace Selector.Model.Migrations
                     b.ToTable("ArtistMapping");
                 });
 
+            modelBuilder.Entity("Selector.Model.SpotifyListen", b =>
+                {
+                    b.Property<int>("Id")
+                        .ValueGeneratedOnAdd()
+                        .HasColumnType("integer");
+
+                    NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property<int>("Id"));
+
+                    b.Property<string>("AlbumName")
+                        .HasColumnType("text")
+                        .UseCollation("case_insensitive");
+
+                    b.Property<string>("ArtistName")
+                        .HasColumnType("text")
+                        .UseCollation("case_insensitive");
+
+                    b.Property<int?>("PlayedDuration")
+                        .HasColumnType("integer");
+
+                    b.Property<DateTime>("Timestamp")
+                        .HasColumnType("timestamp with time zone");
+
+                    b.Property<string>("TrackName")
+                        .HasColumnType("text")
+                        .UseCollation("case_insensitive");
+
+                    b.Property<string>("TrackUri")
+                        .HasColumnType("text");
+
+                    b.Property<string>("UserId")
+                        .HasColumnType("text");
+
+                    b.HasKey("Id");
+
+                    b.HasIndex("UserId");
+
+                    b.ToTable("SpotifyListen");
+                });
+
             modelBuilder.Entity("Selector.Model.TrackLastfmSpotifyMapping", b =>
                 {
                     b.Property<string>("SpotifyUri")
@@ -409,6 +448,15 @@ namespace Selector.Model.Migrations
                         .IsRequired();
                 });
 
+            modelBuilder.Entity("Selector.Model.SpotifyListen", b =>
+                {
+                    b.HasOne("Selector.Model.ApplicationUser", "User")
+                        .WithMany()
+                        .HasForeignKey("UserId");
+
+                    b.Navigation("User");
+                });
+
             modelBuilder.Entity("Selector.Model.UserScrobble", b =>
                 {
                     b.HasOne("Selector.Model.ApplicationUser", "User")

Selector.Model/Scrobble/DBPlayCountPuller.cs

@@ -9,12 +9,12 @@ namespace Selector.Cache
 {
     public class DBPlayCountPuller
     {
-        protected readonly IScrobbleRepository ScrobbleRepository;
+        protected readonly IListenRepository ScrobbleRepository;
         private readonly IOptions<NowPlayingOptions> nowOptions;
 
         public DBPlayCountPuller(
             IOptions<NowPlayingOptions> options,
-            IScrobbleRepository scrobbleRepository
+            IListenRepository scrobbleRepository
         )
         {
             ScrobbleRepository = scrobbleRepository;

Selector.Model/Scrobble/IScrobbleRepository.cs

@@ -6,17 +6,17 @@ using System.Threading.Tasks;
 
 namespace Selector.Model
 {
-    public interface IScrobbleRepository
+    public interface IScrobbleRepository: IListenRepository
     {
         void Add(UserScrobble item);
         void AddRange(IEnumerable<UserScrobble> item);
-        IEnumerable<UserScrobble> GetAll(string include = null, string userId = null, string username = null, string trackName = null, string albumName = null, string artistName = null, DateTime? from = null, DateTime? to = null);
+        //IEnumerable<UserScrobble> GetAll(string include = null, string userId = null, string username = null, string trackName = null, string albumName = null, string artistName = null, DateTime? from = null, DateTime? to = null);
         UserScrobble Find(int key, string include = null);
         void Remove(int key);
         public void Remove(UserScrobble scrobble);
         public void RemoveRange(IEnumerable<UserScrobble> scrobbles);
         void Update(UserScrobble item);
         Task<int> Save();
-        int Count(string include = null, string userId = null, string username = null, string trackName = null, string albumName = null, string artistName = null, DateTime? from = null, DateTime? to = null);
+        //int Count(string userId = null, string username = null, string trackName = null, string albumName = null, string artistName = null, DateTime? from = null, DateTime? to = null);
     }
 }

Selector.Model/Scrobble/ScrobbleRepository.cs

@@ -95,7 +95,7 @@ namespace Selector.Model
             return scrobbles;
         }
 
-        public IEnumerable<UserScrobble> GetAll(string include = null, string userId = null, string username = null, string trackName = null, string albumName = null, string artistName = null, DateTime? from = null, DateTime? to = null)
+        public IEnumerable<IListen> GetAll(string include = null, string userId = null, string username = null, string trackName = null, string albumName = null, string artistName = null, DateTime? from = null, DateTime? to = null)
             => GetAllQueryable(include: include, userId: userId, username: username, trackName: trackName, albumName: albumName, artistName: artistName, from: from, to: to).AsEnumerable();
 
         public void Remove(int key)
@@ -123,7 +123,7 @@ namespace Selector.Model
             return db.SaveChangesAsync();
         }
 
-        public int Count(string include = null, string userId = null, string username = null, string trackName = null, string albumName = null, string artistName = null, DateTime? from = null, DateTime? to = null)
-            => GetAllQueryable(include: include, userId: userId, username: username, trackName: trackName, albumName: albumName, artistName: artistName, from: from, to: to).Count();
+        public int Count(string userId = null, string username = null, string trackName = null, string albumName = null, string artistName = null, DateTime? from = null, DateTime? to = null)
+            => GetAllQueryable(userId: userId, username: username, trackName: trackName, albumName: albumName, artistName: artistName, from: from, to: to).Count();
     }
 }

Selector.Model/Scrobble/UserScrobble.cs

@@ -2,7 +2,7 @@
 
 namespace Selector.Model
 {
-    public class UserScrobble: Scrobble
+    public class UserScrobble: Scrobble, IUserListen
     {
         public int Id { get; set; }
         public string UserId { get; set; }

Selector.Model/Selector.Model.csproj

@@ -30,6 +30,10 @@
 
   <ItemGroup>
     <Folder Include="Migrations\" />
+    <Folder Include="Listen\" />
   </ItemGroup>
 
+  <ItemGroup>
+    <None Remove="Listen\" />
+  </ItemGroup>
 </Project>

Selector.Web/Startup.cs

@@ -64,7 +64,11 @@ namespace Selector.Web
                 options.UseNpgsql(Configuration.GetConnectionString("Default"))
             );
             services.AddDBPlayCountPuller();
-            services.AddTransient<IScrobbleRepository, ScrobbleRepository>();
+            services.AddTransient<IScrobbleRepository, ScrobbleRepository>()
+                    .AddTransient<ISpotifyListenRepository, SpotifyListenRepository>();
+
+            services.AddTransient<IListenRepository, MetaListenRepository>();
+            //services.AddTransient<IListenRepository, SpotifyListenRepository>();
 
             services.AddIdentity<ApplicationUser, IdentityRole>()
                 .AddEntityFrameworkStores<ApplicationDbContext>()

Selector.sln

@@ -24,6 +24,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Selector.Cache", "Selector.
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Selector.Event", "Selector.Event\Selector.Event.csproj", "{C2FF1673-CB1A-43B7-A814-07BB3CB3A0D6}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Selector.Data", "Selector.Data\Selector.Data.csproj", "{CB62ACCB-94F1-4B78-A195-8B108B9E800D}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -58,6 +60,10 @@ Global
 		{C2FF1673-CB1A-43B7-A814-07BB3CB3A0D6}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{C2FF1673-CB1A-43B7-A814-07BB3CB3A0D6}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{C2FF1673-CB1A-43B7-A814-07BB3CB3A0D6}.Release|Any CPU.Build.0 = Release|Any CPU
+		{CB62ACCB-94F1-4B78-A195-8B108B9E800D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{CB62ACCB-94F1-4B78-A195-8B108B9E800D}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{CB62ACCB-94F1-4B78-A195-8B108B9E800D}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{CB62ACCB-94F1-4B78-A195-8B108B9E800D}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE

Selector/Listen/IListen.cs

@@ -0,0 +1,13 @@
+using System;
+
+namespace Selector;
+
+public interface IListen
+{
+    DateTime Timestamp { get; set; }
+
+    string TrackName { get; set; }
+    string AlbumName { get; set; }
+    string ArtistName { get; set; }
+}
+

Selector/Listen/Listen.cs

@@ -0,0 +1,13 @@
+using System;
+
+namespace Selector;
+
+public class Listen: IListen
+{
+    public string TrackName { get; set; }
+    public string AlbumName { get; set; }
+    public string ArtistName { get; set; }
+
+    public DateTime Timestamp { get; set; }
+}
+

Selector/Scrobble/PlayDensity.cs

@@ -6,9 +6,9 @@ namespace Selector
 {
 	public static class PlayDensity
 	{
-		public static decimal Density(this IEnumerable<Scrobble> scrobbles, TimeSpan window) => scrobbles.Density(DateTime.UtcNow - window, DateTime.UtcNow);
+		public static decimal Density(this IEnumerable<IListen> scrobbles, TimeSpan window) => scrobbles.Density(DateTime.UtcNow - window, DateTime.UtcNow);
 
-		public static decimal Density(this IEnumerable<Scrobble> scrobbles, DateTime from, DateTime to)
+		public static decimal Density(this IEnumerable<IListen> scrobbles, DateTime from, DateTime to)
         {
 			var filteredScrobbles = scrobbles.Where(s => s.Timestamp > from && s.Timestamp < to);
 
@@ -17,7 +17,7 @@ namespace Selector
 			return filteredScrobbles.Count() / dayDelta;
         }
 
-		public static decimal Density(this IEnumerable<Scrobble> scrobbles)
+		public static decimal Density(this IEnumerable<IListen> scrobbles)
 		{
 			var minDate = scrobbles.Select(s => s.Timestamp).Min();
 			var maxDate = scrobbles.Select(s => s.Timestamp).Max();

Selector/Scrobble/Resampler.cs

@@ -11,7 +11,7 @@ namespace Selector
 
 	public static class Resampler
 	{
-		public static IEnumerable<CountSample> Resample(this IEnumerable<Scrobble> scrobbles, TimeSpan window)
+		public static IEnumerable<CountSample> Resample(this IEnumerable<IListen> scrobbles, TimeSpan window)
 		{
 			var sortedScrobbles = scrobbles.OrderBy(s => s.Timestamp).ToList();
 
@@ -68,7 +68,7 @@ namespace Selector
 			}
 		}
 
-		public static IEnumerable<CountSample> ResampleByMonth(this IEnumerable<Scrobble> scrobbles)
+		public static IEnumerable<CountSample> ResampleByMonth(this IEnumerable<IListen> scrobbles)
 		{
 			var sortedScrobbles = scrobbles.OrderBy(s => s.Timestamp).ToList();
 

Selector/Scrobble/Scrobble.cs

@@ -2,7 +2,7 @@
 
 namespace Selector
 {
-    public class Scrobble
+    public class Scrobble: IListen
     {
         public DateTime Timestamp { get; set; }
         public string TrackName { get; set; }

Selector/Scrobble/ScrobbleMatcher.cs

@@ -6,22 +6,22 @@ using System.Linq;
 
 namespace Selector
 {
-    public static class ScrobbleMatcher
+    public static class ListenMatcher
     {
-        public static bool MatchTime(Scrobble nativeScrobble, LastTrack serviceScrobble) 
+        public static bool MatchTime(IListen nativeScrobble, LastTrack serviceScrobble) 
             => serviceScrobble.TimePlayed.Equals(nativeScrobble);
 
-        public static bool MatchTime(Scrobble nativeScrobble, Scrobble serviceScrobble)
+        public static bool MatchTime(IListen nativeScrobble, IListen serviceScrobble)
             => serviceScrobble.Timestamp.Equals(nativeScrobble.Timestamp);
 
-        public static (IEnumerable<Scrobble>, IEnumerable<Scrobble>) IdentifyDiffs(IEnumerable<Scrobble> existing, IEnumerable<Scrobble> toApply, bool matchContents = true)
+        public static (IEnumerable<IListen>, IEnumerable<IListen>) IdentifyDiffs(IEnumerable<IListen> existing, IEnumerable<IListen> toApply, bool matchContents = true)
         {
             existing = existing.OrderBy(s => s.Timestamp);
             toApply = toApply.OrderBy(s => s.Timestamp);
             var toApplyIter = toApply.GetEnumerator();
 
-            var toAdd = new List<Scrobble>();
-            var toRemove = new List<Scrobble>();
+            var toAdd = new List<IListen>();
+            var toRemove = new List<IListen>();
 
             var toApplyOverrun = false;
 
@@ -79,7 +79,7 @@ namespace Selector
             return (toAdd, toRemove);
         }
 
-        public static void MatchData(Scrobble currentExisting, Scrobble toApply)
+        public static void MatchData(IListen currentExisting, IListen toApply)
         {
             if (!currentExisting.TrackName.Equals(toApply.TrackName, StringComparison.InvariantCultureIgnoreCase))
             {
@@ -97,19 +97,19 @@ namespace Selector
             }
         }
 
-        public static (IEnumerable<Scrobble>, IEnumerable<Scrobble>) IdentifyDiffsContains(IEnumerable<Scrobble> existing, IEnumerable<Scrobble> toApply)
+        public static (IEnumerable<IListen>, IEnumerable<IListen>) IdentifyDiffsContains(IEnumerable<IListen> existing, IEnumerable<IListen> toApply)
         {
-            var toAdd = toApply.Where(s => !existing.Contains(s, new ScrobbleComp()));
-            var toRemove = existing.Where(s => !toApply.Contains(s, new ScrobbleComp()));
+            var toAdd = toApply.Where(s => !existing.Contains(s, new ListenComp()));
+            var toRemove = existing.Where(s => !toApply.Contains(s, new ListenComp()));
 
             return (toAdd, toRemove);
         }
 
-        public class ScrobbleComp : IEqualityComparer<Scrobble>
+        public class ListenComp : IEqualityComparer<IListen>
         {
-            public bool Equals(Scrobble x, Scrobble y) => x.Timestamp == y.Timestamp;
+            public bool Equals(IListen x, IListen y) => x.Timestamp == y.Timestamp;
 
-            public int GetHashCode([DisallowNull] Scrobble obj) => obj.Timestamp.GetHashCode();
+            public int GetHashCode([DisallowNull] IListen obj) => obj.Timestamp.GetHashCode();
         }
     }
 }

spotify-data.ipynb

@@ -0,0 +1,109 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import json\n",
+    "import os\n",
+    "from datetime import datetime\n",
+    "import random\n",
+    "from pprint import pprint"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 15,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Total: 445582\n",
+      "Total Populated: 442034 (99.20%)\n",
+      "Total New: 130411 (29.27%)\n",
+      "{'conn_country': 'GB',\n",
+      " 'episode_name': None,\n",
+      " 'episode_show_name': None,\n",
+      " 'incognito_mode': False,\n",
+      " 'ip_addr_decrypted': '82.132.244.90',\n",
+      " 'master_metadata_album_album_name': 'Tales Told By Dead Friends',\n",
+      " 'master_metadata_album_artist_name': 'Mayday Parade',\n",
+      " 'master_metadata_track_name': 'Three Cheers For Five Years',\n",
+      " 'ms_played': 33274,\n",
+      " 'offline': False,\n",
+      " 'offline_timestamp': 0,\n",
+      " 'platform': 'iOS 9.2.1 (iPhone7,1)',\n",
+      " 'reason_end': 'fwdbtn',\n",
+      " 'reason_start': 'fwdbtn',\n",
+      " 'shuffle': True,\n",
+      " 'skipped': True,\n",
+      " 'spotify_episode_uri': None,\n",
+      " 'spotify_track_uri': 'spotify:track:1aw8gphDUzqalHEi9Z8M38',\n",
+      " 'ts': '2016-02-11T08:13:55Z',\n",
+      " 'user_agent_decrypted': 'unknown',\n",
+      " 'username': 'sarsoo'}\n"
+     ]
+    }
+   ],
+   "source": [
+    "data = []\n",
+    "data_with_names = []\n",
+    "new_data = []\n",
+    "\n",
+    "folder = '/Users/andy/lab/backups/spotify-2022-03-07'\n",
+    "\n",
+    "for i in os.listdir(folder):\n",
+    "    if i.startswith('endsong_'):\n",
+    "        with open(f'{folder}/{i}') as f:\n",
+    "            data += json.loads(f.read())\n",
+    "\n",
+    "data.sort(key = lambda a: a['ts'])\n",
+    "data_with_names = [i for i in data if i['master_metadata_track_name'] is not None]\n",
+    "new_data = [i for i in data_with_names if datetime.fromisoformat(i['ts'].split('T')[0]) < datetime(2017, 11, 3)]\n",
+    "\n",
+    "print(f'Total: {len(data)}')\n",
+    "print(f'Total Populated: {len(data_with_names)} ({len(data_with_names)/len(data)*100:.2f}%)')\n",
+    "print(f'Total New: {len(new_data)} ({len(new_data)/len(data)*100:.2f}%)')\n",
+    "\n",
+    "pprint(random.choice(new_data))\n",
+    "# print(min(i[0] for i in data))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "interpreter": {
+   "hash": "a0a5145e6c304e2a9afaf5b930a2955b950bd4b81fe94f7c42930f43f42762eb"
+  },
+  "kernelspec": {
+   "display_name": "Python 3.10.7 64-bit",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.4"
+  },
+  "orig_nbformat": 4
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}