split refresh into separate tasks for track/album/artist

music/api/spotfm.py

@@ -6,7 +6,9 @@ import datetime
 
 from music.api.decorators import admin_required, login_or_basic_auth, lastfm_username_required, spotify_link_required, cloud_task, gae_cron
 import music.db.database as database
-from music.tasks.refresh_lastfm_stats import refresh_lastfm_stats
+from music.tasks.refresh_lastfm_stats import refresh_lastfm_track_stats, \
+    refresh_lastfm_album_stats, \
+    refresh_lastfm_artist_stats
 
 from spotfm.maths.counter import Counter
 from spotframework.model.uri import Uri
@@ -82,7 +84,9 @@ def playlist_refresh(username=None):
         if os.environ.get('DEPLOY_DESTINATION', None) == 'PROD':
             create_refresh_playlist_task(username, playlist_name)
         else:
-            refresh_lastfm_stats(username, playlist_name)
+            refresh_lastfm_track_stats(username, playlist_name)
+            refresh_lastfm_album_stats(username, playlist_name)
+            refresh_lastfm_artist_stats(username, playlist_name)
 
         return jsonify({'message': 'execution requested', 'status': 'success'}), 200
 
@@ -91,17 +95,47 @@ def playlist_refresh(username=None):
         return jsonify({"error": 'no name requested'}), 400
 
 
-@blueprint.route('/playlist/refresh/task', methods=['POST'])
+@blueprint.route('/playlist/refresh/task/track', methods=['POST'])
 @cloud_task
-def run_playlist_task():
+def run_playlist_track_task():
 
     payload = request.get_data(as_text=True)
     if payload:
         payload = json.loads(payload)
 
-        logger.info(f'running {payload["username"]} / {payload["name"]}')
+        logger.info(f'refreshing tracks {payload["username"]} / {payload["name"]}')
 
-        refresh_lastfm_stats(payload['username'], payload['name'])
+        refresh_lastfm_track_stats(payload['username'], payload['name'])
+
+        return jsonify({'message': 'executed playlist', 'status': 'success'}), 200
+
+
+@blueprint.route('/playlist/refresh/task/album', methods=['POST'])
+@cloud_task
+def run_playlist_album_task():
+
+    payload = request.get_data(as_text=True)
+    if payload:
+        payload = json.loads(payload)
+
+        logger.info(f'refreshing albums {payload["username"]} / {payload["name"]}')
+
+        refresh_lastfm_album_stats(payload['username'], payload['name'])
+
+        return jsonify({'message': 'executed playlist', 'status': 'success'}), 200
+
+
+@blueprint.route('/playlist/refresh/task/artist', methods=['POST'])
+@cloud_task
+def run_playlist_artist_task():
+
+    payload = request.get_data(as_text=True)
+    if payload:
+        payload = json.loads(payload)
+
+        logger.info(f'refreshing artists {payload["username"]} / {payload["name"]}')
+
+        refresh_lastfm_artist_stats(payload['username'], payload['name'])
 
         return jsonify({'message': 'executed playlist', 'status': 'success'}), 200
 
@@ -185,7 +219,7 @@ def execute_user(username):
                 if os.environ.get('DEPLOY_DESTINATION', None) == 'PROD':
                     create_refresh_playlist_task(username, iterate_playlist['name'], seconds_delay)
                 else:
-                    refresh_lastfm_stats(username, iterate_playlist['name'])
+                    refresh_lastfm_track_stats(username, iterate_playlist['name'])
 
                 seconds_delay += 1200
     else:
@@ -217,26 +251,55 @@ def create_refresh_user_task(username, delay=0):
 
 def create_refresh_playlist_task(username, playlist_name, delay=0):
 
-    task = {
+    track_task = {
         'app_engine_http_request': {  # Specify the type of request.
             'http_method': 'POST',
-            'relative_uri': '/api/spotfm/playlist/refresh/task',
+            'relative_uri': '/api/spotfm/playlist/refresh/task/track',
             'body': json.dumps({
                 'username': username,
                 'name': playlist_name
             }).encode()
         }
     }
-
     if delay > 0:
-
         d = datetime.datetime.utcnow() + datetime.timedelta(seconds=delay)
-
-        # Create Timestamp protobuf.
         timestamp = timestamp_pb2.Timestamp()
         timestamp.FromDatetime(d)
 
-        # Add the timestamp to the tasks.
+        track_task['schedule_time'] = timestamp
-        task['schedule_time'] = timestamp
 
-    tasker.create_task(task_path, task)
+    album_task = {
+        'app_engine_http_request': {  # Specify the type of request.
+            'http_method': 'POST',
+            'relative_uri': '/api/spotfm/playlist/refresh/task/album',
+            'body': json.dumps({
+                'username': username,
+                'name': playlist_name
+            }).encode()
+        }
+    }
+    d = datetime.datetime.utcnow() + datetime.timedelta(seconds=delay + 180)
+    timestamp = timestamp_pb2.Timestamp()
+    timestamp.FromDatetime(d)
+
+    album_task['schedule_time'] = timestamp
+
+    artist_task = {
+        'app_engine_http_request': {  # Specify the type of request.
+            'http_method': 'POST',
+            'relative_uri': '/api/spotfm/playlist/refresh/task/artist',
+            'body': json.dumps({
+                'username': username,
+                'name': playlist_name
+            }).encode()
+        }
+    }
+    d = datetime.datetime.utcnow() + datetime.timedelta(seconds=delay + 360)
+    timestamp = timestamp_pb2.Timestamp()
+    timestamp.FromDatetime(d)
+
+    artist_task['schedule_time'] = timestamp
+
+    tasker.create_task(task_path, track_task)
+    tasker.create_task(task_path, album_task)
+    tasker.create_task(task_path, artist_task)

music/tasks/refresh_lastfm_stats.py

@@ -13,7 +13,7 @@ db = firestore.Client()
 logger = logging.getLogger(__name__)
 
 
-def refresh_lastfm_stats(username, playlist_name):
+def refresh_lastfm_track_stats(username, playlist_name):
 
     logger.info(f'refreshing {playlist_name} stats for {username}')
 
@@ -27,21 +27,64 @@ def refresh_lastfm_stats(username, playlist_name):
 
     spotify_playlist = spotnet.get_playlist(uri=Uri(playlist_dict['uri']))
     track_count = counter.count_playlist(playlist=spotify_playlist)
-    album_count = counter.count_playlist(playlist=spotify_playlist, query_album=True)
-    artist_count = counter.count_playlist(playlist=spotify_playlist, query_artist=True)
 
     user_count = fmnet.get_user_scrobble_count()
     percent = round((track_count * 100) / user_count, 2)
-    album_percent = round((album_count * 100) / user_count, 2)
-    artist_percent = round((artist_count * 100) / user_count, 2)
 
     database_ref.update({
         'lastfm_stat_count': track_count,
-        'lastfm_stat_album_count': album_count,
-        'lastfm_stat_artist_count': artist_count,
-
         'lastfm_stat_percent': percent,
+
+        'lastfm_stat_last_refresh': datetime.utcnow()
+    })
+
+
+def refresh_lastfm_album_stats(username, playlist_name):
+
+    logger.info(f'refreshing {playlist_name} stats for {username}')
+
+    fmnet = database.get_authed_lastfm_network(username=username)
+    spotnet = database.get_authed_spotify_network(username=username)
+    counter = Counter(fmnet=fmnet, spotnet=spotnet)
+
+    database_ref = database.get_user_playlist_ref_by_username(user=username, playlist=playlist_name)
+
+    playlist_dict = database_ref.get().to_dict()
+
+    spotify_playlist = spotnet.get_playlist(uri=Uri(playlist_dict['uri']))
+    album_count = counter.count_playlist(playlist=spotify_playlist, query_album=True)
+
+    user_count = fmnet.get_user_scrobble_count()
+    album_percent = round((album_count * 100) / user_count, 2)
+
+    database_ref.update({
+        'lastfm_stat_album_count': album_count,
         'lastfm_stat_album_percent': album_percent,
+
+        'lastfm_stat_last_refresh': datetime.utcnow()
+    })
+
+
+def refresh_lastfm_artist_stats(username, playlist_name):
+
+    logger.info(f'refreshing {playlist_name} stats for {username}')
+
+    fmnet = database.get_authed_lastfm_network(username=username)
+    spotnet = database.get_authed_spotify_network(username=username)
+    counter = Counter(fmnet=fmnet, spotnet=spotnet)
+
+    database_ref = database.get_user_playlist_ref_by_username(user=username, playlist=playlist_name)
+
+    playlist_dict = database_ref.get().to_dict()
+
+    spotify_playlist = spotnet.get_playlist(uri=Uri(playlist_dict['uri']))
+    artist_count = counter.count_playlist(playlist=spotify_playlist, query_artist=True)
+
+    user_count = fmnet.get_user_scrobble_count()
+    artist_percent = round((artist_count * 100) / user_count, 2)
+
+    database_ref.update({
+        'lastfm_stat_artist_count': artist_count,
         'lastfm_stat_artist_percent': artist_percent,
 
         'lastfm_stat_last_refresh': datetime.utcnow()