improved logging, added support for optional this month/last month

spotify/__init__.py

@@ -1 +1,30 @@
 from .spotify import app
+
+import logging
+import os
+
+logger = logging.getLogger(__name__)
+logger.setLevel('DEBUG')
+
+if os.environ.get('DEPLOY_DESTINATION', None) == 'PROD':
+    from google.cloud.logging.handlers import CloudLoggingHandler
+    from google.cloud import logging as glogging
+
+    log_format = '%(funcName)s - %(message)s'
+    formatter = logging.Formatter(log_format)
+
+    client = glogging.Client()
+    handler = CloudLoggingHandler(client, name='playlist-manager')
+
+    handler.setFormatter(formatter)
+
+    logger.addHandler(handler)
+
+else:
+    log_format = '%(levelname)s %(name)s:%(funcName)s - %(message)s'
+    formatter = logging.Formatter(log_format)
+
+    stream_handler = logging.StreamHandler()
+    stream_handler.setFormatter(formatter)
+
+    logger.addHandler(stream_handler)

spotify/api/api.py

@@ -3,6 +3,7 @@ from flask import Blueprint, session, request, jsonify
 import os
 import datetime
 import json
+import logging
 
 from google.cloud import firestore
 from google.cloud import pubsub_v1
@@ -18,12 +19,13 @@ import spotify.api.database as database
 blueprint = Blueprint('api', __name__)
 db = firestore.Client()
 publisher = pubsub_v1.PublisherClient()
+
 tasker = tasks_v2.CloudTasksClient()
-
 task_path = tasker.queue_path('sarsooxyz', 'europe-west2', 'spotify-executions')
-
 run_playlist_topic_path = publisher.topic_path('sarsooxyz', 'run_user_playlist')
 
+logger = logging.getLogger(__name__)
+
 
 @blueprint.route('/playlists', methods=['GET'])
 def get_playlists():
@@ -41,6 +43,7 @@ def get_playlists():
         return jsonify(response), 200
 
     else:
+        logger.warning('user not logged in')
         return jsonify({'error': 'not logged in'}), 401
 
 
@@ -70,6 +73,7 @@ def playlist():
 
                 elif request.method == 'DELETE':
 
+                    logger.info(f'deleted {session["username"]} / {queried_playlist[0].to_dict()["name"]}')
                     playlists.document(queried_playlist[0].id).delete()
 
                     return jsonify({"message": 'playlist deleted', "status": "success"}), 200
@@ -94,6 +98,8 @@ def playlist():
             playlist_type = request_json.get('type', None)
 
             playlist_day_boundary = request_json.get('day_boundary', None)
+            playlist_add_this_month = request_json.get('add_this_month', None)
+            playlist_add_last_month = request_json.get('add_last_month', None)
 
             playlist_recommendation = request_json.get('include_recommendations', None)
             playlist_recommendation_sample = request_json.get('recommendation_sample', None)
@@ -127,8 +133,11 @@ def playlist():
 
                 if playlist_type == 'recents':
                     to_add['day_boundary'] = playlist_day_boundary if playlist_day_boundary is not None else 21
+                    to_add['add_this_month'] = playlist_add_this_month if playlist_add_this_month is not None else False
+                    to_add['add_last_month'] = playlist_add_last_month if playlist_add_last_month is not None else False
 
                 playlists.document().set(to_add)
+                logger.info(f'added {session["username"]} / {playlist_name}')
 
                 return jsonify({"message": 'playlist added', "status": "success"}), 201
 
@@ -165,6 +174,12 @@ def playlist():
                 if playlist_day_boundary is not None:
                     dic['day_boundary'] = playlist_day_boundary
 
+                if playlist_add_this_month is not None:
+                    dic['add_this_month'] = playlist_add_this_month
+
+                if playlist_add_last_month is not None:
+                    dic['add_last_month'] = playlist_add_last_month
+
                 if playlist_recommendation is not None:
                     dic['include_recommendations'] = playlist_recommendation
 
@@ -175,13 +190,16 @@ def playlist():
                     dic['type'] = playlist_type
 
                 if len(dic) == 0:
+                    logger.warning(f'no changes to make for {session["username"]} / {playlist_name}')
                     return jsonify({"message": 'no changes to make', "status": "error"}), 400
 
                 playlist_doc.update(dic)
+                logger.info(f'updated {session["username"]} / {playlist_name}')
 
                 return jsonify({"message": 'playlist updated', "status": "success"}), 200
 
     else:
+        logger.warning('user not logged in')
         return jsonify({'error': 'not logged in'}), 401
 
 
@@ -221,9 +239,11 @@ def user():
             dic = {}
 
             if 'locked' in request_json:
+                logger.info(f'updating lock {request_json["username"]} / {request_json["locked"]}')
                 dic['locked'] = request_json['locked']
 
             if 'spotify_linked' in request_json:
+                logger.info(f'deauthing {request_json["username"]}')
                 if request_json['spotify_linked'] is False:
                     dic.update({
                         'access_token': None,
@@ -232,13 +252,16 @@ def user():
                     })
 
             if len(dic) == 0:
+                logger.warning(f'no updates for {request_json["username"]}')
                 return jsonify({"message": 'no changes to make', "status": "error"}), 400
 
             actionable_user.update(dic)
+            logger.info(f'updated {request_json["username"]}')
 
             return jsonify({'message': 'account updated', 'status': 'succeeded'}), 200
 
     else:
+        logger.warning('user not logged in')
         return jsonify({'error': 'not logged in'}), 401
 
 
@@ -292,10 +315,12 @@ def change_password():
             if check_password_hash(current_user.get().to_dict()['password'], request_json['current_password']):
 
                 current_user.update({'password': generate_password_hash(request_json['new_password'])})
+                logger.info(f'password udpated {session["username"]}')
 
                 return jsonify({"message": 'password changed', "status": "success"}), 200
 
             else:
+                logger.warning(f"incorrect password {session['username']}")
                 return jsonify({'error': 'wrong password provided'}), 401
 
         else:
@@ -319,11 +344,15 @@ def play_playlist():
         request_include_recommendations = request_json.get('include_recommendations', True)
         request_recommendation_sample = request_json.get('recommendation_sample', 10)
         request_day_boundary = request_json.get('day_boundary', 10)
+        request_add_this_month = request_json.get('add_this_month', False)
+        request_add_last_month = request_json.get('add_last_month', False)
+
+        logger.info(f'playing {session["username"]}')
 
         if request_parts or request_playlists:
             if len(request_parts) > 0 or len(request_playlists) > 0:
 
-                if os.environ.get('DEPLOY_DESTINATION', None) and os.environ['DEPLOY_DESTINATION'] == 'PROD':
+                if os.environ.get('DEPLOY_DESTINATION', None) == 'PROD':
                     create_play_user_playlist_task(session['username'],
                                                    parts=request_parts,
                                                    playlist_type=request_playlist_type,
@@ -331,7 +360,9 @@ def play_playlist():
                                                    shuffle=request_shuffle,
                                                    include_recommendations=request_include_recommendations,
                                                    recommendation_sample=request_recommendation_sample,
-                                                   day_boundary=request_day_boundary)
+                                                   day_boundary=request_day_boundary,
+                                                   add_this_month=request_add_this_month,
+                                                   add_last_month=request_add_last_month)
                 else:
                     play_user_playlist(session['username'],
                                        parts=request_parts,
@@ -340,17 +371,22 @@ def play_playlist():
                                        shuffle=request_shuffle,
                                        include_recommendations=request_include_recommendations,
                                        recommendation_sample=request_recommendation_sample,
-                                       day_boundary=request_day_boundary)
+                                       day_boundary=request_day_boundary,
+                                       add_this_month=request_add_this_month,
+                                       add_last_month=request_add_last_month)
 
                 return jsonify({'message': 'execution requested', 'status': 'success'}), 200
 
             else:
+                logger.error(f'insufficient playlist/part lengths {session["username"]}')
                 return jsonify({'error': 'insufficient playlist sources'}), 400
 
         else:
+            logger.error(f'no playlists/parts {session["username"]}')
             return jsonify({'error': 'insufficient playlist sources'}), 400
 
     else:
+        logger.warning('user not logged in')
         return jsonify({'error': 'not logged in'}), 401
 
 
@@ -360,6 +396,7 @@ def play_playlist_task():
         payload = request.get_data(as_text=True)
         if payload:
             payload = json.loads(payload)
+            logger.info(f'playing {payload["username"]}')
 
             play_user_playlist(payload['username'],
                                parts=payload['parts'],
@@ -368,10 +405,13 @@ def play_playlist_task():
                                shuffle=payload['shuffle'],
                                include_recommendations=payload['include_recommendations'],
                                recommendation_sample=payload['recommendation_sample'],
-                               day_boundary=payload['day_boundary'])
+                               day_boundary=payload['day_boundary'],
+                               add_this_month=payload['add_this_month'],
+                               add_last_month=payload['add_last_month'])
 
             return jsonify({'message': 'executed playlist', 'status': 'success'}), 200
     else:
+        logger.warning('non tasks request')
         return jsonify({'error': 'unauthorized'}), 401
 
 
@@ -384,7 +424,7 @@ def run_playlist():
 
         if playlist_name:
 
-            if os.environ.get('DEPLOY_DESTINATION', None) and os.environ['DEPLOY_DESTINATION'] == 'PROD':
+            if os.environ.get('DEPLOY_DESTINATION', None) == 'PROD':
                 create_run_user_playlist_task(session['username'], playlist_name)
             else:
                 run_user_playlist(session['username'], playlist_name)
@@ -392,9 +432,11 @@ def run_playlist():
             return jsonify({'message': 'execution requested', 'status': 'success'}), 200
 
         else:
+            logger.warning('no playlist requested')
             return jsonify({"error": 'no name requested'}), 400
 
     else:
+        logger.warning('user not logged in')
         return jsonify({'error': 'not logged in'}), 401
 
 
@@ -406,10 +448,13 @@ def run_playlist_task():
         if payload:
             payload = json.loads(payload)
 
+            logger.info(f'running {payload["username"]} / {payload["name"]}')
+
             run_user_playlist(payload['username'], payload['name'])
 
             return jsonify({'message': 'executed playlist', 'status': 'success'}), 200
     else:
+        logger.warning('non tasks request')
         return jsonify({'error': 'unauthorized'}), 401
 
 
@@ -428,6 +473,7 @@ def run_user():
         return jsonify({'message': 'executed user', 'status': 'success'}), 200
 
     else:
+        logger.warning('user not logged in')
         return jsonify({'error': 'not logged in'}), 401
 
 
@@ -440,6 +486,7 @@ def run_user_task():
             execute_user(payload)
             return jsonify({'message': 'executed user', 'status': 'success'}), 200
     else:
+        logger.warning('non tasks request')
         return jsonify({'error': 'unauthorized'}), 401
 
 
@@ -456,6 +503,7 @@ def run_users():
         return jsonify({'message': 'executed all users', 'status': 'success'}), 200
 
     else:
+        logger.warning('user not logged in')
         return jsonify({'error': 'not logged in'}), 401
 
 
@@ -466,12 +514,14 @@ def run_users_cron():
         execute_all_users()
         return jsonify({'status': 'success'}), 200
     else:
+        logger.warning('user not logged in')
         return jsonify({'status': 'error', 'message': 'unauthorised'}), 401
 
 
 def execute_all_users():
 
     seconds_delay = 0
+    logger.info('running')
 
     for iter_user in [i.to_dict() for i in db.collection(u'spotify_users').stream()]:
 
@@ -505,12 +555,13 @@ def execute_user(username):
                  database.get_user_playlists_collection(database.get_user_query_stream(username)[0].id).stream()]
 
     seconds_delay = 0
+    logger.info(f'running {username}')
 
     for iterate_playlist in playlists:
         if len(iterate_playlist['parts']) > 0 or len(iterate_playlist['playlist_references']) > 0:
             if iterate_playlist.get('playlist_id', None):
 
-                if os.environ.get('DEPLOY_DESTINATION', None) and os.environ['DEPLOY_DESTINATION'] == 'PROD':
+                if os.environ.get('DEPLOY_DESTINATION', None) == 'PROD':
                     create_run_user_playlist_task(username, iterate_playlist['name'], seconds_delay)
                 else:
                     run_playlist(username, iterate_playlist['name'])
@@ -553,6 +604,8 @@ def create_play_user_playlist_task(username,
                                    include_recommendations=False,
                                    recommendation_sample=10,
                                    day_boundary=10,
+                                   add_this_month=False,
+                                   add_last_month=False,
                                    delay=0):
     task = {
         'app_engine_http_request': {  # Specify the type of request.
@@ -566,7 +619,9 @@ def create_play_user_playlist_task(username,
                 'shuffle': shuffle,
                 'include_recommendations': include_recommendations,
                 'recommendation_sample': recommendation_sample,
-                'day_boundary': day_boundary
+                'day_boundary': day_boundary,
+                'add_this_month': add_this_month,
+                'add_last_month': add_last_month
             }).encode()
         }
     }

spotify/api/spotify.py

@@ -1,13 +1,19 @@
 from google.cloud import firestore
 
+import logging
+
 from spotframework.net.user import User
 from spotframework.net.network import Network
 
 db = firestore.Client()
 
+logger = logging.getLogger(__name__)
+
 
 def create_playlist(username, name):
 
+    logger.info(f'creating {username} / {name}')
+
     users = [i for i in db.collection(u'spotify_users').where(u'username', u'==', username).stream()]
 
     if len(users) == 1:
@@ -22,10 +28,12 @@ def create_playlist(username, name):
 
         resp = net.create_playlist(net.user.username, name)
 
-        if 'id' in resp:
+        if resp and resp.get('id', None):
             return resp['id']
         else:
-            raise Exception('Error creating playlist')
+            logger.error(f'no response received {username} / {name}')
+            return None
 
     else:
-        raise ValueError('no/multiple username(s)')
+        logger.error(f'{len(users)} users found')
+        return None

spotify/auth/auth.py

@@ -4,6 +4,7 @@ from werkzeug.security import check_password_hash, generate_password_hash
 
 import urllib
 import datetime
+import logging
 from base64 import b64encode
 import requests
 
@@ -13,6 +14,8 @@ blueprint = Blueprint('authapi', __name__)
 
 db = firestore.Client()
 
+logger = logging.getLogger(__name__)
+
 
 @blueprint.route('/login', methods=['GET', 'POST'])
 def login():
@@ -48,15 +51,18 @@ def login():
         if check_password_hash(doc['password'], password):
 
             if doc['locked']:
+                logger.warning(f'locked account attempt {username}')
                 flash('account locked')
                 return redirect(url_for('index'))
 
             user_reference = db.collection(u'spotify_users').document(u'{}'.format(users[0].id))
             user_reference.update({'last_login': datetime.datetime.utcnow()})
 
+            logger.info(f'success {username}')
             session['username'] = username
             return redirect(url_for('app_route'))
         else:
+            logger.warning(f'failed attempt {username}')
             flash('incorrect password')
             return redirect(url_for('index'))
 
@@ -66,6 +72,8 @@ def login():
 
 @blueprint.route('/logout', methods=['GET', 'POST'])
 def logout():
+    if 'username' in session:
+        logger.info(f'logged out {session["username"]}')
     session.pop('username', None)
     flash('logged out')
     return redirect(url_for('index'))
@@ -113,6 +121,7 @@ def register():
             'validated': True
         })
 
+        logger.info(f'new user {username}')
         session['username'] = username
         return redirect(url_for('authapi.auth'))
 

spotify/spotify.py

@@ -2,37 +2,12 @@ from flask import Flask, render_template, redirect, request, session, flash, url
 from google.cloud import firestore
 
 import os
-import logging
 
 from spotify.auth import auth_blueprint
 from spotify.api import api_blueprint
 
-from google.cloud.logging.handlers import CloudLoggingHandler
-from google.cloud import logging as glogging
-
-# Project ID is determined by the GCLOUD_PROJECT environment variable
 db = firestore.Client()
 
-logger = logging.getLogger(__name__)
-logger.setLevel('INFO')
-
-log_format = '%(levelname)s %(name)s:%(funcName)s - %(message)s'
-formatter = logging.Formatter(log_format)
-
-if os.environ.get('DEPLOY_DESTINATION', None) and os.environ['DEPLOY_DESTINATION'] == 'PROD':
-    client = glogging.Client()
-    handler = CloudLoggingHandler(client)
-
-    handler.setFormatter(formatter)
-
-    logger.addHandler(handler)
-
-else:
-    stream_handler = logging.StreamHandler()
-    stream_handler.setFormatter(formatter)
-
-    logger.addHandler(stream_handler)
-
 app = Flask(__name__, static_folder=os.path.join(os.path.dirname(__file__), '..', 'build'), template_folder="templates")
 app.secret_key = db.collection(u'spotify').document(u'config').get().to_dict()['secret_key']
 app.register_blueprint(auth_blueprint, url_prefix='/auth')

spotify/tasks/play_user_playlist.py

@@ -15,6 +15,8 @@ db = firestore.Client()
 
 captured_playlists = []
 
+logger = logging.getLogger(__name__)
+
 
 def play_user_playlist(username,
                        playlist_type='default',
@@ -23,25 +25,31 @@ def play_user_playlist(username,
                        shuffle=False,
                        include_recommendations=True,
                        recommendation_sample=10,
-                       day_boundary=10):
-
-    logger = logging.getLogger(__name__)
+                       day_boundary=10,
+                       add_this_month=False,
+                       add_last_month=False):
 
     users = [i for i in db.collection(u'spotify_users').where(u'username', u'==', username).stream()]
 
-    logger.info(f'{username}')
+    logger.info(f'playing for {username}')
 
     if len(users) == 1:
 
         user_dict = users[0].to_dict()
 
-        if not parts and not playlists:
+        if parts is None and playlists is None:
             logger.critical(f'no playlists to use for creation ({username})')
-            return
+            return None
+
+        if parts is None:
+            parts = []
+
+        if playlists is None:
+            playlists = []
 
         if len(parts) == 0 and len(playlists) == 0:
             logger.critical(f'no playlists to use for creation ({username})')
-            return
+            return None
 
         spotify_keys = db.document('key/spotify').get().to_dict()
 
@@ -63,9 +71,6 @@ def play_user_playlist(username,
         global captured_playlists
         captured_playlists = []
 
-        if not parts:
-            parts = []
-
         submit_parts = parts
 
         for part in playlists:
@@ -79,7 +84,9 @@ def play_user_playlist(username,
                                                 submit_parts,
                                                 processors,
                                                 include_recommendations=include_recommendations,
-                                                recommendation_limit=int(recommendation_sample))
+                                                recommendation_limit=int(recommendation_sample),
+                                                add_this_month=add_this_month,
+                                                add_last_month=add_last_month)
         else:
             tracks = engine.make_playlist(submit_parts,
                                           processors,
@@ -90,7 +97,7 @@ def play_user_playlist(username,
 
     else:
         logger.critical(f'multiple/no user(s) found ({username})')
-        return
+        return None
 
 
 def generate_parts(user_id, name):

spotify/tasks/run_user_playlist.py

@@ -15,14 +15,14 @@ db = firestore.Client()
 
 captured_playlists = []
 
+logger = logging.getLogger(__name__)
+
 
 def run_user_playlist(username, playlist_name):
 
-    logger = logging.getLogger(__name__)
-
     users = [i for i in db.collection(u'spotify_users').where(u'username', u'==', username).stream()]
 
-    logger.info(f'{username} / {playlist_name}')
+    logger.info(f'running {username} / {playlist_name}')
 
     if len(users) == 1:
 
@@ -38,11 +38,11 @@ def run_user_playlist(username, playlist_name):
 
             if playlist_dict['playlist_id'] is None:
                 logger.critical(f'no playlist id to populate ({username}/{playlist_name})')
-                return
+                return None
 
             if len(playlist_dict['parts']) == 0 and len(playlist_dict['playlist_references']) == 0:
                 logger.critical(f'no playlists to use for creation ({username}/{playlist_name})')
-                return
+                return None
 
             spotify_keys = db.document('key/spotify').get().to_dict()
 
@@ -74,7 +74,9 @@ def run_user_playlist(username, playlist_name):
                                                     submit_parts,
                                                     processors,
                                                     include_recommendations=playlist_dict['include_recommendations'],
-                                                    recommendation_limit=int(playlist_dict['recommendation_sample']))
+                                                    recommendation_limit=int(playlist_dict['recommendation_sample']),
+                                                    add_this_month=playlist_dict.get('add_this_month', False),
+                                                    add_last_month=playlist_dict.get('add_last_month', False))
             else:
                 tracks = engine.make_playlist(submit_parts,
                                               processors,
@@ -86,11 +88,11 @@ def run_user_playlist(username, playlist_name):
 
         else:
             logger.critical(f'multiple/no playlists found ({username}/{playlist_name})')
-            return
+            return None
 
     else:
         logger.critical(f'multiple/no user(s) found ({username}/{playlist_name})')
-        return
+        return None
 
 
 def generate_parts(user_id, name):