admin script, live playlist updating with functions or tasks, config option

admin

@@ -0,0 +1,110 @@
+#!/usr/bin/env python3
+import shutil
+import os
+import sys
+from cmd import Cmd
+
+stage_dir = '_playlist-manager'
+scss_rel_path = os.path.join('src', 'scss', 'style.scss')
+css_rel_path = os.path.join('build', 'style.css')
+
+
+class Admin(Cmd):
+    intro = 'Music Tools Admin... ? for help'
+    prompt = '> '
+
+    def prepare_stage(self):
+        print('>> backing up a directory')
+        os.chdir('..')
+
+        print('>> deleting old deployment stage')
+        shutil.rmtree(stage_dir, ignore_errors=True)
+
+        print('>> copying main source')
+        shutil.copytree('playlist-manager', stage_dir)
+
+        for dependency in ['spotframework', 'fmframework', 'spotfm']:
+            print(f'>> injecting {dependency}')
+            shutil.copytree(
+                os.path.join(dependency, dependency),
+                os.path.join(stage_dir, dependency)
+            )
+
+        os.chdir(stage_dir)
+        os.system('gcloud config set project sarsooxyz')
+
+    def prepare_frontend(self):
+        print('>> building css')
+        os.system(f'sass --style=compressed {scss_rel_path} {css_rel_path}')
+
+        print('>> building javascript')
+        os.system('npm run build')
+
+    def prepare_main(self, path):
+        print('>> preparing main.py')
+        shutil.copy(f'main.{path}.py', 'main.py')
+
+    def deploy_function(self, name, timeout: int = 60):
+        os.system(f'gcloud functions deploy {name} '
+                  f'--runtime=python38 '
+                  f'--set-env-vars DEPLOY_DESTINATION=PROD '
+                  f'--timeout={timeout}s')
+
+    def do_api(self, args):
+        self.prepare_stage()
+        self.prepare_frontend()
+        self.prepare_main('api')
+
+        print('>> deploying')
+        os.system('gcloud app deploy')
+
+    def do_tag(self, args):
+        self.prepare_stage()
+        self.prepare_main('update_tag')
+
+        print('>> deploying')
+        self.deploy_function('update_tag')
+
+    def do_playlist(self, args):
+        self.prepare_stage()
+        self.prepare_main('run_playlist')
+
+        print('>> deploying')
+        self.deploy_function('run_user_playlist')
+
+    def do_exit(self, args):
+        exit(0)
+
+    def do_sass(self, args):
+        os.system(f'sass --style=compressed {scss_rel_path} {css_rel_path}')
+
+    def do_watchsass(self, args):
+        os.system(f'sass --style=compressed --watch {scss_rel_path} {css_rel_path}')
+
+    def do_rename(self, args):
+        from music.model.user import User
+        from music.model.playlist import Playlist
+
+        username = input('enter username: ')
+        user = User.collection.filter('username', '==', username).get()
+
+        if user is None:
+            print('>> user not found')
+
+        name = input('enter playlist name: ')
+        playlist = Playlist.collection.parent(user.key).filter('name', '==', name).get()
+
+        if playlist is None:
+            print('>> playlist not found')
+
+        new_name = input('enter new name: ')
+        playlist.name = new_name
+        playlist.update()
+
+
+if __name__ == '__main__':
+    console = Admin()
+    if len(sys.argv) > 1:
+        console.onecmd(' '.join(sys.argv[1:]))
+    else:
+        console.cmdloop()

app.yaml

@@ -1,4 +1,4 @@
-runtime: python37
+runtime: python38
 service: spotify
 
 instance_class: F2

deploy

@@ -1,55 +0,0 @@
-#!/usr/bin/env bash
-
-stage_dir=_playlist-manager
-
-echo '>> backing up a directory'
-cd ..
-
-echo '>> deleting old deployment stage'
-rm -rf $stage_dir
-
-echo '>> copying main source'
-cp -r playlist-manager $stage_dir
-
-echo '>> injecting spotframework'
-cp -r spotframework/spotframework $stage_dir/
-
-echo '>> injecting fmframework'
-cp -r fmframework/fmframework $stage_dir/
-
-echo '>> injecting spotfm'
-cp -r spotfm/spotfm $stage_dir/
-
-cd $stage_dir
-
-gcloud config set project sarsooxyz
-
-echo '>>> Target?'
-echo ''
-echo '(0) > api'
-echo '(1) > update_tag'
-read deploy_target
-
-
-case "$deploy_target" in
-        0)
-            echo '>> building css'
-            sass --style=compressed src/scss/style.scss build/style.css
-
-            echo '>> building javascript'
-            npm run build
-
-            echo '>> deploying'
-            gcloud app deploy
-            ;;
-
-        1)
-            echo '>> deploying update_tag'
-            gcloud functions deploy update_tag
-            ;;
-
-esac
-
-
-
-

main.api.py

@@ -0,0 +1,6 @@
+from music.music import create_app
+
+app = create_app()
+
+if __name__ == '__main__':
+    app.run(host='127.0.0.1', port=8080, debug=True)

main.run_playlist.py

@@ -0,0 +1,15 @@
+def run_user_playlist(event, context):
+    import logging
+
+    logger = logging.getLogger('music')
+
+    if event.get('attributes'):
+        if 'username' in event['attributes'] and 'name' in event['attributes']:
+
+            from music.tasks.run_user_playlist import run_user_playlist as do_run_user_playlist
+            do_run_user_playlist(username=event['attributes']['username'], playlist_name=event['attributes']["name"])
+
+        else:
+            logger.error('no parameters in event attributes')
+    else:
+        logger.error('no attributes in event')

main.update_tag.py

@@ -1,9 +1,3 @@
-from music import app
-from music.tasks.update_tag import update_tag as do_update_tag
-
-app = app
-
-
 def update_tag(event, context):
     import logging
 
@@ -11,12 +5,11 @@ def update_tag(event, context):
 
     if event.get('attributes'):
         if 'username' in event['attributes'] and 'tag_id' in event['attributes']:
+
+            from music.tasks.update_tag import update_tag as do_update_tag
             do_update_tag(username=event['attributes']['username'], tag_id=event['attributes']["tag_id"])
+
         else:
             logger.error('no parameters in event attributes')
     else:
         logger.error('no attributes in event')
-
-
-if __name__ == '__main__':
-    app.run(host='127.0.0.1', port=8080, debug=True)

music/__init__.py

@@ -1,5 +1,3 @@
-from .music import app
-
 import logging
 import os
 

music/api/api.py

@@ -8,7 +8,8 @@ import logging
 from datetime import datetime
 
 from music.api.decorators import login_required, login_or_basic_auth, admin_required, gae_cron, cloud_task
-from music.cloud.tasks import update_all_user_playlists, update_playlists, run_user_playlist_task
+from music.cloud import queue_run_user_playlist, offload_or_run_user_playlist
+from music.cloud.tasks import update_all_user_playlists, update_playlists
 from music.tasks.run_user_playlist import run_user_playlist
 
 from music.model.user import User
@@ -284,9 +285,9 @@ def run_playlist(user=None):
     if playlist_name:
 
         if os.environ.get('DEPLOY_DESTINATION', None) == 'PROD':
-            run_user_playlist_task(user.username, playlist_name)
+            queue_run_user_playlist(user.username, playlist_name)  # pass to either cloud tasks or functions
         else:
-            run_user_playlist(user.username, playlist_name)
+            run_user_playlist(user.username, playlist_name)  # update synchronously
 
         return jsonify({'message': 'execution requested', 'status': 'success'}), 200
 
@@ -297,7 +298,7 @@ def run_playlist(user=None):
 
 @blueprint.route('/playlist/run/task', methods=['POST'])
 @cloud_task
-def run_playlist_task():
+def run_playlist_task():  # receives cloud tasks request for update
 
     payload = request.get_data(as_text=True)
     if payload:
@@ -305,7 +306,7 @@ def run_playlist_task():
 
         logger.info(f'running {payload["username"]} / {payload["name"]}')
 
-        run_user_playlist(payload['username'], payload['name'])
+        offload_or_run_user_playlist(payload['username'], payload['name'])  # check whether offloading to cloud function
 
         return jsonify({'message': 'executed playlist', 'status': 'success'}), 200
 

music/auth/auth.py

@@ -1,7 +1,7 @@
 from flask import Blueprint, session, flash, request, redirect, url_for, render_template
-from google.cloud import firestore
 from werkzeug.security import generate_password_hash
 from music.model.user import User
+from music.model.config import Config
 
 import urllib
 import datetime
@@ -11,8 +11,6 @@ import requests
 
 blueprint = Blueprint('authapi', __name__)
 
-db = firestore.Client()
-
 logger = logging.getLogger(__name__)
 
 
@@ -98,7 +96,7 @@ def register():
         user = User()
         user.username = username
         user.password = generate_password_hash(password)
-        user.last_login = datetime.utcnow()
+        user.last_login = datetime.datetime.utcnow()
 
         user.save()
 
@@ -112,10 +110,10 @@ def auth():
 
     if 'username' in session:
 
-        client_id = db.document('key/spotify').get().to_dict()['clientid']
+        config = Config.collection.get("config/music-tools")
         params = urllib.parse.urlencode(
             {
-                'client_id': client_id,
+                'client_id': config.spotify_client_id,
                 'response_type': 'code',
                 'scope': 'playlist-modify-public playlist-modify-private playlist-read-private user-read-playback-state user-modify-playback-state user-library-read',
                 'redirect_uri': 'https://music.sarsoo.xyz/auth/spotify/token'
@@ -137,9 +135,11 @@ def token():
             flash('authorization failed')
             return redirect('app_route')
         else:
-            app_credentials = db.document('key/spotify').get().to_dict()
+            config = Config.collection.get("config/music-tools")
 
-            idsecret = b64encode(bytes(app_credentials['clientid'] + ':' + app_credentials['clientsecret'], "utf-8")).decode("ascii")
+            idsecret = b64encode(
+                bytes(config.spotify_client_id + ':' + config.spotify_client_secret, "utf-8")
+            ).decode("ascii")
             headers = {'Authorization': 'Basic %s' % idsecret}
 
             data = {

music/cloud/__init__.py

@@ -0,0 +1,49 @@
+import logging
+
+from music.model.config import Config
+from music.tasks.run_user_playlist import run_user_playlist as run_now
+from .function import run_user_playlist_function
+from .tasks import run_user_playlist_task
+
+logger = logging.getLogger(__name__)
+
+
+def queue_run_user_playlist(username: str, playlist_name: str):
+    config = Config.collection.get("config/music-tools")
+
+    if config is None:
+        logger.error(f'no config object returned, passing to cloud function {username} / {playlist_name}')
+        run_user_playlist_function(username=username, playlist_name=playlist_name)
+
+    if config.playlist_cloud_operating_mode == 'task':
+        logger.debug(f'passing {username} / {playlist_name} to cloud tasks')
+        run_user_playlist_task(username=username, playlist_name=playlist_name)
+
+    elif config.playlist_cloud_operating_mode == 'function':
+        logger.debug(f'passing {username} / {playlist_name} to cloud function')
+        run_user_playlist_function(username=username, playlist_name=playlist_name)
+
+    else:
+        logger.critical(f'invalid operating mode {username} / {playlist_name}, '
+                        f'{config.playlist_cloud_operating_mode}, passing to cloud function')
+        run_user_playlist_function(username=username, playlist_name=playlist_name)
+
+
+def offload_or_run_user_playlist(username: str, playlist_name: str):
+    config = Config.collection.get("config/music-tools")
+
+    if config is None:
+        logger.error(f'no config object returned, passing to cloud function {username} / {playlist_name}')
+        run_user_playlist_function(username=username, playlist_name=playlist_name)
+
+    if config.playlist_cloud_operating_mode == 'task':
+        run_now(username=username, playlist_name=playlist_name)
+
+    elif config.playlist_cloud_operating_mode == 'function':
+        logger.debug(f'offloading {username} / {playlist_name} to cloud function')
+        run_user_playlist_function(username=username, playlist_name=playlist_name)
+
+    else:
+        logger.critical(f'invalid operating mode {username} / {playlist_name}, '
+                        f'{config.playlist_cloud_operating_mode}, passing to cloud function')
+        run_user_playlist_function(username=username, playlist_name=playlist_name)

music/cloud/function.py

@@ -9,12 +9,27 @@ def update_tag(username, tag_id):
     """Queue serverless tag update for user"""
     logger.info(f'queuing {tag_id} update for {username}')
 
-    if username is None:
+    if username is None or tag_id is None:
-        logger.error(f'no username provided')
+        logger.error(f'less than two args provided, {username} / {tag_id}')
         return
 
-    if tag_id is None:
+    if not isinstance(username, str) or not isinstance(tag_id, str):
-        logger.error(f'no tag_id provided for {username}')
+        logger.error(f'less than two strings provided, {type(username)} / {type(tag_id)}')
         return
 
     publisher.publish('projects/sarsooxyz/topics/update_tag', b'', tag_id=tag_id, username=username)
+
+
+def run_user_playlist_function(username, playlist_name):
+    """Queue serverless playlist update for user"""
+    logger.info(f'queuing {playlist_name} update for {username}')
+
+    if username is None or playlist_name is None:
+        logger.error(f'less than two args provided, {username} / {playlist_name}')
+        return
+
+    if not isinstance(username, str) or not isinstance(playlist_name, str):
+        logger.error(f'less than two strings provided, {type(username)} / {type(playlist_name)}')
+        return
+
+    publisher.publish('projects/sarsooxyz/topics/run_user_playlist', b'', name=playlist_name, username=username)

music/db/database.py

@@ -1,13 +1,12 @@
-from google.cloud import firestore
+from dataclasses import dataclass
 import logging
 from datetime import timedelta, datetime, timezone
 
 from spotframework.net.network import Network as SpotifyNetwork, SpotifyNetworkException
+from spotframework.net.user import NetworkUser
 from fmframework.net.network import Network as FmNetwork
-from music.db.user import DatabaseUser
 from music.model.user import User
-
+from music.model.config import Config
-db = firestore.Client()
 
 logger = logging.getLogger(__name__)
 
@@ -33,10 +32,10 @@ def refresh_token_database_callback(user):
 def get_authed_spotify_network(user):
     if user is not None:
         if user.spotify_linked:
-            spotify_keys = db.document('key/spotify').get().to_dict()
+            config = Config.collection.get("config/music-tools")
 
-            user_obj = DatabaseUser(client_id=spotify_keys['clientid'],
+            user_obj = DatabaseUser(client_id=config.spotify_client_id,
-                                    client_secret=spotify_keys['clientsecret'],
+                                    client_secret=config.spotify_client_secret,
                                     refresh_token=user.refresh_token,
                                     user_id=user.username,
                                     access_token=user.access_token)
@@ -66,9 +65,15 @@ def get_authed_spotify_network(user):
 def get_authed_lastfm_network(user):
     if user is not None:
         if user.lastfm_username:
-            fm_keys = db.document('key/fm').get().to_dict()
+            config = Config.collection.get("config/music-tools")
-            return FmNetwork(username=user.lastfm_username, api_key=fm_keys['clientid'])
+            return FmNetwork(username=user.lastfm_username, api_key=config.last_fm_client_id)
         else:
             logger.error(f'{user.username} has no last.fm username')
     else:
         logger.error(f'no user provided')
+
+
+@dataclass
+class DatabaseUser(NetworkUser):
+    """adding music tools username to spotframework network user"""
+    user_id: str = None

music/db/user.py

@@ -1,14 +0,0 @@
-from spotframework.util.console import Color
-from spotframework.net.user import NetworkUser
-
-
-class DatabaseUser(NetworkUser):
-
-    def __init__(self, client_id, client_secret, refresh_token, user_id, access_token=None):
-        super().__init__(client_id=client_id, client_secret=client_secret,
-                         refresh_token=refresh_token, access_token=access_token)
-        self.user_id = user_id
-
-    def __repr__(self):
-        return Color.RED + Color.BOLD + 'DatabaseUser' + Color.END + \
-               f': {self.user_id}, {self.username}, {self.display_name}, {self.uri}'

music/model/config.py

@@ -0,0 +1,14 @@
+from fireo.models import Model
+from fireo.fields import TextField, BooleanField, DateTime, NumberField, ListField
+
+
+class Config(Model):
+    class Meta:
+        collection_name = 'config'
+
+    spotify_client_id = TextField()
+    spotify_client_secret = TextField()
+    last_fm_client_id = TextField()
+
+    playlist_cloud_operating_mode = TextField()  # task, function
+    secret_key = TextField()

music/music.py

@@ -1,28 +1,36 @@
 from flask import Flask, render_template, redirect, session, flash, url_for
-from google.cloud import firestore
 
+import logging
 import os
 
 from music.auth import auth_blueprint
 from music.api import api_blueprint, player_blueprint, fm_blueprint, \
     spotfm_blueprint, spotify_blueprint, admin_blueprint, tag_blueprint
+from music.model.config import Config
 
-db = firestore.Client()
+logger = logging.getLogger(__name__)
-
-app = Flask(__name__, static_folder=os.path.join(os.path.dirname(__file__), '..', 'build'), template_folder="templates")
-app.secret_key = db.collection(u'spotify').document(u'config').get().to_dict()['secret_key']
-app.register_blueprint(auth_blueprint, url_prefix='/auth')
-app.register_blueprint(api_blueprint, url_prefix='/api')
-app.register_blueprint(player_blueprint, url_prefix='/api/player')
-app.register_blueprint(fm_blueprint, url_prefix='/api/fm')
-app.register_blueprint(spotfm_blueprint, url_prefix='/api/spotfm')
-app.register_blueprint(spotify_blueprint, url_prefix='/api/spotify')
-app.register_blueprint(admin_blueprint, url_prefix='/api/admin')
-app.register_blueprint(tag_blueprint, url_prefix='/api')
 
 
-@app.route('/')
+def create_app():
-def index():
+    app = Flask(__name__, static_folder=os.path.join(os.path.dirname(__file__), '..', 'build'), template_folder="templates")
+
+    config = Config.collection.get("config/music-tools")
+    if config is not None:
+        app.secret_key = config.secret_key
+    else:
+        logger.error('no config returned, skipping secret key')
+
+    app.register_blueprint(auth_blueprint, url_prefix='/auth')
+    app.register_blueprint(api_blueprint, url_prefix='/api')
+    app.register_blueprint(player_blueprint, url_prefix='/api/player')
+    app.register_blueprint(fm_blueprint, url_prefix='/api/fm')
+    app.register_blueprint(spotfm_blueprint, url_prefix='/api/spotfm')
+    app.register_blueprint(spotify_blueprint, url_prefix='/api/spotify')
+    app.register_blueprint(admin_blueprint, url_prefix='/api/admin')
+    app.register_blueprint(tag_blueprint, url_prefix='/api')
+
+    @app.route('/')
+    def index():
 
         if 'username' in session:
             logged_in = True
@@ -32,10 +40,9 @@ def index():
 
         return render_template('login.html', logged_in=logged_in)
 
-
+    @app.route('/app', defaults={'path': ''})
-@app.route('/app', defaults={'path': ''})
+    @app.route('/app/<path:path>')
-@app.route('/app/<path:path>')
+    def app_route(path):
-def app_route(path):
 
         if 'username' not in session:
             flash('please log in')
@@ -43,4 +50,6 @@ def app_route(path):
 
         return render_template('app.html')
 
+    return app
+
 # [END gae_python37_app]

music/tasks/create_playlist.py

@@ -1,12 +1,8 @@
-from google.cloud import firestore
-
 import logging
 
 import music.db.database as database
 from spotframework.net.network import SpotifyNetworkException
 
-db = firestore.Client()
-
 logger = logging.getLogger(__name__)
 
 

music/tasks/refresh_lastfm_stats.py

@@ -1,5 +1,3 @@
-from google.cloud import firestore
-
 import logging
 from datetime import datetime
 
@@ -13,8 +11,6 @@ from spotframework.net.network import SpotifyNetworkException
 
 from fmframework.net.network import LastFMNetworkException
 
-db = firestore.Client()
-
 logger = logging.getLogger(__name__)
 
 

rename_playlist.py

@@ -1,14 +0,0 @@
-from music.model.user import User
-from music.model.playlist import Playlist
-
-user = User.collection.filter('username', '==', 'andy').get()
-
-name = input('enter playlist name: ')
-playlist = Playlist.collection.parent(user.key).filter('name', '==', name).get()
-
-if playlist is not None:
-    new_name = input('enter new name: ')
-    playlist.name = new_name
-    playlist.update()
-else:
-    print('playlist not found')

requirements.txt

@@ -15,15 +15,17 @@ google-cloud-tasks==1.5.0
 googleapis-common-protos==1.52.0
 grpc-google-iam-v1==0.12.3
 grpcio==1.30.0
+gunicorn==20.0.4
 idna==2.9
 isort==4.3.21
 itsdangerous==1.1.0
 Jinja2==2.11.2
-lazy-object-proxy==1.5.0
+lazy-object-proxy==1.4.3
 MarkupSafe==1.1.1
 mccabe==0.6.1
 numpy==1.19.0
 opencv-python==4.2.0.34
+pathtools==0.1.2
 protobuf==3.12.2
 pyasn1==0.4.8
 pyasn1-modules==0.2.8
@@ -35,5 +37,6 @@ six==1.15.0
 tabulate==0.8.7
 toml==0.10.1
 urllib3==1.25.9
+watchdog==0.10.3
 Werkzeug==1.0.1
 wrapt==1.12.1

sass

@@ -1 +0,0 @@
-sass --style=compressed src/scss/style.scss build/style.css

watchsass

@@ -1 +0,0 @@
-sass --style=compressed --watch src/scss/style.scss build/style.css