From b549cf4ca6bcd5fc21dcd4510e59ecae78f3df25 Mon Sep 17 00:00:00 2001 From: RecursiveGreen Date: Tue, 21 May 2019 10:33:48 -0400 Subject: [PATCH 01/30] Upgraded requirements. --- requirements-dev.txt | Bin 502 -> 514 bytes requirements.txt | 14 ++++++++------ 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 0599b76eb3468b5ea1aec7b329480a7ebd55ba31..c6c120e2a9bc99c5374a59bed9c87c91108caf59 100644 GIT binary patch delta 127 zcmeyy+{7}$iPeZfkHK`JyCS2>#7tXeBL;(s%k_cm>+y`Hlg$}rS&f16hLa;1wHYlZ zw=x=gm@@D(a50oHR5BDWq%x#1=rSZS6a)Eo47Lo0K(!$C5H;ltML@X%BsoK{3PZ37 E0O1H10ssI2 delta 91 zcmZo-`NllKiPexnkHKJ~yCS3U#7tX8lZh+!fvlVHj3$#U8D&|GfU-uDqZzdsO((ZA m8cUln@G@{Q6f=18.3.0 -cffi>=1.11.5 +argon2-cffi>=19.1.0 +cffi>=1.12.3 dj-database-url>=0.5.0 -Django>=2.1.5 +Django>=2.2.1 django-authtools>=1.6.0 -djangorestframework>=3.9.0 -psycopg2>=2.7.6.1 +djangorestframework>=3.9.4 +psycopg2>=2.8.2 pycparser>=2.19 python-decouple>=3.1 -pytz>=2018.9 +pytz>=2019.1 +six>=1.12.0 +sqlparse>=0.3.0 From 385dc746056f1d75dcd1ad4a731f42afe24050a8 Mon Sep 17 00:00:00 2001 From: RecursiveGreen Date: Tue, 21 May 2019 11:47:16 -0400 Subject: [PATCH 02/30] Move back to standard settings file. --- .../{settings/base.py => settings.py} | 14 +++++++ .../savepointradio/settings/__init__.py | 0 .../savepointradio/settings/development.py | 38 ------------------- 3 files changed, 14 insertions(+), 38 deletions(-) rename savepointradio/savepointradio/{settings/base.py => settings.py} (90%) delete mode 100644 savepointradio/savepointradio/settings/__init__.py delete mode 100644 savepointradio/savepointradio/settings/development.py diff --git a/savepointradio/savepointradio/settings/base.py b/savepointradio/savepointradio/settings.py similarity index 90% rename from savepointradio/savepointradio/settings/base.py rename to savepointradio/savepointradio/settings.py index 19e3497..21be075 100644 --- a/savepointradio/savepointradio/settings/base.py +++ b/savepointradio/savepointradio/settings.py @@ -1,6 +1,7 @@ import os from decouple import config +from dj_database_url import parse as db_url SETTINGS_DIR = os.path.dirname(os.path.abspath(__file__)) @@ -8,6 +9,8 @@ CONFIG_DIR = os.path.dirname(SETTINGS_DIR) PROJECT_DIR = os.path.dirname(CONFIG_DIR) BASE_DIR = os.path.dirname(PROJECT_DIR) +DEBUG = config('DEBUG', default=False, cast=bool) + # # Django-specific settings # @@ -29,6 +32,17 @@ AUTH_PASSWORD_VALIDATORS = [ AUTH_USER_MODEL = 'core.RadioUser' +DATABASES = { + 'default': config( + 'DATABASE_URL', + default='sqlite:///' + os.path.join(PROJECT_DIR, 'spradio.sqlite3'), + cast=db_url + ) +} + +if DEBUG: + EMAIL_BACKEND = 'django.core.mail.backends.smtp.EmailBackend' + INSTALLED_APPS = [ 'django.contrib.admin', 'django.contrib.auth', diff --git a/savepointradio/savepointradio/settings/__init__.py b/savepointradio/savepointradio/settings/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/savepointradio/savepointradio/settings/development.py b/savepointradio/savepointradio/settings/development.py deleted file mode 100644 index d2b5975..0000000 --- a/savepointradio/savepointradio/settings/development.py +++ /dev/null @@ -1,38 +0,0 @@ -from decouple import config -from dj_database_url import parse as db_url - -from .base import * - - -ALLOWED_HOSTS = [] - -DATABASES = { - 'default': config( - 'DATABASE_URL', - default='sqlite:///' + os.path.join(PROJECT_DIR, 'testdb.sqlite3'), - cast=db_url - ) -} - -DEBUG = True - -EMAIL_BACKEND = 'django.core.mail.backends.console.EmailBackend' - -''' -LOGGING = { - 'version': 1, - 'disable_existing_loggers': False, - 'handlers': { - 'console': { - 'level': 'DEBUG', - 'class': 'logging.StreamHandler', - } - }, - 'loggers': { - 'django.db.backends': { - 'handlers': ['console'], - 'level': 'DEBUG', - }, - } -} -''' From 7e0b2a5a45bbd5984ba296de9e2ec9d4c079e070 Mon Sep 17 00:00:00 2001 From: RecursiveGreen Date: Mon, 3 Jun 2019 10:24:57 -0400 Subject: [PATCH 03/30] New script for exporting existing db to json file. --- contrib/export_playlist/export_playlist.py | 213 +++++++++++++++++++++ contrib/export_playlist/requirements.txt | 1 + 2 files changed, 214 insertions(+) create mode 100644 contrib/export_playlist/export_playlist.py create mode 100644 contrib/export_playlist/requirements.txt diff --git a/contrib/export_playlist/export_playlist.py b/contrib/export_playlist/export_playlist.py new file mode 100644 index 0000000..8d0f71b --- /dev/null +++ b/contrib/export_playlist/export_playlist.py @@ -0,0 +1,213 @@ +''' +export_playlist.py + +This is the helper script that exports old playlist databases to be reimported +by the new database later. +''' + +import argparse +from decimal import Decimal, getcontext +import json +import mimetypes +import os +import sqlite3 +import sys +import unicodedata + +import magic + + +def scrub(text): + ''' + Forcing a Unicode NFC normalization to remove combining marks that mess + with certain Python fucntions. + ''' + if text: + return unicodedata.normalize('NFC', text) + return None + + +def detect_mime(path): + ''' + Guess a file's mimetype from it's magic number. If inconclusive, then + guess based on it's file extension. + ''' + mimetype = magic.from_file(path, mime=True) + if mimetype == 'application/octet-stream': + return mimetypes.guess_type(path, strict=True)[0] + return mimetype + + +def adapt_decimal(number): + '''Sqlite3 adapter for Decimal types''' + return str(number) + + +def convert_decimal(text): + '''Sqlite3 converter for Decimal types''' + return float(text.decode('utf8')) + + +def import_sqlite3(db_file): + ''' + Imports a playlist from an SQLite3 database file and exports to a + JSON file. + ''' + totals = { + 'albums': 0, + 'artists': 0, + 'games': 0, + 'songs': 0, + 'jingles': 0 + } + + if not os.path.isfile(db_file): + raise FileNotFoundError + + detect_types = sqlite3.PARSE_DECLTYPES | sqlite3.PARSE_COLNAMES + con = sqlite3.connect(db_file, detect_types=detect_types) + cur = con.cursor() + + # Fetching albums first + albums = list() + for album in con.execute('SELECT title, enabled FROM albums'): + albums.append({ + 'title': scrub(album[0]), + 'disabled': not bool(album[1]) + }) + totals['albums'] += 1 + print('Exported {} albums'.format(str(totals['albums']))) + + # Next up, artists + artists = list() + artist_query = 'SELECT alias, firstname, lastname, enabled FROM artists' + for artist in con.execute(artist_query): + artists.append({ + 'alias': scrub(artist[0]) or '', + 'first_name': scrub(artist[1]) or '', + 'last_name': scrub(artist[2]) or '', + 'disabled': not bool(artist[3]) + }) + totals['artists'] += 1 + print('Exported {} artists'.format(str(totals['artists']))) + + # On to games + games = list() + for game in con.execute('SELECT title, enabled FROM games'): + games.append({ + 'title': scrub(game[0]), + 'disabled': not bool(game[1]) + }) + totals['games'] += 1 + print('Exported {} games'.format(str(totals['games']))) + + # Now the songs + songs = list() + songs_query = '''SELECT + songs.songs_id AS id, + games.title AS game, + albums.title AS album, + songs.enabled AS enabled, + songs.type AS type, + songs.title AS title, + songs.length AS length, + songs.path AS path + FROM songs + LEFT JOIN games + ON (songs.game = games.games_id) + LEFT JOIN albums + ON (songs.album = albums.albums_id)''' + cur.execute(songs_query) + old_songs = cur.fetchall() + for song in old_songs: + # Deal with the list of artists + song_artists = list() + song_artist_query = '''SELECT + ifnull(alias, "") AS alias, + ifnull(firstname, "") AS firstname, + ifnull(lastname, "") AS lastname + FROM artists + WHERE artists_id + IN (SELECT artists_artists_id + FROM songs_have_artists + WHERE songs_songs_id = ?)''' + cur.execute(song_artist_query, [song[0]]) + song_artist_results = cur.fetchall() + for artist in song_artist_results: + song_artists.append({ + 'alias': scrub(artist[0]), + 'first_name': scrub(artist[1]), + 'last_name': scrub(artist[2]) + }) + store = {'path': scrub(song[7]), + 'mime': detect_mime(scrub(song[7])), + 'filesize': os.stat(scrub(song[7])).st_size, + 'length': song[6]} + songs.append({'album': scrub(song[2]), + 'artists': song_artists, + 'game': scrub(song[1]), + 'disabled': not bool(song[3]), + 'type': song[4], + 'title': scrub(song[5]), + 'store': store}) + if song[4] == 'S': + totals['songs'] += 1 + else: + totals['jingles'] += 1 + print('Exported {} requestables ({} songs, {} jingles)'.format( + str(totals['songs'] + totals['jingles']), + str(totals['songs']), + str(totals['jingles']) + )) + + return {'albums': albums, + 'artists': artists, + 'games': games, + 'songs': songs} + + +def main(): + '''Main loop of the program''' + getcontext().prec = 8 + + sqlite3.register_adapter(Decimal, adapt_decimal) + sqlite3.register_converter("decimal", convert_decimal) + + description = 'Exports old playlist to a file for reimporting later.' + + parser = argparse.ArgumentParser(description=description) + subparsers = parser.add_subparsers(dest='command') + + parser_sqlite3 = subparsers.add_parser( + 'sqlite3', + help='Imports old Sqlite3 database.' + ) + parser_sqlite3.add_argument( + 'db_file', + help='Path to the sqlite3 database file.', + nargs=1 + ) + + if len(sys.argv) == 1: + sys.stderr.write('Error: please specify a command\n\n') + parser.print_help(sys.stderr) + sys.exit(1) + + results = None + + args = parser.parse_args() + + if args.command == 'sqlite3': + results = import_sqlite3(args.db_file[0]) + + if results: + with open('playlist.json', 'w', encoding='utf8') as file: + json.dump(results, + file, + ensure_ascii=False, + sort_keys=True, + indent=4) + + +if __name__ == '__main__': + main() diff --git a/contrib/export_playlist/requirements.txt b/contrib/export_playlist/requirements.txt new file mode 100644 index 0000000..e7a74f7 --- /dev/null +++ b/contrib/export_playlist/requirements.txt @@ -0,0 +1 @@ +python-magic>=0.4.15 From 9fa3a408b14abbd020824af059b06aec2a3134ee Mon Sep 17 00:00:00 2001 From: RecursiveGreen Date: Mon, 3 Jun 2019 10:59:18 -0400 Subject: [PATCH 04/30] Initial changes to Song model using Stores. --- savepointradio/radio/fields.py | 39 +++ savepointradio/radio/forms.py | 21 ++ .../management/commands/importoldradio.py | 233 ++++++++---------- savepointradio/radio/managers.py | 97 ++++---- .../0004_new_song_path_structure.py | 49 ++++ savepointradio/radio/models.py | 102 +++++--- 6 files changed, 325 insertions(+), 216 deletions(-) create mode 100644 savepointradio/radio/fields.py create mode 100644 savepointradio/radio/forms.py create mode 100644 savepointradio/radio/migrations/0004_new_song_path_structure.py diff --git a/savepointradio/radio/fields.py b/savepointradio/radio/fields.py new file mode 100644 index 0000000..8cc2b5d --- /dev/null +++ b/savepointradio/radio/fields.py @@ -0,0 +1,39 @@ +''' +Custom model fields for the Save Point Radio project. +''' + +from django.core import validators +from django.db import models +from django.utils.translation import ugettext_lazy as _ + +from .forms import ALLOWED_SCHEMES, RadioIRIFormField + + +class RadioIRIField(models.TextField): + ''' + A custom URL model field that allows schemes that match those from + Liquidsoap. This is necessary due to a bug in how Django currently + handles custom URLFields: + + https://code.djangoproject.com/ticket/25594 + https://stackoverflow.com/questions/41756572/ + ''' + default_validators = [validators.URLValidator(schemes=ALLOWED_SCHEMES)] + description = _("Long IRI") + + def __init__(self, verbose_name=None, name=None, **kwargs): + # This is a limit for Internet Explorer URLs + kwargs.setdefault('max_length', 2000) + super().__init__(verbose_name, name, **kwargs) + + def deconstruct(self): + name, path, args, kwargs = super().deconstruct() + if kwargs.get("max_length") == 2000: + del kwargs['max_length'] + return name, path, args, kwargs + + def formfield(self, **kwargs): + return super().formfield(**{ + 'form_class': RadioIRIFormField, + **kwargs, + }) diff --git a/savepointradio/radio/forms.py b/savepointradio/radio/forms.py new file mode 100644 index 0000000..48c0667 --- /dev/null +++ b/savepointradio/radio/forms.py @@ -0,0 +1,21 @@ +''' +Custom forms/formfields for the Save Point Radio project. +''' + +from django.core import validators +from django.forms.fields import URLField + + +ALLOWED_SCHEMES = ['http', 'https', 'file', 'ftp', 'ftps', 's3'] + + +class RadioIRIFormField(URLField): + ''' + A custom URL form field that allows schemes that match those from + Liquidsoap. This is necessary due to a bug in how Django currently + handles custom URLFields: + + https://code.djangoproject.com/ticket/25594 + https://stackoverflow.com/questions/41756572/ + ''' + default_validators = [validators.URLValidator(schemes=ALLOWED_SCHEMES)] diff --git a/savepointradio/radio/management/commands/importoldradio.py b/savepointradio/radio/management/commands/importoldradio.py index 971e8ca..60015e2 100644 --- a/savepointradio/radio/management/commands/importoldradio.py +++ b/savepointradio/radio/management/commands/importoldradio.py @@ -1,160 +1,129 @@ -from decimal import * +''' +Django management command to import old playlist data. This should only be used +for seeding a newly created database. +''' + +import decimal +import json import os -import sqlite3 from django.core.management.base import BaseCommand, CommandError -from radio.models import Album, Artist, Game, Song +from core.utils import path_to_iri +from radio.models import Album, Artist, Game, Store, Song -getcontext().prec = 8 - - -def adapt_decimal(d): - return str(d) - - -def convert_decimal(s): - return Decimal(s.decode('utf8')) - - -sqlite3.register_adapter(Decimal, adapt_decimal) -sqlite3.register_converter("decimal", convert_decimal) +decimal.getcontext().prec = 8 class Command(BaseCommand): - help = 'Imports the old radio data from the original sqlite3 database' + '''Main "importoldreadio" command class''' + help = 'Imports the old radio data from an exported playlist' def add_arguments(self, parser): - parser.add_argument('sqlite3_db_file', nargs=1) + parser.add_argument('playlist_file', nargs=1) def handle(self, *args, **options): - if not os.path.isfile(options['sqlite3_db_file'][0]): + playlist_file = options['playlist_file'][0] + if not os.path.isfile(playlist_file): raise CommandError('File does not exist') - else: - total_albums = 0 - total_artists = 0 - total_games = 0 - total_songs = 0 - total_jingles = 0 - detect_types = sqlite3.PARSE_DECLTYPES | sqlite3.PARSE_COLNAMES - con = sqlite3.connect(options['sqlite3_db_file'][0], - detect_types=detect_types) - cur = con.cursor() + with open(playlist_file, 'r', encoding='utf8') as pfile: + playlist = json.load(pfile, parse_float=decimal.Decimal) - # Fetching albums first - for album in con.execute('SELECT title, enabled FROM albums'): - album_disabled = not bool(album[1]) - Album.objects.create(title=album[0], disabled=album_disabled) - total_albums += 1 + totals = { + 'albums': 0, + 'artists': 0, + 'games': 0, + 'songs': 0, + 'jingles': 0 + } - self.stdout.write('Imported {} albums'.format(str(total_albums))) + # Fetching albums first + for album in playlist['albums']: + Album.objects.create(title=album['title'], + disabled=album['disabled']) + total_albums += 1 - # Next up, artists - cur.execute('''SELECT - artists_id, - alias, - firstname, - lastname, - enabled - FROM artists''') - artists = cur.fetchall() + self.stdout.write('Imported {} albums'.format(str(totals['albums']))) - for artist in artists: - artist_disabled = not bool(artist[4]) - Artist.objects.create(alias=artist[1] or '', - first_name=artist[2] or '', - last_name=artist[3] or '', - disabled=artist_disabled) - total_artists += 1 + # Next up, artists + for artist in playlist['artists']: + Artist.objects.create(alias=artist['alias'] or '', + first_name=artist['first_name'] or '', + last_name=artist['last_name'] or '', + disabled=artist['disabled']) + total_artists += 1 - self.stdout.write('Imported {} artists'.format(str(total_artists))) + self.stdout.write('Imported {} artists'.format(str(totals['artists']))) - # On to games - for game in con.execute('SELECT title, enabled FROM games'): - game_disabled = not bool(game[1]) - Game.objects.create(title=game[0], disabled=game_disabled) - total_games += 1 + # On to games + for game in playlist['games']: + Game.objects.create(title=game['title'], + disabled=game['disabled']) + total_games += 1 - self.stdout.write('Imported {} games'.format(str(total_games))) + self.stdout.write('Imported {} games'.format(str(totals['games']))) - # Followed by the songs - cur.execute('''SELECT - songs.songs_id AS id, - games.title AS game, - albums.title AS album, - songs.enabled AS enabled, - songs.type AS type, - songs.title AS title, - songs.length AS length, - songs.path AS path - FROM songs - LEFT JOIN games - ON (songs.game = games.games_id) - LEFT JOIN albums - ON (songs.album = albums.albums_id)''') - songs = cur.fetchall() + # Followed by the songs + for song in playlist['songs']: + try: + album = Album.objects.get(title__exact=song['album']) + except Album.DoesNotExist: + album = None - for song in songs: - try: - album = Album.objects.get(title__exact=song[2]) - except Album.DoesNotExist: - album = None + try: + game = Game.objects.get(title__exact=song['game']) + except Game.DoesNotExist: + game = None - try: - game = Game.objects.get(title__exact=song[1]) - except Game.DoesNotExist: - game = None + new_song = Song.objects.create(album=album, + game=game, + disabled=song['disabled'], + song_type=song['type'], + title=song['title']) - song_disabled = not bool(song[3]) - new_song = Song.objects.create(album=album, - game=game, - disabled=song_disabled, - song_type=song[4], - title=song[5], - length=song[6], - path=song[7]) - if song[4] == 'S': - total_songs += 1 - else: - total_jingles += 1 - - cur.execute('''SELECT - ifnull(alias, "") AS alias, - ifnull(firstname, "") AS firstname, - ifnull(lastname, "") AS lastname - FROM artists - WHERE artists_id - IN (SELECT artists_artists_id - FROM songs_have_artists - WHERE songs_songs_id = ?)''', [song[0]]) - old_artists = cur.fetchall() - for old_artist in old_artists: - new_artist = Artist.objects.get(alias__exact=old_artist[0], - first_name__exact=old_artist[1], - last_name__exact=old_artist[2]) - new_song.artists.add(new_artist) - - self.stdout.write( - 'Imported {} requestables ({} songs, {} jingles)'.format( - str(total_songs + total_jingles), - str(total_songs), - str(total_jingles) + for artist in song['artists']: + new_artist = Artist.objects.get( + alias__exact=artist['alias'] or '', + first_name__exact=artist['first_name'] or '', + last_name__exact=artist['last_name'] or '' ) + new_song.artists.add(new_artist) + + new_store = Store.objects.create( + iri=path_to_iri(song['store']['path']), + mime_type=song['store']['mime'], + file_size=song['store']['filesize'], + length=song['store']['length'] ) - - pub = input('Do you want to publish all imported objects as well? ' - '[Y/N] ') - - if pub == 'Y' or pub == 'y': - for al in Album.objects.all(): - al.publish() - for ar in Artist.objects.all(): - ar.publish() - for g in Game.objects.all(): - g.publish() - for s in Song.objects.all(): - s.publish() - self.stdout.write('Published imported objects successfully') + new_song.stores.add(new_store) + new_song.current_store = new_store + new_song.save() + if song['type'] == 'S': + totals['songs'] += 1 else: - self.stdout.write('Skipped publishing songs') + totals['jingles'] += 1 + + self.stdout.write( + 'Imported {} requestables ({} songs, {} jingles)'.format( + str(totals['songs'] + totals['jingles']), + str(totals['songs']), + str(totals['jingles']) + ) + ) + + pub = input('Do you want to publish all imported objects as well? ' + '[Y/N] ') + + if pub in ('Y', 'y'): + for album in Album.objects.all(): + album.publish() + for artist in Artist.objects.all(): + artist.publish() + for game in Game.objects.all(): + game.publish() + for song in Song.objects.all(): + song.publish() + self.stdout.write('Published imported objects successfully') + else: + self.stdout.write('Skipped publishing songs') diff --git a/savepointradio/radio/managers.py b/savepointradio/radio/managers.py index 5a7c898..0d6dda7 100644 --- a/savepointradio/radio/managers.py +++ b/savepointradio/radio/managers.py @@ -1,3 +1,7 @@ +''' +Django Model Managers for the Radio application. +''' + from datetime import timedelta from decimal import getcontext, Decimal, ROUND_UP from random import randint @@ -15,123 +19,126 @@ getcontext().prec = 16 class RadioManager(models.Manager): - """ + ''' Custom object manager for filtering out common behaviors for radio objects. - """ + ''' def get_queryset(self): - """ + ''' Return customized default QuerySet. - """ + ''' return RadioQuerySet(self.model, using=self._db) def disabled(self): - """ + ''' Radio objects that are marked as disabled. - """ + ''' return self.get_queryset().disabled() def enabled(self): - """ + ''' Radio objects that are marked as enabled. - """ + ''' return self.get_queryset().enabled() def published(self): - """ + ''' Radio objects that are marked as published. - """ + ''' return self.get_queryset().published() def unpublished(self): - """ + ''' Radio objects that are marked as unpublished. - """ + ''' return self.get_queryset().unpublished() def available(self): - """ + ''' Radio objects that are enabled and published. - """ + ''' return self.enabled().published() class SongManager(RadioManager): - """ + ''' Custom object manager for filtering out common behaviors for Song objects. - """ + ''' def get_queryset(self): - """ + ''' Return customized default QuerySet for Songs. - """ + ''' return SongQuerySet(self.model, using=self._db) def available_jingles(self): - """ + ''' Jingles that are currently published and are enabled. - """ + ''' return self.available().jingles() def available_songs(self): - """ + ''' Songs that are currently published and are enabled. - """ + ''' return self.available().songs() def playlist_length(self): - """ + ''' Total length of available songs in the playlist (in seconds). - """ - length = self.available_songs().aggregate(models.Sum('length')) - return length['length__sum'] + ''' + a_songs = self.available_songs() + length = a_songs.aggregate( + total_time=models.Sum('current_store__length') + ) + return length['total_time'] def wait_total(self, adjusted_ratio=0.0): - """ + ''' Default length in seconds before a song can be played again. This is based on the replay ratio set in the application settings. - """ + ''' total_ratio = get_setting('replay_ratio') + adjusted_ratio wait = self.playlist_length() * Decimal(total_ratio) wait = wait.quantize(Decimal('.01'), rounding=ROUND_UP) return timedelta(seconds=float(wait)) def datetime_from_wait(self): - """ + ''' Datetime of now minus the default wait time for played songs. - """ + ''' return timezone.now() - self.wait_total() def playable(self): - """ + ''' Songs that are playable because they are available (enabled & published) and they have not been played within the default wait time (or at all). - """ + ''' return self.available_songs().filter( - models.Q(next_play__lt=timezone.now()) | - models.Q(next_play__isnull=True) - ) + models.Q(next_play__lt=timezone.now()) | + models.Q(next_play__isnull=True) + ) def requestable(self): - """ + ''' Songs that can be placed in the request queue for playback. - """ + ''' # Import SongRequest here to get rid of circular dependencies - SongRequest = apps.get_model(app_label='profiles', - model_name='SongRequest') - requests = SongRequest.music.unplayed().values_list('song__id', - flat=True) + song_request = apps.get_model(app_label='profiles', + model_name='SongRequest') + requests = song_request.music.unplayed().values_list('song__id', + flat=True) return self.playable().exclude(id__in=requests) def get_random_requestable_song(self): - """ + ''' Pick a random requestable song and return it. - """ + ''' return self.requestable()[randint(0, self.requestable().count() - 1)] def get_random_jingle(self): - """ + ''' Pick a random jingle and return it. - """ + ''' random_index = randint(0, self.available_jingles().count() - 1) return self.available_jingles()[random_index] diff --git a/savepointradio/radio/migrations/0004_new_song_path_structure.py b/savepointradio/radio/migrations/0004_new_song_path_structure.py new file mode 100644 index 0000000..691ed25 --- /dev/null +++ b/savepointradio/radio/migrations/0004_new_song_path_structure.py @@ -0,0 +1,49 @@ +# Generated by Django 2.2.1 on 2019-05-31 03:00 + +import django.core.validators +from django.db import migrations, models +import django.db.models.deletion +import radio.fields + + +class Migration(migrations.Migration): + + dependencies = [ + ('radio', '0003_song_next_play'), + ] + + operations = [ + migrations.CreateModel( + name='Store', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('created_date', models.DateTimeField(auto_now_add=True, verbose_name='added on')), + ('modified_date', models.DateTimeField(auto_now=True, verbose_name='last modified')), + ('iri', radio.fields.RadioIRIField()), + ('mime_type', models.CharField(blank=True, max_length=64, verbose_name='file MIME type')), + ('file_size', models.BigIntegerField(blank=True, null=True, validators=[django.core.validators.MinValueValidator(0)], verbose_name='file size')), + ('length', models.DecimalField(blank=True, decimal_places=2, max_digits=8, null=True, verbose_name='song length (in seconds)')), + ], + options={ + 'abstract': False, + }, + ), + migrations.RemoveField( + model_name='song', + name='length', + ), + migrations.RemoveField( + model_name='song', + name='path', + ), + migrations.AddField( + model_name='song', + name='current_store', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='current_of', to='radio.Store'), + ), + migrations.AddField( + model_name='song', + name='stores', + field=models.ManyToManyField(blank=True, related_name='song', to='radio.Store'), + ), + ] diff --git a/savepointradio/radio/models.py b/savepointradio/radio/models.py index 7b0cd0d..0610577 100644 --- a/savepointradio/radio/models.py +++ b/savepointradio/radio/models.py @@ -1,13 +1,19 @@ +''' +Django Models for the Radio application. +''' + from datetime import timedelta from decimal import getcontext, Decimal, ROUND_UP from django.apps import apps +from django.core.validators import MinValueValidator from django.db import models from django.utils import timezone from django.utils.translation import ugettext_lazy as _ from core.behaviors import Disableable, Publishable, Timestampable from core.utils import get_setting +from .fields import RadioIRIField from .managers import RadioManager, SongManager @@ -16,9 +22,9 @@ getcontext().prec = 16 class Album(Disableable, Publishable, Timestampable, models.Model): - """ + ''' A model for a music album. - """ + ''' title = models.CharField(_('title'), max_length=255, unique=True) sorted_title = models.CharField(_('naturalized title'), @@ -37,9 +43,9 @@ class Album(Disableable, Publishable, Timestampable, models.Model): class Artist(Disableable, Publishable, Timestampable, models.Model): - """ + ''' A model for a music artist. - """ + ''' alias = models.CharField(_('alias'), max_length=127, blank=True) first_name = models.CharField(_('first name'), max_length=127, blank=True) last_name = models.CharField(_('last name'), max_length=127, blank=True) @@ -57,10 +63,10 @@ class Artist(Disableable, Publishable, Timestampable, models.Model): @property def full_name(self): - """ + ''' String representing the artist's full name including an alias, if available. - """ + ''' if self.alias: if self.first_name or self.last_name: return '{} "{}" {}'.format(self.first_name, @@ -74,9 +80,9 @@ class Artist(Disableable, Publishable, Timestampable, models.Model): class Game(Disableable, Publishable, Timestampable, models.Model): - """ + ''' A model for a game. - """ + ''' title = models.CharField(_('title'), max_length=255, unique=True) sorted_title = models.CharField(_('naturalized title'), @@ -94,10 +100,29 @@ class Game(Disableable, Publishable, Timestampable, models.Model): return self.title +class Store(Timestampable, models.Model): + ''' + A model to represent various data locations (stores) for the song. + ''' + iri = RadioIRIField() + mime_type = models.CharField(_('file MIME type'), + max_length=64, + blank=True) + file_size = models.BigIntegerField(_('file size'), + validators=[MinValueValidator(0)], + blank=True, + null=True) + length = models.DecimalField(_('song length (in seconds)'), + max_digits=8, + decimal_places=2, + null=True, + blank=True) + + class Song(Disableable, Publishable, Timestampable, models.Model): - """ + ''' A model for a song. - """ + ''' JINGLE = 'J' SONG = 'S' TYPE_CHOICES = ( @@ -128,13 +153,12 @@ class Song(Disableable, Publishable, Timestampable, models.Model): null=True, blank=True, editable=False) - length = models.DecimalField(_('song length (in seconds)'), - max_digits=8, - decimal_places=2, - null=True, - blank=True) - path = models.TextField(_('absolute path to song file')) - + stores = models.ManyToManyField(Store, blank=True, related_name='song') + current_store = models.ForeignKey(Store, + on_delete=models.SET_NULL, + null=True, + blank=True, + related_name='current_of') sorted_title = models.CharField(_('naturalized title'), db_index=True, editable=False, @@ -147,34 +171,34 @@ class Song(Disableable, Publishable, Timestampable, models.Model): ordering = ['sorted_title', ] def _is_jingle(self): - """ + ''' Is the object a jingle? - """ + ''' return self.song_type == 'J' _is_jingle.boolean = True is_jingle = property(_is_jingle) def _is_song(self): - """ + ''' Is the object a song? - """ + ''' return self.song_type == 'S' _is_song.boolean = True is_song = property(_is_song) def _is_available(self): - """ + ''' Is the object both enabled and published? - """ + ''' return self._is_enabled() and self._is_published() _is_available.boolean = True is_available = property(_is_available) def _full_title(self): - """ + ''' String representing the entire song title, including the game and artists involved. - """ + ''' if self._is_song(): enabled_artists = self.artists.all().filter(disabled=False) all_artists = ', '.join([a.full_name for a in enabled_artists]) @@ -185,9 +209,9 @@ class Song(Disableable, Publishable, Timestampable, models.Model): full_title = property(_full_title) def _average_rating(self): - """ + ''' Decimal number of the average rating of a song from 1 - 5. - """ + ''' ratings = self.rating_set.all() if ratings: avg = Decimal(ratings.aggregate(avg=models.Avg('value'))['avg']) @@ -196,9 +220,9 @@ class Song(Disableable, Publishable, Timestampable, models.Model): average_rating = property(_average_rating) def get_time_until_requestable(self): - """ + ''' Length of time before a song can be requested again. - """ + ''' if self._is_song() and self._is_available(): if self.last_played: allowed_datetime = Song.music.datetime_from_wait() @@ -209,9 +233,9 @@ class Song(Disableable, Publishable, Timestampable, models.Model): return None def get_date_when_requestable(self, last_play=None): - """ + ''' Datetime when a song can be requested again. - """ + ''' last = self.last_played if last_play is None else last_play if self._is_song() and self._is_available(): @@ -232,10 +256,10 @@ class Song(Disableable, Publishable, Timestampable, models.Model): return None def _is_playable(self): - """ + ''' Is the song available and not been played within the default waiting period (or at all)? - """ + ''' if self._is_song() and self._is_available(): return self.get_date_when_requestable() <= timezone.now() return False @@ -243,14 +267,14 @@ class Song(Disableable, Publishable, Timestampable, models.Model): is_playable = property(_is_playable) def _is_requestable(self): - """ + ''' Is the song playable and has it not already been requested? - """ + ''' if self._is_playable(): - SongRequest = apps.get_model(app_label='profiles', - model_name='SongRequest') - requests = SongRequest.music.unplayed().values_list('song__id', - flat=True) + song_request = apps.get_model(app_label='profiles', + model_name='SongRequest') + requests = song_request.music.unplayed().values_list('song__id', + flat=True) return self.pk not in requests return False _is_requestable.boolean = True From d7360887cd9f776e9ec61e4b288b3e2ed0dd38c1 Mon Sep 17 00:00:00 2001 From: RecursiveGreen Date: Mon, 3 Jun 2019 11:01:32 -0400 Subject: [PATCH 05/30] Add docstring. --- savepointradio/savepointradio/settings.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/savepointradio/savepointradio/settings.py b/savepointradio/savepointradio/settings.py index 21be075..1f92390 100644 --- a/savepointradio/savepointradio/settings.py +++ b/savepointradio/savepointradio/settings.py @@ -1,3 +1,7 @@ +''' +Django settings file. +''' + import os from decouple import config From 4c7c2e0dc333d0c9857b3a9b7887639ab2340065 Mon Sep 17 00:00:00 2001 From: RecursiveGreen Date: Mon, 3 Jun 2019 11:29:18 -0400 Subject: [PATCH 06/30] Add string representation for Store. --- savepointradio/radio/models.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/savepointradio/radio/models.py b/savepointradio/radio/models.py index 0610577..d2d6187 100644 --- a/savepointradio/radio/models.py +++ b/savepointradio/radio/models.py @@ -118,6 +118,9 @@ class Store(Timestampable, models.Model): null=True, blank=True) + def __str__(self): + return self.iri + class Song(Disableable, Publishable, Timestampable, models.Model): ''' From c48f848bbacd859d0ced1aeefba80f943a53a5ad Mon Sep 17 00:00:00 2001 From: RecursiveGreen Date: Mon, 3 Jun 2019 14:37:28 -0400 Subject: [PATCH 07/30] Comment cleanup with proper RFC numbers. --- savepointradio/core/utils.py | 77 +++++++++++++++++++++++++++++------- 1 file changed, 63 insertions(+), 14 deletions(-) diff --git a/savepointradio/core/utils.py b/savepointradio/core/utils.py index 920c53e..f9af9b0 100644 --- a/savepointradio/core/utils.py +++ b/savepointradio/core/utils.py @@ -1,25 +1,42 @@ +''' +Various utlity functions that are independant of any Django app or +model. +''' + +from nturl2path import pathname2url as ntpathname2url +from nturl2path import url2pathname as url2ntpathname import random import re import string from unicodedata import normalize +from urllib.parse import urljoin, urlparse +from urllib.request import pathname2url, url2pathname from django.core.exceptions import ObjectDoesNotExist from django.db import connection +from django.utils.encoding import iri_to_uri, uri_to_iri from .models import Setting def generate_password(length=32): + ''' + Quick and dirty random password generator. + + ***WARNING*** - Although this is likely "good enough" to create a secure + password, there are no validations (suitible entropy, dictionary words, + etc.) and should not be completely trusted. YOU HAVE BEEN WARNED. + ''' chars = string.ascii_letters + string.digits + string.punctuation rng = random.SystemRandom() return ''.join([rng.choice(chars) for i in range(length)]) def get_len(rawqueryset): - """ + ''' Adds/Overrides a dynamic implementation of the length protocol to the definition of RawQuerySet. - """ + ''' def __len__(self): params = ['{}'.format(p) for p in self.params] sql = ''.join(('SELECT COUNT(*) FROM (', @@ -33,11 +50,13 @@ def get_len(rawqueryset): def get_setting(name): + '''Helper function to get dynamic settings from the database.''' setting = Setting.objects.get(name=name) return setting.get() def set_setting(name, value, setting_type=None): + '''Helper function to set dynamic settings from the database.''' setting_types = {'Integer': 0, 'Float': 1, 'String': 2, 'Bool': 3} try: setting = Setting.objects.get(name=name) @@ -57,13 +76,13 @@ def set_setting(name, value, setting_type=None): def naturalize(text): - """ + ''' Return a normalized unicode string, with removed starting articles, for use in natural sorting. Code was inspired by 'django-naturalsortfield' from Nathan Reynolds: https://github.com/nathforge/django-naturalsortfield - """ + ''' def naturalize_int_match(match): return '{:08d}'.format(int(match.group(0))) @@ -79,9 +98,9 @@ def naturalize(text): def quantify(quantity, model): - """ + ''' A message based on the quantity and singular/plural name of the model. - """ + ''' if quantity == 1: message = '1 {}'.format(model._meta.verbose_name) else: @@ -92,21 +111,20 @@ def quantify(quantity, model): def create_success_message(parent_model, parent_quantity, child_model, child_quantity, remove=False): - """ + ''' Creates a message for displaying the success of model modification. - """ + ''' p_message = quantify(parent_quantity, parent_model) c_message = quantify(child_quantity, child_model) if remove: return '{} successfully removed from {}'.format(c_message, p_message) - else: - return '{} successfully added to {}.'.format(c_message, p_message) + return '{} successfully added to {}.'.format(c_message, p_message) def get_pretty_time(seconds): - """ + ''' Displays a human-readable representation of time. - """ + ''' if seconds > 0: periods = [ ('year', 60*60*24*365.25), @@ -123,5 +141,36 @@ def get_pretty_time(seconds): period_name, ('s', '')[period_value == 1])) return ', '.join(strings) - else: - return 'Now' + return 'Now' + + +def path_to_iri(path): + ''' + OS-independant attempt at converting any OS absolute path to an + RFC3987-defined IRI along with the file scheme from RFC8089. + ''' + # Looking to see if the path starts with a drive letter or UNC path + # (eg. 'D:\' or '\\') + windows = re.match(r'^(?:[A-Za-z]:|\\)\\', path) + if windows: + return uri_to_iri(urljoin('file:', ntpathname2url(path))) + return uri_to_iri(urljoin('file:', pathname2url(path))) + + +def iri_to_path(iri): + ''' + OS-independant attempt at converting an RFC3987-defined IRI with a file + scheme from RFC8089 to an OS-specific absolute path. + ''' + # Drive letter IRI will have three slashes followed by the drive letter + # UNC path IRI will have two slashes followed by the UNC path + uri = iri_to_uri(iri) + patt = r'^(?:file:///[A-Za-z]:/|file://[A-Za-z0-9!@#$%^&\'\)\(\.\-_{}~]+/)' + windows = re.match(patt, uri) + if windows: + parse = urlparse(uri) + # UNC path URIs put the server name in the 'netloc' parameter. + if parse.netloc: + return '\\' + url2ntpathname('/' + parse.netloc + parse.path) + return url2ntpathname(parse.path) + return url2pathname(urlparse(uri).path) From eed7aee5960430ef2bc416e75717f29b27ef6fdb Mon Sep 17 00:00:00 2001 From: RecursiveGreen Date: Mon, 3 Jun 2019 14:41:23 -0400 Subject: [PATCH 08/30] Temporarily disable custom ModelField for IRI. --- savepointradio/radio/fields.py | 7 +++++++ savepointradio/radio/forms.py | 7 +++++++ .../migrations/0004_new_song_path_structure.py | 5 ++--- savepointradio/radio/models.py | 14 ++++++++++++-- 4 files changed, 28 insertions(+), 5 deletions(-) diff --git a/savepointradio/radio/fields.py b/savepointradio/radio/fields.py index 8cc2b5d..ae70834 100644 --- a/savepointradio/radio/fields.py +++ b/savepointradio/radio/fields.py @@ -18,6 +18,13 @@ class RadioIRIField(models.TextField): https://code.djangoproject.com/ticket/25594 https://stackoverflow.com/questions/41756572/ ''' + + # TODO: Because of a shortcoming of Django's URLValidator code, the + # 'file://' scheme does not validate properly on most cases due to + # incompatibilities with optional hostnames. Disabling the custom field + # for now until I can figure out a non-lethal way of handling this. + # https://code.djangoproject.com/ticket/25595 + default_validators = [validators.URLValidator(schemes=ALLOWED_SCHEMES)] description = _("Long IRI") diff --git a/savepointradio/radio/forms.py b/savepointradio/radio/forms.py index 48c0667..0ae9eb8 100644 --- a/savepointradio/radio/forms.py +++ b/savepointradio/radio/forms.py @@ -18,4 +18,11 @@ class RadioIRIFormField(URLField): https://code.djangoproject.com/ticket/25594 https://stackoverflow.com/questions/41756572/ ''' + + # TODO: Because of a shortcoming of Django's URLValidator code, the + # 'file://' scheme does not validate properly on most cases due to + # incompatibilities with optional hostnames. Disabling the custom field + # for now until I can figure out a non-lethal way of handling this. + # https://code.djangoproject.com/ticket/25595 + default_validators = [validators.URLValidator(schemes=ALLOWED_SCHEMES)] diff --git a/savepointradio/radio/migrations/0004_new_song_path_structure.py b/savepointradio/radio/migrations/0004_new_song_path_structure.py index 691ed25..8ba7077 100644 --- a/savepointradio/radio/migrations/0004_new_song_path_structure.py +++ b/savepointradio/radio/migrations/0004_new_song_path_structure.py @@ -1,9 +1,8 @@ -# Generated by Django 2.2.1 on 2019-05-31 03:00 +# Generated by Django 2.2.1 on 2019-06-03 18:39 import django.core.validators from django.db import migrations, models import django.db.models.deletion -import radio.fields class Migration(migrations.Migration): @@ -19,7 +18,7 @@ class Migration(migrations.Migration): ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), ('created_date', models.DateTimeField(auto_now_add=True, verbose_name='added on')), ('modified_date', models.DateTimeField(auto_now=True, verbose_name='last modified')), - ('iri', radio.fields.RadioIRIField()), + ('iri', models.TextField(verbose_name='IRI path to song file')), ('mime_type', models.CharField(blank=True, max_length=64, verbose_name='file MIME type')), ('file_size', models.BigIntegerField(blank=True, null=True, validators=[django.core.validators.MinValueValidator(0)], verbose_name='file size')), ('length', models.DecimalField(blank=True, decimal_places=2, max_digits=8, null=True, verbose_name='song length (in seconds)')), diff --git a/savepointradio/radio/models.py b/savepointradio/radio/models.py index d2d6187..9d020f8 100644 --- a/savepointradio/radio/models.py +++ b/savepointradio/radio/models.py @@ -13,7 +13,7 @@ from django.utils.translation import ugettext_lazy as _ from core.behaviors import Disableable, Publishable, Timestampable from core.utils import get_setting -from .fields import RadioIRIField +# from .fields import RadioIRIField from .managers import RadioManager, SongManager @@ -104,7 +104,17 @@ class Store(Timestampable, models.Model): ''' A model to represent various data locations (stores) for the song. ''' - iri = RadioIRIField() + # TODO: Because of a shortcoming of Django's URLValidator code, the + # 'file://' scheme does not validate properly on most cases due to + # incompatibilities with optional hostnames. Disabling the custom field + # for now until I can figure out a non-lethal way of handling this. + # https://code.djangoproject.com/ticket/25594 + # https://code.djangoproject.com/ticket/25595 + # https://stackoverflow.com/questions/41756572/ + + # iri = RadioIRIField() + + iri = models.TextField(_('IRI path to song file')) mime_type = models.CharField(_('file MIME type'), max_length=64, blank=True) From ba1f6f05da6a23122b9232038fc067bb9d52b465 Mon Sep 17 00:00:00 2001 From: RecursiveGreen Date: Mon, 3 Jun 2019 15:02:43 -0400 Subject: [PATCH 09/30] Fixed total variables. --- savepointradio/radio/management/commands/importoldradio.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/savepointradio/radio/management/commands/importoldradio.py b/savepointradio/radio/management/commands/importoldradio.py index 60015e2..c9c6e58 100644 --- a/savepointradio/radio/management/commands/importoldradio.py +++ b/savepointradio/radio/management/commands/importoldradio.py @@ -42,7 +42,7 @@ class Command(BaseCommand): for album in playlist['albums']: Album.objects.create(title=album['title'], disabled=album['disabled']) - total_albums += 1 + totals['albums'] += 1 self.stdout.write('Imported {} albums'.format(str(totals['albums']))) @@ -52,7 +52,7 @@ class Command(BaseCommand): first_name=artist['first_name'] or '', last_name=artist['last_name'] or '', disabled=artist['disabled']) - total_artists += 1 + totals['artists'] += 1 self.stdout.write('Imported {} artists'.format(str(totals['artists']))) @@ -60,7 +60,7 @@ class Command(BaseCommand): for game in playlist['games']: Game.objects.create(title=game['title'], disabled=game['disabled']) - total_games += 1 + totals['games'] += 1 self.stdout.write('Imported {} games'.format(str(totals['games']))) From 3afc5c7a3a6350cd0df13a3eb90e84ade03d36df Mon Sep 17 00:00:00 2001 From: RecursiveGreen Date: Mon, 3 Jun 2019 15:03:23 -0400 Subject: [PATCH 10/30] Implemented Store into admin pages. --- savepointradio/radio/admin.py | 50 +++++++++++++++++++++++++++++------ 1 file changed, 42 insertions(+), 8 deletions(-) diff --git a/savepointradio/radio/admin.py b/savepointradio/radio/admin.py index e5fefa4..2a10ede 100644 --- a/savepointradio/radio/admin.py +++ b/savepointradio/radio/admin.py @@ -3,7 +3,7 @@ from django.db import models from django.forms import TextInput from .actions import change_items, publish_items, remove_items -from .models import Album, Artist, Game, Song +from .models import Album, Artist, Game, Song, Store class ArtistInline(admin.TabularInline): @@ -13,6 +13,13 @@ class ArtistInline(admin.TabularInline): extra = 0 +class StoreInline(admin.TabularInline): + model = Song.stores.through + verbose_name = 'data store' + verbose_name_plural = 'data stores' + extra = 0 + + @admin.register(Album) class AlbumAdmin(admin.ModelAdmin): # Detail List display @@ -101,6 +108,28 @@ class GameAdmin(admin.ModelAdmin): publish_games.short_description = "Publish selected games" +@admin.register(Store) +class StoreAdmin(admin.ModelAdmin): + # Detail List display + list_display = ('iri', + 'mime_type', + 'file_size', + 'length') + search_fields = ['iri'] + + # Edit Form display + readonly_fields = ('created_date', 'modified_date') + fieldsets = ( + ('Main', { + 'fields': ('iri', 'mime_type', 'file_size', 'length') + }), + ('Stats', { + 'classes': ('collapse',), + 'fields': ('created_date', 'modified_date') + }) + ) + + @admin.register(Song) class SongAdmin(admin.ModelAdmin): formfield_overrides = { @@ -123,8 +152,7 @@ class SongAdmin(admin.ModelAdmin): # Edit Form display exclude = ('artists',) - readonly_fields = ('length', - 'last_played', + readonly_fields = ('last_played', 'num_played', 'created_date', 'modified_date', @@ -137,8 +165,8 @@ class SongAdmin(admin.ModelAdmin): ('Main', { 'fields': ('song_type', 'title', - 'path', - 'published_date') + 'published_date', + 'current_store') }), ('Stats', { 'classes': ('collapse',), @@ -146,8 +174,7 @@ class SongAdmin(admin.ModelAdmin): 'modified_date', 'last_played', 'num_played', - 'next_play', - 'length') + 'next_play') }), ('Album', { 'fields': ('album',) @@ -156,7 +183,14 @@ class SongAdmin(admin.ModelAdmin): 'fields': ('game',) }) ) - inlines = [ArtistInline] + inlines = [ArtistInline, StoreInline] + + def formfield_for_foreignkey(self, db_field, request, **kwargs): + if db_field.name == 'current_store': + kwargs['queryset'] = Store.objects.filter( + song__pk=request.resolver_match.kwargs['object_id'] + ) + return super().formfield_for_foreignkey(db_field, request, **kwargs) def artist_list(self, obj): return ', '.join([a.full_name for a in obj.artists.all()]) From c305a32717fd4b4b0d6657f0316c1d0f061a4e57 Mon Sep 17 00:00:00 2001 From: RecursiveGreen Date: Tue, 4 Jun 2019 12:03:12 -0400 Subject: [PATCH 11/30] Updated API for new Stores and general cleanup. --- savepointradio/api/serializers/profiles.py | 6 +- savepointradio/api/serializers/radio.py | 80 ++++++++++++++++++---- savepointradio/api/views/profiles.py | 6 +- savepointradio/api/views/radio.py | 26 ++++--- 4 files changed, 91 insertions(+), 27 deletions(-) diff --git a/savepointradio/api/serializers/profiles.py b/savepointradio/api/serializers/profiles.py index f61db15..5ab8cdb 100644 --- a/savepointradio/api/serializers/profiles.py +++ b/savepointradio/api/serializers/profiles.py @@ -4,7 +4,7 @@ from rest_framework.serializers import (IntegerField, ModelSerializer, Serializer) from profiles.models import RadioProfile, SongRequest, Rating -from .radio import BasicSongRetrieveSerializer +from .radio import SongMinimalSerializer User = get_user_model() @@ -44,7 +44,7 @@ class RateSongSerializer(Serializer): class HistorySerializer(ModelSerializer): profile = BasicProfileSerializer() - song = BasicSongRetrieveSerializer() + song = SongMinimalSerializer() class Meta: model = SongRequest @@ -52,7 +52,7 @@ class HistorySerializer(ModelSerializer): class BasicProfileRatingsSerializer(ModelSerializer): - song = BasicSongRetrieveSerializer() + song = SongMinimalSerializer() class Meta: model = Rating diff --git a/savepointradio/api/serializers/radio.py b/savepointradio/api/serializers/radio.py index 09f64a2..f6a7bab 100644 --- a/savepointradio/api/serializers/radio.py +++ b/savepointradio/api/serializers/radio.py @@ -1,71 +1,127 @@ -from rest_framework.serializers import (IntegerField, ListField, +from rest_framework.serializers import (DecimalField, IntegerField, ListField, ModelSerializer, Serializer, + SerializerMethodField, StringRelatedField) +from core.utils import iri_to_path from radio.models import Album, Artist, Game, Song class AlbumSerializer(ModelSerializer): + '''A base serializer for an album model.''' class Meta: model = Album fields = ('id', 'title') class ArtistSerializer(ModelSerializer): + '''A base serializer for an artist model.''' class Meta: model = Artist fields = ('id', 'alias', 'first_name', 'last_name') class ArtistFullnameSerializer(ModelSerializer): + ''' + A base serializer for an artist model, but combining all name + attributes into one field. + ''' class Meta: model = Artist fields = ('id', 'full_name') class GameSerializer(ModelSerializer): + '''A base serializer for a game model.''' class Meta: model = Game fields = ('id', 'title') -class BasicSongSerializer(ModelSerializer): - class Meta: - model = Song - fields = ('id', 'album', 'artists', 'game', 'title', 'average_rating', - 'is_requestable') +class SongSerializer(ModelSerializer): + '''A base serializer for a song model.''' + length = DecimalField( + max_digits=10, + decimal_places=2, + source='current_store.length' + ) - -class FullSongSerializer(ModelSerializer): class Meta: model = Song fields = ('id', 'album', 'artists', 'published_date', 'game', - 'num_played', 'last_played', 'length', 'song_type', 'title', - 'average_rating', 'is_requestable') + 'num_played', 'last_played', 'length', 'next_play', + 'song_type', 'title', 'average_rating', 'is_requestable') -class BasicSongRetrieveSerializer(BasicSongSerializer): +class SongMinimalSerializer(ModelSerializer): + '''Minimal song information, usually appended to favorites/ratings.''' album = AlbumSerializer() artists = ArtistFullnameSerializer(many=True) game = GameSerializer() + class Meta: + model = Song + fields = ('id', 'album', 'artists', 'game', 'title') -class FullSongRetrieveSerializer(FullSongSerializer): + +class SongListSerializer(ModelSerializer): + '''Song information used in large listings.''' + album = AlbumSerializer() + artists = ArtistFullnameSerializer(many=True) + game = GameSerializer() + length = DecimalField( + max_digits=10, + decimal_places=2, + source='current_store.length' + ) + + class Meta: + model = Song + fields = ('id', 'album', 'artists', 'game', 'title', 'average_rating', + 'length', 'is_requestable') + + +class SongRetrieveSerializer(SongSerializer): + ''' + An almost complete listing of a song's information, based on a single + object retrieval. + ''' album = AlbumSerializer() artists = ArtistSerializer(many=True) game = GameSerializer() class RadioSongSerializer(ModelSerializer): + ''' + A song serializer that is specific to the radio DJ and the underlying + audio manipulation application. + ''' album = StringRelatedField() artists = StringRelatedField(many=True) game = StringRelatedField() + length = DecimalField( + max_digits=10, + decimal_places=2, + source='current_store.length' + ) + path = SerializerMethodField() class Meta: model = Song fields = ('album', 'artists', 'game', 'song_type', 'title', 'length', 'path') + def get_path(self, obj): + '''Converts the IRI into a filesystem path.''' + iri = str(obj.current_store.iri) + if iri.startswith('file://'): + return iri_to_path(iri) + return iri + class SongArtistsListSerializer(Serializer): + ''' + A serializer for adding or removing artists from a song based on + the song's id number. + ''' artists = ListField(child=IntegerField(), min_length=1, max_length=10) diff --git a/savepointradio/api/views/profiles.py b/savepointradio/api/views/profiles.py index 2b4e568..52fe553 100644 --- a/savepointradio/api/views/profiles.py +++ b/savepointradio/api/views/profiles.py @@ -11,7 +11,7 @@ from ..serializers.profiles import (BasicProfileSerializer, FullProfileSerializer, HistorySerializer, BasicProfileRatingsSerializer) -from ..serializers.radio import BasicSongRetrieveSerializer +from ..serializers.radio import SongListSerializer class ProfileViewSet(viewsets.ModelViewSet): @@ -52,10 +52,10 @@ class ProfileViewSet(viewsets.ModelViewSet): page = self.paginate_queryset(favorites) if page is not None: - serializer = BasicSongRetrieveSerializer(page, many=True) + serializer = SongListSerializer(page, many=True) return self.get_paginated_response(serializer.data) - serializer = BasicSongRetrieveSerializer(favorites, many=True) + serializer = SongListSerializer(favorites, many=True) return Response(serializer.data) @action(detail=True, permission_classes=[AllowAny]) diff --git a/savepointradio/api/views/radio.py b/savepointradio/api/views/radio.py index 4287fb0..80ac724 100644 --- a/savepointradio/api/views/radio.py +++ b/savepointradio/api/views/radio.py @@ -10,9 +10,9 @@ from ..serializers.profiles import (BasicProfileSerializer, BasicSongRatingsSerializer, RateSongSerializer) from ..serializers.radio import (AlbumSerializer, ArtistSerializer, - GameSerializer, FullSongSerializer, - SongArtistsListSerializer, - FullSongRetrieveSerializer) + GameSerializer, SongSerializer, + SongListSerializer, SongRetrieveSerializer, + SongArtistsListSerializer) class AlbumViewSet(viewsets.ModelViewSet): @@ -83,9 +83,11 @@ class SongViewSet(viewsets.ModelViewSet): (Thanks to https://stackoverflow.com/questions/22616973/) ''' - if self.action in ['list', 'retrieve']: - return FullSongRetrieveSerializer - return FullSongSerializer + if self.action == 'list': + return SongListSerializer + if self.action == 'retrieve': + return SongRetrieveSerializer + return SongSerializer def _artists_change(self, request, remove=False): song = self.get_object() @@ -101,20 +103,21 @@ class SongViewSet(viewsets.ModelViewSet): message = 'Artists {} song.'.format(('added to', 'removed from')[remove]) return Response({'detail': message}) - else: - return Response(serializer.errors, - status=status.HTTP_400_BAD_REQUEST) + return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST) @action(methods=['post'], detail=True, permission_classes=[IsAdminUser]) def artists_add(self, request, pk=None): + '''Adds an artist to a song.''' return self._artists_change(request) @action(methods=['post'], detail=True, permission_classes=[IsAdminUser]) def artists_remove(self, request, pk=None): + '''Removes an artist from a song.''' return self._artists_change(request, remove=True) @action(detail=True, permission_classes=[AllowAny]) def favorites(self, request, pk=None): + '''Get a list of users who added this song to their favorites list.''' song = self.get_object() profiles = song.song_favorites.all().order_by('user__name') @@ -130,6 +133,7 @@ class SongViewSet(viewsets.ModelViewSet): detail=True, permission_classes=[IsAuthenticatedAndNotDJ]) def favorite(self, request, pk=None): + '''Add a song to the user's favorites list.''' song = self.get_object() profile = RadioProfile.objects.get(user=request.user) if song not in profile.favorites.all(): @@ -144,6 +148,7 @@ class SongViewSet(viewsets.ModelViewSet): detail=True, permission_classes=[IsAuthenticatedAndNotDJ]) def unfavorite(self, request, pk=None): + '''Remove a song from the user's favorites list.''' song = self.get_object() profile = RadioProfile.objects.get(user=request.user) if song in profile.favorites.all(): @@ -157,6 +162,7 @@ class SongViewSet(viewsets.ModelViewSet): @action(detail=True, permission_classes=[AllowAny]) def ratings(self, request, pk=None): + '''Get a list of a song's ratings.''' song = self.get_object() ratings = song.rating_set.all().order_by('-created_date') @@ -172,6 +178,7 @@ class SongViewSet(viewsets.ModelViewSet): detail=True, permission_classes=[IsAuthenticatedAndNotDJ]) def rate(self, request, pk=None): + '''Add a user's rating to a song.''' serializer = RateSongSerializer(data=request.data) if serializer.is_valid(): song = self.get_object() @@ -195,6 +202,7 @@ class SongViewSet(viewsets.ModelViewSet): detail=True, permission_classes=[IsAuthenticatedAndNotDJ]) def unrate(self, request, pk=None): + '''Remove a user's rating from a song.''' song = self.get_object() profile = RadioProfile.objects.get(user=request.user) rating = song.rating_set.filter(profile=profile) From c428a5a3e9f2d0b9d62a539070509eebe131d431 Mon Sep 17 00:00:00 2001 From: RecursiveGreen Date: Tue, 4 Jun 2019 12:22:58 -0400 Subject: [PATCH 12/30] Add Store Serializer and changed related_name. --- savepointradio/api/serializers/radio.py | 25 +++++++++++++++---- savepointradio/radio/admin.py | 4 +-- .../management/commands/importoldradio.py | 2 +- savepointradio/radio/managers.py | 2 +- .../0004_new_song_path_structure.py | 4 +-- savepointradio/radio/models.py | 10 ++++---- 6 files changed, 31 insertions(+), 16 deletions(-) diff --git a/savepointradio/api/serializers/radio.py b/savepointradio/api/serializers/radio.py index f6a7bab..1c2b67c 100644 --- a/savepointradio/api/serializers/radio.py +++ b/savepointradio/api/serializers/radio.py @@ -4,7 +4,7 @@ from rest_framework.serializers import (DecimalField, IntegerField, ListField, StringRelatedField) from core.utils import iri_to_path -from radio.models import Album, Artist, Game, Song +from radio.models import Album, Artist, Game, Song, Store class AlbumSerializer(ModelSerializer): @@ -38,12 +38,27 @@ class GameSerializer(ModelSerializer): fields = ('id', 'title') +class StoreSerializer(ModelSerializer): + '''A base serializer for a data store model.''' + active = SerializerMethodField() + + class Meta: + model = Store + fields = ('id', 'iri', 'file_size', 'length', 'mime_type') + + def get_active(self, obj): + '''Checks to see if this store is active for a song.''' + if obj.active_for: + return True + return False + + class SongSerializer(ModelSerializer): '''A base serializer for a song model.''' length = DecimalField( max_digits=10, decimal_places=2, - source='current_store.length' + source='active_store.length' ) class Meta: @@ -72,7 +87,7 @@ class SongListSerializer(ModelSerializer): length = DecimalField( max_digits=10, decimal_places=2, - source='current_store.length' + source='active_store.length' ) class Meta: @@ -102,7 +117,7 @@ class RadioSongSerializer(ModelSerializer): length = DecimalField( max_digits=10, decimal_places=2, - source='current_store.length' + source='active_store.length' ) path = SerializerMethodField() @@ -113,7 +128,7 @@ class RadioSongSerializer(ModelSerializer): def get_path(self, obj): '''Converts the IRI into a filesystem path.''' - iri = str(obj.current_store.iri) + iri = str(obj.active_store.iri) if iri.startswith('file://'): return iri_to_path(iri) return iri diff --git a/savepointradio/radio/admin.py b/savepointradio/radio/admin.py index 2a10ede..6c5b006 100644 --- a/savepointradio/radio/admin.py +++ b/savepointradio/radio/admin.py @@ -166,7 +166,7 @@ class SongAdmin(admin.ModelAdmin): 'fields': ('song_type', 'title', 'published_date', - 'current_store') + 'active_store') }), ('Stats', { 'classes': ('collapse',), @@ -186,7 +186,7 @@ class SongAdmin(admin.ModelAdmin): inlines = [ArtistInline, StoreInline] def formfield_for_foreignkey(self, db_field, request, **kwargs): - if db_field.name == 'current_store': + if db_field.name == 'active_store': kwargs['queryset'] = Store.objects.filter( song__pk=request.resolver_match.kwargs['object_id'] ) diff --git a/savepointradio/radio/management/commands/importoldradio.py b/savepointradio/radio/management/commands/importoldradio.py index c9c6e58..0d0915b 100644 --- a/savepointradio/radio/management/commands/importoldradio.py +++ b/savepointradio/radio/management/commands/importoldradio.py @@ -97,7 +97,7 @@ class Command(BaseCommand): length=song['store']['length'] ) new_song.stores.add(new_store) - new_song.current_store = new_store + new_song.active_store = new_store new_song.save() if song['type'] == 'S': totals['songs'] += 1 diff --git a/savepointradio/radio/managers.py b/savepointradio/radio/managers.py index 0d6dda7..41ae5e8 100644 --- a/savepointradio/radio/managers.py +++ b/savepointradio/radio/managers.py @@ -88,7 +88,7 @@ class SongManager(RadioManager): ''' a_songs = self.available_songs() length = a_songs.aggregate( - total_time=models.Sum('current_store__length') + total_time=models.Sum('active_store__length') ) return length['total_time'] diff --git a/savepointradio/radio/migrations/0004_new_song_path_structure.py b/savepointradio/radio/migrations/0004_new_song_path_structure.py index 8ba7077..5ae3fd0 100644 --- a/savepointradio/radio/migrations/0004_new_song_path_structure.py +++ b/savepointradio/radio/migrations/0004_new_song_path_structure.py @@ -37,8 +37,8 @@ class Migration(migrations.Migration): ), migrations.AddField( model_name='song', - name='current_store', - field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='current_of', to='radio.Store'), + name='active_store', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='active_for', to='radio.Store'), ), migrations.AddField( model_name='song', diff --git a/savepointradio/radio/models.py b/savepointradio/radio/models.py index 9d020f8..1704e26 100644 --- a/savepointradio/radio/models.py +++ b/savepointradio/radio/models.py @@ -167,11 +167,11 @@ class Song(Disableable, Publishable, Timestampable, models.Model): blank=True, editable=False) stores = models.ManyToManyField(Store, blank=True, related_name='song') - current_store = models.ForeignKey(Store, - on_delete=models.SET_NULL, - null=True, - blank=True, - related_name='current_of') + active_store = models.ForeignKey(Store, + on_delete=models.SET_NULL, + null=True, + blank=True, + related_name='active_for') sorted_title = models.CharField(_('naturalized title'), db_index=True, editable=False, From 5317b4574879858206182d393f351f063ab1261d Mon Sep 17 00:00:00 2001 From: RecursiveGreen Date: Tue, 4 Jun 2019 12:43:04 -0400 Subject: [PATCH 13/30] Forgot to add 'active' to fields. --- savepointradio/api/serializers/radio.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/savepointradio/api/serializers/radio.py b/savepointradio/api/serializers/radio.py index 1c2b67c..3313159 100644 --- a/savepointradio/api/serializers/radio.py +++ b/savepointradio/api/serializers/radio.py @@ -44,7 +44,7 @@ class StoreSerializer(ModelSerializer): class Meta: model = Store - fields = ('id', 'iri', 'file_size', 'length', 'mime_type') + fields = ('id', 'active', 'iri', 'file_size', 'length', 'mime_type') def get_active(self, obj): '''Checks to see if this store is active for a song.''' From 52ad3c8600b6379054d0867f2876a503011ad91f Mon Sep 17 00:00:00 2001 From: RecursiveGreen Date: Tue, 4 Jun 2019 14:26:09 -0400 Subject: [PATCH 14/30] Removed unused import. --- savepointradio/api/serializers/controls.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/savepointradio/api/serializers/controls.py b/savepointradio/api/serializers/controls.py index 4323e04..a05cbd7 100644 --- a/savepointradio/api/serializers/controls.py +++ b/savepointradio/api/serializers/controls.py @@ -1,7 +1,6 @@ from rest_framework.serializers import (IntegerField, ModelSerializer, - Serializer, - StringRelatedField) + Serializer) from profiles.models import SongRequest from .radio import RadioSongSerializer From 58dddd2d0d6b343a61f1f7968062657ac1c977ed Mon Sep 17 00:00:00 2001 From: RecursiveGreen Date: Tue, 4 Jun 2019 14:58:52 -0400 Subject: [PATCH 15/30] Stores now accessible through API. --- savepointradio/api/serializers/radio.py | 16 +++++- savepointradio/api/urls.py | 5 +- savepointradio/api/views/radio.py | 74 +++++++++++++++++++++++-- 3 files changed, 87 insertions(+), 8 deletions(-) diff --git a/savepointradio/api/serializers/radio.py b/savepointradio/api/serializers/radio.py index 3313159..a0fdaf4 100644 --- a/savepointradio/api/serializers/radio.py +++ b/savepointradio/api/serializers/radio.py @@ -1,4 +1,5 @@ -from rest_framework.serializers import (DecimalField, IntegerField, ListField, +from rest_framework.serializers import (BooleanField, DecimalField, + IntegerField, ListField, ModelSerializer, Serializer, SerializerMethodField, StringRelatedField) @@ -48,7 +49,7 @@ class StoreSerializer(ModelSerializer): def get_active(self, obj): '''Checks to see if this store is active for a song.''' - if obj.active_for: + if obj.active_for.all(): return True return False @@ -139,4 +140,15 @@ class SongArtistsListSerializer(Serializer): A serializer for adding or removing artists from a song based on the song's id number. ''' + # TODO: Probably should move to PrimaryKeyRelatedField. artists = ListField(child=IntegerField(), min_length=1, max_length=10) + + +class SongStoresSerializer(Serializer): + ''' + A serializer for adding or removing a data store from a song based on + the song's id number. + ''' + # TODO: Probably should move to PrimaryKeyRelatedField. + store = IntegerField() + set_active = BooleanField(default=False) diff --git a/savepointradio/api/urls.py b/savepointradio/api/urls.py index 6049595..2c53d47 100644 --- a/savepointradio/api/urls.py +++ b/savepointradio/api/urls.py @@ -4,8 +4,8 @@ from rest_framework.routers import DefaultRouter from api.views.controls import JustPlayed, MakeRequest, NextRequest from api.views.profiles import HistoryViewSet, ProfileViewSet -from api.views.radio import (AlbumViewSet, ArtistViewSet, - GameViewSet, SongViewSet) +from api.views.radio import (AlbumViewSet, ArtistViewSet, GameViewSet, + StoreViewSet, SongViewSet) class OptionalSlashRouter(DefaultRouter): @@ -28,6 +28,7 @@ router.register(r'profiles', ProfileViewSet, base_name='profile') router.register(r'albums', AlbumViewSet, base_name='album') router.register(r'artists', ArtistViewSet, base_name='artist') router.register(r'games', GameViewSet, base_name='game') +router.register(r'stores', StoreViewSet, base_name='store') router.register(r'songs', SongViewSet, base_name='song') urlpatterns = [ diff --git a/savepointradio/api/views/radio.py b/savepointradio/api/views/radio.py index 80ac724..f6a1a52 100644 --- a/savepointradio/api/views/radio.py +++ b/savepointradio/api/views/radio.py @@ -4,15 +4,17 @@ from rest_framework.permissions import AllowAny, IsAdminUser from rest_framework.response import Response from profiles.models import RadioProfile, Rating -from radio.models import Album, Artist, Game, Song +from radio.models import Album, Artist, Game, Song, Store from ..permissions import IsAdminOrReadOnly, IsAuthenticatedAndNotDJ from ..serializers.profiles import (BasicProfileSerializer, BasicSongRatingsSerializer, RateSongSerializer) from ..serializers.radio import (AlbumSerializer, ArtistSerializer, - GameSerializer, SongSerializer, - SongListSerializer, SongRetrieveSerializer, - SongArtistsListSerializer) + GameSerializer, StoreSerializer, + SongSerializer, SongListSerializer, + SongRetrieveSerializer, + SongArtistsListSerializer, + SongStoresSerializer) class AlbumViewSet(viewsets.ModelViewSet): @@ -63,6 +65,12 @@ class GameViewSet(viewsets.ModelViewSet): return Game.music.available() +class StoreViewSet(viewsets.ModelViewSet): + queryset = Store.objects.all() + permission_classes = [IsAdminUser] + serializer_class = StoreSerializer + + class SongViewSet(viewsets.ModelViewSet): permission_classes = [IsAdminOrReadOnly] @@ -94,12 +102,18 @@ class SongViewSet(viewsets.ModelViewSet): serializer = SongArtistsListSerializer(data=request.data) if serializer.is_valid(): artists = Artist.objects.filter(pk__in=serializer.data['artists']) + for artist in artists: if remove: song.artists.remove(artist) else: song.artists.add(artist) + song.save() + + if song.artists.count() == 0: + song.disable('No artists specified for song.') + message = 'Artists {} song.'.format(('added to', 'removed from')[remove]) return Response({'detail': message}) @@ -115,6 +129,58 @@ class SongViewSet(viewsets.ModelViewSet): '''Removes an artist from a song.''' return self._artists_change(request, remove=True) + def _store_change(self, request, remove=False): + song = self.get_object() + serializer = SongStoresSerializer(data=request.data) + if serializer.is_valid(): + try: + store = Store.objects.get(pk=serializer.data['store']) + except Store.DoesNotExist: + return Response({'detail': 'Store does not exist.'}, + status=status.HTTP_400_BAD_REQUEST) + + if remove: + song.stores.remove(store) + else: + song.stores.add(store) + + if serializer.data['set_active'] and not remove: + song.active_store = store + + song.save() + + if song.stores.count() == 0: + song.disable('No stores specified for song.') + + message = 'Store {} song.'.format(('added to', + 'removed from')[remove]) + return Response({'detail': message}) + return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST) + + @action(methods=['post'], detail=True, permission_classes=[IsAdminUser]) + def store_add(self, request, pk=None): + '''Adds a data store to a song.''' + return self._store_change(request) + + @action(methods=['post'], detail=True, permission_classes=[IsAdminUser]) + def store_remove(self, request, pk=None): + '''Removes a data store from a song.''' + return self._store_change(request, remove=True) + + @action(detail=True, permission_classes=[IsAdminUser]) + def stores(self, request, pk=None): + '''Get a list of data stores associate with this song.''' + song = self.get_object() + stores = song.stores.all().order_by('-created_date') + + page = self.paginate_queryset(stores) + if page is not None: + serializer = StoreSerializer(page, many=True) + return self.get_paginated_response(serializer.data) + + serializer = StoreSerializer(stores, many=True) + return Response(serializer.data) + @action(detail=True, permission_classes=[AllowAny]) def favorites(self, request, pk=None): '''Get a list of users who added this song to their favorites list.''' From f8ff8fa4a862bab15284b6b9a462c8fc5c306fca Mon Sep 17 00:00:00 2001 From: RecursiveGreen Date: Wed, 5 Jun 2019 13:13:12 -0400 Subject: [PATCH 16/30] Fix typo. --- contrib/export_playlist/export_playlist.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/export_playlist/export_playlist.py b/contrib/export_playlist/export_playlist.py index 8d0f71b..f5cf2f7 100644 --- a/contrib/export_playlist/export_playlist.py +++ b/contrib/export_playlist/export_playlist.py @@ -20,7 +20,7 @@ import magic def scrub(text): ''' Forcing a Unicode NFC normalization to remove combining marks that mess - with certain Python fucntions. + with certain Python functions. ''' if text: return unicodedata.normalize('NFC', text) From 32ef9afae37377e54b9b14d99ef54a68f23090c5 Mon Sep 17 00:00:00 2001 From: RecursiveGreen Date: Wed, 5 Jun 2019 13:13:38 -0400 Subject: [PATCH 17/30] Enable file logging and code cleanup. --- contrib/djcontrol/djcontrol.py | 131 ++++++++++++++++++++++++--------- 1 file changed, 97 insertions(+), 34 deletions(-) diff --git a/contrib/djcontrol/djcontrol.py b/contrib/djcontrol/djcontrol.py index 9889cb6..b42ed17 100644 --- a/contrib/djcontrol/djcontrol.py +++ b/contrib/djcontrol/djcontrol.py @@ -8,6 +8,8 @@ when a song has been played. import argparse import json +import logging +from logging.handlers import RotatingFileHandler import requests @@ -25,6 +27,22 @@ HEADERS = { ANNOTATE = 'annotate:req_id="{}",type="{}",artist="{}",title="{}",game="{}":{}' +logging.basicConfig( + handlers=[ + RotatingFileHandler( + './song_requests.log', + maxBytes=1000000, + backupCount=5, + encoding='utf8' + ) + ], + level=logging.INFO, + format=('[%(asctime)s] [%(levelname)s]' + ' [%(name)s.%(funcName)s] === %(message)s'), + datefmt='%Y-%m-%dT%H:%M:%S' + ) +LOGGER = logging.getLogger('djcontrol') + def clean_quotes(unclean_string): ''' @@ -43,23 +61,12 @@ def beautify_artists(artists): return clean_quotes(output.join(artists)) -description = 'Lets the DJ control the radio.' - -parser = argparse.ArgumentParser(description=description) -subparsers = parser.add_subparsers(dest='command') - -parser_next = subparsers.add_parser('next', - help='Gets the next song from the radio.') -parser_played = subparsers.add_parser('played', - help='Tells the radio which song just played.') -parser_played.add_argument('request', - help='Song request ID number.', - nargs=1, - type=int) - -args = parser.parse_args() - -if args.command == 'next': +def next_request(): + ''' + Sends an HTTP[S] request to the radio web service to retrieve the next + requested song. + ''' + LOGGER.debug('Received command to get next song request.') try: r = requests.get(API_URL + 'next/', headers=HEADERS, @@ -67,14 +74,15 @@ if args.command == 'next': r.encoding = 'utf-8' r.raise_for_status() except requests.exceptions.HTTPError as errh: - print('Http Error: {}'.format(errh)) + LOGGER.error('Http Error: %s', errh) except requests.exceptions.ConnectionError as errc: - print('Error Connecting: {}'.format(errc)) + LOGGER.error('Error Connecting: %s', errc) except requests.exceptions.Timeout as errt: - print('Timeout Error: {}'.format(errt)) + LOGGER.error('Timeout Error: %s', errt) except requests.exceptions.RequestException as err: - print('Error: {}'.format(err)) + LOGGER.error('Error: %s', err) else: + LOGGER.debug('Received JSON response: %s', r.text) req = json.loads(r.text) song = req['song'] if song['song_type'] == 'J': @@ -85,15 +93,34 @@ if args.command == 'next': artist = beautify_artists(song['artists']) title = clean_quotes(song['title']) game = clean_quotes(song['game']) - print(ANNOTATE.format(req['id'], - song['song_type'], - artist, - title, - game, - song['path'])) -elif args.command == 'played': + LOGGER.info( + 'Req_ID: %s, Artist[s]: %s, Title: %s, Game: %s, Path: %s', + req['id'], + artist, + title, + game, + song['path'] + ) + annotate_string = ANNOTATE.format( + req['id'], + song['song_type'], + artist, + title, + game, + song['path'] + ) + LOGGER.debug(annotate_string) + print(annotate_string) + + +def just_played(request_id): + ''' + Sends an HTTP[S] request to the radio web service to let it know that a + song has been played. + ''' + LOGGER.debug('Received command to report a song was just played.') try: - req_played = json.dumps({'song_request': args.request[0]}) + req_played = json.dumps({'song_request': request_id}) r = requests.post(API_URL + 'played/', headers=HEADERS, data=req_played, @@ -101,11 +128,47 @@ elif args.command == 'played': r.encoding = 'utf-8' r.raise_for_status() except requests.exceptions.HTTPError as errh: - print('Http Error: {}'.format(errh)) - print(r.text) + LOGGER.error('Http Error: %s', errh) except requests.exceptions.ConnectionError as errc: - print('Error Connecting: {}'.format(errc)) + LOGGER.error('Error Connecting: %s', errc) except requests.exceptions.Timeout as errt: - print('Timeout Error: {}'.format(errt)) + LOGGER.error('Timeout Error: %s', errt) except requests.exceptions.RequestException as err: - print('Error: {}'.format(err)) + LOGGER.error('Error: %s', err) + else: + LOGGER.info('Req_ID: %s', request_id) + + +def main(): + '''Main loop of the program''' + description = 'Lets the DJ control the radio.' + + parser = argparse.ArgumentParser(description=description) + subparsers = parser.add_subparsers(dest='command') + + parser_next = subparsers.add_parser( + 'next', + help='Gets the next song from the radio.' + ) + + parser_played = subparsers.add_parser( + 'played', + help='Tells the radio which song just played.' + ) + parser_played.add_argument( + 'request', + help='Song request ID number.', + nargs=1, + type=int + ) + + args = parser.parse_args() + + if args.command == 'next': + next_request() + elif args.command == 'played': + just_played(args.request[0]) + + +if __name__ == '__main__': + main() From 789d58f1b9097268c3efdf89c2ace728aa096ac4 Mon Sep 17 00:00:00 2001 From: RecursiveGreen Date: Wed, 5 Jun 2019 15:28:35 -0400 Subject: [PATCH 18/30] Code cleanup. --- contrib/djcontrol/djcontrol.py | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/contrib/djcontrol/djcontrol.py b/contrib/djcontrol/djcontrol.py index b42ed17..c4c2c27 100644 --- a/contrib/djcontrol/djcontrol.py +++ b/contrib/djcontrol/djcontrol.py @@ -68,11 +68,9 @@ def next_request(): ''' LOGGER.debug('Received command to get next song request.') try: - r = requests.get(API_URL + 'next/', - headers=HEADERS, - timeout=5) - r.encoding = 'utf-8' - r.raise_for_status() + resp = requests.get(API_URL + 'next/', headers=HEADERS, timeout=5) + resp.encoding = 'utf-8' + resp.raise_for_status() except requests.exceptions.HTTPError as errh: LOGGER.error('Http Error: %s', errh) except requests.exceptions.ConnectionError as errc: @@ -82,9 +80,9 @@ def next_request(): except requests.exceptions.RequestException as err: LOGGER.error('Error: %s', err) else: - LOGGER.debug('Received JSON response: %s', r.text) - req = json.loads(r.text) - song = req['song'] + LOGGER.debug('Received JSON response: %s', resp.text) + song_request = json.loads(resp.text) + song = song_request['song'] if song['song_type'] == 'J': artist = RADIO_NAME title = 'Jingle' @@ -95,14 +93,14 @@ def next_request(): game = clean_quotes(song['game']) LOGGER.info( 'Req_ID: %s, Artist[s]: %s, Title: %s, Game: %s, Path: %s', - req['id'], + song_request['id'], artist, title, game, song['path'] ) annotate_string = ANNOTATE.format( - req['id'], + song_request['id'], song['song_type'], artist, title, @@ -120,13 +118,15 @@ def just_played(request_id): ''' LOGGER.debug('Received command to report a song was just played.') try: - req_played = json.dumps({'song_request': request_id}) - r = requests.post(API_URL + 'played/', - headers=HEADERS, - data=req_played, - timeout=5) - r.encoding = 'utf-8' - r.raise_for_status() + request_played = json.dumps({'song_request': request_id}) + resp = requests.post( + API_URL + 'played/', + headers=HEADERS, + data=request_played, + timeout=5 + ) + resp.encoding = 'utf-8' + resp.raise_for_status() except requests.exceptions.HTTPError as errh: LOGGER.error('Http Error: %s', errh) except requests.exceptions.ConnectionError as errc: From 6f76b7c6113ef6de90fbdae95fe725f930739a77 Mon Sep 17 00:00:00 2001 From: RecursiveGreen Date: Thu, 6 Jun 2019 15:44:56 -0400 Subject: [PATCH 19/30] ugettext is deprecated--use gettext instead. --- savepointradio/core/behaviors.py | 2 +- savepointradio/core/models.py | 2 +- savepointradio/profiles/models.py | 2 +- savepointradio/radio/behaviors.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/savepointradio/core/behaviors.py b/savepointradio/core/behaviors.py index 7322e54..e869ea0 100644 --- a/savepointradio/core/behaviors.py +++ b/savepointradio/core/behaviors.py @@ -1,6 +1,6 @@ from django.db import models from django.utils import timezone -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ class Disableable(models.Model): diff --git a/savepointradio/core/models.py b/savepointradio/core/models.py index 6ebc84d..cb389e1 100644 --- a/savepointradio/core/models.py +++ b/savepointradio/core/models.py @@ -1,5 +1,5 @@ from django.db import models -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from authtools.models import AbstractNamedUser diff --git a/savepointradio/profiles/models.py b/savepointradio/profiles/models.py index b80d423..898ff2b 100644 --- a/savepointradio/profiles/models.py +++ b/savepointradio/profiles/models.py @@ -2,7 +2,7 @@ from django.conf import settings from django.core.validators import (MaxLengthValidator, MinValueValidator, MaxValueValidator) from django.db import models -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from core.behaviors import Disableable, Timestampable from core.utils import get_setting diff --git a/savepointradio/radio/behaviors.py b/savepointradio/radio/behaviors.py index c269dd0..3632368 100644 --- a/savepointradio/radio/behaviors.py +++ b/savepointradio/radio/behaviors.py @@ -1,6 +1,6 @@ from django.db import models from django.utils import timezone -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ class Disableable(models.Model): From f3ec6a1ae3813e4a29b6ed90a66f7b169b4d5344 Mon Sep 17 00:00:00 2001 From: RecursiveGreen Date: Thu, 6 Jun 2019 15:45:25 -0400 Subject: [PATCH 20/30] Create a custom validator for the radio IRI. --- savepointradio/core/utils.py | 3 +- savepointradio/core/validators.py | 58 +++++++++++++++++++++++++++++++ 2 files changed, 60 insertions(+), 1 deletion(-) create mode 100644 savepointradio/core/validators.py diff --git a/savepointradio/core/utils.py b/savepointradio/core/utils.py index f9af9b0..0d904db 100644 --- a/savepointradio/core/utils.py +++ b/savepointradio/core/utils.py @@ -17,6 +17,7 @@ from django.db import connection from django.utils.encoding import iri_to_uri, uri_to_iri from .models import Setting +from .validators import GROUP_NT_DRIVE_LETTER, GROUP_NT_UNC def generate_password(length=32): @@ -165,7 +166,7 @@ def iri_to_path(iri): # Drive letter IRI will have three slashes followed by the drive letter # UNC path IRI will have two slashes followed by the UNC path uri = iri_to_uri(iri) - patt = r'^(?:file:///[A-Za-z]:/|file://[A-Za-z0-9!@#$%^&\'\)\(\.\-_{}~]+/)' + patt = r'^(?:' + GROUP_NT_DRIVE_LETTER + r'|' + GROUP_NT_UNC + r')' windows = re.match(patt, uri) if windows: parse = urlparse(uri) diff --git a/savepointradio/core/validators.py b/savepointradio/core/validators.py new file mode 100644 index 0000000..6030762 --- /dev/null +++ b/savepointradio/core/validators.py @@ -0,0 +1,58 @@ +''' +Custom Django model/form field validators for the Save Point Radio project. +''' + +import re + +from django.core import validators +from django.core.exceptions import ValidationError +from django.utils.translation import gettext_lazy as _ + + +GROUP_NT_UNC = r'file://[A-Za-z0-9!@#$%^&\'\)\(\.\-_{}~]+/' + +GROUP_NT_DRIVE_LETTER = r'file:///[A-Za-z](?:\:|\|)/' + +GROUP_NON_AUTH = r'file:///[A-Za-z0-9!@#$%^&\'\)\(\.\-_{}~]+' + +FILE_IRI_PATTERN = ( + r'^(?P' + + GROUP_NT_UNC + + r')|(?P' + + GROUP_NT_DRIVE_LETTER + + r')|(?P' + + GROUP_NON_AUTH + + r')' +) + + +class RadioIRIValidator(validators.URLValidator): + ''' + Validates an RFC3987-defined IRI along with RFC8089 for file:// and other + custom schemes. + ''' + + message = _('Enter a valid IRI.') + schemes = ['http', 'https', 'file', 'ftp', 'ftps', 's3'] + + def __init__(self, schemes=None, **kwargs): + super().__init__(**kwargs) + if schemes is not None: + self.schemes = schemes + + def __call__(self, value): + # Check the schemes first + scheme = value.split('://')[0].lower() + if scheme not in self.schemes: + raise ValidationError(self.message, code=self.code) + + # Ignore the non-standard IRI + if scheme == 'file': + pattern = re.compile(FILE_IRI_PATTERN) + if not pattern.match(value): + raise ValidationError(self.message, code=self.code) + elif scheme == 's3': + # Nothing to validate, really. . . + return + else: + super().__call__(value) From d866ce7cda8a45a999484e8526150297322c1411 Mon Sep 17 00:00:00 2001 From: RecursiveGreen Date: Thu, 6 Jun 2019 15:45:56 -0400 Subject: [PATCH 21/30] Re-enable the custom Radio IRI model form/field. --- savepointradio/radio/fields.py | 15 +++++---------- savepointradio/radio/forms.py | 12 ++---------- .../migrations/0004_new_song_path_structure.py | 5 +++-- savepointradio/radio/models.py | 16 +++------------- 4 files changed, 13 insertions(+), 35 deletions(-) diff --git a/savepointradio/radio/fields.py b/savepointradio/radio/fields.py index ae70834..bbf3fea 100644 --- a/savepointradio/radio/fields.py +++ b/savepointradio/radio/fields.py @@ -2,11 +2,12 @@ Custom model fields for the Save Point Radio project. ''' -from django.core import validators from django.db import models -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ -from .forms import ALLOWED_SCHEMES, RadioIRIFormField +from core.validators import RadioIRIValidator + +from .forms import RadioIRIFormField class RadioIRIField(models.TextField): @@ -19,13 +20,7 @@ class RadioIRIField(models.TextField): https://stackoverflow.com/questions/41756572/ ''' - # TODO: Because of a shortcoming of Django's URLValidator code, the - # 'file://' scheme does not validate properly on most cases due to - # incompatibilities with optional hostnames. Disabling the custom field - # for now until I can figure out a non-lethal way of handling this. - # https://code.djangoproject.com/ticket/25595 - - default_validators = [validators.URLValidator(schemes=ALLOWED_SCHEMES)] + default_validators = [RadioIRIValidator()] description = _("Long IRI") def __init__(self, verbose_name=None, name=None, **kwargs): diff --git a/savepointradio/radio/forms.py b/savepointradio/radio/forms.py index 0ae9eb8..68639f4 100644 --- a/savepointradio/radio/forms.py +++ b/savepointradio/radio/forms.py @@ -2,11 +2,9 @@ Custom forms/formfields for the Save Point Radio project. ''' -from django.core import validators from django.forms.fields import URLField - -ALLOWED_SCHEMES = ['http', 'https', 'file', 'ftp', 'ftps', 's3'] +from core.validators import RadioIRIValidator class RadioIRIFormField(URLField): @@ -19,10 +17,4 @@ class RadioIRIFormField(URLField): https://stackoverflow.com/questions/41756572/ ''' - # TODO: Because of a shortcoming of Django's URLValidator code, the - # 'file://' scheme does not validate properly on most cases due to - # incompatibilities with optional hostnames. Disabling the custom field - # for now until I can figure out a non-lethal way of handling this. - # https://code.djangoproject.com/ticket/25595 - - default_validators = [validators.URLValidator(schemes=ALLOWED_SCHEMES)] + default_validators = [RadioIRIValidator()] diff --git a/savepointradio/radio/migrations/0004_new_song_path_structure.py b/savepointradio/radio/migrations/0004_new_song_path_structure.py index 5ae3fd0..2f94ed4 100644 --- a/savepointradio/radio/migrations/0004_new_song_path_structure.py +++ b/savepointradio/radio/migrations/0004_new_song_path_structure.py @@ -1,8 +1,9 @@ -# Generated by Django 2.2.1 on 2019-06-03 18:39 +# Generated by Django 2.2.1 on 2019-06-06 19:06 import django.core.validators from django.db import migrations, models import django.db.models.deletion +import radio.fields class Migration(migrations.Migration): @@ -18,7 +19,7 @@ class Migration(migrations.Migration): ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), ('created_date', models.DateTimeField(auto_now_add=True, verbose_name='added on')), ('modified_date', models.DateTimeField(auto_now=True, verbose_name='last modified')), - ('iri', models.TextField(verbose_name='IRI path to song file')), + ('iri', radio.fields.RadioIRIField(verbose_name='IRI path to song file')), ('mime_type', models.CharField(blank=True, max_length=64, verbose_name='file MIME type')), ('file_size', models.BigIntegerField(blank=True, null=True, validators=[django.core.validators.MinValueValidator(0)], verbose_name='file size')), ('length', models.DecimalField(blank=True, decimal_places=2, max_digits=8, null=True, verbose_name='song length (in seconds)')), diff --git a/savepointradio/radio/models.py b/savepointradio/radio/models.py index 1704e26..e0cd628 100644 --- a/savepointradio/radio/models.py +++ b/savepointradio/radio/models.py @@ -9,11 +9,11 @@ from django.apps import apps from django.core.validators import MinValueValidator from django.db import models from django.utils import timezone -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from core.behaviors import Disableable, Publishable, Timestampable from core.utils import get_setting -# from .fields import RadioIRIField +from .fields import RadioIRIField from .managers import RadioManager, SongManager @@ -104,17 +104,7 @@ class Store(Timestampable, models.Model): ''' A model to represent various data locations (stores) for the song. ''' - # TODO: Because of a shortcoming of Django's URLValidator code, the - # 'file://' scheme does not validate properly on most cases due to - # incompatibilities with optional hostnames. Disabling the custom field - # for now until I can figure out a non-lethal way of handling this. - # https://code.djangoproject.com/ticket/25594 - # https://code.djangoproject.com/ticket/25595 - # https://stackoverflow.com/questions/41756572/ - - # iri = RadioIRIField() - - iri = models.TextField(_('IRI path to song file')) + iri = RadioIRIField(_('IRI path to song file')) mime_type = models.CharField(_('file MIME type'), max_length=64, blank=True) From 4d24ab956fa3d602a09414bf55804baf19b25e69 Mon Sep 17 00:00:00 2001 From: Josh Date: Fri, 7 Jun 2019 16:16:01 -0400 Subject: [PATCH 22/30] Update requirements to latest versions. --- requirements-dev.txt | Bin 514 -> 514 bytes requirements.txt | 4 ++-- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index c6c120e2a9bc99c5374a59bed9c87c91108caf59..bed47ded0bb0a4e21a328e627f146ef17b23e0c5 100644 GIT binary patch delta 17 YcmZo-X=2%Mi;>lYL65;;@_j~S04+iU#Q*>R delta 17 YcmZo-X=2%Mi;>lsL65<7@_j~S04-7k$p8QV diff --git a/requirements.txt b/requirements.txt index f32f97e..e6ac912 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,10 +1,10 @@ argon2-cffi>=19.1.0 cffi>=1.12.3 dj-database-url>=0.5.0 -Django>=2.2.1 +Django>=2.2.2 django-authtools>=1.6.0 djangorestframework>=3.9.4 -psycopg2>=2.8.2 +psycopg2-binary>=2.8.2 pycparser>=2.19 python-decouple>=3.1 pytz>=2019.1 From c7811ddd11c16f26ceb8ef6d675efcd3ba634600 Mon Sep 17 00:00:00 2001 From: RecursiveGreen Date: Mon, 10 Jun 2019 07:37:22 -0400 Subject: [PATCH 23/30] Code shuffle. --- savepointradio/core/utils.py | 18 +++++++++++++++++- savepointradio/core/validators.py | 17 +---------------- 2 files changed, 18 insertions(+), 17 deletions(-) diff --git a/savepointradio/core/utils.py b/savepointradio/core/utils.py index 0d904db..24d73eb 100644 --- a/savepointradio/core/utils.py +++ b/savepointradio/core/utils.py @@ -17,7 +17,23 @@ from django.db import connection from django.utils.encoding import iri_to_uri, uri_to_iri from .models import Setting -from .validators import GROUP_NT_DRIVE_LETTER, GROUP_NT_UNC + + +GROUP_NT_UNC = r'file://[A-Za-z0-9!@#$%^&\'\)\(\.\-_{}~]+/' + +GROUP_NT_DRIVE_LETTER = r'file:///[A-Za-z](?:\:|\|)/' + +GROUP_NON_AUTH = r'file:///[A-Za-z0-9!@#$%^&\'\)\(\.\-_{}~]+' + +FILE_IRI_PATTERN = ( + r'^(?P' + + GROUP_NT_UNC + + r')|(?P' + + GROUP_NT_DRIVE_LETTER + + r')|(?P' + + GROUP_NON_AUTH + + r')' +) def generate_password(length=32): diff --git a/savepointradio/core/validators.py b/savepointradio/core/validators.py index 6030762..bcdc7da 100644 --- a/savepointradio/core/validators.py +++ b/savepointradio/core/validators.py @@ -8,22 +8,7 @@ from django.core import validators from django.core.exceptions import ValidationError from django.utils.translation import gettext_lazy as _ - -GROUP_NT_UNC = r'file://[A-Za-z0-9!@#$%^&\'\)\(\.\-_{}~]+/' - -GROUP_NT_DRIVE_LETTER = r'file:///[A-Za-z](?:\:|\|)/' - -GROUP_NON_AUTH = r'file:///[A-Za-z0-9!@#$%^&\'\)\(\.\-_{}~]+' - -FILE_IRI_PATTERN = ( - r'^(?P' + - GROUP_NT_UNC + - r')|(?P' + - GROUP_NT_DRIVE_LETTER + - r')|(?P' + - GROUP_NON_AUTH + - r')' -) +from .utils import FILE_IRI_PATTERN class RadioIRIValidator(validators.URLValidator): From 490e37a2f582654ee4fddad93fc4e44a69a95ba9 Mon Sep 17 00:00:00 2001 From: RecursiveGreen Date: Mon, 10 Jun 2019 15:48:55 -0400 Subject: [PATCH 24/30] Add file hashing to the export. --- contrib/export_playlist/export_playlist.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/contrib/export_playlist/export_playlist.py b/contrib/export_playlist/export_playlist.py index f5cf2f7..5b0dbeb 100644 --- a/contrib/export_playlist/export_playlist.py +++ b/contrib/export_playlist/export_playlist.py @@ -7,6 +7,7 @@ by the new database later. import argparse from decimal import Decimal, getcontext +import hashlib import json import mimetypes import os @@ -38,6 +39,19 @@ def detect_mime(path): return mimetype +def hash_file(path): + ''' + Run a music file through a hashing algorithm (SHA3_256) and return the + hexidecimal digest. + ''' + try: + with open(path, 'rb') as file: + filehash = hashlib.sha3_256(file.read()).hexdigest() + except OSError: + filehash = None + return filehash + + def adapt_decimal(number): '''Sqlite3 adapter for Decimal types''' return str(number) @@ -142,6 +156,7 @@ def import_sqlite3(db_file): store = {'path': scrub(song[7]), 'mime': detect_mime(scrub(song[7])), 'filesize': os.stat(scrub(song[7])).st_size, + 'filehash': hash_file(scrub(song[7])), 'length': song[6]} songs.append({'album': scrub(song[2]), 'artists': song_artists, From 0907ea72bd77d2ed1dc108b2109cc2cde86ad092 Mon Sep 17 00:00:00 2001 From: RecursiveGreen Date: Wed, 12 Jun 2019 13:51:36 -0400 Subject: [PATCH 25/30] Support uploading to Amazon S3 (and equivalent). --- contrib/upload_s3/requirements.txt | 9 + contrib/upload_s3/upload_s3.py | 196 ++++++++++++++++++ .../management/commands/importoldradio.py | 13 +- 3 files changed, 217 insertions(+), 1 deletion(-) create mode 100644 contrib/upload_s3/requirements.txt create mode 100644 contrib/upload_s3/upload_s3.py diff --git a/contrib/upload_s3/requirements.txt b/contrib/upload_s3/requirements.txt new file mode 100644 index 0000000..1d5f2ac --- /dev/null +++ b/contrib/upload_s3/requirements.txt @@ -0,0 +1,9 @@ +boto3>=1.9.166 +botocore>=1.12.166 +docutils>=0.14 +jmespath>=0.9.4 +python-dateutil>=2.8.0 +python-decouple>=3.1 +s3transfer>=0.2.1 +six>=1.12.0 +urllib3>=1.25.3 diff --git a/contrib/upload_s3/upload_s3.py b/contrib/upload_s3/upload_s3.py new file mode 100644 index 0000000..31f6b59 --- /dev/null +++ b/contrib/upload_s3/upload_s3.py @@ -0,0 +1,196 @@ +''' +upload_s3.py + +This is the helper script that uploads songs from an exported playlist into +an Amazon S3 instance (or other implementations, like DigialOcean Spaces). +''' + +import argparse +import json +import os +import sys +import threading +from unicodedata import normalize + +from decouple import config +import boto3 + +# If these four are not defined, then boto3 will look for defaults in the +# ~/.aws configurations +S3_REGION = config('S3_REGION', default=None) +S3_ENDPOINT = config('S3_ENDPOINT', default=None) +S3_ACCESS_KEY = config('S3_ACCESS_KEY', default=None) +S3_SECRET_KEY = config('S3_SECRET_KEY', default=None) + +# This has to be defined regardless. +S3_BUCKET = config('S3_BUCKET') + +# Radio name for metadata +RADIO_NAME = config('RADIO_NAME', default='Save Point Radio') + + +class Progress(object): + ''' + A callback class for the Amazon S3 upload to detect how far along in an + upload we are. + ''' + def __init__(self, filepath): + self._filepath = filepath + self._filename = os.path.basename(filepath) + self._size = float(os.path.getsize(filepath)) + self._seen_so_far = 0 + self._lock = threading.Lock() + + def __call__(self, bytes_amount): + with self._lock: + self._seen_so_far += bytes_amount + percentage = (self._seen_so_far / self._size) * 100 + sys.stdout.write( + "\r%s %s / %s (%.2f%%)" % ( + self._filename, self._seen_so_far, self._size, + percentage + ) + ) + sys.stdout.flush() + + +def asciify(text): + ''' + Converts a unicode string to pure ascii. + ''' + normal = normalize('NFKC', text) + return normal.encode('ascii', 'backslashreplace').decode('ascii') + + +def get_fullname(artist): + ''' + String representing the artist's full name including an alias, if + available. + ''' + if artist['alias']: + if artist['first_name'] or artist['last_name']: + return '{} "{}" {}'.format(artist['first_name'], + artist['alias'], + artist['last_name']) + return artist['alias'] + return '{} {}'.format(artist['first_name'], artist['last_name']) + + +def beautify_artists(artists): + ''' + Turns a list of one or more artists into a proper English listing. + ''' + fullnames = [get_fullname(artist) for artist in artists] + output = ', ' + if len(fullnames) == 2: + output = ' & ' + return output.join(fullnames) + + +def import_playlist(playlist_file): + ''' + Imports a playlist from a JSON file, uploads the files to an S3[-like] + instance, and exports a new JSON file with the updated paths. + ''' + if not os.path.isfile(playlist_file): + raise FileNotFoundError + + with open(playlist_file, 'r', encoding='utf8') as pfile: + playlist = json.load(pfile) + + session = boto3.session.Session() + client = session.client( + 's3', + region_name=S3_REGION, + endpoint_url=S3_ENDPOINT, + aws_access_key_id=S3_ACCESS_KEY, + aws_secret_access_key=S3_SECRET_KEY + ) + + for song in playlist['songs']: + old_path = song['store']['path'] + + if song['type'] == 'S': + prefix = 'songs' + metadata = { + 'album': asciify(song['album']), + 'artists': asciify(beautify_artists(song['artists'])), + 'game': asciify(song['game']), + 'title': asciify(song['title']), + 'length': str(song['store']['length']), + 'original-path': asciify(old_path) + } + else: + prefix = 'jingles' + metadata = { + 'artists': asciify(RADIO_NAME), + 'title': asciify(song['title']), + 'length': str(song['store']['length']), + 'original-path': asciify(old_path) + } + file_hash = song['store']['filehash'] + ext = os.path.splitext(old_path)[1] + new_path = '{}/{}{}'.format(prefix, file_hash, ext) + + client.upload_file( + old_path, + S3_BUCKET, + new_path, + ExtraArgs={ + 'Metadata': metadata, + 'ContentType': song['store']['mime'] + }, + Callback=Progress(old_path) + ) + + song['store']['path'] = 's3://{}/{}'.format(S3_BUCKET, new_path) + + sys.stdout.write("\r\n") + sys.stdout.flush() + + return playlist + + +def main(): + '''Main loop of the program''' + + description = 'Uploads song files to an Amazon S3 (or similar) instance.' + + parser = argparse.ArgumentParser(description=description) + subparsers = parser.add_subparsers(dest='command') + + parser_playlist = subparsers.add_parser( + 'playlist', + help='Import playlist song data.' + ) + parser_playlist.add_argument( + 'filepath', + help='Path to the playlist file.', + nargs=1 + ) + + if len(sys.argv) == 1: + sys.stderr.write('Error: please specify a command\n\n') + parser.print_help(sys.stderr) + sys.exit(1) + + results = None + + args = parser.parse_args() + + if args.command == 'playlist': + results = import_playlist(args.filepath[0]) + + if results: + with open('playlist_s3.json', 'w', encoding='utf8') as file: + json.dump( + results, + file, + ensure_ascii=False, + sort_keys=True, + indent=4 + ) + + +if __name__ == '__main__': + main() diff --git a/savepointradio/radio/management/commands/importoldradio.py b/savepointradio/radio/management/commands/importoldradio.py index 0d0915b..3e62e46 100644 --- a/savepointradio/radio/management/commands/importoldradio.py +++ b/savepointradio/radio/management/commands/importoldradio.py @@ -6,6 +6,7 @@ for seeding a newly created database. import decimal import json import os +import re from django.core.management.base import BaseCommand, CommandError @@ -90,8 +91,18 @@ class Command(BaseCommand): ) new_song.artists.add(new_artist) + localfile = re.match( + r'^(?:(?:[A-Za-z]:|\\)\\|\/)', + song['store']['path'] + ) + + if localfile: + iri = path_to_iri(song['store']['path']) + else: + iri = song['store']['path'] + new_store = Store.objects.create( - iri=path_to_iri(song['store']['path']), + iri=iri, mime_type=song['store']['mime'], file_size=song['store']['filesize'], length=song['store']['length'] From b48c2eece54b3b817cda69150b03bbfb6d3387ac Mon Sep 17 00:00:00 2001 From: RecursiveGreen Date: Wed, 12 Jun 2019 15:53:06 -0400 Subject: [PATCH 26/30] Add logging to the uploads. --- contrib/upload_s3/upload_s3.py | 60 +++++++++++++++++++++++++++------- 1 file changed, 49 insertions(+), 11 deletions(-) diff --git a/contrib/upload_s3/upload_s3.py b/contrib/upload_s3/upload_s3.py index 31f6b59..87fbe88 100644 --- a/contrib/upload_s3/upload_s3.py +++ b/contrib/upload_s3/upload_s3.py @@ -7,9 +7,11 @@ an Amazon S3 instance (or other implementations, like DigialOcean Spaces). import argparse import json +import logging import os import sys import threading +import traceback from unicodedata import normalize from decouple import config @@ -28,6 +30,15 @@ S3_BUCKET = config('S3_BUCKET') # Radio name for metadata RADIO_NAME = config('RADIO_NAME', default='Save Point Radio') +logging.basicConfig( + handlers=[logging.FileHandler('./s3_uploads.log', encoding='utf8')], + level=logging.INFO, + format=('[%(asctime)s] [%(levelname)s]' + ' [%(name)s.%(funcName)s] === %(message)s'), + datefmt='%Y-%m-%dT%H:%M:%S' + ) +LOGGER = logging.getLogger('upload_s3') + class Progress(object): ''' @@ -107,6 +118,8 @@ def import_playlist(playlist_file): aws_secret_access_key=S3_SECRET_KEY ) + totals = {'success': 0, 'fail': 0} + for song in playlist['songs']: old_path = song['store']['path'] @@ -132,22 +145,45 @@ def import_playlist(playlist_file): ext = os.path.splitext(old_path)[1] new_path = '{}/{}{}'.format(prefix, file_hash, ext) - client.upload_file( - old_path, - S3_BUCKET, - new_path, - ExtraArgs={ - 'Metadata': metadata, - 'ContentType': song['store']['mime'] - }, - Callback=Progress(old_path) - ) + LOGGER.info('Begin upload of: %s', old_path) - song['store']['path'] = 's3://{}/{}'.format(S3_BUCKET, new_path) + try: + client.upload_file( + old_path, + S3_BUCKET, + new_path, + ExtraArgs={ + 'Metadata': metadata, + 'ContentType': song['store']['mime'] + }, + Callback=Progress(old_path) + ) + except Exception: + LOGGER.error( + 'Upload failed for: %s -- %s', + old_path, + traceback.print_exc() + ) + totals['fail'] += 1 + else: + song['store']['path'] = 's3://{}/{}'.format(S3_BUCKET, new_path) + LOGGER.info( + 'Successful upload of: %s to %s', + old_path, + song['store']['path'] + ) + totals['success'] += 1 sys.stdout.write("\r\n") sys.stdout.flush() + result_message = 'Uploads complete -- {} successful, {} failures'.format( + totals['success'], + totals['fail'] + ) + print(result_message) + LOGGER.info(result_message) + return playlist @@ -182,6 +218,7 @@ def main(): results = import_playlist(args.filepath[0]) if results: + LOGGER.info('Exporting new playlist file to \'playlist_s3.json\'') with open('playlist_s3.json', 'w', encoding='utf8') as file: json.dump( results, @@ -190,6 +227,7 @@ def main(): sort_keys=True, indent=4 ) + LOGGER.info('Program finished. Exiting.') if __name__ == '__main__': From 80b37c96f4e55d405b211e683c26689086153225 Mon Sep 17 00:00:00 2001 From: RecursiveGreen Date: Mon, 1 Jul 2019 15:01:47 -0400 Subject: [PATCH 27/30] Decoupled settings from the DJ Control script. --- contrib/djcontrol/djcontrol.py | 7 ++++--- contrib/djcontrol/requirements.txt | Bin 164 -> 208 bytes 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/contrib/djcontrol/djcontrol.py b/contrib/djcontrol/djcontrol.py index c4c2c27..b4c9ab8 100644 --- a/contrib/djcontrol/djcontrol.py +++ b/contrib/djcontrol/djcontrol.py @@ -11,14 +11,15 @@ import json import logging from logging.handlers import RotatingFileHandler +from decouple import config import requests -DJ_TOKEN = 'place_generated_token_here' +DJ_TOKEN = config('DJ_TOKEN') -API_URL = 'https://savepointradio.net/api/' +API_URL = config('API_URL') # With trailing slash -RADIO_NAME = 'Save Point Radio' +RADIO_NAME = config('RADIO_NAME') HEADERS = { 'Content-Type': 'application/json; charset=utf-8', diff --git a/contrib/djcontrol/requirements.txt b/contrib/djcontrol/requirements.txt index 175851aa05b26363d40bcd264aba83ad76f033b3..070121fbe8a61708b148079160565bbe55c92ebb 100644 GIT binary patch delta 49 zcmZ3&c!6<3oO%I6B|`~A217nW9)m7J3PUPGGLT)$P{5D_WZN;=G8i-HF&Iva^acPe C(+chY delta 9 Qcmcb>xP);++{7+7024R^#sB~S From b30003847d8debbe54f1a5ab2b651d3c89fd5389 Mon Sep 17 00:00:00 2001 From: RecursiveGreen Date: Wed, 3 Jul 2019 11:57:25 -0400 Subject: [PATCH 28/30] Better request display in the admin interface. --- savepointradio/profiles/admin.py | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/savepointradio/profiles/admin.py b/savepointradio/profiles/admin.py index 8dd5c37..f12007d 100644 --- a/savepointradio/profiles/admin.py +++ b/savepointradio/profiles/admin.py @@ -40,7 +40,30 @@ class ProfileAdmin(admin.ModelAdmin): @admin.register(SongRequest) class RequestAdmin(admin.ModelAdmin): - model = SongRequest + # Detail List display + list_display = ('get_user', + 'song', + 'created_date', + 'queued_at', + 'played_at') + search_fields = ['song', 'profile'] + + # Edit Form display + readonly_fields = ( + 'created_date', + 'modified_date', + 'profile', + 'song', + 'queued_at', + 'played_at' + ) + verbose_name = 'request' verbose_name_plural = 'requests' extra = 0 + + def get_user(self, obj): + '''Returns the username from the profile.''' + return obj.profile.user + get_user.admin_order_field = 'profile' + get_user.short_description = 'User Name' From 813531be26b2ccc62e1795463e5b43959087baa1 Mon Sep 17 00:00:00 2001 From: RecursiveGreen Date: Wed, 3 Jul 2019 11:57:46 -0400 Subject: [PATCH 29/30] Add replaygain support. --- contrib/djcontrol/djcontrol.py | 13 +++++++++-- savepointradio/api/serializers/radio.py | 5 ++-- savepointradio/radio/admin.py | 10 ++++++-- .../management/commands/importoldradio.py | 15 +++++++++++- .../radio/migrations/0005_replaygain_data.py | 23 +++++++++++++++++++ savepointradio/radio/models.py | 21 +++++++++++++++++ 6 files changed, 80 insertions(+), 7 deletions(-) create mode 100644 savepointradio/radio/migrations/0005_replaygain_data.py diff --git a/contrib/djcontrol/djcontrol.py b/contrib/djcontrol/djcontrol.py index b4c9ab8..70d32b8 100644 --- a/contrib/djcontrol/djcontrol.py +++ b/contrib/djcontrol/djcontrol.py @@ -26,7 +26,14 @@ HEADERS = { 'Authorization': 'Token {}'.format(DJ_TOKEN) } -ANNOTATE = 'annotate:req_id="{}",type="{}",artist="{}",title="{}",game="{}":{}' +ANNOTATE = ( + 'annotate:req_id="{}",' + 'type="{}",' + 'artist="{}",' + 'title="{}",' + 'game="{}",' + 'replay_gain="{}":{}' +) logging.basicConfig( handlers=[ @@ -93,11 +100,12 @@ def next_request(): title = clean_quotes(song['title']) game = clean_quotes(song['game']) LOGGER.info( - 'Req_ID: %s, Artist[s]: %s, Title: %s, Game: %s, Path: %s', + 'ID: %s, Artist[s]: %s, Title: %s, Game: %s, Gain: %s, Path: %s', song_request['id'], artist, title, game, + song['replaygain'], song['path'] ) annotate_string = ANNOTATE.format( @@ -106,6 +114,7 @@ def next_request(): artist, title, game, + song['replaygain'], song['path'] ) LOGGER.debug(annotate_string) diff --git a/savepointradio/api/serializers/radio.py b/savepointradio/api/serializers/radio.py index a0fdaf4..cf216b8 100644 --- a/savepointradio/api/serializers/radio.py +++ b/savepointradio/api/serializers/radio.py @@ -1,4 +1,4 @@ -from rest_framework.serializers import (BooleanField, DecimalField, +from rest_framework.serializers import (BooleanField, CharField, DecimalField, IntegerField, ListField, ModelSerializer, Serializer, SerializerMethodField, @@ -120,12 +120,13 @@ class RadioSongSerializer(ModelSerializer): decimal_places=2, source='active_store.length' ) + replaygain = CharField(source='active_store.replaygain') path = SerializerMethodField() class Meta: model = Song fields = ('album', 'artists', 'game', 'song_type', 'title', 'length', - 'path') + 'replaygain', 'path') def get_path(self, obj): '''Converts the IRI into a filesystem path.''' diff --git a/savepointradio/radio/admin.py b/savepointradio/radio/admin.py index 6c5b006..d05e06d 100644 --- a/savepointradio/radio/admin.py +++ b/savepointradio/radio/admin.py @@ -114,14 +114,20 @@ class StoreAdmin(admin.ModelAdmin): list_display = ('iri', 'mime_type', 'file_size', - 'length') + 'length', + '_replaygain') search_fields = ['iri'] # Edit Form display readonly_fields = ('created_date', 'modified_date') fieldsets = ( ('Main', { - 'fields': ('iri', 'mime_type', 'file_size', 'length') + 'fields': ('iri', + 'mime_type', + 'file_size', + 'length', + 'track_gain', + 'track_peak') }), ('Stats', { 'classes': ('collapse',), diff --git a/savepointradio/radio/management/commands/importoldradio.py b/savepointradio/radio/management/commands/importoldradio.py index 3e62e46..2c643ce 100644 --- a/savepointradio/radio/management/commands/importoldradio.py +++ b/savepointradio/radio/management/commands/importoldradio.py @@ -101,11 +101,24 @@ class Command(BaseCommand): else: iri = song['store']['path'] + if song['store']['track_gain']: + gain_str = re.sub(r'[dB\+ ]', '', song['store']['track_gain']) + gain = decimal.Decimal(gain_str) + else: + gain = None + + if song['store']['track_peak']: + peak = decimal.Decimal(song['store']['track_peak']) + else: + peak = None + new_store = Store.objects.create( iri=iri, mime_type=song['store']['mime'], file_size=song['store']['filesize'], - length=song['store']['length'] + length=song['store']['length'], + track_gain=gain, + track_peak=peak ) new_song.stores.add(new_store) new_song.active_store = new_store diff --git a/savepointradio/radio/migrations/0005_replaygain_data.py b/savepointradio/radio/migrations/0005_replaygain_data.py new file mode 100644 index 0000000..8481d49 --- /dev/null +++ b/savepointradio/radio/migrations/0005_replaygain_data.py @@ -0,0 +1,23 @@ +# Generated by Django 2.2.2 on 2019-07-03 13:16 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('radio', '0004_new_song_path_structure'), + ] + + operations = [ + migrations.AddField( + model_name='store', + name='track_gain', + field=models.DecimalField(blank=True, decimal_places=2, max_digits=6, null=True, verbose_name='recommended replaygain adjustment'), + ), + migrations.AddField( + model_name='store', + name='track_peak', + field=models.DecimalField(blank=True, decimal_places=6, max_digits=10, null=True, verbose_name='highest volume level in the track'), + ), + ] diff --git a/savepointradio/radio/models.py b/savepointradio/radio/models.py index e0cd628..d9dd320 100644 --- a/savepointradio/radio/models.py +++ b/savepointradio/radio/models.py @@ -117,6 +117,27 @@ class Store(Timestampable, models.Model): decimal_places=2, null=True, blank=True) + track_gain = models.DecimalField(_('recommended replaygain adjustment'), + max_digits=6, + decimal_places=2, + null=True, + blank=True) + track_peak = models.DecimalField(_('highest volume level in the track'), + max_digits=10, + decimal_places=6, + null=True, + blank=True) + + def _replaygain(self): + ''' + String representation of the recommended amplitude adjustment. + ''' + if self.track_gain is None: + return '+0.00 dB' + if self.track_gain > 0: + return '+{} dB'.format(str(self.track_gain)) + return '{} dB'.format(str(self.track_gain)) + replaygain = property(_replaygain) def __str__(self): return self.iri From 5e432cc721f46f906e70bbd8426637128885e1cc Mon Sep 17 00:00:00 2001 From: RecursiveGreen Date: Mon, 15 Jul 2019 08:32:32 -0400 Subject: [PATCH 30/30] Add aws-cli alternative in case of future issues. --- contrib/download_s3/download_s3.py | 109 +++++++++++++++++++++++++++++ 1 file changed, 109 insertions(+) create mode 100644 contrib/download_s3/download_s3.py diff --git a/contrib/download_s3/download_s3.py b/contrib/download_s3/download_s3.py new file mode 100644 index 0000000..c5fc4a6 --- /dev/null +++ b/contrib/download_s3/download_s3.py @@ -0,0 +1,109 @@ +''' +download_s3.py + +This is the helper script that downloads songs from an Amazon S3 instance +(or other implementations, like DigialOcean Spaces). Currently used as a +workaround for a pipe-leaking issue with the "aws-cli" client. +''' + +import argparse +import logging +import sys +import traceback + +from decouple import config +import boto3 + +# If these four are not defined, then boto3 will look for defaults in the +# ~/.aws configurations +S3_REGION = config('S3_REGION', default=None) +S3_ENDPOINT = config('S3_ENDPOINT', default=None) +S3_ACCESS_KEY = config('S3_ACCESS_KEY', default=None) +S3_SECRET_KEY = config('S3_SECRET_KEY', default=None) + +# Radio name for metadata +RADIO_NAME = config('RADIO_NAME', default='Save Point Radio') + +logging.basicConfig( + handlers=[logging.FileHandler('./s3_downloads.log', encoding='utf8')], + level=logging.INFO, + format=('[%(asctime)s] [%(levelname)s]' + ' [%(name)s.%(funcName)s] === %(message)s'), + datefmt='%Y-%m-%dT%H:%M:%S' + ) +LOGGER = logging.getLogger('download_s3') + + +def download_file(s3path, filepath): + ''' + Downloads a file from an S3 instance and saves it to a specified path. + ''' + + obj_parts = s3path[5:].split('/') + obj_bucket = obj_parts[0] + obj_key = '/'.join(obj_parts[1:]) + + session = boto3.session.Session() + client = session.client( + 's3', + region_name=S3_REGION, + endpoint_url=S3_ENDPOINT, + aws_access_key_id=S3_ACCESS_KEY, + aws_secret_access_key=S3_SECRET_KEY + ) + + try: + client.download_file(obj_bucket, obj_key, filepath) + except Exception: + LOGGER.error( + 'Download failed for: %s -- %s', + s3path, + traceback.print_exc() + ) + result = 1 + else: + LOGGER.info( + 'Successful download of: %s to %s', + s3path, + filepath + ) + result = 0 + + return result + + +def main(): + '''Main loop of the program''' + + description = 'Downloads songs from an Amazon S3 (or similar) instance.' + + parser = argparse.ArgumentParser(description=description) + + parser.add_argument( + 's3path', + help='Path to the S3 object', + nargs=1 + ) + + parser.add_argument( + 'filepath', + help='Path to place the downloaded file', + nargs=1 + ) + + if len(sys.argv) == 1: + sys.stderr.write('Error: please specify a command\n\n') + parser.print_help(sys.stderr) + sys.exit(1) + + args = parser.parse_args() + + if args.s3path and args.filepath: + result = download_file(args.s3path[0], args.filepath[0]) + + LOGGER.info('Program finished. Exiting.') + sys.exit(result) + + +if __name__ == '__main__': + main()