diff --git a/contrib/upload_s3/requirements.txt b/contrib/upload_s3/requirements.txt new file mode 100644 index 0000000..1d5f2ac --- /dev/null +++ b/contrib/upload_s3/requirements.txt @@ -0,0 +1,9 @@ +boto3>=1.9.166 +botocore>=1.12.166 +docutils>=0.14 +jmespath>=0.9.4 +python-dateutil>=2.8.0 +python-decouple>=3.1 +s3transfer>=0.2.1 +six>=1.12.0 +urllib3>=1.25.3 diff --git a/contrib/upload_s3/upload_s3.py b/contrib/upload_s3/upload_s3.py new file mode 100644 index 0000000..31f6b59 --- /dev/null +++ b/contrib/upload_s3/upload_s3.py @@ -0,0 +1,196 @@ +''' +upload_s3.py + +This is the helper script that uploads songs from an exported playlist into +an Amazon S3 instance (or other implementations, like DigialOcean Spaces). +''' + +import argparse +import json +import os +import sys +import threading +from unicodedata import normalize + +from decouple import config +import boto3 + +# If these four are not defined, then boto3 will look for defaults in the +# ~/.aws configurations +S3_REGION = config('S3_REGION', default=None) +S3_ENDPOINT = config('S3_ENDPOINT', default=None) +S3_ACCESS_KEY = config('S3_ACCESS_KEY', default=None) +S3_SECRET_KEY = config('S3_SECRET_KEY', default=None) + +# This has to be defined regardless. +S3_BUCKET = config('S3_BUCKET') + +# Radio name for metadata +RADIO_NAME = config('RADIO_NAME', default='Save Point Radio') + + +class Progress(object): + ''' + A callback class for the Amazon S3 upload to detect how far along in an + upload we are. + ''' + def __init__(self, filepath): + self._filepath = filepath + self._filename = os.path.basename(filepath) + self._size = float(os.path.getsize(filepath)) + self._seen_so_far = 0 + self._lock = threading.Lock() + + def __call__(self, bytes_amount): + with self._lock: + self._seen_so_far += bytes_amount + percentage = (self._seen_so_far / self._size) * 100 + sys.stdout.write( + "\r%s %s / %s (%.2f%%)" % ( + self._filename, self._seen_so_far, self._size, + percentage + ) + ) + sys.stdout.flush() + + +def asciify(text): + ''' + Converts a unicode string to pure ascii. + ''' + normal = normalize('NFKC', text) + return normal.encode('ascii', 'backslashreplace').decode('ascii') + + +def get_fullname(artist): + ''' + String representing the artist's full name including an alias, if + available. + ''' + if artist['alias']: + if artist['first_name'] or artist['last_name']: + return '{} "{}" {}'.format(artist['first_name'], + artist['alias'], + artist['last_name']) + return artist['alias'] + return '{} {}'.format(artist['first_name'], artist['last_name']) + + +def beautify_artists(artists): + ''' + Turns a list of one or more artists into a proper English listing. + ''' + fullnames = [get_fullname(artist) for artist in artists] + output = ', ' + if len(fullnames) == 2: + output = ' & ' + return output.join(fullnames) + + +def import_playlist(playlist_file): + ''' + Imports a playlist from a JSON file, uploads the files to an S3[-like] + instance, and exports a new JSON file with the updated paths. + ''' + if not os.path.isfile(playlist_file): + raise FileNotFoundError + + with open(playlist_file, 'r', encoding='utf8') as pfile: + playlist = json.load(pfile) + + session = boto3.session.Session() + client = session.client( + 's3', + region_name=S3_REGION, + endpoint_url=S3_ENDPOINT, + aws_access_key_id=S3_ACCESS_KEY, + aws_secret_access_key=S3_SECRET_KEY + ) + + for song in playlist['songs']: + old_path = song['store']['path'] + + if song['type'] == 'S': + prefix = 'songs' + metadata = { + 'album': asciify(song['album']), + 'artists': asciify(beautify_artists(song['artists'])), + 'game': asciify(song['game']), + 'title': asciify(song['title']), + 'length': str(song['store']['length']), + 'original-path': asciify(old_path) + } + else: + prefix = 'jingles' + metadata = { + 'artists': asciify(RADIO_NAME), + 'title': asciify(song['title']), + 'length': str(song['store']['length']), + 'original-path': asciify(old_path) + } + file_hash = song['store']['filehash'] + ext = os.path.splitext(old_path)[1] + new_path = '{}/{}{}'.format(prefix, file_hash, ext) + + client.upload_file( + old_path, + S3_BUCKET, + new_path, + ExtraArgs={ + 'Metadata': metadata, + 'ContentType': song['store']['mime'] + }, + Callback=Progress(old_path) + ) + + song['store']['path'] = 's3://{}/{}'.format(S3_BUCKET, new_path) + + sys.stdout.write("\r\n") + sys.stdout.flush() + + return playlist + + +def main(): + '''Main loop of the program''' + + description = 'Uploads song files to an Amazon S3 (or similar) instance.' + + parser = argparse.ArgumentParser(description=description) + subparsers = parser.add_subparsers(dest='command') + + parser_playlist = subparsers.add_parser( + 'playlist', + help='Import playlist song data.' + ) + parser_playlist.add_argument( + 'filepath', + help='Path to the playlist file.', + nargs=1 + ) + + if len(sys.argv) == 1: + sys.stderr.write('Error: please specify a command\n\n') + parser.print_help(sys.stderr) + sys.exit(1) + + results = None + + args = parser.parse_args() + + if args.command == 'playlist': + results = import_playlist(args.filepath[0]) + + if results: + with open('playlist_s3.json', 'w', encoding='utf8') as file: + json.dump( + results, + file, + ensure_ascii=False, + sort_keys=True, + indent=4 + ) + + +if __name__ == '__main__': + main() diff --git a/savepointradio/radio/management/commands/importoldradio.py b/savepointradio/radio/management/commands/importoldradio.py index 0d0915b..3e62e46 100644 --- a/savepointradio/radio/management/commands/importoldradio.py +++ b/savepointradio/radio/management/commands/importoldradio.py @@ -6,6 +6,7 @@ for seeding a newly created database. import decimal import json import os +import re from django.core.management.base import BaseCommand, CommandError @@ -90,8 +91,18 @@ class Command(BaseCommand): ) new_song.artists.add(new_artist) + localfile = re.match( + r'^(?:(?:[A-Za-z]:|\\)\\|\/)', + song['store']['path'] + ) + + if localfile: + iri = path_to_iri(song['store']['path']) + else: + iri = song['store']['path'] + new_store = Store.objects.create( - iri=path_to_iri(song['store']['path']), + iri=iri, mime_type=song['store']['mime'], file_size=song['store']['filesize'], length=song['store']['length']