Support uploading to Amazon S3 (and equivalent).

This commit is contained in:
RecursiveGreen 2019-06-12 13:51:36 -04:00
parent 490e37a2f5
commit 0907ea72bd
3 changed files with 217 additions and 1 deletions

View file

@ -0,0 +1,9 @@
boto3>=1.9.166
botocore>=1.12.166
docutils>=0.14
jmespath>=0.9.4
python-dateutil>=2.8.0
python-decouple>=3.1
s3transfer>=0.2.1
six>=1.12.0
urllib3>=1.25.3

View file

@ -0,0 +1,196 @@
'''
upload_s3.py
This is the helper script that uploads songs from an exported playlist into
an Amazon S3 instance (or other implementations, like DigialOcean Spaces).
'''
import argparse
import json
import os
import sys
import threading
from unicodedata import normalize
from decouple import config
import boto3
# If these four are not defined, then boto3 will look for defaults in the
# ~/.aws configurations
S3_REGION = config('S3_REGION', default=None)
S3_ENDPOINT = config('S3_ENDPOINT', default=None)
S3_ACCESS_KEY = config('S3_ACCESS_KEY', default=None)
S3_SECRET_KEY = config('S3_SECRET_KEY', default=None)
# This has to be defined regardless.
S3_BUCKET = config('S3_BUCKET')
# Radio name for metadata
RADIO_NAME = config('RADIO_NAME', default='Save Point Radio')
class Progress(object):
'''
A callback class for the Amazon S3 upload to detect how far along in an
upload we are.
'''
def __init__(self, filepath):
self._filepath = filepath
self._filename = os.path.basename(filepath)
self._size = float(os.path.getsize(filepath))
self._seen_so_far = 0
self._lock = threading.Lock()
def __call__(self, bytes_amount):
with self._lock:
self._seen_so_far += bytes_amount
percentage = (self._seen_so_far / self._size) * 100
sys.stdout.write(
"\r%s %s / %s (%.2f%%)" % (
self._filename, self._seen_so_far, self._size,
percentage
)
)
sys.stdout.flush()
def asciify(text):
'''
Converts a unicode string to pure ascii.
'''
normal = normalize('NFKC', text)
return normal.encode('ascii', 'backslashreplace').decode('ascii')
def get_fullname(artist):
'''
String representing the artist's full name including an alias, if
available.
'''
if artist['alias']:
if artist['first_name'] or artist['last_name']:
return '{} "{}" {}'.format(artist['first_name'],
artist['alias'],
artist['last_name'])
return artist['alias']
return '{} {}'.format(artist['first_name'], artist['last_name'])
def beautify_artists(artists):
'''
Turns a list of one or more artists into a proper English listing.
'''
fullnames = [get_fullname(artist) for artist in artists]
output = ', '
if len(fullnames) == 2:
output = ' & '
return output.join(fullnames)
def import_playlist(playlist_file):
'''
Imports a playlist from a JSON file, uploads the files to an S3[-like]
instance, and exports a new JSON file with the updated paths.
'''
if not os.path.isfile(playlist_file):
raise FileNotFoundError
with open(playlist_file, 'r', encoding='utf8') as pfile:
playlist = json.load(pfile)
session = boto3.session.Session()
client = session.client(
's3',
region_name=S3_REGION,
endpoint_url=S3_ENDPOINT,
aws_access_key_id=S3_ACCESS_KEY,
aws_secret_access_key=S3_SECRET_KEY
)
for song in playlist['songs']:
old_path = song['store']['path']
if song['type'] == 'S':
prefix = 'songs'
metadata = {
'album': asciify(song['album']),
'artists': asciify(beautify_artists(song['artists'])),
'game': asciify(song['game']),
'title': asciify(song['title']),
'length': str(song['store']['length']),
'original-path': asciify(old_path)
}
else:
prefix = 'jingles'
metadata = {
'artists': asciify(RADIO_NAME),
'title': asciify(song['title']),
'length': str(song['store']['length']),
'original-path': asciify(old_path)
}
file_hash = song['store']['filehash']
ext = os.path.splitext(old_path)[1]
new_path = '{}/{}{}'.format(prefix, file_hash, ext)
client.upload_file(
old_path,
S3_BUCKET,
new_path,
ExtraArgs={
'Metadata': metadata,
'ContentType': song['store']['mime']
},
Callback=Progress(old_path)
)
song['store']['path'] = 's3://{}/{}'.format(S3_BUCKET, new_path)
sys.stdout.write("\r\n")
sys.stdout.flush()
return playlist
def main():
'''Main loop of the program'''
description = 'Uploads song files to an Amazon S3 (or similar) instance.'
parser = argparse.ArgumentParser(description=description)
subparsers = parser.add_subparsers(dest='command')
parser_playlist = subparsers.add_parser(
'playlist',
help='Import playlist song data.'
)
parser_playlist.add_argument(
'filepath',
help='Path to the playlist file.',
nargs=1
)
if len(sys.argv) == 1:
sys.stderr.write('Error: please specify a command\n\n')
parser.print_help(sys.stderr)
sys.exit(1)
results = None
args = parser.parse_args()
if args.command == 'playlist':
results = import_playlist(args.filepath[0])
if results:
with open('playlist_s3.json', 'w', encoding='utf8') as file:
json.dump(
results,
file,
ensure_ascii=False,
sort_keys=True,
indent=4
)
if __name__ == '__main__':
main()

View file

@ -6,6 +6,7 @@ for seeding a newly created database.
import decimal
import json
import os
import re
from django.core.management.base import BaseCommand, CommandError
@ -90,8 +91,18 @@ class Command(BaseCommand):
)
new_song.artists.add(new_artist)
localfile = re.match(
r'^(?:(?:[A-Za-z]:|\\)\\|\/)',
song['store']['path']
)
if localfile:
iri = path_to_iri(song['store']['path'])
else:
iri = song['store']['path']
new_store = Store.objects.create(
iri=path_to_iri(song['store']['path']),
iri=iri,
mime_type=song['store']['mime'],
file_size=song['store']['filesize'],
length=song['store']['length']