maxenceg2m/iTunes
1
0
Fork 0
Code Issues 2 Pull Requests Projects Releases Wiki Activity

Refactor send data argparse

Browse Source
This commit is contained in:
Maxence G. de Montauzan
2021-09-05 02:21:24 +02:00
parent 0230bf260b
commit 95534c92b2
1 changed files with 52 additions and 52 deletions
Download Patch File Download Diff File Expand all files Collapse all files

104
send_data.py
View File

@@ -24,13 +24,13 @@ class bcolors:
UNDERLINE = '\033[4m' UNDERLINE = '\033[4m'
# Default file names # Default file names
DEFAULT_SONG_FILE = 'es-songs.json' SONG_FILE = 'es-songs.json'
DEFAULT_ALBUM_FILE = 'es-albums.json' ALBUM_FILE = 'es-albums.json'
DEFAULT_ARTIST_FILE = 'es-artists.json' ARTIST_FILE = 'es-artists.json'
DEFAULT_MAPPING_SONGS_FILE = 'mapping.songs.json' MAPPING_SONGS_FILE = 'mapping.songs.json'
DEFAULT_MAPPING_ARTISTS_FILE = 'mapping.artists.json' MAPPING_ARTISTS_FILE = 'mapping.artists.json'
DEFAULT_MAPPING_ALBUMS_FILE = 'mapping.albums.json' MAPPING_ALBUMS_FILE = 'mapping.albums.json'
DEFAULT_MAPPING_SUGGEST_FILE = 'mapping.suggest.json' MAPPING_SUGGEST_FILE = 'mapping.suggest.json'
SONG_INDEX = 'itunes-songs' SONG_INDEX = 'itunes-songs'
ALBUM_INDEX = 'itunes-albums' ALBUM_INDEX = 'itunes-albums'
@@ -51,8 +51,8 @@ def main():
args = create_args_parser().parse_args() args = create_args_parser().parse_args()
if not args.song and args.ALL: if args.ALL and args.no_song:
print(__file__ + ': error: argument -A/--ALL: not allowed with argument -s/--song') print(__file__ + ': error: argument -A/--ALL: not allowed with argument --no-song')
sys.exit(-1) sys.exit(-1)
# Overloaded setting value # Overloaded setting value
@@ -72,16 +72,16 @@ def main():
check_is_ok = [] check_is_ok = []
# Send song data # Send song data
if args.song or args.ALL: if not args.no_song:
if args.DELETE: if args.DELETE:
mapping_song = load_file(args.mapping_song, DEFAULT_MAPPING_SONGS_FILE) mapping_song = load_file(args.mapping_song, MAPPING_SONGS_FILE)
if not args.quiet: if not args.quiet:
print("Mapping of song index file: '{}'".format(mapping_song.name)) print("Mapping of song index file: '{}'".format(mapping_song.name))
delete_index(SONG_INDEX, args.quiet) delete_index(SONG_INDEX, args.quiet)
put_mapping(SONG_INDEX, mapping_song, args.quiet) put_mapping(SONG_INDEX, mapping_song, args.quiet)
song_file = load_file(args.song_file, DEFAULT_SONG_FILE) song_file = load_file(args.song_file, SONG_FILE)
if not args.quiet: if not args.quiet:
print("Song file: '{}'".format(song_file.name)) print("Song file: '{}'".format(song_file.name))
@@ -100,7 +100,7 @@ def main():
# Send artist data # Send artist data
if args.artist_file or args.ALL: if args.artist_file or args.ALL:
if args.DELETE: if args.DELETE:
mapping_artist = load_file(args.mapping_artist, DEFAULT_MAPPING_ARTISTS_FILE) mapping_artist = load_file(args.mapping_artist, MAPPING_ARTISTS_FILE)
if not args.quiet: if not args.quiet:
print("Mapping of artist index file: '{}'".format(mapping_artist.name)) print("Mapping of artist index file: '{}'".format(mapping_artist.name))
@@ -111,7 +111,7 @@ def main():
if not artist_file: if not artist_file:
if not args.quiet: if not args.quiet:
print('No artist file specified, take default file...') print('No artist file specified, take default file...')
artist_file = open(DEFAULT_ARTIST_FILE, 'r') artist_file = open(ARTIST_FILE, 'r')
if not args.quiet: if not args.quiet:
print("Artist file: '{}'".format(artist_file.name)) print("Artist file: '{}'".format(artist_file.name))
@@ -126,7 +126,7 @@ def main():
if args.album_file or args.ALL: if args.album_file or args.ALL:
if args.DELETE: if args.DELETE:
mapping_album = load_file(args.mapping_album, DEFAULT_MAPPING_ALBUMS_FILE) mapping_album = load_file(args.mapping_album, MAPPING_ALBUMS_FILE)
if not args.quiet: if not args.quiet:
print("Mapping of artist index file: '{}'".format(mapping_album.name)) print("Mapping of artist index file: '{}'".format(mapping_album.name))
@@ -137,7 +137,7 @@ def main():
if not album_file: if not album_file:
if not args.quiet: if not args.quiet:
print('No album file specified, take default file...') print('No album file specified, take default file...')
album_file = open(DEFAULT_ALBUM_FILE, 'r') album_file = open(ALBUM_FILE, 'r')
if not args.quiet: if not args.quiet:
print("Take file '{}' to send song data".format(album_file.name)) print("Take file '{}' to send song data".format(album_file.name))
@@ -149,26 +149,26 @@ def main():
else: else:
print('Album sent') print('Album sent')
if args.ALL or args.no_suggest: if not args.no_suggest:
print("Process suggestion:") print("Process suggestion:")
if args.DELETE: # TODO Do a method? if args.DELETE:
delete_index(SUGGEST_INDEX, args.quiet) delete_index(SUGGEST_INDEX, args.quiet)
if not args.ALL and not args.album_file and not args.artist_file: if not args.ALL and not args.album_file and not args.artist_file:
print('Only song file processed. No suggestion to process.') print('Only song file processed. No suggestion to process.')
else: else:
if args.DELETE: if args.DELETE:
mapping_suggest = load_file(args.mapping_suggest, DEFAULT_MAPPING_SUGGEST_FILE) mapping_suggest = load_file(args.mapping_suggest, MAPPING_SUGGEST_FILE)
if not args.quiet: if not args.quiet:
print("Mapping of suggest index file: '{}'".format(mapping_suggest.name)) print("Mapping of suggest index file: '{}'".format(mapping_suggest.name))
put_mapping(SUGGEST_INDEX, mapping_suggest, args.quiet) put_mapping(SUGGEST_INDEX, mapping_suggest, args.quiet)
suggs_docs = 0 suggs_docs = 0
if args.album_file or args.ALL: if args.album_file or args.ALL:
suggs_docs += process_file(DEFAULT_ALBUM_FILE, 'Album') suggs_docs += process_file(ALBUM_FILE, 'Album')
print('Created suggestion documents: ' + str(suggs_docs)) print('Created suggestion documents: ' + str(suggs_docs))
if args.artist_file or args.ALL: if args.artist_file or args.ALL:
suggs_docs += process_file(DEFAULT_ARTIST_FILE, 'Artist', 'Album Artist') suggs_docs += process_file(ARTIST_FILE, 'Artist', 'Album Artist')
print('Created suggestion documents: ' + str(suggs_docs)) print('Created suggestion documents: ' + str(suggs_docs))
print("I'm done!") print("I'm done!")
@@ -199,15 +199,14 @@ def create_args_parser():
# TODO rewrit description with multi-index phylosophie # TODO rewrit description with multi-index phylosophie
parser = argparse.ArgumentParser( parser = argparse.ArgumentParser(
description=''' description='''
Send JSON files formated for bulk Elasticsearch operation to an Elasticsearch. Send JSON files formated for bulk Elasticsearch operation to an Elasticsearch.
By default: send song data enable, send album & artist data disabled. By default: send only song data. See option to send album/artist/suggest data.
Check that all the data has been sent.
Detect if index doesn't exist and create it with a mapping file (see -map and -idx argument). Create index if -D option activated with a mapping file (see -map).
Remeber : it's cumulative! If you want to remove songs/artits/albums,
you have to delete and re-create the index (use -D option). It's cumulative! If you want to remove songs/artits/albums, you have to delete and re-create the index (use -D option).''',
''' formatter_class=argparse.RawTextHelpFormatter
) )
# Bulk # Bulk
parser.add_argument('-q', '--quiet', action='store_true', parser.add_argument('-q', '--quiet', action='store_true',
@@ -216,42 +215,43 @@ def create_args_parser():
sending_group = parser.add_argument_group("Sending options") sending_group = parser.add_argument_group("Sending options")
song_group = sending_group.add_mutually_exclusive_group() song_group = sending_group.add_mutually_exclusive_group()
song_group.add_argument('-sf', '--song-file', type=argparse.FileType('r'), song_group.add_argument('-sf', '--song-file', type=argparse.FileType('r'),
help='Song file data to send (default: \'{}\').'.format(DEFAULT_SONG_FILE)) help='Song file data to send (default: \'{}\').'.format(SONG_FILE))
sending_group.add_argument('-al', '--album-file', nargs='?', type=argparse.FileType('r'), const=DEFAULT_ALBUM_FILE, sending_group.add_argument('-al', '--album-file', nargs='?', type=argparse.FileType('r'), const=ALBUM_FILE,
help='Enable sending album data. Optionally, precise the album data file (default: \'{}\')' help='Enable sending album data. Optionally, precise the album data file (default: \'{}\')'
.format(DEFAULT_ALBUM_FILE)) .format(ALBUM_FILE))
sending_group.add_argument('-ar', '--artist-file', nargs='?', type=argparse.FileType('r'), const=DEFAULT_ARTIST_FILE, sending_group.add_argument('-ar', '--artist-file', nargs='?', type=argparse.FileType('r'), const=ARTIST_FILE,
help='Enable sending artist data. Optionally, precise the artist data file (default: \'{}\')' help='Enable sending artist data. Optionally, precise the artist data file (default: \'{}\')'
.format(DEFAULT_ARTIST_FILE)) .format(ARTIST_FILE))
song_group.add_argument('-s', '--song', action='store_false',
help='Disable sending song data')
song_group.add_argument('--no-suggest', action='store_false',
help='Disable sending suggest data')
# Mode # Mode
mode_group = parser.add_argument_group('Mode') mode_group = parser.add_argument_group('Mode')
mode_group.add_argument('-A', '--ALL', action='store_true', mode_group.add_argument('-A', '--ALL', action='store_true',
help='Send all possible data: song, artist and album') help='Send all possible data: song, artist, album and suggest. Use default file if not specified')
mode_group.add_argument('-D', '--DELETE', action='store_true', mode_group.add_argument('-D', '--DELETE', action='store_true',
help='''Delete old index and create a new. help='Delete index and create new. See -map arguement to set mapping file')
See -idx argument to set index name. mode_group.add_argument('--no-song', action='store_true',
See -map arguement to set mapping file.''') help='''Disable sending song data.
Not allowed with -A option.''')
mode_group.add_argument('--no-suggest', action='store_true',
help='Disable sending suggest data. Allowed with -A option')
# Mapping # Mapping
mapping_group = parser.add_argument_group('Mapping files') mapping_group = parser.add_argument_group('Mapping files')
mapping_group.add_argument('-ms', '--mapping-song', type=argparse.FileType('r'), const=DEFAULT_MAPPING_SONGS_FILE, nargs='?', # CAUTION default values cannot be used because they necessarily activate the option
help='Mapping file for songs (default: \'{}\')'.format(DEFAULT_MAPPING_SONGS_FILE)) # QUESTION Use a for with a list of default mapping file?
mapping_group.add_argument('-mr', '--mapping-artist', type=argparse.FileType('r'), const=DEFAULT_ARTIST_FILE, nargs='?', mapping_group.add_argument('-ms', '--mapping-song', type=argparse.FileType('r'), const=MAPPING_SONGS_FILE, nargs='?',
help='Mapping file for artists (default: \'{}\')'.format(DEFAULT_MAPPING_ARTISTS_FILE)) help='Mapping file for songs (default: \'{}\')'.format(MAPPING_SONGS_FILE))
mapping_group.add_argument('-ml', '--mapping-album', type=argparse.FileType('r'), const=DEFAULT_MAPPING_ALBUMS_FILE, nargs='?', mapping_group.add_argument('-mr', '--mapping-artist', type=argparse.FileType('r'), const=ARTIST_FILE, nargs='?',
help='Mapping file for albums (default: \'{}\')'.format(DEFAULT_MAPPING_ALBUMS_FILE)) help='Mapping file for artists (default: \'{}\')'.format(MAPPING_ARTISTS_FILE))
mapping_group.add_argument('-mg', '--mapping-suggest', type=argparse.FileType('r'), const=DEFAULT_MAPPING_SUGGEST_FILE, nargs='?', mapping_group.add_argument('-ml', '--mapping-album', type=argparse.FileType('r'), const=MAPPING_ALBUMS_FILE, nargs='?',
help='Mapping file for suggest (default: \'{}\')'.format(DEFAULT_MAPPING_SUGGEST_FILE)) help='Mapping file for albums (default: \'{}\')'.format(MAPPING_ALBUMS_FILE))
mapping_group.add_argument('-mg', '--mapping-suggest', type=argparse.FileType('r'), const=MAPPING_SUGGEST_FILE, nargs='?',
help='Mapping file for suggest (default: \'{}\')'.format(MAPPING_SUGGEST_FILE))
# Global Settings # Global Settings
g_settings_group = parser.add_argument_group('Global Settings') g_settings_group = parser.add_argument_group('Global Settings')
g_settings_group.add_argument('-els', '--elasticsearch-url', default=ELASTICSEARCH_URL, nargs='?', g_settings_group.add_argument('-els', '--elasticsearch-url', default=ELASTICSEARCH_URL, nargs='?',
help="Elasticsearch URL (default: \'{}\')".format(ELASTICSEARCH_URL)) help="Elasticsearch URL.")
return parser return parser
@@ -283,7 +283,7 @@ def delete_index(index_name, quiet=False):
res = requests.delete(url=ELASTICSEARCH_URL + index_name) res = requests.delete(url=ELASTICSEARCH_URL + index_name)
if res.status_code == 200: if res.status_code == 200:
if not quiet: if not quiet:
print(bcolors.OKGREEN + "Index deleted!" + bcolors.ENDC) print(bcolors.OKGREEN + "Index '{}' deleted!".format(index_name) + bcolors.ENDC)
else: else:
print(bcolors.FAIL + "An error occured" + bcolors.ENDC) print(bcolors.FAIL + "An error occured" + bcolors.ENDC)
if res.json()['error']['type'] == 'index_not_found_exception': if res.json()['error']['type'] == 'index_not_found_exception':
@@ -306,7 +306,7 @@ def put_mapping(index_name, mapping_file, quiet=False):
print(res.text + bcolors.ENDC) print(res.text + bcolors.ENDC)
else: else:
if not quiet: if not quiet:
print(bcolors.OKGREEN + "Mapping sent" + bcolors.ENDC) print(bcolors.OKGREEN + "Mapping for '{}' sent".format(index_name) + bcolors.ENDC)
put_setting(index_name, 0, quiet) put_setting(index_name, 0, quiet)