maxenceg2m/iTunes
1
0
Fork 0
Code Issues 2 Pull Requests Projects Releases Wiki Activity

Refactor send data argparse

Browse Source
This commit is contained in:
Maxence G. de Montauzan
2021-09-05 02:21:24 +02:00
parent 0230bf260b
commit 95534c92b2
1 changed files with 52 additions and 52 deletions
Download Patch File Download Diff File Expand all files Collapse all files

104
send_data.py
View File

@@ -24,13 +24,13 @@ class bcolors:
UNDERLINE = '\033[4m'
# Default file names
DEFAULT_SONG_FILE = 'es-songs.json'
DEFAULT_ALBUM_FILE = 'es-albums.json'
DEFAULT_ARTIST_FILE = 'es-artists.json'
DEFAULT_MAPPING_SONGS_FILE = 'mapping.songs.json'
DEFAULT_MAPPING_ARTISTS_FILE = 'mapping.artists.json'
DEFAULT_MAPPING_ALBUMS_FILE = 'mapping.albums.json'
DEFAULT_MAPPING_SUGGEST_FILE = 'mapping.suggest.json'
SONG_FILE = 'es-songs.json'
ALBUM_FILE = 'es-albums.json'
ARTIST_FILE = 'es-artists.json'
MAPPING_SONGS_FILE = 'mapping.songs.json'
MAPPING_ARTISTS_FILE = 'mapping.artists.json'
MAPPING_ALBUMS_FILE = 'mapping.albums.json'
MAPPING_SUGGEST_FILE = 'mapping.suggest.json'
SONG_INDEX = 'itunes-songs'
ALBUM_INDEX = 'itunes-albums'
@@ -51,8 +51,8 @@ def main():
args = create_args_parser().parse_args()
if not args.song and args.ALL:
print(__file__ + ': error: argument -A/--ALL: not allowed with argument -s/--song')
if args.ALL and args.no_song:
print(__file__ + ': error: argument -A/--ALL: not allowed with argument --no-song')
sys.exit(-1)
# Overloaded setting value
@@ -72,16 +72,16 @@ def main():
check_is_ok = []
# Send song data
if args.song or args.ALL:
if not args.no_song:
if args.DELETE:
mapping_song = load_file(args.mapping_song, DEFAULT_MAPPING_SONGS_FILE)
mapping_song = load_file(args.mapping_song, MAPPING_SONGS_FILE)
if not args.quiet:
print("Mapping of song index file: '{}'".format(mapping_song.name))
delete_index(SONG_INDEX, args.quiet)
put_mapping(SONG_INDEX, mapping_song, args.quiet)
song_file = load_file(args.song_file, DEFAULT_SONG_FILE)
song_file = load_file(args.song_file, SONG_FILE)
if not args.quiet:
print("Song file: '{}'".format(song_file.name))
@@ -100,7 +100,7 @@ def main():
# Send artist data
if args.artist_file or args.ALL:
if args.DELETE:
mapping_artist = load_file(args.mapping_artist, DEFAULT_MAPPING_ARTISTS_FILE)
mapping_artist = load_file(args.mapping_artist, MAPPING_ARTISTS_FILE)
if not args.quiet:
print("Mapping of artist index file: '{}'".format(mapping_artist.name))
@@ -111,7 +111,7 @@ def main():
if not artist_file:
if not args.quiet:
print('No artist file specified, take default file...')
artist_file = open(DEFAULT_ARTIST_FILE, 'r')
artist_file = open(ARTIST_FILE, 'r')
if not args.quiet:
print("Artist file: '{}'".format(artist_file.name))
@@ -126,7 +126,7 @@ def main():
if args.album_file or args.ALL:
if args.DELETE:
mapping_album = load_file(args.mapping_album, DEFAULT_MAPPING_ALBUMS_FILE)
mapping_album = load_file(args.mapping_album, MAPPING_ALBUMS_FILE)
if not args.quiet:
print("Mapping of artist index file: '{}'".format(mapping_album.name))
@@ -137,7 +137,7 @@ def main():
if not album_file:
if not args.quiet:
print('No album file specified, take default file...')
album_file = open(DEFAULT_ALBUM_FILE, 'r')
album_file = open(ALBUM_FILE, 'r')
if not args.quiet:
print("Take file '{}' to send song data".format(album_file.name))
@@ -149,26 +149,26 @@ def main():
else:
print('Album sent')
if args.ALL or args.no_suggest:
if not args.no_suggest:
print("Process suggestion:")
if args.DELETE: # TODO Do a method?
if args.DELETE:
delete_index(SUGGEST_INDEX, args.quiet)
if not args.ALL and not args.album_file and not args.artist_file:
print('Only song file processed. No suggestion to process.')
else:
if args.DELETE:
mapping_suggest = load_file(args.mapping_suggest, DEFAULT_MAPPING_SUGGEST_FILE)
mapping_suggest = load_file(args.mapping_suggest, MAPPING_SUGGEST_FILE)
if not args.quiet:
print("Mapping of suggest index file: '{}'".format(mapping_suggest.name))
put_mapping(SUGGEST_INDEX, mapping_suggest, args.quiet)
suggs_docs = 0
if args.album_file or args.ALL:
suggs_docs += process_file(DEFAULT_ALBUM_FILE, 'Album')
suggs_docs += process_file(ALBUM_FILE, 'Album')
print('Created suggestion documents: ' + str(suggs_docs))
if args.artist_file or args.ALL:
suggs_docs += process_file(DEFAULT_ARTIST_FILE, 'Artist', 'Album Artist')
suggs_docs += process_file(ARTIST_FILE, 'Artist', 'Album Artist')
print('Created suggestion documents: ' + str(suggs_docs))
print("I'm done!")
@@ -199,15 +199,14 @@ def create_args_parser():
# TODO rewrit description with multi-index phylosophie
parser = argparse.ArgumentParser(
description='''
Send JSON files formated for bulk Elasticsearch operation to an Elasticsearch.
Send JSON files formated for bulk Elasticsearch operation to an Elasticsearch.
By default: send song data enable, send album & artist data disabled.
Check that all the data has been sent.
By default: send only song data. See option to send album/artist/suggest data.
Detect if index doesn't exist and create it with a mapping file (see -map and -idx argument).
Remeber : it's cumulative! If you want to remove songs/artits/albums,
you have to delete and re-create the index (use -D option).
'''
Create index if -D option activated with a mapping file (see -map).
It's cumulative! If you want to remove songs/artits/albums, you have to delete and re-create the index (use -D option).''',
formatter_class=argparse.RawTextHelpFormatter
)
# Bulk
parser.add_argument('-q', '--quiet', action='store_true',
@@ -216,42 +215,43 @@ def create_args_parser():
sending_group = parser.add_argument_group("Sending options")
song_group = sending_group.add_mutually_exclusive_group()
song_group.add_argument('-sf', '--song-file', type=argparse.FileType('r'),
help='Song file data to send (default: \'{}\').'.format(DEFAULT_SONG_FILE))
sending_group.add_argument('-al', '--album-file', nargs='?', type=argparse.FileType('r'), const=DEFAULT_ALBUM_FILE,
help='Song file data to send (default: \'{}\').'.format(SONG_FILE))
sending_group.add_argument('-al', '--album-file', nargs='?', type=argparse.FileType('r'), const=ALBUM_FILE,
help='Enable sending album data. Optionally, precise the album data file (default: \'{}\')'
.format(DEFAULT_ALBUM_FILE))
sending_group.add_argument('-ar', '--artist-file', nargs='?', type=argparse.FileType('r'), const=DEFAULT_ARTIST_FILE,
.format(ALBUM_FILE))
sending_group.add_argument('-ar', '--artist-file', nargs='?', type=argparse.FileType('r'), const=ARTIST_FILE,
help='Enable sending artist data. Optionally, precise the artist data file (default: \'{}\')'
.format(DEFAULT_ARTIST_FILE))
song_group.add_argument('-s', '--song', action='store_false',
help='Disable sending song data')
song_group.add_argument('--no-suggest', action='store_false',
help='Disable sending suggest data')
.format(ARTIST_FILE))
# Mode
mode_group = parser.add_argument_group('Mode')
mode_group.add_argument('-A', '--ALL', action='store_true',
help='Send all possible data: song, artist and album')
help='Send all possible data: song, artist, album and suggest. Use default file if not specified')
mode_group.add_argument('-D', '--DELETE', action='store_true',
help='''Delete old index and create a new.
See -idx argument to set index name.
See -map arguement to set mapping file.''')
help='Delete index and create new. See -map arguement to set mapping file')
mode_group.add_argument('--no-song', action='store_true',
help='''Disable sending song data.
Not allowed with -A option.''')
mode_group.add_argument('--no-suggest', action='store_true',
help='Disable sending suggest data. Allowed with -A option')
# Mapping
mapping_group = parser.add_argument_group('Mapping files')
mapping_group.add_argument('-ms', '--mapping-song', type=argparse.FileType('r'), const=DEFAULT_MAPPING_SONGS_FILE, nargs='?',
help='Mapping file for songs (default: \'{}\')'.format(DEFAULT_MAPPING_SONGS_FILE))
mapping_group.add_argument('-mr', '--mapping-artist', type=argparse.FileType('r'), const=DEFAULT_ARTIST_FILE, nargs='?',
help='Mapping file for artists (default: \'{}\')'.format(DEFAULT_MAPPING_ARTISTS_FILE))
mapping_group.add_argument('-ml', '--mapping-album', type=argparse.FileType('r'), const=DEFAULT_MAPPING_ALBUMS_FILE, nargs='?',
help='Mapping file for albums (default: \'{}\')'.format(DEFAULT_MAPPING_ALBUMS_FILE))
mapping_group.add_argument('-mg', '--mapping-suggest', type=argparse.FileType('r'), const=DEFAULT_MAPPING_SUGGEST_FILE, nargs='?',
help='Mapping file for suggest (default: \'{}\')'.format(DEFAULT_MAPPING_SUGGEST_FILE))
# CAUTION default values cannot be used because they necessarily activate the option
# QUESTION Use a for with a list of default mapping file?
mapping_group.add_argument('-ms', '--mapping-song', type=argparse.FileType('r'), const=MAPPING_SONGS_FILE, nargs='?',
help='Mapping file for songs (default: \'{}\')'.format(MAPPING_SONGS_FILE))
mapping_group.add_argument('-mr', '--mapping-artist', type=argparse.FileType('r'), const=ARTIST_FILE, nargs='?',
help='Mapping file for artists (default: \'{}\')'.format(MAPPING_ARTISTS_FILE))
mapping_group.add_argument('-ml', '--mapping-album', type=argparse.FileType('r'), const=MAPPING_ALBUMS_FILE, nargs='?',
help='Mapping file for albums (default: \'{}\')'.format(MAPPING_ALBUMS_FILE))
mapping_group.add_argument('-mg', '--mapping-suggest', type=argparse.FileType('r'), const=MAPPING_SUGGEST_FILE, nargs='?',
help='Mapping file for suggest (default: \'{}\')'.format(MAPPING_SUGGEST_FILE))
# Global Settings
g_settings_group = parser.add_argument_group('Global Settings')
g_settings_group.add_argument('-els', '--elasticsearch-url', default=ELASTICSEARCH_URL, nargs='?',
help="Elasticsearch URL (default: \'{}\')".format(ELASTICSEARCH_URL))
help="Elasticsearch URL.")
return parser
@@ -283,7 +283,7 @@ def delete_index(index_name, quiet=False):
res = requests.delete(url=ELASTICSEARCH_URL + index_name)
if res.status_code == 200:
if not quiet:
print(bcolors.OKGREEN + "Index deleted!" + bcolors.ENDC)
print(bcolors.OKGREEN + "Index '{}' deleted!".format(index_name) + bcolors.ENDC)
else:
print(bcolors.FAIL + "An error occured" + bcolors.ENDC)
if res.json()['error']['type'] == 'index_not_found_exception':
@@ -306,7 +306,7 @@ def put_mapping(index_name, mapping_file, quiet=False):
print(res.text + bcolors.ENDC)
else:
if not quiet:
print(bcolors.OKGREEN + "Mapping sent" + bcolors.ENDC)
print(bcolors.OKGREEN + "Mapping for '{}' sent".format(index_name) + bcolors.ENDC)
put_setting(index_name, 0, quiet)