110 lines
4.1 KiB
Python
110 lines
4.1 KiB
Python
#!/usr/bin/env python
|
|
|
|
"""
|
|
Send JSON files to ELS
|
|
"""
|
|
|
|
import argparse
|
|
import requests
|
|
|
|
def send_data(file, quiet=False):
|
|
"""
|
|
Send a data bulk file to ELS.
|
|
|
|
'file' should be (readable) file object.
|
|
"""
|
|
|
|
if not quiet:
|
|
print("Sending '{}' data file...".format(file.name))
|
|
res = requests.post(url=ELASTICSEARCH_URL + '_bulk',
|
|
data=file,
|
|
headers={'Content-Type': 'application/x-ndjson'})
|
|
if res.status_code != 200:
|
|
print("An error occured")
|
|
# TODO Catch "index_not_found_exception"
|
|
print(res.text)
|
|
else:
|
|
if not quiet:
|
|
print("File '{} sended to Elasticsearch!".format(file.name))
|
|
|
|
|
|
def delete_index(index_name, quiet=False):
|
|
"""
|
|
Delete an index in ELS
|
|
"""
|
|
if not quiet:
|
|
print('Deleting index {}...'.format(index_name))
|
|
res = requests.delete(url=ELASTICSEARCH_URL + INDEX_NAME)
|
|
if res.status_code == 200:
|
|
if not quiet:
|
|
print("Deleted!")
|
|
else:
|
|
print("An error occured")
|
|
print(res.text)
|
|
|
|
#### main block ####
|
|
|
|
ELASTICSEARCH_URL = 'http://localhost:9200/'
|
|
INDEX_NAME = "itunessongs"
|
|
|
|
# Get options
|
|
parser = argparse.ArgumentParser(
|
|
description='''
|
|
Send JSON files formated for bulk Elasticsearch operation to an Elasticsearch.
|
|
|
|
Send song data is enable by default, send album & artist data is disabled default.
|
|
'''
|
|
)
|
|
parser.add_argument('-s', '--song', action='store_false',
|
|
help='Disable send song data')
|
|
parser.add_argument('-sf', '--song-file', default='es-music-data.json', type=argparse.FileType('r'),
|
|
help='Song file data to send (default: es-music-data.json). Need to enable send song data.')
|
|
# FIXME If song is disable and default file not exist ==> Fail
|
|
parser.add_argument('-al', '--album', action='store_true',
|
|
help='Enable send album data')
|
|
parser.add_argument('-alf', '--album-file', default='es-albums-data.json', type=argparse.FileType('r'),
|
|
help='Album file data to send (default: es-albums-data.json). Need to enable send album data.')
|
|
parser.add_argument('-ar', '--artist', action='store_true',
|
|
help='Enable send music data')
|
|
parser.add_argument('-arf', '--artist-file', default='es-artist-data.json', type=argparse.FileType('r'),
|
|
help='Album file data to send (default: es-artist-data.json). Need to enable send artist data.')
|
|
parser.add_argument('-A', '--ALL', action='store_true',
|
|
help='Send all')
|
|
parser.add_argument('-D', '--DELETE', action='store_true',
|
|
help='Delete old index')
|
|
parser.add_argument('-els', '--elasticsearch_url', default=ELASTICSEARCH_URL, nargs='?',
|
|
help="Elasticsearch URL to send data (default: {})".format(ELASTICSEARCH_URL))
|
|
parser.add_argument('-i', '--index_name', default=INDEX_NAME,
|
|
help="Index name in Elasticsearch for data (default: {})".format(INDEX_NAME))
|
|
parser.add_argument('-q', '--quiet', action='store_true',
|
|
help="Disable main output")
|
|
|
|
if __name__ == '__main__':
|
|
args = parser.parse_args()
|
|
INDEX_NAME = args.index_name
|
|
ELASTICSEARCH_URL = args.elasticsearch_url
|
|
|
|
if not args.quiet:
|
|
print("Elasticsearch URL:\t" + ELASTICSEARCH_URL)
|
|
print("Index name:\t\t" + INDEX_NAME)
|
|
|
|
if args.DELETE:
|
|
delete_index(INDEX_NAME, args.quiet)
|
|
|
|
if args.song or args.ALL:
|
|
song_file = args.song_file
|
|
if not args.quiet:
|
|
print("Take file '{}' to send song data".format(song_file.name))
|
|
send_data(song_file, args.quiet)
|
|
if args.artist or args.ALL:
|
|
artist_file = args.artist_file
|
|
if not args.quiet:
|
|
print("Take file '{}' to send song data".format(artist_file.name))
|
|
send_data(artist_file, args.quiet)
|
|
|
|
if args.album or args.ALL:
|
|
album_file = args.album_file
|
|
if not args.quiet:
|
|
print("Take file '{}' to send song data".format(album_file.name))
|
|
send_data(album_file, args.quiet)
|