Maxence G. de Montauzan
8121f3d751
(cherry picked from commit dd322405d047d49e51d528341cbd008d7a98b6ab)
67 lines
1.9 KiB
Python
67 lines
1.9 KiB
Python
import requests
|
|
import json
|
|
|
|
ELS_URL ='http://localhost:9200'
|
|
INDEX = 'itunes-suggest'
|
|
|
|
class NoGoodDataException(Exception):
|
|
def __init__(self, message):
|
|
super().__init__(message)
|
|
|
|
def get_tokens(data: str) -> list:
|
|
if not data:
|
|
return []
|
|
query = {
|
|
"analyzer": "names",
|
|
"text" : data
|
|
}
|
|
|
|
url = '{}/{}/_analyze'.format(ELS_URL, INDEX)
|
|
r = requests.get(url, json=query)
|
|
|
|
if not 'tokens' in r.json():
|
|
print('ERROR: Not tokens in result')
|
|
print('Input: ' + str(data))
|
|
print('Request: ' + str(r.json()))
|
|
raise NoGoodDataException('Data is not correct to get tokens')
|
|
return [t['token'] for t in r.json()['tokens']]
|
|
|
|
def post_document(artist: str, artist_sugget: list, album: str, album_suggest: list) -> bool:
|
|
element = {
|
|
"artist_suggest" : artist_sugget,
|
|
"artist": artist,
|
|
"album": album,
|
|
"album_suggest": album_suggest}
|
|
|
|
# Filter empty keys
|
|
element = {k: v for k, v in element.items() if v}
|
|
|
|
url = '{}/{}/_doc'.format(ELS_URL, INDEX)
|
|
resp = requests.post(url, json=element)
|
|
if resp.status_code != 201:
|
|
print('ELS Response KO')
|
|
print(resp.status_code)
|
|
print(resp.text)
|
|
return
|
|
|
|
el_id = resp.json()['_id']
|
|
# print('Post_element - Element created: ' + el_id)
|
|
return el_id
|
|
|
|
|
|
# Using readlines()
|
|
itunes_file = open('/home/budd/workspace/iTunes/es-artists.json', 'r')
|
|
lines = itunes_file.readlines()
|
|
|
|
# Strips the newline character
|
|
for line in lines:
|
|
data = json.loads(line)
|
|
if "Artist" in data:
|
|
try :
|
|
artist_input = get_tokens(data['Artist'])
|
|
album_input = get_tokens(data['Album'])
|
|
post_document(data['Artist'], artist_input, data['Album'], album_input)
|
|
except NoGoodDataException:
|
|
print('ERROR WITH DATA')
|
|
print(str(data))
|