From 928efb659ecd8f3d6ce4c8159021955d79081055 Mon Sep 17 00:00:00 2001 From: "Maxence G. de Montauzan" Date: Tue, 31 Aug 2021 02:31:32 +0200 Subject: [PATCH] Use send data to process suggestion Not all suggested document is created in ELS Need to refactor send data --- mapping.suggest.json | 48 +++++++++++++++++++++++++++++++++++++++++++ send_data.py | 30 ++++++++++++++++++++++++--- suggester.es | 49 +------------------------------------------- suggester.py | 1 + 4 files changed, 77 insertions(+), 51 deletions(-) create mode 100644 mapping.suggest.json diff --git a/mapping.suggest.json b/mapping.suggest.json new file mode 100644 index 0000000..bed1b6c --- /dev/null +++ b/mapping.suggest.json @@ -0,0 +1,48 @@ +{ + "settings": { + "index": { + "number_of_replicas": 0 + }, + "analysis": { + "filter": { + "french_stop": { + "type": "stop", + "stopwords": "_french_" + }, + "english_stop": { + "type": "stop", + "stopwords": "_english_" + } + }, + "analyzer": { + "names": { + "tokenizer": "standard", + "filter": [ + "lowercase", + "asciifolding", + "french_stop", + "english_stop" + ] + } + } + } + }, + "mappings": { + "properties": { + "artist_suggest": { + "type": "completion", + "search_analyzer": "names" + }, + "artist": { + "type": "keyword" + }, + "album_suggest": { + "type": "completion", + "search_analyzer": "names" + }, + "album": { + "type": "keyword" + } + } + } +} diff --git a/send_data.py b/send_data.py index 35c573d..356f679 100644 --- a/send_data.py +++ b/send_data.py @@ -10,6 +10,8 @@ import json import time import requests +from suggester import process_file + class bcolors: HEADER = '\033[95m' OKBLUE = '\033[94m' @@ -28,10 +30,12 @@ DEFAULT_ARTIST_FILE = 'es-artists.json' DEFAULT_MAPPING_SONGS_FILE = 'mapping.songs.json' DEFAULT_MAPPING_ARTISTS_FILE = 'mapping.artists.json' DEFAULT_MAPPING_ALBUMS_FILE = 'mapping.albums.json' +DEFAULT_MAPPING_SUGGEST_FILE = 'mapping.suggest.json' SONG_INDEX = 'itunes-songs' ALBUM_INDEX = 'itunes-albums' ARTIST_INDEX = 'itunes-artists' +SUGGEST_INDEX = 'itunes-suggest' # TODO Put variables in a config files or in a python library # Global values / set as default values @@ -145,6 +149,22 @@ def main(): else: print('Album sent') + if args.ALL or args.no_suggest: + print("Process suggestion:") + if args.DELETE: # TODO Do a method? + mapping_suggest = load_file(args.mapping_suggest, DEFAULT_MAPPING_SUGGEST_FILE) + if not args.quiet: + print("Mapping of suggest index file: '{}'".format(mapping_suggest.name)) + + delete_index(SUGGEST_INDEX, args.quiet) + put_mapping(SUGGEST_INDEX, mapping_suggest, args.quiet) + + suggs_docs = 0 + suggs_docs += process_file('/home/budd/workspace/iTunes/es-albums.json', 'Album') + print('Created documents: ' + str(suggs_docs)) + suggs_docs += process_file('/home/budd/workspace/iTunes/es-artists.json', 'Artist', 'Album Artist') + print('Created documents: ' + str(suggs_docs)) + print("I'm done!") if check_is_ok.count(False) > 0: print('Some problems occurs') @@ -199,6 +219,8 @@ def create_args_parser(): .format(DEFAULT_ARTIST_FILE)) song_group.add_argument('-s', '--song', action='store_false', help='Disable sending song data') + song_group.add_argument('--no-suggest', action='store_false', + help='Disable sending suggest data') # Mode mode_group = parser.add_argument_group('Mode') @@ -211,12 +233,14 @@ def create_args_parser(): # Mapping mapping_group = parser.add_argument_group('Mapping files') - mode_group.add_argument('-ms', '--mapping-song', type=argparse.FileType('r'), const=DEFAULT_MAPPING_SONGS_FILE, nargs='?', + mapping_group.add_argument('-ms', '--mapping-song', type=argparse.FileType('r'), const=DEFAULT_MAPPING_SONGS_FILE, nargs='?', help='Mapping file for songs (default: \'{}\')'.format(DEFAULT_MAPPING_SONGS_FILE)) - mode_group.add_argument('-mr', '--mapping-artist', type=argparse.FileType('r'), const=DEFAULT_ARTIST_FILE, nargs='?', + mapping_group.add_argument('-mr', '--mapping-artist', type=argparse.FileType('r'), const=DEFAULT_ARTIST_FILE, nargs='?', help='Mapping file for artists (default: \'{}\')'.format(DEFAULT_MAPPING_ARTISTS_FILE)) - mode_group.add_argument('-ml', '--mapping-album', type=argparse.FileType('r'), const=DEFAULT_MAPPING_ALBUMS_FILE, nargs='?', + mapping_group.add_argument('-ml', '--mapping-album', type=argparse.FileType('r'), const=DEFAULT_MAPPING_ALBUMS_FILE, nargs='?', help='Mapping file for albums (default: \'{}\')'.format(DEFAULT_MAPPING_ALBUMS_FILE)) + mapping_group.add_argument('-mg', '--mapping-suggest', type=argparse.FileType('r'), const=DEFAULT_MAPPING_SUGGEST_FILE, nargs='?', + help='Mapping file for suggest (default: \'{}\')'.format(DEFAULT_MAPPING_SUGGEST_FILE)) # Global Settings g_settings_group = parser.add_argument_group('Global Settings') diff --git a/suggester.es b/suggester.es index c329ce0..660144c 100644 --- a/suggester.es +++ b/suggester.es @@ -1,54 +1,7 @@ DELETE itunes-suggest PUT /itunes-suggest -{ - "settings": { - "index": { - "number_of_replicas": 0 - }, - "analysis": { - "filter": { - "french_stop": { - "type": "stop", - "stopwords": "_french_" - }, - "english_stop": { - "type": "stop", - "stopwords": "_english_" - } - }, - "analyzer": { - "names": { - "tokenizer": "standard", - "filter": [ - "lowercase", - "asciifolding", - "french_stop", - "english_stop" - ] - } - } - } - }, - "mappings": { - "properties": { - "artist_suggest": { - "type": "completion", - "search_analyzer": "names" - }, - "artist": { - "type": "keyword" - }, - "album_suggest": { - "type": "completion", - "search_analyzer": "names" - }, - "album": { - "type": "keyword" - } - } - } -} +!./mapping.suggest.json // Also possible to specify analyze for ingesting => https://stackoverflow.com/questions/48304499/elasticsearch-completion-suggester-not-working-with-whitespace-analyzer diff --git a/suggester.py b/suggester.py index 6ebbc90..5fd07eb 100644 --- a/suggester.py +++ b/suggester.py @@ -139,3 +139,4 @@ if __name__ == '__main__': print('Created documents: ' + str(created_docs)) created_docs += process_file('/home/budd/workspace/iTunes/es-artists.json', 'Artist', 'Album Artist') print('Created documents: ' + str(created_docs)) + # TODO Created doc <> nb doc in ELS