maxenceg2m/iTunes
1
0
Fork 0
Code Issues 2 Pull Requests Projects Releases Wiki Activity

Use send data to process suggestion

Browse Source 
Not all suggested document is created in ELS
Need to refactor send data
This commit is contained in:
Maxence G. de Montauzan
2021-08-31 02:31:32 +02:00
parent 88025347ec
commit 928efb659e
4 changed files with 77 additions and 51 deletions
Download Patch File Download Diff File Expand all files Collapse all files

48
mapping.suggest.json Normal file
View File

@@ -0,0 +1,48 @@
{
"settings": {
"index": {
"number_of_replicas": 0
},
"analysis": {
"filter": {
"french_stop": {
"type": "stop",
"stopwords": "_french_"
},
"english_stop": {
"type": "stop",
"stopwords": "_english_"
}
},
"analyzer": {
"names": {
"tokenizer": "standard",
"filter": [
"lowercase",
"asciifolding",
"french_stop",
"english_stop"
]
}
}
}
},
"mappings": {
"properties": {
"artist_suggest": {
"type": "completion",
"search_analyzer": "names"
},
"artist": {
"type": "keyword"
},
"album_suggest": {
"type": "completion",
"search_analyzer": "names"
},
"album": {
"type": "keyword"
}
}
}
}

30
send_data.py
View File

@@ -10,6 +10,8 @@ import json
import time
import requests
from suggester import process_file
class bcolors:
HEADER = '\033[95m'
OKBLUE = '\033[94m'
@@ -28,10 +30,12 @@ DEFAULT_ARTIST_FILE = 'es-artists.json'
DEFAULT_MAPPING_SONGS_FILE = 'mapping.songs.json'
DEFAULT_MAPPING_ARTISTS_FILE = 'mapping.artists.json'
DEFAULT_MAPPING_ALBUMS_FILE = 'mapping.albums.json'
DEFAULT_MAPPING_SUGGEST_FILE = 'mapping.suggest.json'
SONG_INDEX = 'itunes-songs'
ALBUM_INDEX = 'itunes-albums'
ARTIST_INDEX = 'itunes-artists'
SUGGEST_INDEX = 'itunes-suggest'
# TODO Put variables in a config files or in a python library
# Global values / set as default values
@@ -145,6 +149,22 @@ def main():
else:
print('Album sent')
if args.ALL or args.no_suggest:
print("Process suggestion:")
if args.DELETE: # TODO Do a method?
mapping_suggest = load_file(args.mapping_suggest, DEFAULT_MAPPING_SUGGEST_FILE)
if not args.quiet:
print("Mapping of suggest index file: '{}'".format(mapping_suggest.name))
delete_index(SUGGEST_INDEX, args.quiet)
put_mapping(SUGGEST_INDEX, mapping_suggest, args.quiet)
suggs_docs = 0
suggs_docs += process_file('/home/budd/workspace/iTunes/es-albums.json', 'Album')
print('Created documents: ' + str(suggs_docs))
suggs_docs += process_file('/home/budd/workspace/iTunes/es-artists.json', 'Artist', 'Album Artist')
print('Created documents: ' + str(suggs_docs))
print("I'm done!")
if check_is_ok.count(False) > 0:
print('Some problems occurs')
@@ -199,6 +219,8 @@ def create_args_parser():
.format(DEFAULT_ARTIST_FILE))
song_group.add_argument('-s', '--song', action='store_false',
help='Disable sending song data')
song_group.add_argument('--no-suggest', action='store_false',
help='Disable sending suggest data')
# Mode
mode_group = parser.add_argument_group('Mode')
@@ -211,12 +233,14 @@ def create_args_parser():
# Mapping
mapping_group = parser.add_argument_group('Mapping files')
mode_group.add_argument('-ms', '--mapping-song', type=argparse.FileType('r'), const=DEFAULT_MAPPING_SONGS_FILE, nargs='?',
mapping_group.add_argument('-ms', '--mapping-song', type=argparse.FileType('r'), const=DEFAULT_MAPPING_SONGS_FILE, nargs='?',
help='Mapping file for songs (default: \'{}\')'.format(DEFAULT_MAPPING_SONGS_FILE))
mode_group.add_argument('-mr', '--mapping-artist', type=argparse.FileType('r'), const=DEFAULT_ARTIST_FILE, nargs='?',
mapping_group.add_argument('-mr', '--mapping-artist', type=argparse.FileType('r'), const=DEFAULT_ARTIST_FILE, nargs='?',
help='Mapping file for artists (default: \'{}\')'.format(DEFAULT_MAPPING_ARTISTS_FILE))
mode_group.add_argument('-ml', '--mapping-album', type=argparse.FileType('r'), const=DEFAULT_MAPPING_ALBUMS_FILE, nargs='?',
mapping_group.add_argument('-ml', '--mapping-album', type=argparse.FileType('r'), const=DEFAULT_MAPPING_ALBUMS_FILE, nargs='?',
help='Mapping file for albums (default: \'{}\')'.format(DEFAULT_MAPPING_ALBUMS_FILE))
mapping_group.add_argument('-mg', '--mapping-suggest', type=argparse.FileType('r'), const=DEFAULT_MAPPING_SUGGEST_FILE, nargs='?',
help='Mapping file for suggest (default: \'{}\')'.format(DEFAULT_MAPPING_SUGGEST_FILE))
# Global Settings
g_settings_group = parser.add_argument_group('Global Settings')

49
suggester.es
View File

@@ -1,54 +1,7 @@
DELETE itunes-suggest
PUT /itunes-suggest
{
"settings": {
"index": {
"number_of_replicas": 0
},
"analysis": {
"filter": {
"french_stop": {
"type": "stop",
"stopwords": "_french_"
},
"english_stop": {
"type": "stop",
"stopwords": "_english_"
}
},
"analyzer": {
"names": {
"tokenizer": "standard",
"filter": [
"lowercase",
"asciifolding",
"french_stop",
"english_stop"
]
}
}
}
},
"mappings": {
"properties": {
"artist_suggest": {
"type": "completion",
"search_analyzer": "names"
},
"artist": {
"type": "keyword"
},
"album_suggest": {
"type": "completion",
"search_analyzer": "names"
},
"album": {
"type": "keyword"
}
}
}
}
!./mapping.suggest.json
// Also possible to specify analyze for ingesting => https://stackoverflow.com/questions/48304499/elasticsearch-completion-suggester-not-working-with-whitespace-analyzer

1
suggester.py
View File

@@ -139,3 +139,4 @@ if __name__ == '__main__':
print('Created documents: ' + str(created_docs))
created_docs += process_file('/home/budd/workspace/iTunes/es-artists.json', 'Artist', 'Album Artist')
print('Created documents: ' + str(created_docs))
# TODO Created doc <> nb doc in ELS