Use send data to process suggestion

Not all suggested document is created in ELS
Need to refactor send data

mapping.suggest.json

@@ -0,0 +1,48 @@
+{
+    "settings": {
+        "index": {
+            "number_of_replicas": 0
+        },
+        "analysis": {
+            "filter": {
+                "french_stop": {
+                    "type": "stop",
+                    "stopwords": "_french_"
+                },
+                "english_stop": {
+                    "type": "stop",
+                    "stopwords": "_english_"
+                }
+            },
+            "analyzer": {
+                "names": {
+                    "tokenizer": "standard",
+                    "filter": [
+                        "lowercase",
+                        "asciifolding",
+                        "french_stop",
+                        "english_stop"
+                    ]
+                }
+            }
+        }
+    },
+    "mappings": {
+        "properties": {
+            "artist_suggest": {
+                "type": "completion",
+                "search_analyzer": "names"
+            },
+            "artist": {
+                "type": "keyword"
+            },
+            "album_suggest": {
+                "type": "completion",
+                "search_analyzer": "names"
+            },
+            "album": {
+                "type": "keyword"
+            }
+        }
+    }
+}

send_data.py

@@ -10,6 +10,8 @@ import json
 import time
 import requests
 
+from suggester import process_file
+
 class bcolors:
     HEADER = '\033[95m'
     OKBLUE = '\033[94m'
@@ -28,10 +30,12 @@ DEFAULT_ARTIST_FILE = 'es-artists.json'
 DEFAULT_MAPPING_SONGS_FILE = 'mapping.songs.json'
 DEFAULT_MAPPING_ARTISTS_FILE = 'mapping.artists.json'
 DEFAULT_MAPPING_ALBUMS_FILE = 'mapping.albums.json'
+DEFAULT_MAPPING_SUGGEST_FILE = 'mapping.suggest.json'
 
 SONG_INDEX = 'itunes-songs'
 ALBUM_INDEX = 'itunes-albums'
 ARTIST_INDEX = 'itunes-artists'
+SUGGEST_INDEX = 'itunes-suggest'
 # TODO Put variables in a config files or in a python library
 
 # Global values / set as default values
@@ -145,6 +149,22 @@ def main():
         else:
             print('Album sent')
 
+    if args.ALL or args.no_suggest:
+        print("Process suggestion:")
+        if args.DELETE: # TODO Do a method?
+            mapping_suggest = load_file(args.mapping_suggest, DEFAULT_MAPPING_SUGGEST_FILE)
+            if not args.quiet:
+                print("Mapping of suggest index file: '{}'".format(mapping_suggest.name))
+
+            delete_index(SUGGEST_INDEX, args.quiet)
+            put_mapping(SUGGEST_INDEX, mapping_suggest, args.quiet)
+
+        suggs_docs = 0
+        suggs_docs += process_file('/home/budd/workspace/iTunes/es-albums.json', 'Album')
+        print('Created documents: ' + str(suggs_docs))
+        suggs_docs += process_file('/home/budd/workspace/iTunes/es-artists.json', 'Artist', 'Album Artist')
+        print('Created documents: ' + str(suggs_docs))
+
     print("I'm done!")
     if check_is_ok.count(False) > 0:
         print('Some problems occurs')
@@ -199,6 +219,8 @@ def create_args_parser():
                                .format(DEFAULT_ARTIST_FILE))
     song_group.add_argument('-s', '--song', action='store_false',
                             help='Disable sending song data')
+    song_group.add_argument('--no-suggest', action='store_false',
+                            help='Disable sending suggest data')
 
     # Mode
     mode_group = parser.add_argument_group('Mode')
@@ -211,12 +233,14 @@ def create_args_parser():
 
     # Mapping
     mapping_group = parser.add_argument_group('Mapping files')
-    mode_group.add_argument('-ms', '--mapping-song', type=argparse.FileType('r'), const=DEFAULT_MAPPING_SONGS_FILE, nargs='?',
+    mapping_group.add_argument('-ms', '--mapping-song', type=argparse.FileType('r'), const=DEFAULT_MAPPING_SONGS_FILE, nargs='?',
                             help='Mapping file for songs (default: \'{}\')'.format(DEFAULT_MAPPING_SONGS_FILE))
-    mode_group.add_argument('-mr', '--mapping-artist', type=argparse.FileType('r'), const=DEFAULT_ARTIST_FILE, nargs='?',
+    mapping_group.add_argument('-mr', '--mapping-artist', type=argparse.FileType('r'), const=DEFAULT_ARTIST_FILE, nargs='?',
                             help='Mapping file for artists (default: \'{}\')'.format(DEFAULT_MAPPING_ARTISTS_FILE))
-    mode_group.add_argument('-ml', '--mapping-album', type=argparse.FileType('r'), const=DEFAULT_MAPPING_ALBUMS_FILE, nargs='?',
+    mapping_group.add_argument('-ml', '--mapping-album', type=argparse.FileType('r'), const=DEFAULT_MAPPING_ALBUMS_FILE, nargs='?',
                             help='Mapping file for albums (default: \'{}\')'.format(DEFAULT_MAPPING_ALBUMS_FILE))
+    mapping_group.add_argument('-mg', '--mapping-suggest', type=argparse.FileType('r'), const=DEFAULT_MAPPING_SUGGEST_FILE, nargs='?',
+                            help='Mapping file for suggest (default: \'{}\')'.format(DEFAULT_MAPPING_SUGGEST_FILE))
 
     # Global Settings
     g_settings_group = parser.add_argument_group('Global Settings')

suggester.es

@@ -1,54 +1,7 @@
 DELETE itunes-suggest
 
 PUT /itunes-suggest
-{
+!./mapping.suggest.json
-    "settings": {
-        "index": {
-            "number_of_replicas": 0
-        },
-        "analysis": {
-            "filter": {
-                "french_stop": {
-                    "type": "stop",
-                    "stopwords": "_french_"
-                },
-                "english_stop": {
-                    "type": "stop",
-                    "stopwords": "_english_"
-                }
-            },
-            "analyzer": {
-                "names": {
-                    "tokenizer": "standard",
-                    "filter": [
-                        "lowercase",
-                        "asciifolding",
-                        "french_stop",
-                        "english_stop"
-                    ]
-                }
-            }
-        }
-    },
-    "mappings": {
-        "properties": {
-            "artist_suggest": {
-                "type": "completion",
-                "search_analyzer": "names"
-            },
-            "artist": {
-                "type": "keyword"
-            },
-            "album_suggest": {
-                "type": "completion",
-                "search_analyzer": "names"
-            },
-            "album": {
-                "type": "keyword"
-            }
-        }
-    }
-}
 
 // Also possible to specify analyze for ingesting => https://stackoverflow.com/questions/48304499/elasticsearch-completion-suggester-not-working-with-whitespace-analyzer
 

suggester.py

@@ -139,3 +139,4 @@ if __name__ == '__main__':
     print('Created documents: ' + str(created_docs))
     created_docs += process_file('/home/budd/workspace/iTunes/es-artists.json', 'Artist', 'Album Artist')
     print('Created documents: ' + str(created_docs))
+    # TODO Created doc <> nb doc in ELS