Suggester - Autocomplete Serch

Merge branch 'poc/suggester/ng-bootstrap' into dev

dashboard/package.json

@@ -19,15 +19,17 @@
     "@angular/platform-browser": "~11.0.4",
     "@angular/platform-browser-dynamic": "~11.0.4",
     "@angular/router": "~11.0.4",
-    "rxjs": "~6.6.0",
-    "zone.js": "~0.10.2",
+    "@ng-bootstrap/ng-bootstrap": "^9.1.3",
     "bootstrap": "^3.3.7",
-    "tslib": "^2.0.0"
+    "rxjs": "~6.6.0",
+    "tslib": "^2.0.0",
+    "zone.js": "~0.10.2"
   },
   "devDependencies": {
     "@angular-devkit/build-angular": "~0.1100.4",
     "@angular/cli": "~11.0.4",
     "@angular/compiler-cli": "~11.0.4",
+    "@angular/localize": "^11.0.4",
     "@types/jasmine": "~3.6.0",
     "@types/node": "^12.11.1",
     "codelyzer": "^6.0.0",

dashboard/src/app/app.module.ts

@@ -1,6 +1,7 @@
 import { NgModule } from '@angular/core';
 import { BrowserModule } from '@angular/platform-browser';
 import { HttpClientModule } from '@angular/common/http';
+import { FormsModule } from '@angular/forms'; // <-- NgModel lives here
 
 import { AppComponent } from './app.component';
 import { DashboardComponent } from './dashboard/dashboard.component';
@@ -23,11 +24,15 @@ import { RoundPipe } from './pipes/round.pipe';
 import { AlbumsComponent } from './albums/albums.component';
 import { ToSortComponent } from './to-sort/to-sort.component';
 
+import {NgbModule} from '@ng-bootstrap/ng-bootstrap';
+
 @NgModule({
   imports: [
     BrowserModule,
     HttpClientModule,
-    AppRoutingModule
+    FormsModule,
+    AppRoutingModule,
+    NgbModule
   ],
   declarations: [
     AppComponent,

dashboard/src/app/dashboard/dashboard.component.css

@@ -34,5 +34,44 @@
   border-radius: 50%;
   background-color: #9b59b6;
   margin: 0 auto;
-
+}
+
+/* Style for autocompletion */
+/* Stolen from https://ng-bootstrap.github.io/#/components/typeahead/examples */
+::ng-deep .dropdown-item {
+    display: block;
+    width: 100%;
+    padding: .25rem 1.5rem;
+    clear: both;
+    font-weight: 400;
+    color: #212529;
+    text-align: inherit;
+    white-space: nowrap;
+    background-color: transparent;
+    border: 0
+}
+
+::ng-deep .dropdown-item:focus,
+::ng-deep .dropdown-item:hover {
+    color: #16181b;
+    text-decoration: none;
+    background-color: #f8f9fa
+}
+
+::ng-deep .dropdown-item.active,
+::ng-deep .dropdown-item:active {
+    color: #fff;
+    text-decoration: none;
+    background-color: #007bff
+}
+
+::ng-deep .dropdown-item.disabled,
+::ng-deep .dropdown-item:disabled {
+    color: #6c757d;
+    pointer-events: none;
+    background-color: transparent
+}
+
+::ng-deep .dropdown-menu.show {
+    display: block
 }

dashboard/src/app/dashboard/dashboard.component.html

@@ -77,6 +77,30 @@
     </div>
 </div>
 
+<ng-template #rt let-r="result" let-t="term">
+    <!-- glyphicon glyphicon-cd -->
+    <span *ngIf="r.type == 'artist'" class="glyphicon glyphicon-user"></span>
+    <span *ngIf="r.type == 'album'" class="glyphicon glyphicon-cd"></span>
+    &nbsp; <ngb-highlight [result]="r.name" [term]="t"></ngb-highlight>
+  </ng-template>
+
+
+<form class="navbar-form">
+    <div class="form-group" style="display:inline;">
+        <div class="input-group" style="display:table;">
+            <span class="input-group-addon" style="width:1%;">
+                <span class="glyphicon glyphicon-search"></span>
+            </span>
+            <input class="form-control" type="text" name="search" placeholder="Search Here" autocomplete="off" autofocus="autofocus"
+                id="searchSuggest"
+                [(ngModel)]="searchTerm"
+                [ngbTypeahead]="search"
+                [resultTemplate]="rt"
+                (selectItem)="onSelectSearch($event)">
+        </div>
+    </div>
+</form>
+
 <div class="col-md-12">
     <table class="table table-striped">
         <thead>

dashboard/src/app/dashboard/dashboard.component.ts

@@ -1,17 +1,19 @@
 import { Component, OnInit } from '@angular/core';
+import { Router } from '@angular/router';
 
 import { ElsService } from './../els.service';
 import { Song } from './../model/song';
 import { Bucket } from './../model/bucket';
-import { Album } from './../model/album';
-import { Artist } from './../model/artist';
+import { Suggested } from '../model/suggested';
+
+import {Observable, of, OperatorFunction} from 'rxjs';
+import {catchError, debounceTime, distinctUntilChanged, map, tap, switchMap} from 'rxjs/operators';
 
 @Component({
   selector: 'app-dashboard',
   templateUrl: './dashboard.component.html',
   styleUrls: [ './dashboard.component.css' ]
 })
-
 export class DashboardComponent implements OnInit {
   totalTime = 0;
   totalSize = 0;
@@ -30,7 +32,10 @@ export class DashboardComponent implements OnInit {
   lastAddedAlbums: Bucket[] = [];
   albumArtists = [];
 
-  constructor(private elsService: ElsService) { }
+  searchTerm = ''
+  suggested : Suggested[] = []
+
+  constructor(private elsService: ElsService, private route: Router) { }
 
   ngOnInit(): void {
     this.elsService.getTime().then(result => {
@@ -113,4 +118,27 @@ export class DashboardComponent implements OnInit {
       }
     });
   }
+
+  onSelectSearch($event) {
+    this.route.navigate(['/' + $event.item.type + '/' + $event.item.name])
+  }
+
+  searching = false;
+  searchFailed = false;
+
+  search: OperatorFunction<string, readonly string[]> = (text$: Observable<string>) =>
+    text$.pipe(
+      debounceTime(300),
+      distinctUntilChanged(),
+      tap(() => this.searching = true),
+      switchMap(term =>
+        this.elsService.getSuggest(term).pipe(
+          tap(() => this.searchFailed = false),
+          catchError(() => {
+            this.searchFailed = true;
+            return of([]);
+          }))
+      ),
+      tap(() => this.searching = false)
+    )
 }

dashboard/src/app/els.service.ts

@@ -8,6 +8,7 @@ import { Song } from './model/song';
 import { Album } from './model/album';
 import { Artist } from './model/artist';
 import { Bucket } from './model/bucket';
+import { Suggested } from './model/suggested';
 
 @Injectable()
 export class ElsService {
@@ -15,6 +16,7 @@ export class ElsService {
   public static readonly SONG_INDEX_NAME = '/itunes-songs';
   public static readonly ARTIST_INDEX_NAME = '/itunes-artists';
   public static readonly ALBUM_INDEX_NAME = '/itunes-albums';
+  public static readonly SUGGEST_INDEX_NAME = '/itunes-suggest';
 
   protected static readonly ACTION_SEARCH = '/_search';
   protected static readonly ACTION_COUNT = '/_count';
@@ -381,6 +383,34 @@ export class ElsService {
     );
   }
 
+  getSuggest(text: string): Observable<Suggested[]> {
+    console.log('search sugget: ' + text);
+    return this.http
+    .post<any>(this.elsUrl + ElsService.SUGGEST_INDEX_NAME + ElsService.ACTION_SEARCH,
+          JSON.stringify({
+            '_source': ['album', 'artist'],
+            'suggest': {
+                'album-suggest': {
+                    'prefix': text,
+                    'completion': {
+                        'field': 'album_suggest'
+                    }
+                },
+                'artist-suggest': {
+                    'prefix': text,
+                    'completion': {
+                        'field': 'artist_suggest'
+                    }
+                }
+            }
+        }), {headers: this.headers})
+    .pipe(
+      map(res => this.responseSuggesterToSuggested(res, 'album-suggest', 'artist-suggest')),
+      catchError(error => this.handleError(error, 'getSuggest(' + text + ')'))
+    );
+
+  }
+
   /** Process a result to return just one result.
    * Used to get an album or an artist.
    * Take a name to put in console output if no result or more than one result.
@@ -462,6 +492,20 @@ export class ElsService {
     return result;
   }
 
+  protected responseSuggesterToSuggested(res: any, ...suggestName: string[]): Suggested[] {
+    const result: Array<Suggested> = []
+    suggestName.forEach(sname => {
+      res['suggest'][sname][0]['options'].forEach(option => {
+        let suggest = new Suggested()
+        // TODO If more than one key, raise exception
+        suggest.type = String(Object.keys(option['_source']))
+        suggest.name = option['_source'][suggest.type]
+        result.push(suggest)
+      });
+    });
+    return result;
+  }
+
   protected handleError(error: any, origin: string): Promise<any> {
     console.error('An error occurred!');
     console.error('Origin function: ', origin);

dashboard/src/app/model/suggested.ts

@@ -0,0 +1,4 @@
+export class Suggested {
+    name: string;
+    type: string;
+}

dashboard/src/polyfills.ts

@@ -1,3 +1,7 @@
+/***************************************************************************************************
+ * Load `$localize` onto the global scope - used if i18n tags appear in Angular templates.
+ */
+import '@angular/localize/init';
 /**
  * This file includes polyfills needed by Angular and is loaded before the app.
  * You can add your own extra polyfills to this file.

suggester.es

@@ -0,0 +1,137 @@
+DELETE itunes-suggest
+
+PUT /itunes-suggest
+{
+    "settings": {
+        "analysis": {
+            "filter": {
+                "french_stop": {
+                    "type": "stop",
+                    "stopwords": "_french_"
+                },
+                "english_stop": {
+                    "type": "stop",
+                    "stopwords": "_english_"
+                }
+            },
+            "analyzer": {
+                "names": {
+                    "tokenizer": "standard",
+                    "filter": [
+                        "lowercase",
+                        "asciifolding",
+                        "french_stop",
+                        "english_stop"
+                    ]
+                }
+            }
+        }
+    },
+    "mappings": {
+        "properties": {
+            "artist_suggest": {
+                "type": "completion",
+                "search_analyzer": "names"
+            },
+            "artist": {
+                "type": "keyword"
+            },
+            "album_suggest": {
+                "type": "completion",
+                "search_analyzer": "names"
+            },
+            "album": {
+                "type": "keyword"
+            }
+        }
+    }
+}
+
+// Also possible to specify analyze for ingesting => https://stackoverflow.com/questions/48304499/elasticsearch-completion-suggester-not-working-with-whitespace-analyzer
+
+// Problem with word EP, SP
+
+GET itunes-suggest/_analyze
+{
+    "analyzer": "names",
+    "text": "the servent"
+}
+
+GET itunes-suggest/_search
+
+POST itunes-suggest/_search
+{
+    "_source" : "artist",
+    "suggest": {
+        "name-suggest": {
+            "prefix": "sou",
+            "completion": {
+                "field": "artist_suggest"
+            }
+        }
+    }
+}
+
+POST itunes-suggest/_search
+{
+    "_source" : "album",
+    "suggest": {
+        "name-suggest": {
+            "prefix": "new",
+            "completion": {
+                "field": "album_suggest",
+                "size": 20
+            }
+        }
+    }
+}
+
+POST itunes-suggest/_search
+{
+    "_source": ["album", "artist"],
+    "suggest": {
+        "alb-suggest": {
+            "prefix": "sou",
+            "completion": {
+                "field": "album_suggest"
+            }
+        },
+        "ar-suggest": {
+            "prefix": "sou",
+            "completion": {
+                "field": "artist_suggest"
+            }
+        }
+    }
+}
+
+POST itunes-suggest/_search
+{
+    "_source": ["album", "artist"],
+    "suggest": {
+        "alb-suggest": {
+            "prefix": "Francois",
+            "completion": {
+                "field": "album_suggest"
+            }
+        },
+        "ar-suggest": {
+            "prefix": "Francois",
+            "completion": {
+                "field": "artist_suggest"
+            }
+        }
+    }
+}
+
+POST itunes-suggest/_search
+{
+    "suggest": {
+        "ar-suggest": {
+            "prefix": "Femme",
+            "completion": {
+                "field": "artist_suggest"
+            }
+        }
+    }
+}

suggester.py

@@ -0,0 +1,83 @@
+import requests
+import json
+import sys
+
+ELS_URL ='http://localhost:9200'
+INDEX = 'itunes-suggest'
+
+class NoGoodDataException(Exception):
+    def __init__(self, message):
+        super().__init__(message)
+
+def get_tokens(data: str) -> list:
+    if not data:
+        return []
+    query = {
+        "analyzer": "names",
+        "text" : data
+    }
+
+    url = '{}/{}/_analyze'.format(ELS_URL, INDEX)
+    r = requests.get(url, json=query)
+
+    if not 'tokens' in r.json():
+        print('ERROR: Not tokens in result')
+        print('Input: ' + str(data))
+        print('Request: ' + str(r.json()))
+        raise NoGoodDataException('Data is not correct to get tokens')
+    return [t['token'] for t in r.json()['tokens']]
+
+def post_document(name: str, input: list, field_name: str) -> bool:
+    suggest_name = field_name + '_suggest'
+    element = {
+        field_name: name,
+        suggest_name: input
+    }
+
+    # Filter empty keys
+    # element = {k: v for k, v in element.items() if v}
+
+    url = '{}/{}/_doc'.format(ELS_URL, INDEX)
+    resp = requests.post(url, json=element)
+    if resp.status_code != 201:
+        print('ELS Response KO')
+        print(resp.status_code)
+        print(resp.text)
+        return
+
+    el_id = resp.json()['_id']
+    # print('Post_element - Element created: ' + el_id)
+    return el_id
+
+def process_file(file_name: str, field_name: str) -> int:
+    print('Process file: ' + file_name)
+    with open(file_name, 'r') as o_file:
+            lines = o_file.readlines()
+
+    count = 0
+    i = 0
+    for line in lines:
+        i += 1
+        sys.stdout.write(str(int((i/len(lines))*100)) + '%')
+        sys.stdout.flush()
+        sys.stdout.write("\b" * (40+1)) # return to start of line, after '['
+        data = json.loads(line)
+        if "Artist" in data:
+            try :
+                input = get_tokens(data[field_name])
+                post_document(name=data[field_name], input=input, field_name=field_name.lower())
+                count += 1
+            except NoGoodDataException:
+                print('ERROR WITH DATA')
+                print(str(data))
+    print('File processed\n')
+
+    return count
+
+
+if __name__ == '__main__':
+    # Using readlines()
+    count = 0
+    count += process_file('/home/budd/workspace/iTunes/es-albums.json', 'Album')
+    count += process_file('/home/budd/workspace/iTunes/es-artists.json', 'Artist')
+    print('Created documents: ' + str(count))