Use send data to process suggestion

Not all suggested document is created in ELS
Need to refactor send data

dashboard/angular.json

@@ -18,7 +18,6 @@
             "main": "src/main.ts",
             "tsConfig": "tsconfig.app.json",
             "polyfills": "src/polyfills.ts",
-            "aot": true,
             "assets": [
               "src/assets",
               "src/favicon.ico"
@@ -27,7 +26,13 @@
               "src/styles.css",
               "node_modules/bootstrap/dist/css/bootstrap.min.css"
             ],
-            "scripts": []
+            "scripts": [],
+            "vendorChunk": true,
+            "extractLicenses": false,
+            "buildOptimizer": false,
+            "sourceMap": true,
+            "optimization": false,
+            "namedChunks": true
           },
           "configurations": {
             "production": {
@@ -57,7 +62,8 @@
                 }
               ]
             }
-          }
+          },
+          "defaultConfiguration": ""
         },
         "serve": {
           "builder": "@angular-devkit/build-angular:dev-server",

dashboard/package.json

@@ -11,29 +11,32 @@
   },
   "private": true,
   "dependencies": {
-    "@angular/animations": "~11.0.4",
-    "@angular/common": "~11.0.4",
-    "@angular/compiler": "~11.0.4",
-    "@angular/core": "~11.0.4",
-    "@angular/forms": "~11.0.4",
-    "@angular/platform-browser": "~11.0.4",
-    "@angular/platform-browser-dynamic": "~11.0.4",
-    "@angular/router": "~11.0.4",
-    "rxjs": "~6.6.0",
-    "zone.js": "~0.10.2",
+    "@angular/animations": "~12.2.2",
+    "@angular/common": "~12.2.2",
+    "@angular/compiler": "~12.2.2",
+    "@angular/core": "~12.2.2",
+    "@angular/forms": "~12.2.2",
+    "@angular/localize": "~12.2.2",
+    "@angular/platform-browser": "~12.2.2",
+    "@angular/platform-browser-dynamic": "~12.2.2",
+    "@angular/router": "~12.2.2",
+    "@ng-bootstrap/ng-bootstrap": "^10.0.0",
     "bootstrap": "^3.3.7",
-    "tslib": "^2.0.0"
+    "rxjs": "~6.6.0",
+    "tslib": "^2.0.0",
+    "zone.js": "~0.11.4"
   },
   "devDependencies": {
-    "@angular-devkit/build-angular": "~0.1100.4",
-    "@angular/cli": "~11.0.4",
-    "@angular/compiler-cli": "~11.0.4",
+    "@angular-devkit/build-angular": "~12.2.2",
+    "@angular/cli": "~12.2.2",
+    "@angular/compiler-cli": "~12.2.2",
+    "@angular/localize": "^12.2.2",
     "@types/jasmine": "~3.6.0",
     "@types/node": "^12.11.1",
     "codelyzer": "^6.0.0",
     "jasmine-core": "~3.6.0",
     "jasmine-spec-reporter": "~5.0.0",
-    "karma": "~5.1.0",
+    "karma": "~6.3.4",
     "karma-chrome-launcher": "~3.1.0",
     "karma-coverage": "~2.0.3",
     "karma-jasmine": "~4.0.0",
@@ -41,6 +44,6 @@
     "protractor": "~7.0.0",
     "ts-node": "~8.3.0",
     "tslint": "~6.1.0",
-    "typescript": "~4.0.2"
+    "typescript": "~4.3.5"
   }
-}
+}

dashboard/src/app/albums/albums.component.ts

@@ -27,7 +27,7 @@ export class AlbumsComponent implements OnInit {
     this.loadData();
   }
 
-  private editQuery(field: string, value: string, type: query_edit_type): void {
+  private editQuery(field: string, value: Album, type: query_edit_type): void {
     // TODO Move this method to a service
     if (value[field] instanceof Array) {
       value[field] = value[field][0]
@@ -49,12 +49,12 @@ export class AlbumsComponent implements OnInit {
     this.queryEdited = true;
   }
 
-  exlude(field: string, value: string): void {
+  exlude(field: string, value: Album): void {
     this.editQuery(field, value, query_edit_type.exclude)
     this.loadData()
   }
 
-  select(field: string, value: string): void {
+  select(field: string, value: Album): void {
     this.editQuery(field, value, query_edit_type.select)
     this.loadData()
   }

dashboard/src/app/app-routing.module.ts

@@ -7,6 +7,7 @@ import { ArtistComponent } from './artist/artist.component';
 import { GenreComponent } from './genre/genre.component';
 import { TopPlayedComponent } from './top-played/top-played.component';
 import { AlbumsComponent } from './albums/albums.component';
+import { ToSortComponent } from './to-sort/to-sort.component';
 
 const routes: Routes = [
   { path: '', redirectTo: '/dashboard', pathMatch: 'full' },
@@ -15,7 +16,8 @@ const routes: Routes = [
   { path: 'album',      component: AlbumsComponent },
   { path: 'artist/:name',      component: ArtistComponent },
   { path: 'genre/:name',      component: GenreComponent },
-  { path: 'top-played',      component: TopPlayedComponent }
+  { path: 'top-played',      component: TopPlayedComponent },
+  { path: 'to-sort',      component: ToSortComponent }
 ];
 
 @NgModule({

dashboard/src/app/app.component.css

@@ -36,7 +36,8 @@ nav a.active {
 
 /* FIXME Code Repetition  */
 .btn-top {
-  display: inline-block;
+  display: flex;
+  flex-direction: column;
   position: fixed;
   z-index: 1;
   -webkit-transition: opacity .3s 0s, visibility 0s 0s;

dashboard/src/app/app.component.html

@@ -1,5 +1,8 @@
 <nav>
-  <a class="btn-top" routerLink="/dashboard" routerLinkActive="active"><span class="glyphicon glyphicon-home"></span></a><br />
-  <a class="btn-top" routerLink="/album" routerLinkActive="active"><span class="glyphicon glyphicon-cd"></span></a>
+  <div class="btn-top">
+  <a routerLink="/dashboard" routerLinkActive="active"><span class="glyphicon glyphicon-home"></span></a><br />
+  <a routerLink="/album" routerLinkActive="active"><span class="glyphicon glyphicon-cd"></span></a>
+  <a routerLink="/to-sort" routerLinkActive="active"><span class="glyphicon glyphicon-sort"></span></a>
+</div>
 </nav>
 <router-outlet></router-outlet>

dashboard/src/app/app.module.ts

@@ -22,13 +22,17 @@ import { SortByPipe } from './pipes/sort-by.pipe';
 import { ConvertSizeToStringPipe } from './pipes/convert-size-to-string.pipe';
 import { RoundPipe } from './pipes/round.pipe';
 import { AlbumsComponent } from './albums/albums.component';
+import { ToSortComponent } from './to-sort/to-sort.component';
+
+import {NgbModule} from '@ng-bootstrap/ng-bootstrap';
 
 @NgModule({
   imports: [
     BrowserModule,
     HttpClientModule,
     FormsModule,
-    AppRoutingModule
+    AppRoutingModule,
+    NgbModule
   ],
   declarations: [
     AppComponent,
@@ -43,7 +47,8 @@ import { AlbumsComponent } from './albums/albums.component';
     SortByPipe,
     ConvertSizeToStringPipe,
     TopPlayedComponent,
-    RoundPipe
+    RoundPipe,
+    ToSortComponent
   ],
   providers: [
     ElsService,

dashboard/src/app/dashboard/dashboard.component.css

@@ -34,5 +34,44 @@
   border-radius: 50%;
   background-color: #9b59b6;
   margin: 0 auto;
-
+}
+
+/* Style for autocompletion */
+/* Stolen from https://ng-bootstrap.github.io/#/components/typeahead/examples */
+::ng-deep .dropdown-item {
+    display: block;
+    width: 100%;
+    padding: .25rem 1.5rem;
+    clear: both;
+    font-weight: 400;
+    color: #212529;
+    text-align: inherit;
+    white-space: nowrap;
+    background-color: transparent;
+    border: 0
+}
+
+::ng-deep .dropdown-item:focus,
+::ng-deep .dropdown-item:hover {
+    color: #16181b;
+    text-decoration: none;
+    background-color: #f8f9fa
+}
+
+::ng-deep .dropdown-item.active,
+::ng-deep .dropdown-item:active {
+    color: #fff;
+    text-decoration: none;
+    background-color: #007bff
+}
+
+::ng-deep .dropdown-item.disabled,
+::ng-deep .dropdown-item:disabled {
+    color: #6c757d;
+    pointer-events: none;
+    background-color: transparent
+}
+
+::ng-deep .dropdown-menu.show {
+    display: block
 }

dashboard/src/app/dashboard/dashboard.component.html

@@ -77,6 +77,14 @@
     </div>
 </div>
 
+<ng-template #rt let-r="result" let-t="term">
+    <!-- glyphicon glyphicon-cd -->
+    <span *ngIf="r.type == 'artist'" class="glyphicon glyphicon-user"></span>
+    <span *ngIf="r.type == 'album'" class="glyphicon glyphicon-cd"></span>
+    &nbsp; <ngb-highlight [result]="r.name" [term]="t"></ngb-highlight>
+  </ng-template>
+
+
 <form class="navbar-form">
     <div class="form-group" style="display:inline;">
         <div class="input-group" style="display:table;">
@@ -85,13 +93,10 @@
             </span>
             <input class="form-control" type="text" name="search" placeholder="Search Here" autocomplete="off" autofocus="autofocus"
                 id="searchSuggest"
-                list="dynmicUserIds"
                 [(ngModel)]="searchTerm"
-                (keyup)="onSearchChange()"
-                (change)="onSearchSelected($event)">
-                <datalist id="dynmicUserIds">
-                   <option *ngFor="let item of suggested" [value]="item.name" [label]="item.type" [id]="item">{{item}}</option>
-                </datalist>
+                [ngbTypeahead]="search"
+                [resultTemplate]="rt"
+                (selectItem)="onSelectSearch($event)">
         </div>
     </div>
 </form>

dashboard/src/app/dashboard/dashboard.component.ts

@@ -4,16 +4,16 @@ import { Router } from '@angular/router';
 import { ElsService } from './../els.service';
 import { Song } from './../model/song';
 import { Bucket } from './../model/bucket';
-import { Album } from './../model/album';
-import { Artist } from './../model/artist';
 import { Suggested } from '../model/suggested';
 
+import {Observable, of, OperatorFunction} from 'rxjs';
+import {catchError, debounceTime, distinctUntilChanged, map, tap, switchMap} from 'rxjs/operators';
+
 @Component({
   selector: 'app-dashboard',
   templateUrl: './dashboard.component.html',
   styleUrls: [ './dashboard.component.css' ]
 })
-
 export class DashboardComponent implements OnInit {
   totalTime = 0;
   totalSize = 0;
@@ -119,18 +119,26 @@ export class DashboardComponent implements OnInit {
     });
   }
 
-  onSearchChange() {
-    this.elsService.getSuggest(this.searchTerm).subscribe(result => this.suggested = result);
+  onSelectSearch($event) {
+    this.route.navigate(['/' + $event.item.type + '/' + $event.item.name])
   }
 
-  onSearchSelected($event) {
-    let selected = $event.target.value
-    // FIXME Not possible to get good element (just value)
-    // Need to use a plugin to do this correctly
-    this.suggested.forEach(element => {
-      if (element.name == selected) {
-        this.route.navigate(['/' + element.type + '/' + element.name])
-      }
-    });
-  }
+  searching = false;
+  searchFailed = false;
+
+  search: OperatorFunction<string, readonly string[]> = (text$: Observable<string>) =>
+    text$.pipe(
+      debounceTime(300),
+      distinctUntilChanged(),
+      tap(() => this.searching = true),
+      switchMap(term =>
+        this.elsService.getSuggest(term).pipe(
+          tap(() => this.searchFailed = false),
+          catchError(() => {
+            this.searchFailed = true;
+            return of([]);
+          }))
+      ),
+      tap(() => this.searching = false)
+    )
 }

dashboard/src/app/els-sort.service.spec.ts

@@ -0,0 +1,16 @@
+import { TestBed } from '@angular/core/testing';
+
+import { ElsSortService } from './els-sort.service';
+
+describe('ElsSortService', () => {
+  let service: ElsSortService;
+
+  beforeEach(() => {
+    TestBed.configureTestingModule({});
+    service = TestBed.inject(ElsSortService);
+  });
+
+  it('should be created', () => {
+    expect(service).toBeTruthy();
+  });
+});

dashboard/src/app/els-sort.service.ts

@@ -0,0 +1,147 @@
+import { HttpClient } from '@angular/common/http';
+import { Injectable } from '@angular/core';
+import { Observable } from 'rxjs';
+import { map, catchError } from 'rxjs/operators';
+import { ElsService } from './els.service';
+import { Album } from './model/album';
+import { Bucket } from './model/bucket';
+import { Song } from './model/song';
+
+@Injectable({
+  providedIn: 'root'
+})
+export class ElsSortService extends ElsService {
+
+  constructor(protected http: HttpClient) {
+    super(http);
+  }
+
+  getTime(): Promise<number> {
+    return this.http
+      .post<any>(this.elsUrl + ElsService.SONG_INDEX_NAME + ElsService.ACTION_SEARCH,
+            JSON.stringify({
+              query: {
+                bool: {
+                  must_not: [
+                    {
+                      term: {
+                        "Location.tree": "/F:/Musique"
+                      }
+                    }
+                  ]
+                }
+              },
+              aggs: {
+                sum_time: {
+                  sum: { field: 'Total Time'}
+                }
+              },
+              'size': 0
+            }), {headers: this.headers})
+      .toPromise()
+      .then(res => res.aggregations.sum_time.value as number)
+      .catch(error => this.handleError(error, 'getTime()'));
+  }
+
+  getSize(): Promise<number> {
+    return this.http
+      .post<any>(this.elsUrl + ElsService.SONG_INDEX_NAME + ElsService.ACTION_SEARCH,
+            JSON.stringify({
+              query: {
+                bool: {
+                  must_not: [
+                    {
+                      term: {
+                        "Location.tree": "/F:/Musique"
+                      }
+                    }
+                  ]
+                }
+              },
+              aggs: {
+                sum_time: {
+                  sum: { field: 'Size' }
+                }
+              },
+              'size': 0
+            }), {headers: this.headers})
+      .toPromise()
+      .then(res => res.aggregations.sum_time.value as number)
+      .catch(error => this.handleError(error, 'getSize()'));
+  }
+
+  getCountSong(): Promise<number> {
+    return this.http
+      .post<any>(this.elsUrl + ElsService.SONG_INDEX_NAME + ElsService.ACTION_COUNT,
+        JSON.stringify({
+          query: {
+              bool: {
+                  must_not: [
+                      {
+                          term: {
+                              "Location.tree": "/F:/Musique"
+                          }
+                      }
+                  ]
+              }
+          }
+        }), {headers: this.headers})
+      .toPromise()
+      .then(res => res.count as number)
+      .catch(error => this.handleError(error, 'getCountSong()'));
+  }
+
+  getCountNeverListenSong(): Promise<number> {
+    return this.http
+      .post<any>(this.elsUrl + ElsService.SONG_INDEX_NAME + ElsService.ACTION_COUNT,
+            JSON.stringify({
+              'query': {
+                'bool': {
+                  'must_not': [
+                    {
+                    'exists': { 'field': 'Play Count'}
+                  }, {
+                  term: {
+                    "Location.tree": "/F:/Musique"
+                  } }
+                ]
+                }
+              }
+            }), {headers: this.headers})
+      .toPromise()
+      .then(res => res.count as number)
+      .catch(error => this.handleError(error, 'getCountNeverListenSong()'));
+  }
+
+  getAlbums(): Observable<Bucket[]> {
+    return this.http
+      .post(this.elsUrl + ElsService.SONG_INDEX_NAME + ElsService.ACTION_SEARCH,
+        JSON.stringify({
+          query: {
+            bool: {
+              must_not: [
+                {
+                  term: {
+                    "Location.tree": "/F:/Musique"
+                  }
+                }
+              ]
+            }
+          },
+          'size': 0,
+          'aggs': {
+            'albums' : {
+              'terms': {
+                'field' : 'Album.raw',
+                'order': { '_term': 'asc' },
+                'size': 50
+              }
+            }
+          }
+        }), {headers: this.headers})
+      .pipe(
+        map(res => this.responseAggregationToBucket(res, "albums")),
+        catchError(error => this.handleError(error, 'getAlbums'))
+      );
+  }
+}

dashboard/src/app/els.service.ts

@@ -339,7 +339,7 @@ export class ElsService {
       // TODO Take in consideration "sum_other_doc_count"
   }
 
-  getArtistFromAlbumName(albumname: string): Observable<Album[]> {
+  getArtistFromAlbumName(albumname: string): Observable<Album[]> { // TODO Rename ?
     return this.http
       .post<any>(this.elsUrl + ElsService.ALBUM_INDEX_NAME + ElsService.ACTION_SEARCH,
             JSON.stringify({
@@ -436,7 +436,7 @@ export class ElsService {
    *
    * @param res Response to process
    */
-  private responseToSongs(res: any): Song[] {
+  protected responseToSongs(res: any): Song[] {
     const result: Array<Song> = [];
     res.hits.hits.forEach((hit) => {
       result.push(hit._source);
@@ -474,7 +474,7 @@ export class ElsService {
    * @param res Response to process
    * @param name Name of aggregation
    */
-  private responseAggregationToBucket(res: any, name: string): Bucket[] {
+  protected responseAggregationToBucket(res: any, name: string): Bucket[] {
     const result: Array<Bucket> = [];
     res.aggregations[name].buckets.forEach((bucket) => {
       result.push(bucket);

dashboard/src/app/model/suggested.ts

@@ -1,8 +1,4 @@
 export class Suggested {
     name: string;
     type: string;
-
-    public toString() : string {
-        return `${this.name} (${this.type})`;
-    }
 }

dashboard/src/app/to-sort/to-sort.component.css

@@ -0,0 +1,38 @@
+.panel-stat {
+  min-height: 102px;
+}
+
+.panel-green {
+background-color: #16a085;
+color:white;
+}
+.panel-yellow {
+background-color: #f1c40f;
+color:white;
+}
+
+.panel-red {
+background-color: #d9534f;
+color:white;
+}
+
+.panel-purple {
+background-color: #9b59b6;
+color:white;
+}
+
+.panel-blue {
+background-color: #3498db;
+color:white;
+}
+
+.stats_icon {
+font-size: 3em;
+}
+
+.circle {
+border-radius: 50%;
+background-color: #9b59b6;
+margin: 0 auto;
+
+}

dashboard/src/app/to-sort/to-sort.component.html

@@ -0,0 +1,130 @@
+<div class="container">
+  <h1>To sort Songs - </h1>
+
+<br />
+<div class="row cardAdmin">
+  <div class="col-lg-3 col-md-3">
+      <div class="panel panel-blue">
+          <div class="panel-heading">
+              <div class="row">
+                  <div class="col-xs-3">
+                      <span class="glyphicon glyphicon-time stats_icon"></span>
+                  </div>
+                  <div class="col-xs-9 text-right">
+                      <div>
+                          <h3 *ngIf="!totalTime"><span class="glyphicon glyphicon-refresh loading"></span></h3>
+                          <h3 *ngIf="totalTime">{{totalTime | convertMs}}</h3>
+                      </div>
+                      <div><br>Total time ({{totalTime | convertMoreExact}})</div>
+                  </div>
+              </div>
+          </div>
+      </div>
+  </div>
+  <div class="col-lg-3 col-md-3">
+      <div class="panel panel-green">
+          <div class="panel-heading">
+              <div class="row">
+                  <div class="col-xs-3">
+                      <span class="glyphicon glyphicon-hdd stats_icon"></span>
+                  </div>
+                  <div class="col-xs-9 text-right">
+                      <div>
+                          <h3 *ngIf="!totalSize"><span class="glyphicon glyphicon-refresh loading"></span></h3>
+                          <h3 *ngIf="totalSize">{{totalSize | convertSizeToString}}</h3>
+                      </div>
+                      <div><br>Total size</div>
+                  </div>
+              </div>
+          </div>
+      </div>
+  </div>
+  <div class="col-lg-3 col-md-3">
+      <div class="panel panel-yellow">
+          <div class="panel-heading">
+              <div class="row">
+                  <div class="col-xs-3">
+                      <span class="glyphicon glyphicon-play stats_icon"></span>
+                  </div>
+                  <div class="col-xs-9 text-right">
+                      <div>
+                          <h3 *ngIf="!trackCountSong"><span class="glyphicon glyphicon-refresh loading"></span></h3>
+                          <h3 *ngIf="trackCountSong">{{trackCountSong}}</h3>
+                      </div>
+                      <div><br>Total Songs</div>
+                  </div>
+              </div>
+          </div>
+      </div>
+  </div>
+  <div class="col-lg-3 col-md-3">
+      <div class="panel panel-red">
+          <div class="panel-heading">
+              <div class="row">
+                  <div class="col-xs-3">
+                      <span class="glyphicon glyphicon-stop stats_icon"></span>
+                  </div>
+                  <div class="col-xs-9 text-right">
+                      <div>
+                          <h3 *ngIf="!neverListenSong"><span class="glyphicon glyphicon-refresh loading"></span></h3>
+                          <h3 *ngIf="neverListenSong">{{neverListenSong}}</h3>
+                      </div>
+                      <div><br>Never list. songs (~{{neverListenSong / trackCountSong * 100 | round}}%)</div>
+                  </div>
+              </div>
+          </div>
+      </div>
+  </div>
+</div>
+
+<div class="col-md-12">
+  <table class="table table-striped" style="white-space: nowrap;">
+    <thead>
+        <tr>
+            <th>Album</th>
+            <th>Track Count</th>
+            <th>Artist/Album Artist</th>
+            <th>Play Count</th>
+            <th>Rating</th>
+        </tr>
+    </thead>
+    <tbody>
+        <tr *ngFor="let album of toSortAlbum">
+            <td>
+              <a [routerLink]="['/album', album.key]">{{album.key}}</a>&nbsp;
+            </td>
+
+            <td>{{albums[album.key]['Track Count']}}</td>
+
+            <ng-template [ngIf]="albums[album.key]['Album Artist']" [ngIfElse]="artistSection">
+              <td>
+                  <a [routerLink]="['/artist', albums[album.key]['Album Artist']]">{{albums[album.key]['Album Artist']}}</a>&nbsp;
+              </td>
+            </ng-template>
+
+            <ng-template #artistSection>
+              <td>
+                  <a [routerLink]="['/artist', albums[album.key].Artist[0]]">{{albums[album.key].Artist}}</a>&nbsp;
+              </td>
+            </ng-template>
+
+            <ng-template #artistSection>
+              <td>
+                  <a [routerLink]="['/artist', albums[album.key].Artist[0]]">{{albums[album.key].Artist}}</a>&nbsp;
+              </td>
+            </ng-template>
+
+            <td>
+              {{albums[album.key]['Play Count']}} ({{albums[album.key]['Play Count']/albums[album.key]['Track Count'] | number:'1.0-0'}}/songs)
+            </td>
+
+            <td class="star" [title]="(albums[album.key]['Album Rating Computed']?'Computed Rating: ':'Rating: ') + albums[album.key]['Album Rating']">
+            <span *ngFor="let item of numberToArray(albums[album.key]['Album Rating'], 20)">
+              <span class="glyphicon" [ngClass]="albums[album.key]['Album Rating Computed']?'glyphicon-star-empty':'glyphicon-star'"></span>
+            </span>
+            </td>
+        </tr>
+    </tbody>
+  </table>
+</div>
+</div>

dashboard/src/app/to-sort/to-sort.component.spec.ts

@@ -0,0 +1,25 @@
+import { ComponentFixture, TestBed } from '@angular/core/testing';
+
+import { ToSortComponent } from './to-sort.component';
+
+describe('ToSortComponent', () => {
+  let component: ToSortComponent;
+  let fixture: ComponentFixture<ToSortComponent>;
+
+  beforeEach(async () => {
+    await TestBed.configureTestingModule({
+      declarations: [ ToSortComponent ]
+    })
+    .compileComponents();
+  });
+
+  beforeEach(() => {
+    fixture = TestBed.createComponent(ToSortComponent);
+    component = fixture.componentInstance;
+    fixture.detectChanges();
+  });
+
+  it('should create', () => {
+    expect(component).toBeTruthy();
+  });
+});

dashboard/src/app/to-sort/to-sort.component.ts

@@ -0,0 +1,89 @@
+import { Component, OnInit } from '@angular/core';
+import { ElsSortService } from '../els-sort.service';
+import { Album } from '../model/album';
+import { Bucket } from '../model/bucket';
+import { Song } from '../model/song';
+
+import { Utils } from '../utils';
+
+@Component({
+  selector: 'app-to-sort',
+  templateUrl: './to-sort.component.html',
+  styleUrls: ['./to-sort.component.css']
+})
+export class ToSortComponent implements OnInit {
+  numberToArray = Utils.numberToArray;
+  // Stats var
+  totalTime = 0;
+  totalSize = 0;
+  trackCountSong = 0;
+  neverListenSong = 0;
+
+  // Global var
+  toSortAlbum: Bucket[] = [];
+  albums = []
+
+  constructor(private elsService: ElsSortService) { }
+
+  ngOnInit(): void {
+    // **** GET STATS ****//
+    this.elsService.getTime().then(result => this.totalTime = result);
+    this.elsService.getSize().then(result => this.totalSize = result);
+    this.elsService.getCountSong().then(result => this.trackCountSong = result);
+    this.elsService.getCountNeverListenSong().then(result => this.neverListenSong = result);
+
+    // **** GET ALBUMS ****//
+    const tmpToSortAlbums: Bucket[] = [];
+    this.elsService.getAlbums().subscribe(buckets => {
+      buckets.forEach(bucket => {
+        if (tmpToSortAlbums.length === 0) {
+          tmpToSortAlbums.push(bucket);
+        } else {
+          let found = false;
+          tmpToSortAlbums.forEach(element => {
+            if (element.key === bucket.key) {
+              element.doc_count += bucket.doc_count;
+              found = true;
+            }
+          });
+          if (!found) {
+            tmpToSortAlbums.push(bucket);
+          }
+        }
+      });
+
+      this.toSortAlbum = tmpToSortAlbums;
+      this.toSortAlbum.forEach(bucket => this.getAlbum(bucket));
+      // console.log(this.toSortAlbum)
+      // console.log(this.albums)
+    });
+  }
+
+  private getAlbum(albumBucket: Bucket) {
+    // For each bucket.key (album name), get Album document
+    // Use track count to compare
+    this.elsService.getArtistFromAlbumName(albumBucket.key).subscribe(albums => {
+      // Identification of the good album
+      let goodAlbum;
+      if (albums.length > 1) {
+        // More than one result for an album name: search good by track count
+        albums.forEach(album => {
+          if (album['Track Count'] === albumBucket.doc_count) {
+            goodAlbum = album;
+          }
+        });
+      } else {
+        // Just one result for album name
+        goodAlbum = albums[0];
+      }
+
+      // TODO Crap security if no good album found
+      if (goodAlbum == undefined) {
+        goodAlbum = albums[0]
+      }
+
+      this.albums[albumBucket.key] = goodAlbum;
+    });
+  }
+
+}

dashboard/src/environments/environment.ts

@@ -13,4 +13,4 @@ export const environment = {
  * This import should be commented out in production mode because it will have a negative impact
  * on performance if an error is thrown.
  */
-// import 'zone.js/dist/zone-error';  // Included with Angular CLI.
+// import 'zone.js/plugins/zone-error';  // Included with Angular CLI.

dashboard/src/polyfills.ts

@@ -1,3 +1,7 @@
+/***************************************************************************************************
+ * Load `$localize` onto the global scope - used if i18n tags appear in Angular templates.
+ */
+import '@angular/localize/init';
 /**
  * This file includes polyfills needed by Angular and is loaded before the app.
  * You can add your own extra polyfills to this file.
@@ -55,7 +59,7 @@
 /***************************************************************************************************
  * Zone JS is required by default for Angular itself.
  */
-import 'zone.js/dist/zone';  // Included with Angular CLI.
+import 'zone.js';  // Included with Angular CLI.
 
 
 /***************************************************************************************************

dashboard/src/test.ts

@@ -1,6 +1,6 @@
 // This file is required by karma.conf.js and loads recursively all the .spec and framework files
 
-import 'zone.js/dist/zone-testing';
+import 'zone.js/testing';
 import { getTestBed } from '@angular/core/testing';
 import {
   BrowserDynamicTestingModule,

dashboard/tsconfig.json

@@ -16,5 +16,8 @@
       "es2018",
       "dom"
     ]
+  },
+  "angularCompilerOptions": {
+    "strictTemplates": true
   }
 }

iTunesParser.py

@@ -147,7 +147,8 @@ class ITunesParser:
                 'Play Count': 0,
                 'Rating': 0,
                 'Genre': set(),
-                'Album': set()
+                'Album': set(),
+                'Album Artist': set()
                 }
 
         # Compute information
@@ -168,6 +169,9 @@ class ITunesParser:
         if 'Album' in track:
             self._artists[akey]['Album'].add(track['Album'])
 
+        if 'Album Artist' in track:
+            self._artists[akey]['Album Artist'].add(track['Artist'])
+
     def _process_album(self, track):
         """
             Process albums in the track part of library and return a JSON formated for a bulk ELS request

mapping.suggest.json

@@ -0,0 +1,48 @@
+{
+    "settings": {
+        "index": {
+            "number_of_replicas": 0
+        },
+        "analysis": {
+            "filter": {
+                "french_stop": {
+                    "type": "stop",
+                    "stopwords": "_french_"
+                },
+                "english_stop": {
+                    "type": "stop",
+                    "stopwords": "_english_"
+                }
+            },
+            "analyzer": {
+                "names": {
+                    "tokenizer": "standard",
+                    "filter": [
+                        "lowercase",
+                        "asciifolding",
+                        "french_stop",
+                        "english_stop"
+                    ]
+                }
+            }
+        }
+    },
+    "mappings": {
+        "properties": {
+            "artist_suggest": {
+                "type": "completion",
+                "search_analyzer": "names"
+            },
+            "artist": {
+                "type": "keyword"
+            },
+            "album_suggest": {
+                "type": "completion",
+                "search_analyzer": "names"
+            },
+            "album": {
+                "type": "keyword"
+            }
+        }
+    }
+}

send_data.py

@@ -10,6 +10,8 @@ import json
 import time
 import requests
 
+from suggester import process_file
+
 class bcolors:
     HEADER = '\033[95m'
     OKBLUE = '\033[94m'
@@ -28,10 +30,12 @@ DEFAULT_ARTIST_FILE = 'es-artists.json'
 DEFAULT_MAPPING_SONGS_FILE = 'mapping.songs.json'
 DEFAULT_MAPPING_ARTISTS_FILE = 'mapping.artists.json'
 DEFAULT_MAPPING_ALBUMS_FILE = 'mapping.albums.json'
+DEFAULT_MAPPING_SUGGEST_FILE = 'mapping.suggest.json'
 
 SONG_INDEX = 'itunes-songs'
 ALBUM_INDEX = 'itunes-albums'
 ARTIST_INDEX = 'itunes-artists'
+SUGGEST_INDEX = 'itunes-suggest'
 # TODO Put variables in a config files or in a python library
 
 # Global values / set as default values
@@ -145,6 +149,22 @@ def main():
         else:
             print('Album sent')
 
+    if args.ALL or args.no_suggest:
+        print("Process suggestion:")
+        if args.DELETE: # TODO Do a method?
+            mapping_suggest = load_file(args.mapping_suggest, DEFAULT_MAPPING_SUGGEST_FILE)
+            if not args.quiet:
+                print("Mapping of suggest index file: '{}'".format(mapping_suggest.name))
+
+            delete_index(SUGGEST_INDEX, args.quiet)
+            put_mapping(SUGGEST_INDEX, mapping_suggest, args.quiet)
+
+        suggs_docs = 0
+        suggs_docs += process_file('/home/budd/workspace/iTunes/es-albums.json', 'Album')
+        print('Created documents: ' + str(suggs_docs))
+        suggs_docs += process_file('/home/budd/workspace/iTunes/es-artists.json', 'Artist', 'Album Artist')
+        print('Created documents: ' + str(suggs_docs))
+
     print("I'm done!")
     if check_is_ok.count(False) > 0:
         print('Some problems occurs')
@@ -199,6 +219,8 @@ def create_args_parser():
                                .format(DEFAULT_ARTIST_FILE))
     song_group.add_argument('-s', '--song', action='store_false',
                             help='Disable sending song data')
+    song_group.add_argument('--no-suggest', action='store_false',
+                            help='Disable sending suggest data')
 
     # Mode
     mode_group = parser.add_argument_group('Mode')
@@ -211,12 +233,14 @@ def create_args_parser():
 
     # Mapping
     mapping_group = parser.add_argument_group('Mapping files')
-    mode_group.add_argument('-ms', '--mapping-song', type=argparse.FileType('r'), const=DEFAULT_MAPPING_SONGS_FILE, nargs='?',
+    mapping_group.add_argument('-ms', '--mapping-song', type=argparse.FileType('r'), const=DEFAULT_MAPPING_SONGS_FILE, nargs='?',
                             help='Mapping file for songs (default: \'{}\')'.format(DEFAULT_MAPPING_SONGS_FILE))
-    mode_group.add_argument('-mr', '--mapping-artist', type=argparse.FileType('r'), const=DEFAULT_ARTIST_FILE, nargs='?',
+    mapping_group.add_argument('-mr', '--mapping-artist', type=argparse.FileType('r'), const=DEFAULT_ARTIST_FILE, nargs='?',
                             help='Mapping file for artists (default: \'{}\')'.format(DEFAULT_MAPPING_ARTISTS_FILE))
-    mode_group.add_argument('-ml', '--mapping-album', type=argparse.FileType('r'), const=DEFAULT_MAPPING_ALBUMS_FILE, nargs='?',
+    mapping_group.add_argument('-ml', '--mapping-album', type=argparse.FileType('r'), const=DEFAULT_MAPPING_ALBUMS_FILE, nargs='?',
                             help='Mapping file for albums (default: \'{}\')'.format(DEFAULT_MAPPING_ALBUMS_FILE))
+    mapping_group.add_argument('-mg', '--mapping-suggest', type=argparse.FileType('r'), const=DEFAULT_MAPPING_SUGGEST_FILE, nargs='?',
+                            help='Mapping file for suggest (default: \'{}\')'.format(DEFAULT_MAPPING_SUGGEST_FILE))
 
     # Global Settings
     g_settings_group = parser.add_argument_group('Global Settings')

suggester.es

@@ -1,51 +1,7 @@
 DELETE itunes-suggest
 
 PUT /itunes-suggest
-{
-    "settings": {
-        "analysis": {
-            "filter": {
-                "french_stop": {
-                    "type": "stop",
-                    "stopwords": "_french_"
-                },
-                "english_stop": {
-                    "type": "stop",
-                    "stopwords": "_english_"
-                }
-            },
-            "analyzer": {
-                "names": {
-                    "tokenizer": "standard",
-                    "filter": [
-                        "lowercase",
-                        "asciifolding",
-                        "french_stop",
-                        "english_stop"
-                    ]
-                }
-            }
-        }
-    },
-    "mappings": {
-        "properties": {
-            "artist_suggest": {
-                "type": "completion",
-                "search_analyzer": "names"
-            },
-            "artist": {
-                "type": "keyword"
-            },
-            "album_suggest": {
-                "type": "completion",
-                "search_analyzer": "names"
-            },
-            "album": {
-                "type": "keyword"
-            }
-        }
-    }
-}
+!./mapping.suggest.json
 
 // Also possible to specify analyze for ingesting => https://stackoverflow.com/questions/48304499/elasticsearch-completion-suggester-not-working-with-whitespace-analyzer
 
@@ -59,7 +15,7 @@ GET itunes-suggest/_analyze
 
 GET itunes-suggest/_search
 
-POST itunes-suggest/_search
+GET itunes-suggest/_search
 {
     "_source" : "artist",
     "suggest": {
@@ -72,7 +28,7 @@ POST itunes-suggest/_search
     }
 }
 
-POST itunes-suggest/_search
+GET itunes-suggest/_search
 {
     "_source" : "album",
     "suggest": {
@@ -86,7 +42,7 @@ POST itunes-suggest/_search
     }
 }
 
-POST itunes-suggest/_search
+GET itunes-suggest/_search
 {
     "_source": ["album", "artist"],
     "suggest": {
@@ -105,7 +61,7 @@ POST itunes-suggest/_search
     }
 }
 
-POST itunes-suggest/_search
+GET itunes-suggest/_search
 {
     "_source": ["album", "artist"],
     "suggest": {
@@ -124,7 +80,7 @@ POST itunes-suggest/_search
     }
 }
 
-POST itunes-suggest/_search
+GET itunes-suggest/_search
 {
     "suggest": {
         "ar-suggest": {

suggester.py

@@ -1,37 +1,77 @@
-import requests
-import json
-import sys
+"""
+    Process files generated by iTunesParser to fill a suggester index.
+    Suggester index in ELS must be created before use.
 
-ELS_URL ='http://localhost:9200'
+    Found suggester.es query to create index.
+"""
+
+import sys
+import json
+import requests
+
+ELS_URL = 'http://localhost:9200'
 INDEX = 'itunes-suggest'
 
 class NoGoodDataException(Exception):
-    def __init__(self, message):
-        super().__init__(message)
+    """ Raise when data can't be correctly analyzed """
 
 def get_tokens(data: str) -> list:
+    """
+        Query Elasticsearch to get token for a string with a specific analyzer.
+        Throw an exception if no token found in ELS response.
+        Parameters
+        ----------
+        data: string
+            String to be analysed to obtain the tokens
+        Returns
+        -------
+        list
+            A list of token
+        Raises
+        ------
+        NoGoodDataException
+            If no tokens are found in the ELS responses, consider that the data is not correct for analysis.
+    """
     if not data:
         return []
     query = {
-        "analyzer": "names",
+        "analyzer": "names", # TODO Parameterize analyzer ?
         "text" : data
     }
 
     url = '{}/{}/_analyze'.format(ELS_URL, INDEX)
-    r = requests.get(url, json=query)
+    req = requests.get(url, json=query)
 
-    if not 'tokens' in r.json():
+    if not 'tokens' in req.json():
         print('ERROR: Not tokens in result')
         print('Input: ' + str(data))
-        print('Request: ' + str(r.json()))
+        print('Request: ' + str(req.json()))
         raise NoGoodDataException('Data is not correct to get tokens')
-    return [t['token'] for t in r.json()['tokens']]
+    return [t['token'] for t in req.json()['tokens']]
 
-def post_document(name: str, input: list, field_name: str) -> bool:
-    suggest_name = field_name + '_suggest'
+def post_document(main_field_value: str, input_terms: list, main_field_name: str) -> str:
+    """
+        Create suggestion document in Elasticsearch.
+
+        Parameters
+        ----------
+        main_field_value : str
+            Value to put in the main field named by `main_field_name`
+        input_terms : list
+            List of suggestion term to put in document
+        main_field_name : str
+            Name of the main field, to fill with `main_field_value`
+
+        Returns
+        -------
+        str
+            Success: ID of created document
+            Fail (ret. status <> 201): None
+    """
+    suggest_name = main_field_name + '_suggest'
     element = {
-        field_name: name,
-        suggest_name: input
+        main_field_name: main_field_value,
+        suggest_name: input_terms
     }
 
     # Filter empty keys
@@ -43,16 +83,29 @@ def post_document(name: str, input: list, field_name: str) -> bool:
         print('ELS Response KO')
         print(resp.status_code)
         print(resp.text)
-        return
+        return None
 
     el_id = resp.json()['_id']
     # print('Post_element - Element created: ' + el_id)
     return el_id
 
-def process_file(file_name: str, field_name: str) -> int:
+def process_file(file_name: str, field_name: str, array_file: str = None) -> int:
+    """
+        Process a JSON file with data
+
+        Parameters
+        ----------
+        file_name: string
+            Path and name of file to analyze
+        field_name: string
+            Name of the field where to find the data to create the suggestion entries
+        array_file: string, Default: None
+            Name of an array field to analyze to create more suggestion entries.
+            Nothing if None
+    """
     print('Process file: ' + file_name)
     with open(file_name, 'r') as o_file:
-            lines = o_file.readlines()
+        lines = o_file.readlines()
 
     count = 0
     i = 0
@@ -62,22 +115,28 @@ def process_file(file_name: str, field_name: str) -> int:
         sys.stdout.flush()
         sys.stdout.write("\b" * (40+1)) # return to start of line, after '['
         data = json.loads(line)
-        if "Artist" in data:
-            try :
-                input = get_tokens(data[field_name])
-                post_document(name=data[field_name], input=input, field_name=field_name.lower())
+        if not "index" in data: # Exclude index line
+            try:
+                suggests_entries = get_tokens(data[field_name])
+
+                if array_file and array_file in data and data[array_file]:
+                    for key in data[array_file]:
+                        suggests_entries.extend(get_tokens(key))
+
+                # TODO Input have the same value several times ==> use to process a score
+                post_document(main_field_value=data[field_name], input_terms=suggests_entries, main_field_name=field_name.lower())
                 count += 1
             except NoGoodDataException:
                 print('ERROR WITH DATA')
                 print(str(data))
     print('File processed\n')
-
     return count
 
 
 if __name__ == '__main__':
-    # Using readlines()
-    count = 0
-    count += process_file('/home/budd/workspace/iTunes/es-albums.json', 'Album')
-    count += process_file('/home/budd/workspace/iTunes/es-artists.json', 'Artist')
-    print('Created documents: ' + str(count))
+    created_docs = 0
+    created_docs += process_file('/home/budd/workspace/iTunes/es-albums.json', 'Album')
+    print('Created documents: ' + str(created_docs))
+    created_docs += process_file('/home/budd/workspace/iTunes/es-artists.json', 'Artist', 'Album Artist')
+    print('Created documents: ' + str(created_docs))
+    # TODO Created doc <> nb doc in ELS