7 Commits

Author SHA1 Message Date
52a0cde401 (front) Autocomplete search on dashboard with Datalist
Create Elasticsearch service to search suggestions

Send to page when choose a suggestion - bad way
It's impossible to use Object with datalist
So it's impossible to distinguish an album from an artist if they have
the same name
2021-08-22 17:04:29 +02:00
5fbcfbae68 (front) Update package-lock 2021-08-22 17:04:25 +02:00
74c6f99fa5 (back) Suggester: Change ELS search analyzer for specific char
Implementing search in dashboard revealed problems when search with specific char (francois perusse)
Note: it's also possible to specify an analyzer for ingestion

(cherry picked from commit 24da127750c995c55d54876a1be8a4ad3bf9ce9c)
2021-08-22 17:02:14 +02:00
60e1fb2e74 (back) Suggester: Improve processor - more generic
Process album & artist with a calculated fields
Separate main
Show progress

(cherry picked from commit fc8407cc6a51fe18b14169b3a3f0e4fc363beb4f)
2021-08-22 17:01:19 +02:00
520d0be595 (back) Suggester V3: Process album in a separate way
(cherry picked from commit ebbeeccfb8535dbb67240d2c68c7dc9a4da7e7f8)
2021-08-22 17:01:04 +02:00
8121f3d751 (back) Suggester V2: Process album data
(cherry picked from commit dd322405d047d49e51d528341cbd008d7a98b6ab)
2021-08-22 17:01:00 +02:00
436edaf3f2 (back) Suggester V1: Process artist data
(cherry picked from commit 02f5705fde37e1aaef5c68de62aafe45fc86d490)
2021-08-22 17:00:01 +02:00
17 changed files with 17025 additions and 492 deletions

16733
dashboard/package-lock.json generated

File diff suppressed because it is too large Load Diff

View File

@@ -7,7 +7,6 @@ import { ArtistComponent } from './artist/artist.component';
import { GenreComponent } from './genre/genre.component';
import { TopPlayedComponent } from './top-played/top-played.component';
import { AlbumsComponent } from './albums/albums.component';
import { ToSortComponent } from './to-sort/to-sort.component';
const routes: Routes = [
{ path: '', redirectTo: '/dashboard', pathMatch: 'full' },
@@ -16,8 +15,7 @@ const routes: Routes = [
{ path: 'album', component: AlbumsComponent },
{ path: 'artist/:name', component: ArtistComponent },
{ path: 'genre/:name', component: GenreComponent },
{ path: 'top-played', component: TopPlayedComponent },
{ path: 'to-sort', component: ToSortComponent }
{ path: 'top-played', component: TopPlayedComponent }
];
@NgModule({

View File

@@ -36,8 +36,7 @@ nav a.active {
/* FIXME Code Repetition */
.btn-top {
display: flex;
flex-direction: column;
display: inline-block;
position: fixed;
z-index: 1;
-webkit-transition: opacity .3s 0s, visibility 0s 0s;

View File

@@ -1,8 +1,5 @@
<nav>
<div class="btn-top">
<a routerLink="/dashboard" routerLinkActive="active"><span class="glyphicon glyphicon-home"></span></a><br />
<a routerLink="/album" routerLinkActive="active"><span class="glyphicon glyphicon-cd"></span></a>
<a routerLink="/to-sort" routerLinkActive="active"><span class="glyphicon glyphicon-sort"></span></a>
</div>
<a class="btn-top" routerLink="/dashboard" routerLinkActive="active"><span class="glyphicon glyphicon-home"></span></a><br />
<a class="btn-top" routerLink="/album" routerLinkActive="active"><span class="glyphicon glyphicon-cd"></span></a>
</nav>
<router-outlet></router-outlet>

View File

@@ -1,6 +1,7 @@
import { NgModule } from '@angular/core';
import { BrowserModule } from '@angular/platform-browser';
import { HttpClientModule } from '@angular/common/http';
import { FormsModule } from '@angular/forms'; // <-- NgModel lives here
import { AppComponent } from './app.component';
import { DashboardComponent } from './dashboard/dashboard.component';
@@ -21,12 +22,12 @@ import { SortByPipe } from './pipes/sort-by.pipe';
import { ConvertSizeToStringPipe } from './pipes/convert-size-to-string.pipe';
import { RoundPipe } from './pipes/round.pipe';
import { AlbumsComponent } from './albums/albums.component';
import { ToSortComponent } from './to-sort/to-sort.component';
@NgModule({
imports: [
BrowserModule,
HttpClientModule,
FormsModule,
AppRoutingModule
],
declarations: [
@@ -42,8 +43,7 @@ import { ToSortComponent } from './to-sort/to-sort.component';
SortByPipe,
ConvertSizeToStringPipe,
TopPlayedComponent,
RoundPipe,
ToSortComponent
RoundPipe
],
providers: [
ElsService,

View File

@@ -77,6 +77,25 @@
</div>
</div>
<form class="navbar-form">
<div class="form-group" style="display:inline;">
<div class="input-group" style="display:table;">
<span class="input-group-addon" style="width:1%;">
<span class="glyphicon glyphicon-search"></span>
</span>
<input class="form-control" type="text" name="search" placeholder="Search Here" autocomplete="off" autofocus="autofocus"
id="searchSuggest"
list="dynmicUserIds"
[(ngModel)]="searchTerm"
(keyup)="onSearchChange()"
(change)="onSearchSelected($event)">
<datalist id="dynmicUserIds">
<option *ngFor="let item of suggested" [value]="item.name" [label]="item.type" [id]="item">{{item}}</option>
</datalist>
</div>
</div>
</form>
<div class="col-md-12">
<table class="table table-striped">
<thead>

View File

@@ -1,10 +1,12 @@
import { Component, OnInit } from '@angular/core';
import { Router } from '@angular/router';
import { ElsService } from './../els.service';
import { Song } from './../model/song';
import { Bucket } from './../model/bucket';
import { Album } from './../model/album';
import { Artist } from './../model/artist';
import { Suggested } from '../model/suggested';
@Component({
selector: 'app-dashboard',
@@ -30,7 +32,10 @@ export class DashboardComponent implements OnInit {
lastAddedAlbums: Bucket[] = [];
albumArtists = [];
constructor(private elsService: ElsService) { }
searchTerm = ''
suggested : Suggested[] = []
constructor(private elsService: ElsService, private route: Router) { }
ngOnInit(): void {
this.elsService.getTime().then(result => {
@@ -113,4 +118,19 @@ export class DashboardComponent implements OnInit {
}
});
}
onSearchChange() {
this.elsService.getSuggest(this.searchTerm).subscribe(result => this.suggested = result);
}
onSearchSelected($event) {
let selected = $event.target.value
// FIXME Not possible to get good element (just value)
// Need to use a plugin to do this correctly
this.suggested.forEach(element => {
if (element.name == selected) {
this.route.navigate(['/' + element.type + '/' + element.name])
}
});
}
}

View File

@@ -1,16 +0,0 @@
import { TestBed } from '@angular/core/testing';
import { ElsSortService } from './els-sort.service';
describe('ElsSortService', () => {
let service: ElsSortService;
beforeEach(() => {
TestBed.configureTestingModule({});
service = TestBed.inject(ElsSortService);
});
it('should be created', () => {
expect(service).toBeTruthy();
});
});

View File

@@ -1,147 +0,0 @@
import { HttpClient } from '@angular/common/http';
import { Injectable } from '@angular/core';
import { Observable } from 'rxjs';
import { map, catchError } from 'rxjs/operators';
import { ElsService } from './els.service';
import { Album } from './model/album';
import { Bucket } from './model/bucket';
import { Song } from './model/song';
@Injectable({
providedIn: 'root'
})
export class ElsSortService extends ElsService {
constructor(protected http: HttpClient) {
super(http);
}
getTime(): Promise<number> {
return this.http
.post<any>(this.elsUrl + ElsService.SONG_INDEX_NAME + ElsService.ACTION_SEARCH,
JSON.stringify({
query: {
bool: {
must_not: [
{
term: {
"Location.tree": "/F:/Musique"
}
}
]
}
},
aggs: {
sum_time: {
sum: { field: 'Total Time'}
}
},
'size': 0
}), {headers: this.headers})
.toPromise()
.then(res => res.aggregations.sum_time.value as number)
.catch(error => this.handleError(error, 'getTime()'));
}
getSize(): Promise<number> {
return this.http
.post<any>(this.elsUrl + ElsService.SONG_INDEX_NAME + ElsService.ACTION_SEARCH,
JSON.stringify({
query: {
bool: {
must_not: [
{
term: {
"Location.tree": "/F:/Musique"
}
}
]
}
},
aggs: {
sum_time: {
sum: { field: 'Size' }
}
},
'size': 0
}), {headers: this.headers})
.toPromise()
.then(res => res.aggregations.sum_time.value as number)
.catch(error => this.handleError(error, 'getSize()'));
}
getCountSong(): Promise<number> {
return this.http
.post<any>(this.elsUrl + ElsService.SONG_INDEX_NAME + ElsService.ACTION_COUNT,
JSON.stringify({
query: {
bool: {
must_not: [
{
term: {
"Location.tree": "/F:/Musique"
}
}
]
}
}
}), {headers: this.headers})
.toPromise()
.then(res => res.count as number)
.catch(error => this.handleError(error, 'getCountSong()'));
}
getCountNeverListenSong(): Promise<number> {
return this.http
.post<any>(this.elsUrl + ElsService.SONG_INDEX_NAME + ElsService.ACTION_COUNT,
JSON.stringify({
'query': {
'bool': {
'must_not': [
{
'exists': { 'field': 'Play Count'}
}, {
term: {
"Location.tree": "/F:/Musique"
} }
]
}
}
}), {headers: this.headers})
.toPromise()
.then(res => res.count as number)
.catch(error => this.handleError(error, 'getCountNeverListenSong()'));
}
getAlbums(): Observable<Bucket[]> {
return this.http
.post(this.elsUrl + ElsService.SONG_INDEX_NAME + ElsService.ACTION_SEARCH,
JSON.stringify({
query: {
bool: {
must_not: [
{
term: {
"Location.tree": "/F:/Musique"
}
}
]
}
},
'size': 0,
'aggs': {
'albums' : {
'terms': {
'field' : 'Album.raw',
'order': { '_term': 'asc' },
'size': 50
}
}
}
}), {headers: this.headers})
.pipe(
map(res => this.responseAggregationToBucket(res, "albums")),
catchError(error => this.handleError(error, 'getAlbums'))
);
}
}

View File

@@ -8,6 +8,7 @@ import { Song } from './model/song';
import { Album } from './model/album';
import { Artist } from './model/artist';
import { Bucket } from './model/bucket';
import { Suggested } from './model/suggested';
@Injectable()
export class ElsService {
@@ -15,6 +16,7 @@ export class ElsService {
public static readonly SONG_INDEX_NAME = '/itunes-songs';
public static readonly ARTIST_INDEX_NAME = '/itunes-artists';
public static readonly ALBUM_INDEX_NAME = '/itunes-albums';
public static readonly SUGGEST_INDEX_NAME = '/itunes-suggest';
protected static readonly ACTION_SEARCH = '/_search';
protected static readonly ACTION_COUNT = '/_count';
@@ -337,7 +339,7 @@ export class ElsService {
// TODO Take in consideration "sum_other_doc_count"
}
getArtistFromAlbumName(albumname: string): Observable<Album[]> { // TODO Rename ?
getArtistFromAlbumName(albumname: string): Observable<Album[]> {
return this.http
.post<any>(this.elsUrl + ElsService.ALBUM_INDEX_NAME + ElsService.ACTION_SEARCH,
JSON.stringify({
@@ -381,6 +383,34 @@ export class ElsService {
);
}
getSuggest(text: string): Observable<Suggested[]> {
console.log('search sugget: ' + text);
return this.http
.post<any>(this.elsUrl + ElsService.SUGGEST_INDEX_NAME + ElsService.ACTION_SEARCH,
JSON.stringify({
'_source': ['album', 'artist'],
'suggest': {
'album-suggest': {
'prefix': text,
'completion': {
'field': 'album_suggest'
}
},
'artist-suggest': {
'prefix': text,
'completion': {
'field': 'artist_suggest'
}
}
}
}), {headers: this.headers})
.pipe(
map(res => this.responseSuggesterToSuggested(res, 'album-suggest', 'artist-suggest')),
catchError(error => this.handleError(error, 'getSuggest(' + text + ')'))
);
}
/** Process a result to return just one result.
* Used to get an album or an artist.
* Take a name to put in console output if no result or more than one result.
@@ -406,7 +436,7 @@ export class ElsService {
*
* @param res Response to process
*/
protected responseToSongs(res: any): Song[] {
private responseToSongs(res: any): Song[] {
const result: Array<Song> = [];
res.hits.hits.forEach((hit) => {
result.push(hit._source);
@@ -444,7 +474,7 @@ export class ElsService {
* @param res Response to process
* @param name Name of aggregation
*/
protected responseAggregationToBucket(res: any, name: string): Bucket[] {
private responseAggregationToBucket(res: any, name: string): Bucket[] {
const result: Array<Bucket> = [];
res.aggregations[name].buckets.forEach((bucket) => {
result.push(bucket);
@@ -462,6 +492,20 @@ export class ElsService {
return result;
}
protected responseSuggesterToSuggested(res: any, ...suggestName: string[]): Suggested[] {
const result: Array<Suggested> = []
suggestName.forEach(sname => {
res['suggest'][sname][0]['options'].forEach(option => {
let suggest = new Suggested()
// TODO If more than one key, raise exception
suggest.type = String(Object.keys(option['_source']))
suggest.name = option['_source'][suggest.type]
result.push(suggest)
});
});
return result;
}
protected handleError(error: any, origin: string): Promise<any> {
console.error('An error occurred!');
console.error('Origin function: ', origin);

View File

@@ -0,0 +1,8 @@
export class Suggested {
name: string;
type: string;
public toString() : string {
return `${this.name} (${this.type})`;
}
}

View File

@@ -1,38 +0,0 @@
.panel-stat {
min-height: 102px;
}
.panel-green {
background-color: #16a085;
color:white;
}
.panel-yellow {
background-color: #f1c40f;
color:white;
}
.panel-red {
background-color: #d9534f;
color:white;
}
.panel-purple {
background-color: #9b59b6;
color:white;
}
.panel-blue {
background-color: #3498db;
color:white;
}
.stats_icon {
font-size: 3em;
}
.circle {
border-radius: 50%;
background-color: #9b59b6;
margin: 0 auto;
}

View File

@@ -1,130 +0,0 @@
<div class="container">
<h1>To sort Songs - </h1>
<br />
<div class="row cardAdmin">
<div class="col-lg-3 col-md-3">
<div class="panel panel-blue">
<div class="panel-heading">
<div class="row">
<div class="col-xs-3">
<span class="glyphicon glyphicon-time stats_icon"></span>
</div>
<div class="col-xs-9 text-right">
<div>
<h3 *ngIf="!totalTime"><span class="glyphicon glyphicon-refresh loading"></span></h3>
<h3 *ngIf="totalTime">{{totalTime | convertMs}}</h3>
</div>
<div><br>Total time ({{totalTime | convertMoreExact}})</div>
</div>
</div>
</div>
</div>
</div>
<div class="col-lg-3 col-md-3">
<div class="panel panel-green">
<div class="panel-heading">
<div class="row">
<div class="col-xs-3">
<span class="glyphicon glyphicon-hdd stats_icon"></span>
</div>
<div class="col-xs-9 text-right">
<div>
<h3 *ngIf="!totalSize"><span class="glyphicon glyphicon-refresh loading"></span></h3>
<h3 *ngIf="totalSize">{{totalSize | convertSizeToString}}</h3>
</div>
<div><br>Total size</div>
</div>
</div>
</div>
</div>
</div>
<div class="col-lg-3 col-md-3">
<div class="panel panel-yellow">
<div class="panel-heading">
<div class="row">
<div class="col-xs-3">
<span class="glyphicon glyphicon-play stats_icon"></span>
</div>
<div class="col-xs-9 text-right">
<div>
<h3 *ngIf="!trackCountSong"><span class="glyphicon glyphicon-refresh loading"></span></h3>
<h3 *ngIf="trackCountSong">{{trackCountSong}}</h3>
</div>
<div><br>Total Songs</div>
</div>
</div>
</div>
</div>
</div>
<div class="col-lg-3 col-md-3">
<div class="panel panel-red">
<div class="panel-heading">
<div class="row">
<div class="col-xs-3">
<span class="glyphicon glyphicon-stop stats_icon"></span>
</div>
<div class="col-xs-9 text-right">
<div>
<h3 *ngIf="!neverListenSong"><span class="glyphicon glyphicon-refresh loading"></span></h3>
<h3 *ngIf="neverListenSong">{{neverListenSong}}</h3>
</div>
<div><br>Never list. songs (~{{neverListenSong / trackCountSong * 100 | round}}%)</div>
</div>
</div>
</div>
</div>
</div>
</div>
<div class="col-md-12">
<table class="table table-striped" style="white-space: nowrap;">
<thead>
<tr>
<th>Album</th>
<th>Track Count</th>
<th>Artist/Album Artist</th>
<th>Play Count</th>
<th>Rating</th>
</tr>
</thead>
<tbody>
<tr *ngFor="let album of toSortAlbum">
<td>
<a [routerLink]="['/album', album.key]">{{album.key}}</a>&nbsp;
</td>
<td>{{albums[album.key]['Track Count']}}</td>
<ng-template [ngIf]="albums[album.key]['Album Artist']" [ngIfElse]="artistSection">
<td>
<a [routerLink]="['/artist', albums[album.key]['Album Artist']]">{{albums[album.key]['Album Artist']}}</a>&nbsp;
</td>
</ng-template>
<ng-template #artistSection>
<td>
<a [routerLink]="['/artist', albums[album.key].Artist[0]]">{{albums[album.key].Artist}}</a>&nbsp;
</td>
</ng-template>
<ng-template #artistSection>
<td>
<a [routerLink]="['/artist', albums[album.key].Artist[0]]">{{albums[album.key].Artist}}</a>&nbsp;
</td>
</ng-template>
<td>
{{albums[album.key]['Play Count']}} ({{albums[album.key]['Play Count']/albums[album.key]['Track Count'] | number:'1.0-0'}}/songs)
</td>
<td class="star" [title]="(albums[album.key]['Album Rating Computed']?'Computed Rating: ':'Rating: ') + albums[album.key]['Album Rating']">
<span *ngFor="let item of numberToArray(albums[album.key]['Album Rating'], 20)">
<span class="glyphicon" [ngClass]="albums[album.key]['Album Rating Computed']?'glyphicon-star-empty':'glyphicon-star'"></span>
</span>
</td>
</tr>
</tbody>
</table>
</div>
</div>

View File

@@ -1,25 +0,0 @@
import { ComponentFixture, TestBed } from '@angular/core/testing';
import { ToSortComponent } from './to-sort.component';
describe('ToSortComponent', () => {
let component: ToSortComponent;
let fixture: ComponentFixture<ToSortComponent>;
beforeEach(async () => {
await TestBed.configureTestingModule({
declarations: [ ToSortComponent ]
})
.compileComponents();
});
beforeEach(() => {
fixture = TestBed.createComponent(ToSortComponent);
component = fixture.componentInstance;
fixture.detectChanges();
});
it('should create', () => {
expect(component).toBeTruthy();
});
});

View File

@@ -1,89 +0,0 @@
import { Component, OnInit } from '@angular/core';
import { ElsSortService } from '../els-sort.service';
import { Album } from '../model/album';
import { Bucket } from '../model/bucket';
import { Song } from '../model/song';
import { Utils } from '../utils';
@Component({
selector: 'app-to-sort',
templateUrl: './to-sort.component.html',
styleUrls: ['./to-sort.component.css']
})
export class ToSortComponent implements OnInit {
numberToArray = Utils.numberToArray;
// Stats var
totalTime = 0;
totalSize = 0;
trackCountSong = 0;
neverListenSong = 0;
// Global var
toSortAlbum: Bucket[] = [];
albums = []
constructor(private elsService: ElsSortService) { }
ngOnInit(): void {
// **** GET STATS ****//
this.elsService.getTime().then(result => this.totalTime = result);
this.elsService.getSize().then(result => this.totalSize = result);
this.elsService.getCountSong().then(result => this.trackCountSong = result);
this.elsService.getCountNeverListenSong().then(result => this.neverListenSong = result);
// **** GET ALBUMS ****//
const tmpToSortAlbums: Bucket[] = [];
this.elsService.getAlbums().subscribe(buckets => {
buckets.forEach(bucket => {
if (tmpToSortAlbums.length === 0) {
tmpToSortAlbums.push(bucket);
} else {
let found = false;
tmpToSortAlbums.forEach(element => {
if (element.key === bucket.key) {
element.doc_count += bucket.doc_count;
found = true;
}
});
if (!found) {
tmpToSortAlbums.push(bucket);
}
}
});
this.toSortAlbum = tmpToSortAlbums;
this.toSortAlbum.forEach(bucket => this.getAlbum(bucket));
// console.log(this.toSortAlbum)
// console.log(this.albums)
});
}
private getAlbum(albumBucket: Bucket) {
// For each bucket.key (album name), get Album document
// Use track count to compare
this.elsService.getArtistFromAlbumName(albumBucket.key).subscribe(albums => {
// Identification of the good album
let goodAlbum;
if (albums.length > 1) {
// More than one result for an album name: search good by track count
albums.forEach(album => {
if (album['Track Count'] === albumBucket.doc_count) {
goodAlbum = album;
}
});
} else {
// Just one result for album name
goodAlbum = albums[0];
}
// TODO Crap security if no good album found
if (goodAlbum == undefined) {
goodAlbum = albums[0]
}
this.albums[albumBucket.key] = goodAlbum;
});
}
}

137
suggester.es Normal file
View File

@@ -0,0 +1,137 @@
DELETE itunes-suggest
PUT /itunes-suggest
{
"settings": {
"analysis": {
"filter": {
"french_stop": {
"type": "stop",
"stopwords": "_french_"
},
"english_stop": {
"type": "stop",
"stopwords": "_english_"
}
},
"analyzer": {
"names": {
"tokenizer": "standard",
"filter": [
"lowercase",
"asciifolding",
"french_stop",
"english_stop"
]
}
}
}
},
"mappings": {
"properties": {
"artist_suggest": {
"type": "completion",
"search_analyzer": "names"
},
"artist": {
"type": "keyword"
},
"album_suggest": {
"type": "completion",
"search_analyzer": "names"
},
"album": {
"type": "keyword"
}
}
}
}
// Also possible to specify analyze for ingesting => https://stackoverflow.com/questions/48304499/elasticsearch-completion-suggester-not-working-with-whitespace-analyzer
// Problem with word EP, SP
GET itunes-suggest/_analyze
{
"analyzer": "names",
"text": "the servent"
}
GET itunes-suggest/_search
POST itunes-suggest/_search
{
"_source" : "artist",
"suggest": {
"name-suggest": {
"prefix": "sou",
"completion": {
"field": "artist_suggest"
}
}
}
}
POST itunes-suggest/_search
{
"_source" : "album",
"suggest": {
"name-suggest": {
"prefix": "new",
"completion": {
"field": "album_suggest",
"size": 20
}
}
}
}
POST itunes-suggest/_search
{
"_source": ["album", "artist"],
"suggest": {
"alb-suggest": {
"prefix": "sou",
"completion": {
"field": "album_suggest"
}
},
"ar-suggest": {
"prefix": "sou",
"completion": {
"field": "artist_suggest"
}
}
}
}
POST itunes-suggest/_search
{
"_source": ["album", "artist"],
"suggest": {
"alb-suggest": {
"prefix": "Francois",
"completion": {
"field": "album_suggest"
}
},
"ar-suggest": {
"prefix": "Francois",
"completion": {
"field": "artist_suggest"
}
}
}
}
POST itunes-suggest/_search
{
"suggest": {
"ar-suggest": {
"prefix": "Femme",
"completion": {
"field": "artist_suggest"
}
}
}
}

83
suggester.py Normal file
View File

@@ -0,0 +1,83 @@
import requests
import json
import sys
ELS_URL ='http://localhost:9200'
INDEX = 'itunes-suggest'
class NoGoodDataException(Exception):
def __init__(self, message):
super().__init__(message)
def get_tokens(data: str) -> list:
if not data:
return []
query = {
"analyzer": "names",
"text" : data
}
url = '{}/{}/_analyze'.format(ELS_URL, INDEX)
r = requests.get(url, json=query)
if not 'tokens' in r.json():
print('ERROR: Not tokens in result')
print('Input: ' + str(data))
print('Request: ' + str(r.json()))
raise NoGoodDataException('Data is not correct to get tokens')
return [t['token'] for t in r.json()['tokens']]
def post_document(name: str, input: list, field_name: str) -> bool:
suggest_name = field_name + '_suggest'
element = {
field_name: name,
suggest_name: input
}
# Filter empty keys
# element = {k: v for k, v in element.items() if v}
url = '{}/{}/_doc'.format(ELS_URL, INDEX)
resp = requests.post(url, json=element)
if resp.status_code != 201:
print('ELS Response KO')
print(resp.status_code)
print(resp.text)
return
el_id = resp.json()['_id']
# print('Post_element - Element created: ' + el_id)
return el_id
def process_file(file_name: str, field_name: str) -> int:
print('Process file: ' + file_name)
with open(file_name, 'r') as o_file:
lines = o_file.readlines()
count = 0
i = 0
for line in lines:
i += 1
sys.stdout.write(str(int((i/len(lines))*100)) + '%')
sys.stdout.flush()
sys.stdout.write("\b" * (40+1)) # return to start of line, after '['
data = json.loads(line)
if "Artist" in data:
try :
input = get_tokens(data[field_name])
post_document(name=data[field_name], input=input, field_name=field_name.lower())
count += 1
except NoGoodDataException:
print('ERROR WITH DATA')
print(str(data))
print('File processed\n')
return count
if __name__ == '__main__':
# Using readlines()
count = 0
count += process_file('/home/budd/workspace/iTunes/es-albums.json', 'Album')
count += process_file('/home/budd/workspace/iTunes/es-artists.json', 'Artist')
print('Created documents: ' + str(count))