diff --git a/.gitignore b/.gitignore index fb814e8..43dc2cc 100644 --- a/.gitignore +++ b/.gitignore @@ -1,8 +1,8 @@ node_modules/ .vscode/ -/es-albums.json -/es-artists.json -/es-songs.json +/es-albums.* +/es-artists.* +/es-songs.* /iTunesLibrary.xml # Working files @@ -17,4 +17,3 @@ sand_box.py rating_test.py iTunesGraphParser.my.py iTunes Library.xml -*.jsonl diff --git a/Justfile b/Justfile index 0e0b84e..5bf1799 100644 --- a/Justfile +++ b/Justfile @@ -1,6 +1,6 @@ parse: - rm -v es-* - python3 iTunesParser.py -f iTunes\ Library.xml + -rm -v es-* + python3 iTunesParser.py -f iTunes\ Library.xml -F json send-all: just send songs diff --git a/iTunesParser.py b/iTunesParser.py index 4851f6b..b1a922b 100644 --- a/iTunesParser.py +++ b/iTunesParser.py @@ -130,13 +130,13 @@ class ITunesParser: return akey = track["Album Artist"] if "Album Artist" in track else track["Artist"] - # Add artist - if akey not in self._artists: - a_id = self.calc_id(akey) + persistent_id = self.calc_id(akey) + + if persistent_id not in self._artists: # Key is used to increment/precise some information # So we use artist name as a key to avoid calculating an ID for each track - self._artists[akey] = { - "Persistent ID": a_id, + self._artists[persistent_id] = { + "Persistent ID": persistent_id, "Name": akey, "Artist": akey, "Track Count": 0, @@ -151,36 +151,38 @@ class ITunesParser: rating = track["Rating"] if "Rating" in track else 0 rating = self.calc_average( - rating, self._artists[akey]["Rating"], self._artists[akey]["Track Count"] + rating, + self._artists[persistent_id]["Rating"], + self._artists[persistent_id]["Track Count"], ) - self._artists[akey]["Track Count"] += 1 - self._artists[akey]["Rating"] = rating - self._artists[akey]["Play Count"] += play_count + self._artists[persistent_id]["Track Count"] += 1 + self._artists[persistent_id]["Rating"] = rating + self._artists[persistent_id]["Play Count"] += play_count if "Genre" in track: # Split up the Genres genre_parts = track["Genre"].split("/") - self._artists[akey]["Genre"] |= set(genre_parts) + self._artists[persistent_id]["Genre"] |= set(genre_parts) if "Album" in track: - self._artists[akey]["Album"].add(track["Album"]) + self._artists[persistent_id]["Album"].add(track["Album"]) def _process_album(self, track): """ Process albums in the track part of library and return a JSON formated for a bulk ELS request """ - if "Album" not in track: return akey = track["Album"] - if akey not in self._albums: - a_id = self.calc_id(akey) + persistent_id = self.calc_id(akey) + + if persistent_id not in self._albums: # Key is used to increment/precise some information # So we use album name as a key to avoid calculating an ID for each track - self._albums[akey] = { - "Persistent ID": a_id, + self._albums[persistent_id] = { + "Persistent ID": persistent_id, "Name": akey, "Album": akey, "Track Count": 0, @@ -201,36 +203,38 @@ class ITunesParser: avg_bitrate = self.calc_average( track["Bit Rate"], - self._albums[akey]["Avg Bit Rate"], - self._albums[akey]["Track Count"], + self._albums[persistent_id]["Avg Bit Rate"], + self._albums[persistent_id]["Track Count"], ) - self._albums[akey]["Avg Bit Rate"] = avg_bitrate - self._albums[akey]["Track Count"] += 1 - self._albums[akey]["Play Count"] += play_count - self._albums[akey]["Total Time"] += total_time + self._albums[persistent_id]["Avg Bit Rate"] = avg_bitrate + self._albums[persistent_id]["Track Count"] += 1 + self._albums[persistent_id]["Play Count"] += play_count + self._albums[persistent_id]["Total Time"] += total_time - self._albums[akey]["Location"] = os.path.dirname(track["Location"]) + self._albums[persistent_id]["Location"] = os.path.dirname(track["Location"]) - if self._albums[akey]["Min Bit Rate"] > track["Bit Rate"]: - self._albums[akey]["Min Bit Rate"] = track["Bit Rate"] + if self._albums[persistent_id]["Min Bit Rate"] > track["Bit Rate"]: + self._albums[persistent_id]["Min Bit Rate"] = track["Bit Rate"] if "Genre" in track: # Split up the Genres genre_parts = track["Genre"].split("/") - self._albums[akey]["Genre"] |= set(genre_parts) + self._albums[persistent_id]["Genre"] |= set(genre_parts) if "Artist" in track: - self._albums[akey]["Artist"].add(track["Artist"]) + self._albums[persistent_id]["Artist"].add(track["Artist"]) if "Album Rating" in track: - self._albums[akey]["Album Rating"] = track["Album Rating"] + self._albums[persistent_id]["Album Rating"] = track["Album Rating"] if "Album Rating Computed" in track: - self._albums[akey]["Album Rating Computed"] = track["Album Rating Computed"] + self._albums[persistent_id]["Album Rating Computed"] = track[ + "Album Rating Computed" + ] if "Album Artist" in track: - self._albums[akey]["Album Artist"] = track["Album Artist"] + self._albums[persistent_id]["Album Artist"] = track["Album Artist"] @classmethod def calc_average(cls, added_value, current_value, nb_values): @@ -251,67 +255,42 @@ class ITunesParser: class WriteElsJson: @staticmethod - def write_artists(artists, output_file): - """ - Write artists data to another JSON file - """ - - file_artist = io.open(output_file, "wb") - for _, artist in artists.items(): - persistent_id = artist["Persistent ID"] - artist["Rating"] = round(artist["Rating"]) - - json_track_index = { - "index": {"_index": ITunesParser.ARTIST_INDEX, "_id": persistent_id} - } - - # file_artist.write(bytes(json.dumps(json_track_index, indent=None, cls=SetEncoder), 'UTF-8')) - # file_artist.write(bytes("\n", 'UTF-8')) - file_artist.write( - bytes(json.dumps(artist, indent=None, cls=SetEncoder), "UTF-8") - ) - file_artist.write(bytes("\n", "UTF-8")) - file_artist.close() - - @staticmethod - def write_albums(albums, output_file): - """ - Write albums data to another JSON file - """ - - file_albums = io.open(output_file, "wb") - for _, album in albums.items(): - persistent_id = album["Persistent ID"] - album["Avg Bit Rate"] = round(album["Avg Bit Rate"]) - - json_track_index = { - "index": {"_index": ITunesParser.ALBUM_INDEX, "_id": persistent_id} - } - - # file_albums.write(bytes(json.dumps(json_track_index, indent=None, cls=SetEncoder), 'UTF-8')) - # file_albums.write(bytes("\n", 'UTF-8')) - file_albums.write( - bytes(json.dumps(album, indent=None, cls=SetEncoder), "UTF-8") - ) - file_albums.write(bytes("\n", "UTF-8")) - file_albums.close() - - @staticmethod - def write_songs(songs, output_file): + def write_elements( + elements: dict, + o_name: str, + json_style: str, + els_index=ITunesParser.SONG_INDEX, + els=False, + ): """ Write songs to a JSON """ - file = io.open(output_file, "wb") - for persistent_id, song in songs.items(): - json_track_index = { - "index": {"_index": ITunesParser.SONG_INDEX, "_id": persistent_id} - } + output_filename = f"{o_name}.{json_style}" - # file.write(bytes(json.dumps(json_track_index, indent=None, cls=SetEncoder), 'UTF-8')) - # file.write(bytes("\n", 'UTF-8')) - file.write(bytes(json.dumps(song, indent=None, cls=SetEncoder), "UTF-8")) - file.write(bytes("\n", "UTF-8")) - file.close() + with io.open(output_filename, "wb") as ofile: + if json_style == "json": + ofile.write(bytes("[\n", "UTF-8")) + for persistent_id, song in elements.items(): + if els: + json_track_index = { + "index": { + "_index": els_index, + "_id": persistent_id, + } + } + # file.write(bytes(json.dumps(json_track_index, indent=None, cls=SetEncoder), 'UTF-8')) + # file.write(bytes("\n", 'UTF-8')) + + ofile.write( + bytes(json.dumps(song, indent=None, cls=SetEncoder), "UTF-8") + ) + if json_style == "json": + ofile.write( + bytes(",", "UTF-8") + ) # TODO Doesn't work -> last line... + ofile.write(bytes("\n", "UTF-8")) + if json_style == "json": + ofile.write(bytes("]\n", "UTF-8")) #### main block #### @@ -345,6 +324,13 @@ parser.add_argument( parser.add_argument( "-c", "--console", action="store_true", help="Output to console instead of file" ) +parser.add_argument( + "-F", + "--format", + choices=["json", "jsonl"], + default="json", + help="Choose JSON style", +) # parser.add_argument('-v', '--verbose', action='store_true', # help='Verbose output') @@ -355,9 +341,13 @@ if __name__ == "__main__": itunes_parser = ITunesParser().parse(args.file) print("Writing JSON files...") - WriteElsJson.write_songs(itunes_parser["songs"], "es-songs.jsonl") - WriteElsJson.write_artists(itunes_parser["artists"], "es-artists.jsonl") - WriteElsJson.write_albums(itunes_parser["albums"], "es-albums.jsonl") + WriteElsJson.write_elements(itunes_parser["songs"], "es-songs", args.format) + WriteElsJson.write_elements( + itunes_parser["artists"], "es-artists", args.format, ITunesParser.ARTIST_INDEX + ) + WriteElsJson.write_elements( + itunes_parser["albums"], "es-albums", args.format, ITunesParser.ARTIST_INDEX + ) print("Done!")