From 7987653b32032b0ecc267e72005e99d5af49efc6 Mon Sep 17 00:00:00 2001 From: nathannathant <74019033+pynathanthomas@users.noreply.github.com> Date: Wed, 24 Feb 2021 14:50:57 -0800 Subject: [PATCH 1/5] added support for other types of links --- qobuz_dl/core.py | 86 +++++++++++++++++++++++++++++++------------- qobuz_dl/metadata.py | 15 ++------ 2 files changed, 63 insertions(+), 38 deletions(-) diff --git a/qobuz_dl/core.py b/qobuz_dl/core.py index 3b7c4d0..553a388 100644 --- a/qobuz_dl/core.py +++ b/qobuz_dl/core.py @@ -21,7 +21,8 @@ WEB_URL = "https://play.qobuz.com/" ARTISTS_SELECTOR = "td.chartlist-artist > a" TITLE_SELECTOR = "td.chartlist-name > a" EXTENSIONS = (".mp3", ".flac") -QUALITIES = {5: "5 - MP3", 6: "6 - FLAC", 7: "7 - 24B<96kHz", 27: "27 - 24B>96kHz"} +QUALITIES = {5: "5 - MP3", 6: "6 - FLAC", + 7: "7 - 24B<96kHz", 27: "27 - 24B>96kHz"} logger = logging.getLogger(__name__) @@ -32,7 +33,8 @@ class PartialFormatter(string.Formatter): def get_field(self, field_name, args, kwargs): try: - val = super(PartialFormatter, self).get_field(field_name, args, kwargs) + val = super(PartialFormatter, self).get_field(field_name, + args, kwargs) except (KeyError, AttributeError): val = None, field_name return val @@ -95,12 +97,29 @@ class QobuzDL: def get_id(self, url): return re.match( - r"https?://(?:w{0,3}|play|open)\.qobuz\.com/(?:(?:album|track|artist" - r"|playlist|label)/|[a-z]{2}-[a-z]{2}/album/-?\w+(?:-\w+)*-?/|user/" - r"library/favorites/)(\w+)", + r"https?://(?:w{0,3}|play|open)\.qobuz\.com/(?:(?:album|track" + r"|artist|playlist|label)/|[a-z]{2}-[a-z]{2}/album/-?\w+(?:-\w+)*" + r"-?/|user/library/favorites/)(\w+)", url, ).group(1) + def get_type(self, url): + if re.match(r'https?', url) is not None: + url_type = url.split('/')[3] + if url_type not in ['album', 'artist', 'playlist', + 'track', 'label']: + if url_type == "user": + url_type = url.split('/')[-1] + else: + # url is from Qobuz store + # e.g. "https://www.qobuz.com/us-en/album/..." + url_type = url.split('/')[4] + else: + # url missing base + # e.g. "/us-en/album/{artist}/{id}" + url_type = url.split('/')[2] + return url_type + def download_from_id(self, item_id, album=True, alt_path=None): if handle_download_id(self.downloads_db, item_id, add_id=False): logger.info( @@ -144,24 +163,27 @@ class QobuzDL: "track": {"album": False, "func": None, "iterable_key": None}, } try: - url_type = url.split("/")[3] + url_type = self.get_type(url) type_dict = possibles[url_type] item_id = self.get_id(url) except (KeyError, IndexError): logger.info( - f'{RED}Invalid url: "{url}". Use urls from https://play.qobuz.com!' + f'{RED}Invalid url: "{url}". Use urls from ' + 'https://play.qobuz.com!' ) return if type_dict["func"]: content = [item for item in type_dict["func"](item_id)] content_name = content[0]["name"] logger.info( - f"{YELLOW}Downloading all the music from {content_name} ({url_type})!" + f"{YELLOW}Downloading all the music from {content_name} " + f"({url_type})!" ) new_path = self.create_dir( os.path.join(self.directory, sanitize_filename(content_name)) ) - items = [item[type_dict["iterable_key"]]["items"] for item in content][0] + items = [item[type_dict["iterable_key"]]["items"] + for item in content][0] logger.info(f"{YELLOW}{len(items)} downloads in queue") for item in items: self.download_from_id( @@ -210,9 +232,11 @@ class QobuzDL: logger.info( f'{YELLOW}Searching {self.lucky_type}s for "{query}".\n' - f"{YELLOW}qobuz-dl will attempt to download the first {self.lucky_limit} results." + f"{YELLOW}qobuz-dl will attempt to download the first " + f"{self.lucky_limit} results." ) - results = self.search_by_type(query, self.lucky_type, self.lucky_limit, True) + results = self.search_by_type(query, self.lucky_type, + self.lucky_limit, True) if download: self.download_list_of_urls(results) @@ -275,7 +299,8 @@ class QobuzDL: ) url = "{}{}/{}".format(WEB_URL, item_type, i.get("id", "")) - item_list.append({"text": text, "url": url} if not lucky else url) + item_list.append({"text": text, "url": url} if not lucky + else url) return item_list except (KeyError, IndexError): logger.info(f"{RED}Invalid type: {item_type}") @@ -287,7 +312,8 @@ class QobuzDL: except (ImportError, ModuleNotFoundError): if os.name == "nt": sys.exit( - 'Please install curses with "pip3 install windows-curses" to continue' + 'Please install curses with ' + '"pip3 install windows-curses" to continue' ) raise @@ -306,13 +332,15 @@ class QobuzDL: try: item_types = ["Albums", "Tracks", "Artists", "Playlists"] - selected_type = pick(item_types, "I'll search for:\n[press Intro]")[0][ - :-1 - ].lower() - logger.info(f"{YELLOW}Ok, we'll search for {selected_type}s{RESET}") + selected_type = pick(item_types, + "I'll search for:\n[press Intro]" + )[0][:-1].lower() + logger.info(f"{YELLOW}Ok, we'll search for " + f"{selected_type}s{RESET}") final_url_list = [] while True: - query = input(f"{CYAN}Enter your search: [Ctrl + c to quit]\n-{DF} ") + query = input(f"{CYAN}Enter your search: [Ctrl + c to quit]\n" + f"-{DF} ") logger.info(f"{YELLOW}Searching...{RESET}") options = self.search_by_type( query, selected_type, self.interactive_limit @@ -334,10 +362,12 @@ class QobuzDL: options_map_func=get_title_text, ) if len(selected_items) > 0: - [final_url_list.append(i[0]["url"]) for i in selected_items] + [final_url_list.append(i[0]["url"]) + for i in selected_items] y_n = pick( ["Yes", "No"], - "Items were added to queue to be downloaded. Keep searching?", + "Items were added to queue to be downloaded. " + "Keep searching?", ) if y_n[0][0] == "N": break @@ -347,7 +377,8 @@ class QobuzDL: if final_url_list: desc = ( "Select [intro] the quality (the quality will " - "be automatically\ndowngraded if the selected is not found)" + "be automatically\ndowngraded if the selected " + "is not found)" ) self.quality = pick( qualities, @@ -389,11 +420,13 @@ class QobuzDL: pl_title = sanitize_filename(soup.select_one("h1").text) pl_directory = os.path.join(self.directory, pl_title) logger.info( - f"{YELLOW}Downloading playlist: {pl_title} ({len(track_list)} tracks)" + f"{YELLOW}Downloading playlist: {pl_title} " + f"({len(track_list)} tracks)" ) for i in track_list: - track_id = self.get_id(self.search_by_type(i, "track", 1, lucky=True)[0]) + track_id = self.get_id(self.search_by_type(i, "track", 1, + lucky=True)[0]) if track_id: self.download_from_id(track_id, False, pl_directory) @@ -410,7 +443,9 @@ class QobuzDL: dirs.sort() audio_rel_files = [ # os.path.abspath(os.path.join(local, file_)) - # os.path.join(rel_folder, os.path.basename(os.path.normpath(local)), file_) + # os.path.join(rel_folder, + # os.path.basename(os.path.normpath(local)), + # file_) os.path.join(os.path.basename(os.path.normpath(local)), file_) for file_ in files if os.path.splitext(file_)[-1] in EXTENSIONS @@ -423,7 +458,8 @@ class QobuzDL: if not audio_files or len(audio_files) != len(audio_rel_files): continue - for audio_rel_file, audio_file in zip(audio_rel_files, audio_files): + for audio_rel_file, audio_file in zip(audio_rel_files, + audio_files): try: pl_item = ( EasyMP3(audio_file) diff --git a/qobuz_dl/metadata.py b/qobuz_dl/metadata.py index 4061df2..b3264e4 100644 --- a/qobuz_dl/metadata.py +++ b/qobuz_dl/metadata.py @@ -44,14 +44,6 @@ def tag_flac(filename, root_dir, final_name, d, album, :param bool istrack :param bool em_image: Embed cover art into file """ - print('in tag_flac d:') - # print(json.dumps(d.keys(), indent=2)) - # print(d.keys()) - print('album:') - # print(album.keys()) - # print(json.dumps(album.keys(), indent=2)) - print(f'{filename=}') - print(f'{istrack=}') audio = FLAC(filename) audio["TITLE"] = get_title(d) @@ -190,13 +182,10 @@ def tag_mp3(filename, root_dir, final_name, d, album, audio['TPOS'] = id3.TPOS(encoding=3, text=str(d["media_number"])) - def lookup_and_set_tags(tag_name, value): - id3tag = id3_legend[tag_name] - audio[id3tag.__name__] = id3tag(encoding=3, text=value) - # write metadata in `tags` to file for k, v in tags.items(): - lookup_and_set_tags(k, v) + id3tag = id3_legend[k] + audio[id3tag.__name__] = id3tag(encoding=3, text=v) if em_image: emb_image = os.path.join(root_dir, "cover.jpg") From 4a57145be7f9807b5454c9b8420ede40d59bebbc Mon Sep 17 00:00:00 2001 From: nathannathant <74019033+pynathanthomas@users.noreply.github.com> Date: Sun, 28 Feb 2021 21:31:43 -0800 Subject: [PATCH 2/5] formatted filepaths --- qobuz_dl/cli.py | 9 +++ qobuz_dl/commands.py | 15 ++++ qobuz_dl/core.py | 7 ++ qobuz_dl/downloader.py | 179 ++++++++++++++++++++++++++++++----------- qobuz_dl/metadata.py | 42 +++++++--- 5 files changed, 194 insertions(+), 58 deletions(-) diff --git a/qobuz_dl/cli.py b/qobuz_dl/cli.py index 78e2526..6df9404 100644 --- a/qobuz_dl/cli.py +++ b/qobuz_dl/cli.py @@ -55,6 +55,9 @@ def reset_config(config_file): spoofer = spoofbuz.Spoofer() config["DEFAULT"]["app_id"] = str(spoofer.getAppId()) config["DEFAULT"]["secrets"] = ",".join(spoofer.getSecrets().values()) + config["DEFAULT"]["folder_format"] = "{artist} - {album} ({year}) " + "[{bit_depth}B-{sampling_rate}kHz]" + config["DEFAULT"]["track_format"] = "{tracknumber}. {tracktitle}" with open(config_file, "w") as configfile: config.write(configfile) logging.info( @@ -98,6 +101,8 @@ def main(): no_cover = config.getboolean("DEFAULT", "no_cover") no_database = config.getboolean("DEFAULT", "no_database") app_id = config["DEFAULT"]["app_id"] + folder_format = config["DEFAULT"]["folder_format"] + track_format = config["DEFAULT"]["track_format"] secrets = [ secret for secret in config["DEFAULT"]["secrets"].split(",") if secret ] @@ -131,6 +136,10 @@ def main(): cover_og_quality=arguments.og_cover or og_cover, no_cover=arguments.no_cover or no_cover, downloads_db=None if no_database or arguments.no_db else QOBUZ_DB, + folder_format=arguments.folder_format + if hasattr(arguments, "folder_format") else folder_format, + track_format=arguments.track_format + if hasattr(arguments, "track_format") else track_format, ) qobuz.initialize_client(email, password, app_id, secrets) diff --git a/qobuz_dl/commands.py b/qobuz_dl/commands.py index 1fcf291..22354a6 100644 --- a/qobuz_dl/commands.py +++ b/qobuz_dl/commands.py @@ -102,6 +102,21 @@ def add_common_arg(custom_parser, default_folder, default_quality): custom_parser.add_argument( "--no-db", action="store_true", help="don't call the database" ) + custom_parser.add_argument( + "-ff", + "--folder-format", + metavar='PATTERN', + help='pattern for formatting folder names, e.g ' + '"{artist} - {album} ({year})". available keys: artist, ' + 'album, year, sampling_rate, bit_rate, tracktitle. ' + 'cannot contain characters used by the system, which includes /:<>', + ) + custom_parser.add_argument( + "-tf", + "--track-format", + metavar='PATTERN', + help='pattern for formatting track names. see `folder-format`.', + ) def qobuz_dl_args( diff --git a/qobuz_dl/core.py b/qobuz_dl/core.py index 553a388..5aa03b4 100644 --- a/qobuz_dl/core.py +++ b/qobuz_dl/core.py @@ -65,6 +65,9 @@ class QobuzDL: cover_og_quality=False, no_cover=False, downloads_db=None, + folder_format='{artist} - {album} ({year}) [{bit_depth}B-' + '{sampling_rate}kHz]', + track_format='{tracknumber}. {tracktitle}', ): self.directory = self.create_dir(directory) self.quality = quality @@ -78,6 +81,8 @@ class QobuzDL: self.cover_og_quality = cover_og_quality self.no_cover = no_cover self.downloads_db = create_db(downloads_db) if downloads_db else None + self.folder_format = folder_format + self.track_format = track_format def initialize_client(self, email, pwd, app_id, secrets): self.client = qopy.Client(email, pwd, app_id, secrets) @@ -140,6 +145,8 @@ class QobuzDL: self.quality_fallback, self.cover_og_quality, self.no_cover, + folder_format=self.folder_format, + track_format=self.track_format ) handle_download_id(self.downloads_db, item_id, add_id=True) except (requests.exceptions.RequestException, NonStreamable) as e: diff --git a/qobuz_dl/downloader.py b/qobuz_dl/downloader.py index ad1a25b..4b28e9a 100644 --- a/qobuz_dl/downloader.py +++ b/qobuz_dl/downloader.py @@ -10,6 +10,9 @@ from qobuz_dl.color import OFF, GREEN, RED, YELLOW, CYAN from qobuz_dl.exceptions import NonStreamable QL_DOWNGRADE = "FormatRestrictedByFormatAvailability" +DEFAULT_MP3_FOLDER_FORMAT = '{artist} - {album} [MP3]' +DEFAULT_MP3_TRACK_FORMAT = '{tracknumber}. {tracktitle}' + logger = logging.getLogger(__name__) @@ -30,16 +33,18 @@ def tqdm_download(url, fname, track_name): def get_description(u: dict, track_title, multiple=None): - downloading_title = f'{track_title} [{u["bit_depth"]}/{u["sampling_rate"]}]' + downloading_title = f'{track_title} ' + f'[{u["bit_depth"]}/{u["sampling_rate"]}]' if multiple: downloading_title = f"[Disc {multiple}] {downloading_title}" return downloading_title -def get_format(client, item_dict, quality, is_track_id=False, track_url_dict=None): +def get_format(client, item_dict, quality, + is_track_id=False, track_url_dict=None) -> tuple: quality_met = True if int(quality) == 5: - return "MP3", quality_met + return ("MP3", quality_met, None, None) track_dict = item_dict if not is_track_id: track_dict = item_dict["tracks"]["items"][0] @@ -53,17 +58,22 @@ def get_format(client, item_dict, quality, is_track_id=False, track_url_dict=Non restrictions = new_track_dict.get("restrictions") if isinstance(restrictions, list): if any( - restriction.get("code") == QL_DOWNGRADE for restriction in restrictions + restriction.get("code") == QL_DOWNGRADE + for restriction in restrictions ): quality_met = False - if ( - new_track_dict["bit_depth"] == 16 - and new_track_dict["sampling_rate"] == 44.1 - ): - return "FLAC", quality_met + + return ( + "FLAC", + quality_met, + new_track_dict["bit_depth"], + new_track_dict["sampling_rate"] + ) return ( - f'{new_track_dict["bit_depth"]}B-{new_track_dict["sampling_rate"]}Khz', + "FLAC", quality_met, + new_track_dict["bit_depth"], + new_track_dict["sampling_rate"], ) except (KeyError, requests.exceptions.HTTPError): return "Unknown", quality_met @@ -104,6 +114,7 @@ def download_and_tag( is_mp3, embed_art=False, multiple=None, + track_format='{tracknumber}. {tracktitle}', ): """ Download and tag a file @@ -112,13 +123,15 @@ def download_and_tag( :param int tmp_count: Temporal download file number :param dict track_url_dict: get_track_url dictionary from Qobuz client :param dict track_metadata: Track item dictionary from Qobuz client - :param dict album_or_track_metadata: Album/track dictionary from Qobuz client + :param dict album_or_track_metadata: Album/track dict from Qobuz client :param bool is_track :param bool is_mp3 :param bool embed_art: Embed cover art into file (FLAC-only) + :param track_format python format-string that determines file naming :param multiple: Multiple disc integer :type multiple: integer or None """ + extension = ".mp3" if is_mp3 else ".flac" try: @@ -134,26 +147,27 @@ def download_and_tag( filename = os.path.join(root_dir, f".{tmp_count:02}.tmp") # Determine the filename - artist = track_metadata.get("performer", {}).get("name") - album_artist = track_metadata.get("album", {}).get("artist", {}).get("name") - new_track_title = track_metadata.get("title") - version = track_metadata.get("version") - - if artist or album_artist: - new_track_title = ( - f"{artist if artist else album_artist}" f' - {track_metadata["title"]}' - ) - if version: - new_track_title = f"{new_track_title} ({version})" - - track_file = f'{track_metadata["track_number"]:02}. {new_track_title}' - final_file = os.path.join(root_dir, sanitize_filename(track_file))[:250] + extension + track_title = track_metadata["title"] + filename_attr = { + 'artist': track_metadata.get("performer", {}).get("name"), + 'albumartist': track_metadata.get("album", {}).get("artist", + {}).get("name"), + 'bit_depth': track_metadata['maximum_bit_depth'], + 'sampling_rate': track_metadata['maximum_sampling_rate'], + 'tracktitle': track_title, + 'version': track_metadata["version"], + 'tracknumber': f"{track_metadata['track_number']:02}" + } + # track_format is a format string + # e.g. '{tracknumber}. {artist} - {tracktitle}' + formatted_path = sanitize_filename(track_format.format(**filename_attr)) + final_file = os.path.join(root_dir, formatted_path)[:250] + extension if os.path.isfile(final_file): - logger.info(f"{OFF}{new_track_title} was already downloaded") + logger.info(f"{OFF}{track_title} was already downloaded") return - desc = get_description(track_url_dict, new_track_title, multiple) + desc = get_description(track_url_dict, track_title, multiple) tqdm_download(url, filename, desc) tag_function = metadata.tag_mp3 if is_mp3 else metadata.tag_flac try: @@ -181,6 +195,9 @@ def download_id_by_type( downgrade_quality=True, cover_og_quality=False, no_cover=False, + folder_format='{artist} - {album} ({year}) ' + '[{bit_depth}B-{sampling_rate}kHz]', + track_format='{tracknumber}. {tracktitle}', ): """ Download and get metadata by ID and type (album or track) @@ -195,9 +212,43 @@ def download_id_by_type( :param bool downgrade: Skip releases not available in set quality :param bool cover_og_quality: Download cover in its original quality :param bool no_cover: Don't download cover art + :param str folder_format: format string that determines folder naming + :param str track_format: format string that determines track naming """ count = 0 + def _clean_format_gen(s: str) -> str: + '''General clean for format strings. Avoids user errors. + ''' + if s.endswith('.mp3'): + s = s[:-4] + elif s.endswith('.flac'): + s = s[:-5] + s = s.strip() + return s + + def _not_mp3_valid(s: str) -> bool: + return 'bit_depth' in s or 'sample_rate' in s + + def clean_format_str(folder: str, track: str, bit_depth) -> tuple: + '''Cleans up the format strings to avoid errors + with MP3 files. + ''' + print(f'in clean_format_str: {folder=}, {track=}, {bit_depth=}') + folder = _clean_format_gen(folder) + track = _clean_format_gen(track) + if bit_depth is None: # if is an mp3, there is no bit depth + if _not_mp3_valid(folder): + logger.error(f'{RED}invalid format string for MP3: "{folder}"' + f'\ndefaulting to "{DEFAULT_MP3_FOLDER_FORMAT}"') + folder = DEFAULT_MP3_FOLDER_FORMAT + if _not_mp3_valid(track): + logger.error(f'{RED}invalid format string for MP3: "{track}"' + f'\ndefaulting to "{DEFAULT_MP3_TRACK_FORMAT}"') + track = DEFAULT_MP3_TRACK_FORMAT + + return folder, track + if album: meta = client.get_album_meta(item_id) @@ -212,35 +263,52 @@ def download_id_by_type( return album_title = get_title(meta) - album_format, quality_met = get_format(client, meta, quality) + + format_info = get_format(client, meta, quality) + file_format, quality_met, bit_depth, sampling_rate = format_info + if not downgrade_quality and not quality_met: logger.info( - f"{OFF}Skipping {album_title} as doesn't met quality requirement" + f"{OFF}Skipping {album_title} as it doesn't " + "meet quality requirement" ) return - logger.info(f"\n{YELLOW}Downloading: {album_title}\nQuality: {album_format}\n") - dirT = ( - meta["artist"]["name"], - album_title, - meta["release_date_original"].split("-")[0], - album_format, + logger.info(f"\n{YELLOW}Downloading: {album_title}\n" + f"Quality: {file_format}\n") + album_attr = { + 'artist': meta["artist"]["name"], + 'album': album_title, + 'year': meta["release_date_original"].split("-")[0], + 'format': file_format, + 'bit_depth': bit_depth, + 'sampling_rate': sampling_rate + } + # TODO: if the quality is MP3, remove `bit_depth` + # and `sampling_rate` tags smartly from format + # instead of defaulting to a pre-chosen format + folder_format, track_format = clean_format_str(folder_format, + track_format, + bit_depth) + sanitized_title = sanitize_filename( + folder_format.format(**album_attr) ) - sanitized_title = sanitize_filename("{} - {} ({}) [{}]".format(*dirT)) dirn = os.path.join(path, sanitized_title) os.makedirs(dirn, exist_ok=True) if no_cover: logger.info(f"{OFF}Skipping cover") else: - get_extra(meta["image"]["large"], dirn, og_quality=cover_og_quality) + get_extra(meta["image"]["large"], dirn, + og_quality=cover_og_quality) if "goodies" in meta: try: get_extra(meta["goodies"][0]["url"], dirn, "booklet.pdf") except: # noqa pass - media_numbers = [track["media_number"] for track in meta["tracks"]["items"]] + media_numbers = [track["media_number"] for track in + meta["tracks"]["items"]] is_multiple = True if len([*{*media_numbers}]) > 1 else False for i in meta["tracks"]["items"]: parse = client.get_track_url(i["id"], quality) @@ -256,6 +324,7 @@ def download_id_by_type( is_mp3, embed_art, i["media_number"] if is_multiple else None, + track_format=track_format, ) else: logger.info(f"{OFF}Demo. Skipping") @@ -267,29 +336,43 @@ def download_id_by_type( meta = client.get_track_meta(item_id) track_title = get_title(meta) logger.info(f"\n{YELLOW}Downloading: {track_title}") - track_format, quality_met = get_format(client, meta, quality, True, parse) + format_info = get_format(client, meta, quality, + is_track_id=True, track_url_dict=parse) + file_format, quality_met, bit_depth, sampling_rate = format_info + + folder_format, track_format = clean_format_str(folder_format, + track_format, + bit_depth) + if not downgrade_quality and not quality_met: logger.info( - f"{OFF}Skipping {track_title} as doesn't met quality requirement" + f"{OFF}Skipping {track_title} as it doesn't " + "meet quality requirement" ) return - dirT = ( - meta["album"]["artist"]["name"], - track_title, - meta["album"]["release_date_original"].split("-")[0], - track_format, + track_attr = { + 'artist': meta["album"]["artist"]["name"], + 'tracktitle': track_title, + 'year': meta["album"]["release_date_original"].split("-")[0], + 'bit_depth': bit_depth, + 'sampling_rate': sampling_rate + } + sanitized_title = sanitize_filename( + folder_format.format(**track_attr) ) - sanitized_title = sanitize_filename("{} - {} [{}] [{}]".format(*dirT)) dirn = os.path.join(path, sanitized_title) os.makedirs(dirn, exist_ok=True) if no_cover: logger.info(f"{OFF}Skipping cover") else: get_extra( - meta["album"]["image"]["large"], dirn, og_quality=cover_og_quality + meta["album"]["image"]["large"], dirn, + og_quality=cover_og_quality ) is_mp3 = True if int(quality) == 5 else False - download_and_tag(dirn, count, parse, meta, meta, True, is_mp3, embed_art) + download_and_tag(dirn, count, parse, meta, + meta, True, is_mp3, embed_art, + track_format=track_format) else: logger.info(f"{OFF}Demo. Skipping") logger.info(f"{GREEN}Completed") diff --git a/qobuz_dl/metadata.py b/qobuz_dl/metadata.py index b3264e4..cea7bdc 100644 --- a/qobuz_dl/metadata.py +++ b/qobuz_dl/metadata.py @@ -10,6 +10,9 @@ logger = logging.getLogger(__name__) # unicode symbols COPYRIGHT, PHON_COPYRIGHT = '\u2117', '\u00a9' +# if a metadata block exceeds this, mutagen will raise error +# and the file won't be tagged +FLAC_MAX_BLOCKSIZE = 16777215 def get_title(track_dict): @@ -30,6 +33,19 @@ def _format_copyright(s: str) -> str: return s +def _format_genres(genres: list) -> str: + '''Fixes the weirdly formatted genre lists returned by the API. + >>> g = ['Pop/Rock', 'Pop/Rock→Rock', 'Pop/Rock→Rock→Alternatif et Indé'] + >>> _format_genres(g) + 'Pop/Rock, Rock, Alternatif et Indé' + ''' + + if genres == []: + return '' + else: + return ', '.join(genres[-1].split('\u2192')) + + # Use KeyError catching instead of dict.get to avoid empty tags def tag_flac(filename, root_dir, final_name, d, album, istrack=True, em_image=False): @@ -72,17 +88,17 @@ def tag_flac(filename, root_dir, final_name, d, album, pass if istrack: - audio["GENRE"] = ", ".join(d["album"]["genres_list"]) # GENRE - audio["ALBUMARTIST"] = d["album"]["artist"]["name"] # ALBUM ARTIST - audio["TRACKTOTAL"] = str(d["album"]["tracks_count"]) # TRACK TOTAL - audio["ALBUM"] = d["album"]["title"] # ALBUM TITLE + audio["GENRE"] = _format_genres(d["album"]["genres_list"]) + audio["ALBUMARTIST"] = d["album"]["artist"]["name"] + audio["TRACKTOTAL"] = str(d["album"]["tracks_count"]) + audio["ALBUM"] = d["album"]["title"] audio["DATE"] = d["album"]["release_date_original"] audio["COPYRIGHT"] = _format_copyright(d["copyright"]) else: - audio["GENRE"] = ", ".join(album["genres_list"]) # GENRE - audio["ALBUMARTIST"] = album["artist"]["name"] # ALBUM ARTIST - audio["TRACKTOTAL"] = str(album["tracks_count"]) # TRACK TOTAL - audio["ALBUM"] = album["title"] # ALBUM TITLE + audio["GENRE"] = _format_genres(album["genres_list"]) + audio["ALBUMARTIST"] = album["artist"]["name"] + audio["TRACKTOTAL"] = str(album["tracks_count"]) + audio["ALBUM"] = album["title"] audio["DATE"] = album["release_date_original"] audio["COPYRIGHT"] = _format_copyright(album["copyright"]) @@ -97,6 +113,12 @@ def tag_flac(filename, root_dir, final_name, d, album, cover_image = multi_emb_image try: + # rest of the metadata still gets embedded + # when the image size is too big + if os.path.getsize(cover_image) > FLAC_MAX_BLOCKSIZE: + raise Exception("downloaded cover size too large to embed. " + "turn off `og_cover` to avoid error") + image = Picture() image.type = 3 image.mime = "image/jpeg" @@ -161,14 +183,14 @@ def tag_mp3(filename, root_dir, final_name, d, album, tags['artist'] = album["artist"]["name"] if istrack: - tags["genre"] = ", ".join(d["album"]["genres_list"]) + tags["genre"] = _format_genres(d["album"]["genres_list"]) tags["albumartist"] = d["album"]["artist"]["name"] tags["album"] = d["album"]["title"] tags["date"] = d["album"]["release_date_original"] tags["copyright"] = _format_copyright(d["copyright"]) tracktotal = str(d["album"]["tracks_count"]) else: - tags["genre"] = ", ".join(album["genres_list"]) + tags["genre"] = _format_genres(album["genres_list"]) tags["albumartist"] = album["artist"]["name"] tags["album"] = album["title"] tags["date"] = album["release_date_original"] From 0d02292823c3b8949b7a9bef5d3ee56ef5d2e1cb Mon Sep 17 00:00:00 2001 From: nathannathant <74019033+pynathanthomas@users.noreply.github.com> Date: Mon, 1 Mar 2021 15:18:08 -0800 Subject: [PATCH 3/5] typing, minor changes --- qobuz_dl/downloader.py | 106 ++++++++++++++++++++++------------------- 1 file changed, 57 insertions(+), 49 deletions(-) diff --git a/qobuz_dl/downloader.py b/qobuz_dl/downloader.py index 4b28e9a..de3130c 100644 --- a/qobuz_dl/downloader.py +++ b/qobuz_dl/downloader.py @@ -1,5 +1,6 @@ import logging import os +from typing import Tuple import requests from pathvalidate import sanitize_filename @@ -12,6 +13,8 @@ from qobuz_dl.exceptions import NonStreamable QL_DOWNGRADE = "FormatRestrictedByFormatAvailability" DEFAULT_MP3_FOLDER_FORMAT = '{artist} - {album} [MP3]' DEFAULT_MP3_TRACK_FORMAT = '{tracknumber}. {tracktitle}' +DEFAULT_UNKNOWN_FOLDER_FORMAT = '{artist} - {album}' +DEFAULT_UNKNOWN_TRACK_FORMAT = '{tracknumber}. {tracktitle}' logger = logging.getLogger(__name__) @@ -40,8 +43,9 @@ def get_description(u: dict, track_title, multiple=None): return downloading_title -def get_format(client, item_dict, quality, - is_track_id=False, track_url_dict=None) -> tuple: +def get_format(client, item_dict, + quality, is_track_id=False, + track_url_dict=None) -> Tuple[str, bool, int, int]: quality_met = True if int(quality) == 5: return ("MP3", quality_met, None, None) @@ -63,12 +67,6 @@ def get_format(client, item_dict, quality, ): quality_met = False - return ( - "FLAC", - quality_met, - new_track_dict["bit_depth"], - new_track_dict["sampling_rate"] - ) return ( "FLAC", quality_met, @@ -76,7 +74,7 @@ def get_format(client, item_dict, quality, new_track_dict["sampling_rate"], ) except (KeyError, requests.exceptions.HTTPError): - return "Unknown", quality_met + return ("Unknown", quality_met, None, None) def get_title(item_dict): @@ -127,7 +125,7 @@ def download_and_tag( :param bool is_track :param bool is_mp3 :param bool embed_art: Embed cover art into file (FLAC-only) - :param track_format python format-string that determines file naming + :param str track_format format-string that determines file naming :param multiple: Multiple disc integer :type multiple: integer or None """ @@ -148,10 +146,10 @@ def download_and_tag( # Determine the filename track_title = track_metadata["title"] + print(track_metadata) filename_attr = { - 'artist': track_metadata.get("performer", {}).get("name"), - 'albumartist': track_metadata.get("album", {}).get("artist", - {}).get("name"), + 'artist': track_metadata["performer"]["name"], + 'albumartist': track_metadata["album"]["artist"]["name"], 'bit_depth': track_metadata['maximum_bit_depth'], 'sampling_rate': track_metadata['maximum_sampling_rate'], 'tracktitle': track_title, @@ -217,38 +215,6 @@ def download_id_by_type( """ count = 0 - def _clean_format_gen(s: str) -> str: - '''General clean for format strings. Avoids user errors. - ''' - if s.endswith('.mp3'): - s = s[:-4] - elif s.endswith('.flac'): - s = s[:-5] - s = s.strip() - return s - - def _not_mp3_valid(s: str) -> bool: - return 'bit_depth' in s or 'sample_rate' in s - - def clean_format_str(folder: str, track: str, bit_depth) -> tuple: - '''Cleans up the format strings to avoid errors - with MP3 files. - ''' - print(f'in clean_format_str: {folder=}, {track=}, {bit_depth=}') - folder = _clean_format_gen(folder) - track = _clean_format_gen(track) - if bit_depth is None: # if is an mp3, there is no bit depth - if _not_mp3_valid(folder): - logger.error(f'{RED}invalid format string for MP3: "{folder}"' - f'\ndefaulting to "{DEFAULT_MP3_FOLDER_FORMAT}"') - folder = DEFAULT_MP3_FOLDER_FORMAT - if _not_mp3_valid(track): - logger.error(f'{RED}invalid format string for MP3: "{track}"' - f'\ndefaulting to "{DEFAULT_MP3_TRACK_FORMAT}"') - track = DEFAULT_MP3_TRACK_FORMAT - - return folder, track - if album: meta = client.get_album_meta(item_id) @@ -284,12 +250,9 @@ def download_id_by_type( 'bit_depth': bit_depth, 'sampling_rate': sampling_rate } - # TODO: if the quality is MP3, remove `bit_depth` - # and `sampling_rate` tags smartly from format - # instead of defaulting to a pre-chosen format folder_format, track_format = clean_format_str(folder_format, track_format, - bit_depth) + file_format) sanitized_title = sanitize_filename( folder_format.format(**album_attr) ) @@ -376,3 +339,48 @@ def download_id_by_type( else: logger.info(f"{OFF}Demo. Skipping") logger.info(f"{GREEN}Completed") + + +# ----------- Utilities ----------- +def _clean_format_gen(s: str) -> str: + '''General clean for format strings. Avoids user errors. + ''' + if s.endswith('.mp3'): + s = s[:-4] + elif s.endswith('.flac'): + s = s[:-5] + s = s.strip() + return s + + +def _not_mp3_valid(s: str) -> bool: + return 'bit_depth' in s or 'sample_rate' in s + + +def clean_format_str(folder: str, track: str, + file_format: str) -> Tuple[str, str]: + '''Cleans up the format strings to avoid errors + with MP3 files. + ''' + folder = _clean_format_gen(folder) + track = _clean_format_gen(track) + if file_format == 'MP3': + if _not_mp3_valid(folder): + logger.error(f'{RED}invalid format string for MP3: "{folder}"' + f'\ndefaulting to "{DEFAULT_MP3_FOLDER_FORMAT}"') + folder = DEFAULT_MP3_FOLDER_FORMAT + if _not_mp3_valid(track): + logger.error(f'{RED}invalid format string for MP3: "{track}"' + f'\ndefaulting to "{DEFAULT_MP3_TRACK_FORMAT}"') + track = DEFAULT_MP3_TRACK_FORMAT + elif file_format == 'Unknown': + if _not_mp3_valid(folder): + logger.error(f'{RED}Error getting format. Defaulting format ' + f'string to "{DEFAULT_UNKNOWN_FOLDER_FORMAT}"') + folder = DEFAULT_UNKNOWN_FOLDER_FORMAT + if _not_mp3_valid(track): + logger.error(f'{RED}Error getting format. Defaulting format ' + f'string to "{DEFAULT_UNKNOWN_TRACK_FORMAT}"') + track = DEFAULT_UNKNOWN_TRACK_FORMAT + + return (folder, track) From 3affd97916ebbb4f0dd49898d88bcffc470d017c Mon Sep 17 00:00:00 2001 From: nathannathant <74019033+pynathanthomas@users.noreply.github.com> Date: Mon, 1 Mar 2021 17:56:52 -0800 Subject: [PATCH 4/5] clean format function improved, config updates --- qobuz_dl/cli.py | 16 +++++- qobuz_dl/commands.py | 2 +- qobuz_dl/downloader.py | 115 ++++++++++++++++++++++------------------- 3 files changed, 77 insertions(+), 56 deletions(-) diff --git a/qobuz_dl/cli.py b/qobuz_dl/cli.py index 6df9404..c71b6c3 100644 --- a/qobuz_dl/cli.py +++ b/qobuz_dl/cli.py @@ -101,8 +101,20 @@ def main(): no_cover = config.getboolean("DEFAULT", "no_cover") no_database = config.getboolean("DEFAULT", "no_database") app_id = config["DEFAULT"]["app_id"] + + if ("folder_format" not in config["DEFAULT"] + or "track_format" not in config["DEFAULT"]): + logging.info(f'{YELLOW}Config file does not include format string,' + ' updating...') + config["DEFAULT"]["folder_format"] = "{artist} - {album} ({year}) " + "[{bit_depth}B-{sampling_rate}kHz]" + config["DEFAULT"]["track_format"] = "{tracknumber}. {tracktitle}" + with open(CONFIG_FILE, 'w') as cf: + config.write(cf) + folder_format = config["DEFAULT"]["folder_format"] track_format = config["DEFAULT"]["track_format"] + secrets = [ secret for secret in config["DEFAULT"]["secrets"].split(",") if secret ] @@ -137,9 +149,9 @@ def main(): no_cover=arguments.no_cover or no_cover, downloads_db=None if no_database or arguments.no_db else QOBUZ_DB, folder_format=arguments.folder_format - if hasattr(arguments, "folder_format") else folder_format, + if arguments.folder_format is not None else folder_format, track_format=arguments.track_format - if hasattr(arguments, "track_format") else track_format, + if arguments.track_format is not None else track_format, ) qobuz.initialize_client(email, password, app_id, secrets) diff --git a/qobuz_dl/commands.py b/qobuz_dl/commands.py index 22354a6..41731e8 100644 --- a/qobuz_dl/commands.py +++ b/qobuz_dl/commands.py @@ -108,7 +108,7 @@ def add_common_arg(custom_parser, default_folder, default_quality): metavar='PATTERN', help='pattern for formatting folder names, e.g ' '"{artist} - {album} ({year})". available keys: artist, ' - 'album, year, sampling_rate, bit_rate, tracktitle. ' + 'albumartist, album, year, sampling_rate, bit_rate, tracktitle. ' 'cannot contain characters used by the system, which includes /:<>', ) custom_parser.add_argument( diff --git a/qobuz_dl/downloader.py b/qobuz_dl/downloader.py index de3130c..11bba76 100644 --- a/qobuz_dl/downloader.py +++ b/qobuz_dl/downloader.py @@ -11,10 +11,17 @@ from qobuz_dl.color import OFF, GREEN, RED, YELLOW, CYAN from qobuz_dl.exceptions import NonStreamable QL_DOWNGRADE = "FormatRestrictedByFormatAvailability" -DEFAULT_MP3_FOLDER_FORMAT = '{artist} - {album} [MP3]' -DEFAULT_MP3_TRACK_FORMAT = '{tracknumber}. {tracktitle}' -DEFAULT_UNKNOWN_FOLDER_FORMAT = '{artist} - {album}' -DEFAULT_UNKNOWN_TRACK_FORMAT = '{tracknumber}. {tracktitle}' +# used in case of error +DEFAULT_FORMATS = { + 'MP3': [ + '{artist} - {album} ({year}) [MP3]', + '{tracknumber}. {tracktitle}', + ], + 'Unknown': [ + '{artist} - {album}', + '{tracknumber}. {tracktitle}', + ] +} logger = logging.getLogger(__name__) @@ -145,15 +152,16 @@ def download_and_tag( filename = os.path.join(root_dir, f".{tmp_count:02}.tmp") # Determine the filename - track_title = track_metadata["title"] - print(track_metadata) + track_title = track_metadata.get("title") + artist = _safe_get(track_metadata, "performer", "name") filename_attr = { - 'artist': track_metadata["performer"]["name"], - 'albumartist': track_metadata["album"]["artist"]["name"], + 'artist': artist, + 'albumartist': _safe_get(track_metadata, "album", "artist", "name", + default=artist), 'bit_depth': track_metadata['maximum_bit_depth'], 'sampling_rate': track_metadata['maximum_sampling_rate'], 'tracktitle': track_title, - 'version': track_metadata["version"], + 'version': track_metadata.get("version"), 'tracknumber': f"{track_metadata['track_number']:02}" } # track_format is a format string @@ -250,9 +258,9 @@ def download_id_by_type( 'bit_depth': bit_depth, 'sampling_rate': sampling_rate } - folder_format, track_format = clean_format_str(folder_format, - track_format, - file_format) + folder_format, track_format = _clean_format_str(folder_format, + track_format, + file_format) sanitized_title = sanitize_filename( folder_format.format(**album_attr) ) @@ -303,9 +311,9 @@ def download_id_by_type( is_track_id=True, track_url_dict=parse) file_format, quality_met, bit_depth, sampling_rate = format_info - folder_format, track_format = clean_format_str(folder_format, - track_format, - bit_depth) + folder_format, track_format = _clean_format_str(folder_format, + track_format, + bit_depth) if not downgrade_quality and not quality_met: logger.info( @@ -342,45 +350,46 @@ def download_id_by_type( # ----------- Utilities ----------- -def _clean_format_gen(s: str) -> str: - '''General clean for format strings. Avoids user errors. - ''' - if s.endswith('.mp3'): - s = s[:-4] - elif s.endswith('.flac'): - s = s[:-5] - s = s.strip() - return s - -def _not_mp3_valid(s: str) -> bool: - return 'bit_depth' in s or 'sample_rate' in s - - -def clean_format_str(folder: str, track: str, - file_format: str) -> Tuple[str, str]: - '''Cleans up the format strings to avoid errors +def _clean_format_str(folder: str, track: str, + file_format: str) -> Tuple[str, str]: + '''Cleans up the format strings, avoids errors with MP3 files. ''' - folder = _clean_format_gen(folder) - track = _clean_format_gen(track) - if file_format == 'MP3': - if _not_mp3_valid(folder): - logger.error(f'{RED}invalid format string for MP3: "{folder}"' - f'\ndefaulting to "{DEFAULT_MP3_FOLDER_FORMAT}"') - folder = DEFAULT_MP3_FOLDER_FORMAT - if _not_mp3_valid(track): - logger.error(f'{RED}invalid format string for MP3: "{track}"' - f'\ndefaulting to "{DEFAULT_MP3_TRACK_FORMAT}"') - track = DEFAULT_MP3_TRACK_FORMAT - elif file_format == 'Unknown': - if _not_mp3_valid(folder): - logger.error(f'{RED}Error getting format. Defaulting format ' - f'string to "{DEFAULT_UNKNOWN_FOLDER_FORMAT}"') - folder = DEFAULT_UNKNOWN_FOLDER_FORMAT - if _not_mp3_valid(track): - logger.error(f'{RED}Error getting format. Defaulting format ' - f'string to "{DEFAULT_UNKNOWN_TRACK_FORMAT}"') - track = DEFAULT_UNKNOWN_TRACK_FORMAT + final = [] + for i, fs in enumerate((folder, track)): + if fs.endswith('.mp3'): + fs = fs[:-4] + elif fs.endswith('.flac'): + fs = fs[:-5] + fs = fs.strip() - return (folder, track) + # default to pre-chosen string if format is invalid + if (file_format in ('MP3', 'Unknown') and + 'bit_depth' in file_format or 'sampling_rate' in file_format): + default = DEFAULT_FORMATS[file_format][i] + logger.error(f'{RED}invalid format string for format {file_format}' + f'. defaulting to {default}') + fs = default + final.append(fs) + + return tuple(final) + + +def _safe_get(d: dict, *keys, default=None): + '''A replacement for chained `get()` statements on dicts: + >>> d = {'foo': {'bar': 'baz'}} + >>> _safe_get(d, 'baz') + None + >>> _safe_get(d, 'foo', 'bar') + 'baz' + ''' + curr = d + res = default + for key in keys: + res = curr.get(key, default) + if res == default or not hasattr(res, '__getitem__'): + return res + else: + curr = res + return res From 91ee9cbde03faaec3aebdb7fc36e85cf4fc1964f Mon Sep 17 00:00:00 2001 From: nathannathant <74019033+pynathanthomas@users.noreply.github.com> Date: Mon, 1 Mar 2021 18:18:10 -0800 Subject: [PATCH 5/5] whitespace --- qobuz_dl/downloader.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/qobuz_dl/downloader.py b/qobuz_dl/downloader.py index 6eef3bd..3aac3b5 100644 --- a/qobuz_dl/downloader.py +++ b/qobuz_dl/downloader.py @@ -50,7 +50,6 @@ def get_description(u: dict, track_title, multiple=None): return downloading_title - def get_format(client, item_dict, quality, is_track_id=False, track_url_dict=None) -> Tuple[str, bool, int, int]: @@ -264,7 +263,6 @@ def download_id_by_type( file_format) sanitized_title = sanitize_filename( folder_format.format(**album_attr) - ) dirn = os.path.join(path, sanitized_title) os.makedirs(dirn, exist_ok=True)