From ad8f567572aca41d8eb1e34f2f1ca885dce2e3fe Mon Sep 17 00:00:00 2001 From: "Xie, Tianshi" Date: Thu, 16 May 2024 16:15:38 -0400 Subject: [PATCH 1/6] fix model validation crash when encountering meta in album. --- src/models/album_meta.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/models/album_meta.py b/src/models/album_meta.py index 818c210..9ee43db 100644 --- a/src/models/album_meta.py +++ b/src/models/album_meta.py @@ -148,12 +148,21 @@ class Relationships(BaseModel): record_labels: RecordLabels = Field(..., alias='record-labels') +class ContentVersion(BaseModel): + MZ_INDEXER: Optional[int] = None + RTCI: Optional[int] = None + + +class Meta(BaseModel): + contentVersion: ContentVersion + + class Datum(BaseModel): id: Optional[str] = None type: Optional[str] = None href: Optional[str] = None attributes: Attributes - relationships: Relationships + meta: Meta class AlbumMeta(BaseModel): From 117a08ee4f47e4408c1a286c840b18e36e3ffbf2 Mon Sep 17 00:00:00 2001 From: "Xie, Tianshi" Date: Thu, 16 May 2024 16:18:38 -0400 Subject: [PATCH 2/6] add support to pulling explicit tag. fix tags on upc. isrc(mp4 not support, use WM tags), and composor missing(possible bug in mp4box, apple documents uses @wrt which belongs to writer in mp4box doc.) --- .gitignore | 2 ++ config.example.toml | 4 ++-- src/api.py | 2 +- src/metadata.py | 9 ++++++++- src/models/song_data.py | 1 + 5 files changed, 14 insertions(+), 4 deletions(-) diff --git a/.gitignore b/.gitignore index c8dc31a..80ff238 100644 --- a/.gitignore +++ b/.gitignore @@ -161,3 +161,5 @@ cython_debug/ #.idea/ config.toml +.python-version +.vscode/* diff --git a/config.example.toml b/config.example.toml index a16e963..d20b0e8 100644 --- a/config.example.toml +++ b/config.example.toml @@ -73,10 +73,10 @@ afterDownloaded = "" # title, artist, album, album_artist, composer, # genre, created, track, tracknum, disk, # record_company, upc, isrc, copyright, -# lyrics, cover +# lyrics, cover, ratings(rtng) embedMetadata = ["title", "artist", "album", "album_artist", "composer", "genre", "created", "track", "tracknum", "disk", "lyrics", "cover", "copyright", - "record_company", "upc", "isrc"] + "record_company", "upc", "isrc","rtng"] [mitm] # The host proxy server listens on diff --git a/src/api.py b/src/api.py index d146318..12bfdc7 100644 --- a/src/api.py +++ b/src/api.py @@ -159,7 +159,7 @@ async def get_cover(url: str, cover_format: str, cover_size: str): async def get_song_info(song_id: str, token: str, storefront: str, lang: str): async with request_lock: req = await client.get(f"https://amp-api.music.apple.com/v1/catalog/{storefront}/songs/{song_id}", - params={"extend": "extendedAssetUrls", "include": "albums", "l": lang}, + params={"extend": "extendedAssetUrls", "include": "albums,explicit", "l": lang}, headers={"Authorization": f"Bearer {token}", "User-Agent": user_agent_itunes, "Origin": "https://music.apple.com"}) song_data_obj = SongData.model_validate(req.json()) diff --git a/src/metadata.py b/src/metadata.py index 0f90b0a..16b3770 100644 --- a/src/metadata.py +++ b/src/metadata.py @@ -41,6 +41,12 @@ class SongMetadata(BaseModel): lrc = ttml_convent_to_lrc(value) tags.append(f"{key}={lrc}") continue + if key.lower() in ('upc', 'isrc'): + tags.append(f"WM/{key.lower()}={value}") + continue + if key == 'composer': + tags.append(f"writer={value}") + continue tags.append(f"{key}={value}") return ":".join(tags) @@ -55,7 +61,8 @@ class SongMetadata(BaseModel): copyright=song_data.relationships.albums.data[0].attributes.copyright, record_company=song_data.relationships.albums.data[0].attributes.recordLabel, upc=song_data.relationships.albums.data[0].attributes.upc, - isrc=song_data.attributes.isrc + isrc=song_data.attributes.isrc, + rtng=1 if song_data.attributes.contentRating and song_data.attributes.contentRating == 'explicit' else 0 ) def set_lyrics(self, lyrics: str): diff --git a/src/models/song_data.py b/src/models/song_data.py index 272b844..f56611c 100644 --- a/src/models/song_data.py +++ b/src/models/song_data.py @@ -58,6 +58,7 @@ class Attributes(BaseModel): previews: List[Preview] artistName: Optional[str] = None extendedAssetUrls: ExtendedAssetUrls + contentRating: Optional[str] = None class Artwork1(BaseModel): From c6bc6e957e57e9cedcaad7fec8036f36cbdb3a07 Mon Sep 17 00:00:00 2001 From: "Xie, Tianshi" Date: Thu, 16 May 2024 17:37:16 -0400 Subject: [PATCH 3/6] fix: crush for album missing relationship data --- src/models/album_meta.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/models/album_meta.py b/src/models/album_meta.py index 9ee43db..09e222c 100644 --- a/src/models/album_meta.py +++ b/src/models/album_meta.py @@ -162,6 +162,7 @@ class Datum(BaseModel): type: Optional[str] = None href: Optional[str] = None attributes: Attributes + relationships: Relationships meta: Meta From 39976d144ba16e295641487fd95b3170df05f546 Mon Sep 17 00:00:00 2001 From: "Xie, Tianshi" Date: Thu, 16 May 2024 17:42:02 -0400 Subject: [PATCH 4/6] fix: sanlitize the meta dict's hard-copy before format the path, otherwise strange album name such as 'xxx \ cccc' would not get cleaned by last process and crush the os. --- src/utils.py | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/src/utils.py b/src/utils.py index dc42f34..f5504cd 100644 --- a/src/utils.py +++ b/src/utils.py @@ -13,6 +13,8 @@ from src.exceptions import NotTimeSyncedLyricsException from src.models import PlaylistInfo from src.types import * +from copy import deepcopy + def check_url(url): pattern = regex.compile( @@ -146,17 +148,26 @@ def playlist_metadata_to_params(playlist: PlaylistInfo): return {"playlistName": playlist.data[0].attributes.name, "playlistCuratorName": playlist.data[0].attributes.curatorName} +def get_path_safe_dict(param: dict): + new_param = deepcopy(param) + for key, val in new_param.items(): + if isinstance(val, str): + new_param[key] = get_valid_filename(str(val)) + return new_param def get_song_name_and_dir_path(codec: str, config: Download, metadata, playlist: PlaylistInfo = None): if playlist: + safe_meta = get_path_safe_dict(metadata.model_dump()) + safe_pl_meta = get_path_safe_dict(playlist_metadata_to_params(playlist)) song_name = config.playlistSongNameFormat.format(codec=codec, playlistSongIndex=metadata.playlistIndex, - **metadata.model_dump()) + **safe_meta) dir_path = Path(config.playlistDirPathFormat.format(codec=codec, - **metadata.model_dump(), - **playlist_metadata_to_params(playlist))) + **safe_meta, + **safe_pl_meta)) else: - song_name = config.songNameFormat.format(codec=codec, **metadata.model_dump()) - dir_path = Path(config.dirPathFormat.format(codec=codec, **metadata.model_dump())) + safe_meta = get_path_safe_dict(metadata.model_dump()) + song_name = config.songNameFormat.format(codec=codec, **safe_meta) + dir_path = Path(config.dirPathFormat.format(codec=codec, **safe_meta)) if sys.platform == "win32": song_name = get_valid_filename(song_name) dir_path = Path(*[get_valid_filename(part) if ":\\" not in part else part for part in dir_path.parts]) From d437975744c3f0730445c4d1e3c2cbf7a0d22ead Mon Sep 17 00:00:00 2001 From: "Xie, Tianshi" Date: Thu, 16 May 2024 20:21:17 -0400 Subject: [PATCH 5/6] fix: in old lyrics some lines would be missing ms digits like '06:06'. And in this case we would manually add dummy one at last. --- src/utils.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/utils.py b/src/utils.py index f5504cd..cc478ef 100644 --- a/src/utils.py +++ b/src/utils.py @@ -84,6 +84,8 @@ def ttml_convent_to_lrc(ttml: str) -> str: lyric_time: str = lyric.get("begin") if not lyric_time: raise NotTimeSyncedLyricsException + if lyric_time.find('.') == -1: + lyric_time += '.000' match lyric_time.count(":"): case 0: split_time = lyric_time.split(".") From 66bdd0fcc8c086723bba2aab82b16c20a00aba42 Mon Sep 17 00:00:00 2001 From: "Xie, Tianshi" Date: Fri, 17 May 2024 00:31:48 -0400 Subject: [PATCH 6/6] fix: crash under album data, sometimes sub-relationship would be empty, eg: data.0.relationships.tracks.data.30.relationships --- src/models/album_meta.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/models/album_meta.py b/src/models/album_meta.py index 09e222c..08ed970 100644 --- a/src/models/album_meta.py +++ b/src/models/album_meta.py @@ -113,7 +113,7 @@ class Datum1(BaseModel): type: Optional[str] = None href: Optional[str] = None attributes: Attributes1 - relationships: Relationships1 + relationships: Optional[Relationships1] = None class Tracks(BaseModel):