From 61b4d0e4296ec3219773a45a51385b4efe9c5e13 Mon Sep 17 00:00:00 2001 From: Joshua Boniface Date: Fri, 7 Apr 2023 05:22:57 -0400 Subject: [PATCH] Add support for filtering on instrument parts --- README.md | 104 ++++++++++++------ c3dbdl/c3dbdl.py | 271 ++++++++++++++++++++++++++++++++++++----------- 2 files changed, 282 insertions(+), 93 deletions(-) diff --git a/README.md b/README.md index b9b820d..66836b7 100644 --- a/README.md +++ b/README.md @@ -105,28 +105,37 @@ guaranteed to be smaller than the total number listed on the C3DB website. ## Searching & Downloading -Once a database has been built, you can start searching for downloading songs. +Once a database has been built, you can start searching for and downloading songs. To search for songs, use the `search` command. This command takes `--filter` arguments in order to show what -song(s) would be downloaded by a given filter, without actually triggering the download. Once you have a valid -filter from a search, you can use it to download. +song(s) would be downloaded by a given filter, along with their basic information, without actually triggering +a download. Once you have a valid filter from a search, you can use it to `download` precisely the song(s) you +want. -To download songs, use the `download` command. See the following sections for more details on the specifics of -the filters and output formatting of the `download` command. +See the following sections for more details on the specifics of the filters and output formatting of the +`search` and `download` commands. By default, when downloading a given song, all possible download links (`dl_links`) will be downloaded; this can be limited by using the `-i`/`--download-id` and `-d`/`--download-descr` options to pick and choose specific -files. +files. A specific example usecase would be to specify `--download-descr 360` to only download Xbox 360 RBCONs. Once a song has been downloaded, assuming that the file structure doesn't change, subsequent `download` runs will not overwrite it and will simply skip downloading the file. ### Filtering -Filtering out the songs in the database is a key part of this tool. You might want to be able to grab only select -genres, artists, authors, etc. to make your custom song packs. +Filtering out the songs in the database is a key part of this tool. You might want to be able to grab only songs +with certain genres, artists, instruments, etc. or by certain authors, to make your custom song packs. -`c3dbdl` is able to filter by several key categories: +If multiple filters are specified, they are treated as a logical AND, i.e. *all* of the give filters must apply +to a song for it to be matched. + +Filtering is always done during the search/download stage; the JSON database will always contain all possible +entries from the C3DB. + +#### Information Filters + +`c3dbdl` is able to filter songs by their general information in several key categories: * `genre`: The genre of the song. * `artist`: The artist of the song. @@ -135,35 +144,68 @@ genres, artists, authors, etc. to make your custom song packs. * `year`: The year of the album/song. * `author`: The author of the file on C3DB. -Note that we *cannot* filter - mostly for parsing difficulty reasons - by intrument type or difficulty, by song -length, or by any other information not mentioned above. +To use information filters, append one or more `--filter` options to your `c3dbdl search` or `download` command. An +information filter option begins with the literal `--filter`, followed by the field (e.g. `genre` or `artist`), then +finally the text value to filter on, for instance `Rock` or `Santana` or `2012`. The text must be quoted if it +contains any whitespace. -Filtering is always done during the search/download stage; the JSON database will always contain all possible entries. +Information filter values are fuzzy. They are case insensitive, and use the `in` construct. So, for example, the +filter string `--filter song "edmund fitzgerald"` would match the song title "The Wreck of the Edmund Fitzgerald". -To use filters, append one or more `--filter` options to your `c3dbdl download` or `search` command. A filter option -begins with the literal `--filter`, followed by the category (e.g. `genre` or `artist`), then finally the text to -filter on, for instance `Rock` or `Santana` or `2012`. The text must be quoted if it contains whitespace. - -If more that one filter is specified, they are treated as a logical AND, i.e. all the listed filters must apply to -a given song for it to be downloaded in that run. - -Filter values are fuzzy. They are case insensitive, and use the `in` construct. So, for example, the filter string -`--filter song "edmund fitzgerald"` would match the song title "The Wreck of the Edmund Fitzgerald". - -Filters allow very specific download selections to be run. For example, let's look for all songs by Rush -from the album Vapor Trails (the remixed version) authored by ejthedj: +For example, to find all songs by Rush from the album Vapor Trails (the remixed version) authored by ejthedj: ``` -c3dbdl download --filter artist Rush --filter album "Vapor Trails [Remixed]" --author ejthedj +c3dbdl search --filter artist Rush --filter album "Vapor Trails [Remixed]" --filter author ejthedj Found 19563 songs from JSON database file 'Downloads/c3db.json' -Downloading 1 song files... -> Downloading song "Rush - Sweet Miracle" by ejthedj... -Downloading file "Rock Band 3 Xbox 360" from https://dl.c3universe.com/s/ejthedj/sweetMiracle... -Successfully downloaded to ../Prog/ejthedj/Rush/Vapor Trails [Remixed]/Sweet Miracle [2002].sweetMiracle +Found 1 matching songs: + +> Song: "Rush - Sweet Miracle" from "Vapor Trails [Remixed] (2002)" by ejthedj + Instruments: guitar [2], bass [3], drums [4], vocals [4], keys [None] + Available downloads: + * Rock Band 3 Xbox 360 ``` -In this case, one song matched and was downloaded. Feel free to experiment with the various filters to find -exactly what you're looking for. +In this case, one song matched; applying the same filter to a `download` would thus download only the single song. + +#### Instrument Filters + +In addition to the information filters, `c3dbdl` can also filter by available instrument parts. There are 5 valid +instruments that can be filtered on: + +* `guitar` +* `bass` +* `drums` +* `vocals` +* `keys` + +To use instrument filters, append one or more `--filter instrument ` options to your `c3dbdl search` or + `download` command. An instrument filter option begins with the literal `--filter instrument`, followed by the +instrument you wish to filter on. + +If a part contains the instrument at any difficulty (from 0-6), it will match the filter; if the instrument part +is missing, it will not match. + +You can also invert the match by adding `no-` to the instrument name. So `--filter instrument no-keys` would +only match songs *without* a keys part. + +For example, to find all songs by Rush which have a keys part but no vocal part: + +``` +c3dbdl search --filter artist Rush --filter instrument keys --filter instrument no-vocals +Found 19562 songs from JSON database file 'Downloads/c3db.json' +Found 1 matching songs: + +> Song: "Rush - La Villa Strangiato" from "Hemispheres (1978)" by DoNotPassGo + Instruments: guitar [6], bass [5], drums [6], vocals [None], keys [1] + Available downloads: + * Rock Band 3 Xbox 360 + * Rock Band 3 Wii + * Rock Band 3 PS3 + * Phase Shift + * Rock Band 3 Xbox 360 (Alternate Version) +``` + +In this case, one song matched; applying the same filter to a `download` would thus download only the single song. ### Output Format diff --git a/c3dbdl/c3dbdl.py b/c3dbdl/c3dbdl.py index 04c722a..8bbd27d 100755 --- a/c3dbdl/c3dbdl.py +++ b/c3dbdl/c3dbdl.py @@ -170,7 +170,6 @@ def fetchSongData(entries): song_entry["dl_links"] = dl_links # Return messages and song entry - print(song_entry) return messages, song_entry @@ -494,11 +493,10 @@ def database(): "-f", "--filter", "_filters", - envvar="C3DBDL_DL_FILTERS", default=[], multiple=True, nargs=2, - help="Add a filter option.", + help="Add a search filter.", ) @click.option( "-l", @@ -513,6 +511,7 @@ def database(): "-i", "--download-id", "_id", + envvar="C3DBDL_DL_ID", default=None, type=int, help='Download only "dl_links" entry N (1 is first, etc.), or all if unspecified.', @@ -521,6 +520,7 @@ def database(): "-d", "--download-descr", "_desc", + envvar="C3DBDL_DL_DESCR", default=None, help='Download only "dl_links" entries with this in their description (fuzzy).', ) @@ -528,6 +528,19 @@ def download(_filters, _id, _desc, _limit, _file_structure): """ Download song(s) from the C3DB webpage. + Filters allow granular selection of the song(s) to download. Multiple filters can be + specified, and a song is selected only if ALL filters match (logical AND). Filters are + specified in the form "--filter ". + + For a full list of and explanation for filters, see the help output for the "search" + command ("c3dbdl search --help"). + + In addition to filters, each song may have more than one download link, to provide + multiple versions of the same song (for example, normal and multitracks, or alternate + charts). For each song, the "-i"/"--download-id" and "-d"/"--download-descr" options + can help filter these out, or both can be left blank to download all possible files + for a given song. + \b The output file structure can be specified as a path format with any of the following fields included, surrounded by curly braces: @@ -543,33 +556,12 @@ def download(_filters, _id, _desc, _limit, _file_structure): The default output file structure is: "{artist}/{album}/{title}.{author}.{orig_name}" - Filters allow granular selection of the song(s) to download. Multiple filters can be - specified, and a song is selected only if ALL filters match (logical AND). Each filter - is in the form "--filter [database_key] [value]". - - The valid "database_key" values are identical to the output file fields above, except - for "orig_name". - - \b - For example, to download all songs in the genre "Rock": - --filter genre Rock - - \b - Or to download all songs by the artist "Rush" and the author "MyName": - --filter artist Rush --filter author MyName - - In addition to filters, each song may have more than one download link, to provide - multiple versions of the same song (for example, normal and multitracks, or alternate - charts). For each song, the "-i"/"--download-id" and "-d"/"--download-descr" options - can help filter these out, or both can be left blank to download all possible files - for a given song. Mostly useful when being extremely restrictive with filters, less - so when downloading many songs at once. - \b The following environment variables can be used for scripting purposes: * C3DBDL_DL_FILE_STRUCTURE: equivalent to "--file-structure" - * C3DBDL_DL_FILTERS: equivalent to "--filter"; limited to one instance * C3DBDL_DL_LIMIT: equivalent to "--limit" + * C3DBDL_DL_ID: equivalent to "--download-id" + * C3DBDL_DL_DESCR: equivalent to "--download-descr" """ with open(config["database_filename"], "r") as fh: @@ -581,18 +573,69 @@ def download(_filters, _id, _desc, _limit, _file_structure): pending_songs = list() for song in all_songs: - if len(_filters) < 1: - add_to_pending = True - else: - try: - pending_filters = [ - _filter[1].lower() in song[_filter[0]].lower() - for _filter in _filters - ] - add_to_pending = all(pending_filters) - except KeyError as e: - click.echo(f"Invalid filter field {e}") - exit(1) + add_to_pending = True + song_filters = _filters + song_information_filters = list() + song_instrument_filters = list() + + if len(_filters) > 0: + # Extract the instrument filters + for _filter in song_filters: + if _filter[0] == "instrument": + song_instrument_filters.append(_filter[1].lower()) + else: + song_information_filters.append(_filter) + + if len(song_information_filters) > 0 or len(song_instrument_filters) > 0: + # Parse the information filters + if len(song_information_filters) > 0: + try: + pending_information_filters = list() + for information_filter in song_information_filters: + filter_field = information_filter[0].lower() + filter_value = information_filter[1].lower() + if re.match("^~", filter_value): + filter_value = filter_value.replace('~', '') + if filter_value in song[filter_field].lower(): + pending_information_filters.append(True) + else: + pending_information_filters.append(False) + else: + if filter_value == song[filter_field].lower(): + pending_information_filters.append(True) + else: + pending_information_filters.append(False) + information_add_to_pending = all(pending_information_filters) + except KeyError as e: + click.echo(f"Invalid filter field {e}") + exit(1) + else: + information_add_to_pending = True + + # Parse the instrument filters + if len(song_instrument_filters) > 0: + try: + pending_instrument_filters = list() + for instrument_filter in song_instrument_filters: + if re.match("^no-", instrument_filter): + instrument_filter = instrument_filter.replace('no-', '') + if song["instruments"][instrument_filter] is None: + pending_instrument_filters.append(True) + else: + pending_instrument_filters.append(False) + else: + if song["instruments"][instrument_filter] is not None: + pending_instrument_filters.append(True) + else: + pending_instrument_filters.append(False) + instrument_add_to_pending = all(pending_instrument_filters) + except KeyError as e: + click.echo(f"Invalid instrument value {e}") + exit(1) + else: + instrument_add_to_pending = True + + add_to_pending = all([information_add_to_pending, instrument_add_to_pending]) if add_to_pending: pending_songs.append(song) @@ -600,7 +643,7 @@ def download(_filters, _id, _desc, _limit, _file_structure): if _limit is not None: pending_songs = pending_songs[0:_limit] - click.echo(f"Downloading {len(pending_songs)} song files...") + click.echo(f"Downloading {len(pending_songs)} songs...") for song in pending_songs: downloadSong(config["download_directory"], _file_structure, song, _id, _desc) @@ -611,26 +654,68 @@ def download(_filters, _id, _desc, _limit, _file_structure): "-f", "--filter", "_filters", - envvar="C3DBDL_DL_FILTERS", default=[], multiple=True, nargs=2, - help="Add a filter option.", + help="Add a search filter.", ) def search(_filters): """ Search for song(s) from the C3DB local database. Filters allow granular selection of the song(s) to download. Multiple filters can be - specified, and a song is selected only if ALL filters match (logical AND). Each filter - is in the form "--filter [database_key] [value]". - - For a full list of and explanation for filters, see the "download" command help - (command "c3dbdl download --help"). + specified, and a song is selected only if ALL filters match (logical AND). Filters are + specified in the form "--filter ". \b - The following environment variables can be used for scripting purposes: - * C3DBDL_DL_FILTERS: equivalent to "--filter"; limited to one instance + The valid fields for the "" value are: + * genre: The genre of the song. + * artist: The artist of the song. + * album: The album of the song. + * title: The title of the song. + * year: The year of the album/song. + * author: The author of the file on C3DB. + * instrument: An instrument chart for the song. + + \b + For example, to download all songs in the genre "Rock": + --filter genre Rock + + \b + Or to download all songs by the artist "Rush" and the author "MyName": + --filter artist Rush --filter author MyName + + Filter values are case insensitive, and non-instrument filters can be made fuzzy by + adding a tilde ("~") to the beginning of the "". + + \b + For example, to match all songs with "Word" in their titles: + --filter title ~word + + Instrument filters allow selection of the presence of instruments. If an instrument + fitler is given, only songs which contain parts for the given instrument(s) will be + shown. + + \b + The valid instruments are: + * guitar + * bass + * drums + * vocals + * keys + + To negate an instrument filter and find only entires without the specified + instrument, append "no-" to the instrument name. + + \b + For example, to download only songs that have a keys part but no vocal part: + --filter instrument keys --filter instrument no-vocals + + Note that while instrument difficulties are displayed in the output of this command, + they can not be filtered on; this is up to the user to do manually. The purpose of + instrument filters is to ensure that songs contain or don't contain given parts, not + to granularly select the difficulty of said parts (that's for the players of the game + to do, not us). """ with open(config["database_filename"], "r") as fh: @@ -642,18 +727,69 @@ def search(_filters): pending_songs = list() for song in all_songs: - if len(_filters) < 1: - add_to_pending = True - else: - try: - pending_filters = [ - _filter[1].lower() in song[_filter[0]].lower() - for _filter in _filters - ] - add_to_pending = all(pending_filters) - except KeyError as e: - click.echo(f"Invalid filter field {e}") - exit(1) + add_to_pending = True + song_filters = _filters + song_information_filters = list() + song_instrument_filters = list() + + if len(_filters) > 0: + # Extract the instrument filters + for _filter in song_filters: + if _filter[0] == "instrument": + song_instrument_filters.append(_filter[1].lower()) + else: + song_information_filters.append(_filter) + + if len(song_information_filters) > 0 or len(song_instrument_filters) > 0: + # Parse the information filters + if len(song_information_filters) > 0: + try: + pending_information_filters = list() + for information_filter in song_information_filters: + filter_field = information_filter[0].lower() + filter_value = information_filter[1].lower() + if re.match("^~", filter_value): + filter_value = filter_value.replace('~', '') + if filter_value in song[filter_field].lower(): + pending_information_filters.append(True) + else: + pending_information_filters.append(False) + else: + if filter_value == song[filter_field].lower(): + pending_information_filters.append(True) + else: + pending_information_filters.append(False) + information_add_to_pending = all(pending_information_filters) + except KeyError as e: + click.echo(f"Invalid filter field {e}") + exit(1) + else: + information_add_to_pending = True + + # Parse the instrument filters + if len(song_instrument_filters) > 0: + try: + pending_instrument_filters = list() + for instrument_filter in song_instrument_filters: + if re.match("^no-", instrument_filter): + instrument_filter = instrument_filter.replace('no-', '') + if song["instruments"][instrument_filter] is None: + pending_instrument_filters.append(True) + else: + pending_instrument_filters.append(False) + else: + if song["instruments"][instrument_filter] is not None: + pending_instrument_filters.append(True) + else: + pending_instrument_filters.append(False) + instrument_add_to_pending = all(pending_instrument_filters) + except KeyError as e: + click.echo(f"Invalid instrument value {e}") + exit(1) + else: + instrument_add_to_pending = True + + add_to_pending = all([information_add_to_pending, instrument_add_to_pending]) if add_to_pending: pending_songs.append(song) @@ -662,10 +798,21 @@ def search(_filters): click.echo() for entry in pending_songs: click.echo( - f"""> "{entry['artist']} - {entry['title']}" from "{entry['album']} ({entry['year']})" by {entry['author']}""" + f"""> Song: "{entry['artist']} - {entry['title']}" from "{entry['album']} ({entry['year']})" by {entry['author']}""" + ) + + instrument_list = list() + for instrument in entry["instruments"]: + instrument_list.append(f"{instrument} [{entry['instruments'][instrument]}]") + click.echo( + f""" Instruments: {', '.join(instrument_list)}""", + ) + + click.echo( + f""" Available downloads:""" ) for link in entry["dl_links"]: - click.echo(f""" * {link['description']}""") + click.echo(f""" * {link['description']}""") click.echo()