Add handler for Mediafile downloads

This commit is contained in:
Joshua Boniface 2023-04-28 18:40:54 -04:00
parent aa680dadc2
commit 447eb4120a
1 changed files with 78 additions and 47 deletions

View File

@ -153,8 +153,6 @@ def fetchSongData(entries):
for link_entry in download_links: for link_entry in download_links:
link = link_entry.get("href") link = link_entry.get("href")
description = link_entry.get_text().strip() description = link_entry.get_text().strip()
if "c3universe.com" not in link:
continue
messages.append(f"Found download link: {link} ({description})") messages.append(f"Found download link: {link} ({description})")
dl_links.append( dl_links.append(
{ {
@ -252,6 +250,72 @@ def buildDatabase(pages, concurrency):
return found_songs return found_songs
def downloadFile(download_url, download_path, download_filename):
attempts = 1
p = None
try:
with requests.get(download_url, stream=True) as r:
while attempts <= 3:
try:
r.raise_for_status()
break
except Exception:
click.echo(
f"Download attempt failed: HTTP {r.status_code}; retrying {attempts}/3"
)
sleep(attempts)
attempts += 1
if r is None or r.status_code != 200:
if r:
code = r.status_code
else:
code = "-1"
raise HTTPError(download_url, code, "", None, None)
if not os.path.exists(download_path):
os.makedirs(download_path)
with open(download_filename, "wb") as f:
for chunk in r.iter_content(chunk_size=8192):
f.write(chunk)
click.echo(f"Successfully downloaded to {download_filename}")
except Exception as e:
click.echo(f"Download attempt failed: {e}")
return None
def parseC3Universe(dl_link):
try:
p = requests.get(dl_link)
if p.status_code != 200:
raise HTTPError(dl_link, p.status_code, "", None, None)
parsed_html = BeautifulSoup(p.text, "html.parser")
download_url = (
parsed_html.body.find("div", attrs={"class": "lock-head"})
.find("a")
.get("href")
)
return download_url
except Exception as e:
click.echo(f"Failed parsing or retrieving HTML link: {e}")
return None
def parseMediafire(dl_link):
try:
p = requests.get(dl_link)
if p.status_code != 200:
raise HTTPError(dl_link, p.status_code, "", None, None)
parsed_html = BeautifulSoup(p.text, "html.parser")
download_url = parsed_html.find(
"a", attrs={"id": "downloadButton", "rel": "nofollow", "aria-label": "Download file"}
).get("href")
return download_url
except Exception as e:
click.echo(f"Failed parsing or retrieving HTML link: {e}")
return None
def downloadSong(destination, filename, entry, dlid, dldesc): def downloadSong(destination, filename, entry, dlid, dldesc):
click.echo( click.echo(
f"""> Downloading song "{entry['artist']} - {entry['title']}" by {entry['author']}...""" f"""> Downloading song "{entry['artist']} - {entry['title']}" by {entry['author']}..."""
@ -278,21 +342,19 @@ def downloadSong(destination, filename, entry, dlid, dldesc):
return return
for dl_link in dl_links: for dl_link in dl_links:
try: if 'dl.c3universe.com' in dl_link['link']:
p = requests.get(dl_link["link"]) download_url = parseC3Universe(dl_link["link"])
if p.status_code != 200: elif 'www.mediafire.com' in dl_link["link"]:
raise HTTPError(dl_link["link"], p.status_code, "", None, None) download_url = parseMediafire(dl_link["link"])
else:
parsed_html = BeautifulSoup(p.text, "html.parser") click.echo("Download URL is not valid for CLI download; skipping...")
download_url = ( click.echo(f"URL: {dl_link['link']}")
parsed_html.body.find("div", attrs={"class": "lock-head"})
.find("a")
.get("href")
)
except Exception as e:
click.echo(f"Failed parsing or retrieving HTML link: {e}")
continue continue
if download_url is None:
continue
print(entry)
download_filename = filename.format( download_filename = filename.format(
genre=entry["genre"], genre=entry["genre"],
artist=entry["artist"], artist=entry["artist"],
@ -312,38 +374,7 @@ def downloadSong(destination, filename, entry, dlid, dldesc):
click.echo(f"File exists at {download_filename}") click.echo(f"File exists at {download_filename}")
continue continue
attempts = 1 downloadFile(download_url, download_path, download_filename)
p = None
try:
with requests.get(download_url, stream=True) as r:
while attempts <= 3:
try:
r.raise_for_status()
break
except Exception:
click.echo(
f"Download attempt failed: HTTP {r.status_code}; retrying {attempts}/3"
)
sleep(attempts)
attempts += 1
if r is None or r.status_code != 200:
if r:
code = r.status_code
else:
code = "-1"
raise HTTPError(download_url, code, "", None, None)
if not os.path.exists(download_path):
os.makedirs(download_path)
with open(download_filename, "wb") as f:
for chunk in r.iter_content(chunk_size=8192):
f.write(chunk)
click.echo(f"Successfully downloaded to {download_filename}")
except Exception as e:
click.echo(f"Download attempt failed: {e}")
continue
@click.command(name="build", short_help="Build the local database.") @click.command(name="build", short_help="Build the local database.")
@click.option( @click.option(