From ca36555e6b92c23d0fbe66fad6f7e27ea900c945 Mon Sep 17 00:00:00 2001 From: Joshua Boniface Date: Thu, 6 Apr 2023 12:54:10 -0400 Subject: [PATCH] Ensure basic HTML parsing is in retry block --- c3dbdl | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/c3dbdl b/c3dbdl index 515e89a..39fc831 100755 --- a/c3dbdl +++ b/c3dbdl @@ -111,14 +111,17 @@ def buildDatabase(pages, concurrency): try: click.echo(f"Parsing page {i} (attempt {attempts}/3)...") p = requests.get(f"{config['base_songs_url']}?page={i}") + if p is None or p.status_code != 200: + raise + parsed_html = BeautifulSoup(p.text, 'html.parser') + if parsed_html.body is None: + raise + if parsed_html.body.find('div', attrs={'class':'portlet-body'}) is None: + raise break except Exception: sleep(attempts) attempts += 1 - if p is None or p.status_code != 200: - break - - parsed_html = BeautifulSoup(p.text, 'html.parser') table_html = parsed_html.body.find('div', attrs={'class':'portlet-body'}).find('tbody')