diff --git a/waternsw_grabber/waternsw_grabber.py b/waternsw_grabber/waternsw_grabber.py index 58c5d06..ea9612a 100644 --- a/waternsw_grabber/waternsw_grabber.py +++ b/waternsw_grabber/waternsw_grabber.py @@ -1,6 +1,7 @@ import os import re import time +import logging import requests import warnings import pandas as pd @@ -167,13 +168,33 @@ def open_browser(download_dir): def telemetered_bore_downloader(bore_ids, start_date, end_date, download_dir): driver = open_browser(download_dir) + # Set up log File + log_name = os.path.join(download_dir, 'errors.log') + logging.basicConfig(filename=log_name, level=logging.ERROR) + # Download bore logs pbar = tqdm(bore_ids) for bore_id in pbar: pbar.set_description(bore_id) try: get_telemetered_bore(driver, bore_id, start_date, end_date) - except ValueError as e: - print(e) + except ValueError: + logging.error('Failed to download {}\n'.format(bore_id)) + + # Tidy up console after tqdm + print('\n') + + # Stop logging + logging.shutdown() + with open(log_name, 'r') as f: + log_data = f.read() + + # Check contents of log file + if log_data: + warnings.warn( + 'Some files failed to download. See log for details.', + stacklevel=2) + else: + os.remove(log_name) driver.quit()