Create log for failed downloads

waternsw_grabber/waternsw_grabber.py

@@ -1,6 +1,7 @@
 import os
 import re
 import time
+import logging
 import requests
 import warnings
 import pandas as pd
@@ -167,13 +168,33 @@ def open_browser(download_dir):
 def telemetered_bore_downloader(bore_ids, start_date, end_date, download_dir):
     driver = open_browser(download_dir)
 
+    # Set up log File
+    log_name = os.path.join(download_dir, 'errors.log')
+    logging.basicConfig(filename=log_name, level=logging.ERROR)
+
     # Download bore logs
     pbar = tqdm(bore_ids)
     for bore_id in pbar:
         pbar.set_description(bore_id)
         try:
             get_telemetered_bore(driver, bore_id, start_date, end_date)
-        except ValueError as e:
+        except ValueError:
-            print(e)
+            logging.error('Failed to download {}\n'.format(bore_id))
+
+    # Tidy up console after tqdm
+    print('\n')
+
+    # Stop logging
+    logging.shutdown()
+    with open(log_name, 'r') as f:
+        log_data = f.read()
+
+    # Check contents of log file
+    if log_data:
+        warnings.warn(
+            'Some files failed to download. See log for details.',
+            stacklevel=2)
+    else:
+        os.remove(log_name)
 
     driver.quit()