diff --git a/waternsw_grabber/__init__.py b/waternsw_grabber/__init__.py index fb51146..16a5b02 100644 --- a/waternsw_grabber/__init__.py +++ b/waternsw_grabber/__init__.py @@ -1 +1 @@ -from .waternsw_grabber import open_browser, get_telemetered_bore +from .waternsw_grabber import open_browser, get_telemetered_bore, telemetered_bore_downloader diff --git a/waternsw_grabber/waternsw_grabber.py b/waternsw_grabber/waternsw_grabber.py index af8c308..223770e 100644 --- a/waternsw_grabber/waternsw_grabber.py +++ b/waternsw_grabber/waternsw_grabber.py @@ -3,6 +3,7 @@ import re import requests import warnings import pandas as pd +from tqdm import tqdm from selenium import webdriver from selenium.webdriver.common.by import By from selenium.webdriver.common.keys import Keys @@ -36,7 +37,7 @@ def return_to_master_frame(driver): driver.switch_to.frame(webhyd) -def wait_for_element(driver, by, x, timeout=10): +def wait_for_element(driver, by, x, timeout=30): try: element_present = EC.presence_of_element_located((by, x)) WebDriverWait(driver, timeout).until(element_present) @@ -54,7 +55,7 @@ def get_telemetered_bore(driver, bore_id, start_date, end_date): # Wait for results frame to load WebDriverWait( - driver, timeout=20).until( + driver, timeout=30).until( EC.frame_to_be_available_and_switch_to_it('gwgwlf_org')) wait_for_element(driver, By.XPATH, '//*[@id="tabstext"]') @@ -62,7 +63,7 @@ def get_telemetered_bore(driver, bore_id, start_date, end_date): driver.execute_script("menuloc.display_frame('gw','gwcf_org','1')") driver.switch_to.parent_frame() WebDriverWait( - driver, timeout=20).until( + driver, timeout=30).until( EC.frame_to_be_available_and_switch_to_it('gwgwcf_org')) wait_for_element(driver, By.ID, 'submit') @@ -140,3 +141,15 @@ def open_browser(download_dir): driver.get(url) return driver + + +def telemetered_bore_downloader(bore_ids, start_date, end_date, download_dir): + driver = open_browser(download_dir) + + # Download bore logs + pbar = tqdm(bore_ids) + for bore_id in pbar: + pbar.set_description(bore_id) + get_telemetered_bore(driver, bore_id, start_date, end_date) + + driver.quit()