Download decomissioned sites correctly

waternsw_grabber/waternsw_grabber.py

@@ -56,6 +56,30 @@ def wait_for_element(driver, by, x, timeout=180):
     WebDriverWait(driver, timeout).until(element_present)
 
 
+def wait_for_body_text(driver):
+    """Wait for body text element on page to load, and not be empty.
+
+    Args:
+        driver:   selenium webdriver object
+
+    Returns
+        Body text
+
+    Raises
+        TimeoutException if element does not load within timeout period
+    """
+    body_text = None
+    while not body_text:
+        try:
+            # Get contents of body text
+            body_text = driver.find_element_by_xpath('//*/body').text
+        except (StaleElementReferenceException, NoSuchElementException):
+            pass
+        time.sleep(0.5)
+
+    return body_text
+
+
 def get_telemetered_bore(driver, bore_id, start_date, end_date):
     """Download single record from telemetered bore.
 
@@ -81,29 +105,29 @@ def get_telemetered_bore(driver, bore_id, start_date, end_date):
     driver.switch_to.frame('gwgwlf_org')
 
     # Wait until body text of iframe has loaded
-    body_text = None
+    body_text = wait_for_body_text(driver)
-    while not body_text:
-        try:
-            # Get contents of body text
-            body_text = driver.find_element_by_xpath('//*/body').text
-        except (StaleElementReferenceException, NoSuchElementException):
-            pass
-        time.sleep(0.5)
 
     # Detect if bore record does not exist
     if 'No SITE record found for site' in body_text:
         raise ValueError('No SITE record found for site {}'.format(bore_id))
-    elif 'No variables data found for this site.' in body_text:
-        raise ValueError('No variables data found for site {}'.format(bore_id))
 
     # Wait for navigation tabs
     wait_for_element(driver, By.XPATH, '//*[@id="tabstext"]')
 
-    # Activate outputs tab, and wait for 'Get Output' button
+    # Activate outputs tab
     driver.execute_script("menuloc.display_frame('gw', 'gwcf_org', '1')")
     driver.switch_to.parent_frame()
     wait_for_element(driver, By.ID, 'gwgwcf_org')
     driver.switch_to.frame('gwgwcf_org')
+
+    # Wait until body text of iframe has loaded
+    body_text = wait_for_body_text(driver)
+
+    # Detect if no variables are available
+    if 'No variables data found for this site.' in body_text:
+        raise ValueError('No variables data found for site {}'.format(bore_id))
+
+    # Wait for 'Get Output' button
     wait_for_element(driver, By.ID, 'submit')
 
     # Get output select controls