From b14c37a1397f9d2450b84427dd22b9bdd8808880 Mon Sep 17 00:00:00 2001
From: Dan Howe <d.howe@wrl.unsw.edu.au>
Date: Mon, 4 Mar 2019 11:27:39 +1100
Subject: [PATCH] Download decomissioned sites correctly

---
 waternsw_grabber/waternsw_grabber.py | 46 +++++++++++++++++++++-------
 1 file changed, 35 insertions(+), 11 deletions(-)

diff --git a/waternsw_grabber/waternsw_grabber.py b/waternsw_grabber/waternsw_grabber.py
index c4ec5a3..8f1e0e5 100644
--- a/waternsw_grabber/waternsw_grabber.py
+++ b/waternsw_grabber/waternsw_grabber.py
@@ -56,6 +56,30 @@ def wait_for_element(driver, by, x, timeout=180):
     WebDriverWait(driver, timeout).until(element_present)
 
 
+def wait_for_body_text(driver):
+    """Wait for body text element on page to load, and not be empty.
+
+    Args:
+        driver:   selenium webdriver object
+
+    Returns
+        Body text
+
+    Raises
+        TimeoutException if element does not load within timeout period
+    """
+    body_text = None
+    while not body_text:
+        try:
+            # Get contents of body text
+            body_text = driver.find_element_by_xpath('//*/body').text
+        except (StaleElementReferenceException, NoSuchElementException):
+            pass
+        time.sleep(0.5)
+
+    return body_text
+
+
 def get_telemetered_bore(driver, bore_id, start_date, end_date):
     """Download single record from telemetered bore.
 
@@ -81,29 +105,29 @@ def get_telemetered_bore(driver, bore_id, start_date, end_date):
     driver.switch_to.frame('gwgwlf_org')
 
     # Wait until body text of iframe has loaded
-    body_text = None
-    while not body_text:
-        try:
-            # Get contents of body text
-            body_text = driver.find_element_by_xpath('//*/body').text
-        except (StaleElementReferenceException, NoSuchElementException):
-            pass
-        time.sleep(0.5)
+    body_text = wait_for_body_text(driver)
 
     # Detect if bore record does not exist
     if 'No SITE record found for site' in body_text:
         raise ValueError('No SITE record found for site {}'.format(bore_id))
-    elif 'No variables data found for this site.' in body_text:
-        raise ValueError('No variables data found for site {}'.format(bore_id))
 
     # Wait for navigation tabs
     wait_for_element(driver, By.XPATH, '//*[@id="tabstext"]')
 
-    # Activate outputs tab, and wait for 'Get Output' button
+    # Activate outputs tab
     driver.execute_script("menuloc.display_frame('gw', 'gwcf_org', '1')")
     driver.switch_to.parent_frame()
     wait_for_element(driver, By.ID, 'gwgwcf_org')
     driver.switch_to.frame('gwgwcf_org')
+
+    # Wait until body text of iframe has loaded
+    body_text = wait_for_body_text(driver)
+
+    # Detect if no variables are available
+    if 'No variables data found for this site.' in body_text:
+        raise ValueError('No variables data found for site {}'.format(bore_id))
+
+    # Wait for 'Get Output' button
     wait_for_element(driver, By.ID, 'submit')
 
     # Get output select controls