Updated tag_registered to read .pkl data and efficiently locate the tide value

coastsnap/tag_registered.py

@@ -44,17 +44,13 @@ def get_site_tide_data(images_parent_dir, site):
     
     if tide_filename == 'NO_TIDE.mat':
         return False
-        
-    mat_path = images_parent_dir + '/Tide Data/' + tide_filename
-    mat = sio.loadmat(mat_path)
-    tide_dict = mat['tide']
-    ts = tide_dict[0][0] # Format of tide/time Matlab data is gross
-    tt = list(ts[0])     # ts[0] = tides, ts[1] = times, ts[2] = 
     
-    print(site + ": " +"Loading tide data... (this may take half a minute)")     
-    tide_times = [datenum_to_datetime(i) for i in tt] # THIS STEP TAKES A LONG TIME
+    tide_filename_pkl = tide_filename[:-4] + '.pkl'
+    tides_path = images_parent_dir + '/Tide Data/Tide_Data_Python/' + tide_filename_pkl
+    tides_df = pd.read_pickle(tides_path)
+    
+    return tides_df   
 
-    return tide_times, ts    
 
 
 def tag_image(image2tag):
@@ -78,9 +74,14 @@ def tag_image(image2tag):
 
     image.save(registered_path)
 
-image2tag = None #initialize this here
+#-----------------------------------------------------------------------------#
+# This is the main function
+
+
+image2tag = None 
 
 for site in os.listdir(images_dir): # Loop through SITES
+    print(site)
     i=0
     site_complete = False # A flag to stop tagging images when found
     tide_data = False
@@ -89,18 +90,20 @@ for site in os.listdir(images_dir): # Loop through SITES
     try: # Check if site contains 'Processed' directory
         years_list = os.listdir(photoshop_path)
         years_list.reverse()
-    except: 
-        continue
+    except: # Site doesn't contain 'Processed'. Go to next site.
+        continue 
     for year in years_list: # Loop through YEARS
         if site_complete: break 
         year_path = photoshop_path + '/' + year
         image_list = os.listdir(year_path)
         image_list.reverse()
+        
         for image_filename in image_list: # Loop through IMAGES
             registered_year_path = year_path.replace('Photoshop', 'Registered')
             registered_image_path = registered_year_path + '/' + image_filename[:-4] + '_registered.jpg' 
             if site not in image_filename: # Check the filename has the site in it
                 continue
+            
             if os.path.isfile(registered_image_path): # Image already tagged. Go to next site.
                 site_complete = True;
                 if i == 0:
@@ -110,6 +113,7 @@ for site in os.listdir(images_dir): # Loop through SITES
                 else:
                     print(site + ": " + str(i) + " images tagged no tide")                  
                 break
+            
             else: # Image not tagged. Tag it.
                 if not os.path.exists(registered_year_path): # Chech that registered/year directory exists
                     os.makedirs(registered_year_path)     # if not, create it
@@ -117,16 +121,16 @@ for site in os.listdir(images_dir): # Loop through SITES
                 pathname = os.path.join(year_path, image_filename)
                 image2tag = RegisteredImage(pathname, image_filename) # Create image object
                 if i == 1: # Retrieve Tide Data once for each site
-                    site_tide_data, ts = get_site_tide_data(images_parent_dir, site)
-
-                font = image2tag.get_font()
-                    
-                if site_tide_data:
-                    image2tag.get_tide(site_tide_data, ts)
+                    tides_df = get_site_tide_data(images_parent_dir, site)
                     
+                font = image2tag.get_font()
+                image2tag.get_tide(tides_df)
                 image2tag.create_tag()
+                
                 tag_image(image2tag)
     
+    # No images previously tagged
+    # print result
     if not site_complete:
         if i == 0:
             print(site + ": " + str(i) + " images tagged")

coastsnap/utils.py

@@ -31,12 +31,11 @@ def datenum_to_datetime(datenum):
 
 
 # Sourced from https://stackoverflow.com/questions/32237862/find-the-closest-date-to-a-given-date
-def nearest(items, pivot):
-    """
-    This function will return the datetime in items 
-    which is the closest to the date pivot
-    """
-    return min(items, key=lambda x: abs(x - pivot))
+def nearest(all_dates, target_date_raw):
+    abs_deltas_from_target_date = np.absolute(all_dates - target_date_raw)
+    index_of_min_delta_from_target_date = np.argmin(abs_deltas_from_target_date)
+    closest_date = all_dates[index_of_min_delta_from_target_date]
+    return closest_date
 
 
 def progressbar(it, prefix="", size=60, out=sys.stdout): # Python3.3+
@@ -79,7 +78,7 @@ class RegisteredImage():
         self.height = None
         self.font = None
 
-    def get_tide(self, site_tide_data, ts):
+    def get_tide(self, tides_df):
 
         # Account for daylight savings
         # ASSUMPTION: All .mat tide files are either AEST or AEDT 
@@ -87,14 +86,11 @@ class RegisteredImage():
             self.hour = str(int(self.hour) - 1)
         date_string = self.year + '-' + self.month + '-' + self.day + ' ' + self.hour + ':' + self.minute + ':' + self.second
         img_datetime = datetime.strptime(date_string, "%Y-%m-%d %H:%M:%S") # Image date/time as a datetime object
-        
-        tide_date = nearest(site_tide_data, img_datetime)
-        mat_index = site_tide_data.index(tide_date)  # Retrieve the index of the .mat tide/time
-        mat_tide = round(ts[1][mat_index][0], 2) # Associated tide
-        
-        #print('Image date/time: ' + date_string)
-        #print('Tide record:     ' + str(tide_date))
-        self.tide = "{:.2f}".format(mat_tide)
+        tide_date = nearest(tides_df['time'], img_datetime)
+        tide = tides_df.loc[tides_df['time'] == tide_date, 'tide level']
+        tide = tide.iloc[0]
+
+        self.tide = "{:.2f}".format(tide)
     
     def create_tag(self):
         # Create image tag