From bc9eaf77ec03e69b1d58428ee42f7e343d3d5f31 Mon Sep 17 00:00:00 2001 From: Jonathan Chan Date: Fri, 24 Jun 2022 11:18:18 +1000 Subject: [PATCH] Updated tag_registered to read .pkl data and efficiently locate the tide value --- coastsnap/tag_registered.py | 40 ++++++++++++++++++++----------------- coastsnap/utils.py | 26 ++++++++++-------------- 2 files changed, 33 insertions(+), 33 deletions(-) diff --git a/coastsnap/tag_registered.py b/coastsnap/tag_registered.py index b9027c3..790cf06 100644 --- a/coastsnap/tag_registered.py +++ b/coastsnap/tag_registered.py @@ -44,17 +44,13 @@ def get_site_tide_data(images_parent_dir, site): if tide_filename == 'NO_TIDE.mat': return False - - mat_path = images_parent_dir + '/Tide Data/' + tide_filename - mat = sio.loadmat(mat_path) - tide_dict = mat['tide'] - ts = tide_dict[0][0] # Format of tide/time Matlab data is gross - tt = list(ts[0]) # ts[0] = tides, ts[1] = times, ts[2] = - print(site + ": " +"Loading tide data... (this may take half a minute)") - tide_times = [datenum_to_datetime(i) for i in tt] # THIS STEP TAKES A LONG TIME + tide_filename_pkl = tide_filename[:-4] + '.pkl' + tides_path = images_parent_dir + '/Tide Data/Tide_Data_Python/' + tide_filename_pkl + tides_df = pd.read_pickle(tides_path) + + return tides_df - return tide_times, ts def tag_image(image2tag): @@ -78,9 +74,14 @@ def tag_image(image2tag): image.save(registered_path) -image2tag = None #initialize this here +#-----------------------------------------------------------------------------# +# This is the main function + + +image2tag = None for site in os.listdir(images_dir): # Loop through SITES + print(site) i=0 site_complete = False # A flag to stop tagging images when found tide_data = False @@ -89,18 +90,20 @@ for site in os.listdir(images_dir): # Loop through SITES try: # Check if site contains 'Processed' directory years_list = os.listdir(photoshop_path) years_list.reverse() - except: - continue + except: # Site doesn't contain 'Processed'. Go to next site. + continue for year in years_list: # Loop through YEARS if site_complete: break year_path = photoshop_path + '/' + year image_list = os.listdir(year_path) image_list.reverse() + for image_filename in image_list: # Loop through IMAGES registered_year_path = year_path.replace('Photoshop', 'Registered') registered_image_path = registered_year_path + '/' + image_filename[:-4] + '_registered.jpg' if site not in image_filename: # Check the filename has the site in it continue + if os.path.isfile(registered_image_path): # Image already tagged. Go to next site. site_complete = True; if i == 0: @@ -110,6 +113,7 @@ for site in os.listdir(images_dir): # Loop through SITES else: print(site + ": " + str(i) + " images tagged no tide") break + else: # Image not tagged. Tag it. if not os.path.exists(registered_year_path): # Chech that registered/year directory exists os.makedirs(registered_year_path) # if not, create it @@ -117,16 +121,16 @@ for site in os.listdir(images_dir): # Loop through SITES pathname = os.path.join(year_path, image_filename) image2tag = RegisteredImage(pathname, image_filename) # Create image object if i == 1: # Retrieve Tide Data once for each site - site_tide_data, ts = get_site_tide_data(images_parent_dir, site) - - font = image2tag.get_font() - - if site_tide_data: - image2tag.get_tide(site_tide_data, ts) + tides_df = get_site_tide_data(images_parent_dir, site) + font = image2tag.get_font() + image2tag.get_tide(tides_df) image2tag.create_tag() + tag_image(image2tag) + # No images previously tagged + # print result if not site_complete: if i == 0: print(site + ": " + str(i) + " images tagged") diff --git a/coastsnap/utils.py b/coastsnap/utils.py index da1fdd6..2869a73 100644 --- a/coastsnap/utils.py +++ b/coastsnap/utils.py @@ -31,12 +31,11 @@ def datenum_to_datetime(datenum): # Sourced from https://stackoverflow.com/questions/32237862/find-the-closest-date-to-a-given-date -def nearest(items, pivot): - """ - This function will return the datetime in items - which is the closest to the date pivot - """ - return min(items, key=lambda x: abs(x - pivot)) +def nearest(all_dates, target_date_raw): + abs_deltas_from_target_date = np.absolute(all_dates - target_date_raw) + index_of_min_delta_from_target_date = np.argmin(abs_deltas_from_target_date) + closest_date = all_dates[index_of_min_delta_from_target_date] + return closest_date def progressbar(it, prefix="", size=60, out=sys.stdout): # Python3.3+ @@ -79,7 +78,7 @@ class RegisteredImage(): self.height = None self.font = None - def get_tide(self, site_tide_data, ts): + def get_tide(self, tides_df): # Account for daylight savings # ASSUMPTION: All .mat tide files are either AEST or AEDT @@ -87,14 +86,11 @@ class RegisteredImage(): self.hour = str(int(self.hour) - 1) date_string = self.year + '-' + self.month + '-' + self.day + ' ' + self.hour + ':' + self.minute + ':' + self.second img_datetime = datetime.strptime(date_string, "%Y-%m-%d %H:%M:%S") # Image date/time as a datetime object - - tide_date = nearest(site_tide_data, img_datetime) - mat_index = site_tide_data.index(tide_date) # Retrieve the index of the .mat tide/time - mat_tide = round(ts[1][mat_index][0], 2) # Associated tide - - #print('Image date/time: ' + date_string) - #print('Tide record: ' + str(tide_date)) - self.tide = "{:.2f}".format(mat_tide) + tide_date = nearest(tides_df['time'], img_datetime) + tide = tides_df.loc[tides_df['time'] == tide_date, 'tide level'] + tide = tide.iloc[0] + + self.tide = "{:.2f}".format(tide) def create_tag(self): # Create image tag