geetools_VH/main_test.py

#==========================================================#
# Create a classifier for satellite images
#==========================================================#

# load modules
import os
import pickle
import warnings
import numpy as np
import matplotlib.cm as cm
warnings.filterwarnings("ignore")
import matplotlib.pyplot as plt
from pylab import ginput

import SDS_download, SDS_preprocess, SDS_shoreline, SDS_tools, SDS_classification

filepath_sites = os.path.join(os.getcwd(), 'polygons')
sites = os.listdir(filepath_sites)

for site in sites:

    polygon = SDS_tools.coords_from_kml(os.path.join(filepath_sites,site))
    
    # load Sentinel-2 images
    inputs = {
        'polygon': polygon,
        'dates': ['2016-10-01', '2016-11-01'],
        'sat_list': ['S2'],
        'sitename': site[:site.find('.')]
            }
    
    satname = inputs['sat_list'][0]
    
    metadata = SDS_download.get_images(inputs)
    metadata = SDS_download.remove_cloudy_images(metadata,inputs,0.2)
    filepath = os.path.join(os.getcwd(), 'data', inputs['sitename'])
    with open(os.path.join(filepath, inputs['sitename'] + '_metadata_' + satname + '.pkl'), 'wb') as f:
        pickle.dump(metadata, f)
    #with open(os.path.join(filepath, inputs['sitename'] + '_metadata_' + satname + '.pkl'), 'rb') as f:
    #    metadata = pickle.load(f)
        
    # settings needed to run the shoreline extraction
    settings = {
           
        # general parameters:
        'cloud_thresh': 0.1,         # threshold on maximum cloud cover
        'output_epsg': 28356,        # epsg code of spatial reference system desired for the output
           
        # shoreline detection parameters:
        'min_beach_size': 20,        # minimum number of connected pixels for a beach
        'buffer_size': 7,            # radius (in pixels) of disk for buffer around sandy pixels
        'min_length_sl': 200,       # minimum length of shoreline perimeter to be kept 
        'max_dist_ref': 100,        # max distance (in meters) allowed from a reference shoreline
        
        # quality control:
        'check_detection': True,    # if True, shows each shoreline detection and lets the user 
                                    # decide which ones are correct and which ones are false due to
                                    # the presence of clouds 
        # also add the inputs 
        'inputs': inputs
    }
    # preprocess images (cloud masking, pansharpening/down-sampling)
    SDS_preprocess.preprocess_all_images(metadata, settings)
        
    training_data = dict([])
    training_data['sand'] = dict([])
    training_data['swash'] = dict([])
    training_data['water'] = dict([])
    training_data['land'] = dict([])
    
    # read images
    filepath = SDS_tools.get_filepath(inputs,satname)
    filenames = metadata[satname]['filenames']
    
    for i in range(len(filenames)):
    
        fn = SDS_tools.get_filenames(filenames[i],filepath,satname)
        im_ms, georef, cloud_mask, im20, imQA = SDS_preprocess.preprocess_single(fn,satname)
        
        nrow = im_ms.shape[0]
        ncol = im_ms.shape[1]
        
        im_RGB = SDS_preprocess.rescale_image_intensity(im_ms[:,:,[2,1,0]], cloud_mask, 99.9)
        plt.figure()
        mng = plt.get_current_fig_manager()                                         
        mng.window.showMaximized()
        plt.imshow(im_RGB)
        plt.axis('off')
        
        # Digitize sandy pixels
        plt.title('Digitize SAND pixels', fontweight='bold', fontsize=15)
        pt = ginput(n=1000, timeout=100000, show_clicks=True)
        
        if len(pt) > 0:
            pt = np.round(pt).astype(int)    
            im_sand = np.zeros((nrow,ncol))
            for k in range(len(pt)):
                im_sand[pt[k,1],pt[k,0]] = 1
                im_RGB[pt[k,1],pt[k,0],0] = 1
                im_RGB[pt[k,1],pt[k,0],1] = 0
                im_RGB[pt[k,1],pt[k,0],2] = 0
            im_sand = im_sand.astype(bool)          
            features = SDS_classification.calculate_features(im_ms, cloud_mask, im_sand)
        else:
            im_sand = np.zeros((nrow,ncol)).astype(bool)
            features = []     
        training_data['sand'][filenames[i]] = {'pixels':im_sand,'features':features}
        
        # Digitize swash pixels
        plt.title('Digitize SWASH pixels', fontweight='bold', fontsize=15)
        plt.draw()
        pt = ginput(n=1000, timeout=100000, show_clicks=True)
    
        if len(pt) > 0:
            pt = np.round(pt).astype(int)
            im_swash = np.zeros((nrow,ncol))
            for k in range(len(pt)):
                im_swash[pt[k,1],pt[k,0]] = 1
                im_RGB[pt[k,1],pt[k,0],0] = 0
                im_RGB[pt[k,1],pt[k,0],1] = 1
                im_RGB[pt[k,1],pt[k,0],2] = 0
            im_swash = im_swash.astype(bool)  
            features = SDS_classification.calculate_features(im_ms, cloud_mask, im_swash)
        else:
            im_swash = np.zeros((nrow,ncol)).astype(bool)
            features = []
        training_data['swash'][filenames[i]] = {'pixels':im_swash,'features':features}
    
        # Digitize rectangle containig water pixels
        plt.title('Click 2 points to draw a rectange in the WATER', fontweight='bold', fontsize=15)
        plt.draw()
        pt = ginput(n=2, timeout=100000, show_clicks=True)
        if len(pt) > 0:
            pt = np.round(pt).astype(int) 
            idx_row = np.arange(np.min(pt[:,1]),np.max(pt[:,1])+1,1) 
            idx_col = np.arange(np.min(pt[:,0]),np.max(pt[:,0])+1,1) 
            xx, yy = np.meshgrid(idx_row,idx_col, indexing='ij')
            rows = xx.reshape(xx.shape[0]*xx.shape[1])
            cols = yy.reshape(yy.shape[0]*yy.shape[1])
            im_water = np.zeros((nrow,ncol)).astype(bool)
            for k in range(len(rows)):
                im_water[rows[k],cols[k]] = 1
                im_RGB[rows[k],cols[k],0] = 0
                im_RGB[rows[k],cols[k],1] = 0
                im_RGB[rows[k],cols[k],2] = 1
            im_water = im_water.astype(bool)        
            features = SDS_classification.calculate_features(im_ms, cloud_mask, im_water)
        else:
            im_water = np.zeros((nrow,ncol)).astype(bool)
            features = []     
        training_data['water'][filenames[i]] = {'pixels':im_water,'features':features}
        
        # Digitize rectangle containig land pixels
        plt.title('Click 2 points to draw a rectange in the LAND', fontweight='bold', fontsize=15)
        plt.draw()
        pt = ginput(n=2, timeout=100000, show_clicks=True)
        plt.close()
        if len(pt) > 0:
            pt = np.round(pt).astype(int)  
            idx_row = np.arange(np.min(pt[:,1]),np.max(pt[:,1])+1,1) 
            idx_col = np.arange(np.min(pt[:,0]),np.max(pt[:,0])+1,1) 
            xx, yy = np.meshgrid(idx_row,idx_col, indexing='ij')
            rows = xx.reshape(xx.shape[0]*xx.shape[1])
            cols = yy.reshape(yy.shape[0]*yy.shape[1]) 
            im_land = np.zeros((nrow,ncol)).astype(bool)
            for k in range(len(rows)):
                im_land[rows[k],cols[k]] = 1
                im_RGB[rows[k],cols[k],0] = 1
                im_RGB[rows[k],cols[k],1] = 1
                im_RGB[rows[k],cols[k],2] = 0
            im_land = im_land.astype(bool) 
            features = SDS_classification.calculate_features(im_ms, cloud_mask, im_land)
        else:
            im_land = np.zeros((nrow,ncol)).astype(bool)
            features = []  
        training_data['land'][filenames[i]] = {'pixels':im_land,'features':features}
         
        plt.figure()
        plt.title('Classified image')
        plt.imshow(im_RGB)
        
    # save training data for each site
    filepath = os.path.join(os.getcwd(), 'data', inputs['sitename'])
    with open(os.path.join(filepath, inputs['sitename'] + '_training_' + satname + '.pkl'), 'wb') as f:
        pickle.dump(training_data, f)
#%%

## load Landsat 5 images
#inputs = {
#    'polygon': polygon,
#    'dates': ['1987-01-01', '1988-01-01'],
#    'sat_list': ['L5'],
#    'sitename': site[:site.find('.')]
#        }
#metadata = SDS_download.get_images(inputs)
#
## load Landsat 7 images
#inputs = {
#    'polygon': polygon,
#    'dates': ['2001-01-01', '2002-01-01'],
#    'sat_list': ['L7'],
#    'sitename': site[:site.find('.')]
#        }
#metadata = SDS_download.get_images(inputs)
#
## load Landsat 8 images
#inputs = {
#    'polygon': polygon,
#    'dates': ['2014-01-01', '2015-01-01'],
#    'sat_list': ['L8'],
#    'sitename': site[:site.find('.')]
#        }
#metadata = SDS_download.get_images(inputs)


#%% clean the Landsat collections

#import ee
#from datetime import datetime, timedelta
#import pytz
#import copy
#ee.Initialize()
#site = sites[0]
#dates = ['2017-12-01', '2017-12-25']
#polygon = SDS_tools.coords_from_kml(os.path.join(filepath_sites,site))
## Landsat collection
#input_col = ee.ImageCollection('LANDSAT/LC08/C01/T1_RT_TOA')
## filter by location and dates
#flt_col = input_col.filterBounds(ee.Geometry.Polygon(polygon)).filterDate(inputs['dates'][0],inputs['dates'][1])
## get all images in the filtered collection
#im_all = flt_col.getInfo().get('features')
#cloud_cover = [_['properties']['CLOUD_COVER'] for _ in im_all]
#if np.any([_ > 90 for _ in cloud_cover]):
#    idx_delete = np.where([_ > 90 for _ in cloud_cover])[0]
#    im_all_cloud = [x for k,x in enumerate(im_all) if k not in idx_delete]


#%% clean the S2 collection

#import ee
#from datetime import datetime, timedelta
#import pytz
#import copy
#ee.Initialize()
## Sentinel2 collection
#input_col = ee.ImageCollection('COPERNICUS/S2')
## filter by location and dates
#flt_col = input_col.filterBounds(ee.Geometry.Polygon(polygon)).filterDate(inputs['dates'][0],inputs['dates'][1])
## get all images in the filtered collection
#im_all = flt_col.getInfo().get('features')
#
## remove duplicates (there are many in S2 collection)
## timestamps
#timestamps = [datetime.fromtimestamp(_['properties']['system:time_start']/1000, tz=pytz.utc) for _ in im_all]
## utm zones
#utm_zones = np.array([int(_['bands'][0]['crs'][5:]) for _ in im_all])
#utm_zone_selected =  np.max(np.unique(utm_zones))
#idx_all = np.arange(0,len(im_all),1)
#idx_covered = np.ones(len(im_all)).astype(bool)
#idx_delete = []
#i = 0
#while 1:
#    same_time = np.abs([(timestamps[i]-_).total_seconds() for _ in timestamps]) < 60*60*24
#    idx_same_time = np.where(same_time)[0]
#    same_utm = utm_zones == utm_zone_selected
#    idx_temp = np.where([same_time[j] == True and same_utm[j] == False for j in idx_all])[0]
#    idx_keep = idx_same_time[[_ not in idx_temp for _ in idx_same_time ]]
#    if len(idx_keep) > 2: # if more than 2 images with same date and same utm, drop the last one
#       idx_temp = np.append(idx_temp,idx_keep[-1])
#    for j in idx_temp:
#        idx_delete.append(j)
#    idx_covered[idx_same_time] = False
#    if np.any(idx_covered):
#        i = np.where(idx_covered)[0][0]
#    else:
#        break
#im_all_updated = [x for k,x in enumerate(im_all) if k not in idx_delete]
#
## remove very cloudy images (>90% cloud)
#cloud_cover = [_['properties']['CLOUDY_PIXEL_PERCENTAGE'] for _ in im_all_updated]
#if np.any([_ > 90 for _ in cloud_cover]):
#    idx_delete = np.where([_ > 90 for _ in cloud_cover])[0]
#    im_all_cloud = [x for k,x in enumerate(im_all_updated) if k not in idx_delete]