forked from kilianv/CoastSat_WRL
Compare commits
10 Commits
developmen
...
master
| Author | SHA1 | Date |
|---|---|---|
kvos
|
62f5c5330f | 7 years ago |
|
kvos
|
cd74f6c39c | 7 years ago |
 Kilian Vos
|
0e18b6d4d0 | 7 years ago |
|
kvos
|
ee556de2fe | 7 years ago |
|
kvos
|
e92fd60ba2 | 7 years ago |
|
kvos
|
2abac763b1 | 7 years ago |
|
Kilian Vos
|
ca90712623 | 7 years ago |
|
kvos
|
1882c98d9b | 7 years ago |
|
Kilian Vos
|
b964426cd4 | 7 years ago |
|
Kilian Vos
|
ea32adf79b | 7 years ago |
@ -1,6 +1,9 @@
|
||||
This toolbox uses the Google Earth Engine Python API to explore collections of publicly available satellite imagery (Landsat, Sentinel).
|
||||
|
||||
It has .py routines to:
|
||||
- read and preprocess satellite images (cloud masking, contrast stretching)
|
||||
- pansharpen Landsat 8 images
|
||||
- extract shorelines with the Marching Squares algorithm
|
||||
- read and preprocess satellite images (cloud masking, contrast stretching)
|
||||
- pansharpen Landsat 8 images
|
||||
- extract shorelines with the Marching Squares algorithm
|
||||
- classify image in 4 classes (sand, whitewater, water, other) using a Neural Network classifier
|
||||
|
||||
Requirements: all the packages contained in py3_environments.txt
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@ -0,0 +1,390 @@
|
||||
#==========================================================#
|
||||
#==========================================================#
|
||||
# Download L5, L7, L8, S2 images of a given area
|
||||
#==========================================================#
|
||||
#==========================================================#
|
||||
|
||||
|
||||
|
||||
#==========================================================#
|
||||
# Initial settings
|
||||
#==========================================================#
|
||||
import os
|
||||
import numpy as np
|
||||
import matplotlib.pyplot as plt
|
||||
import pdb
|
||||
import ee
|
||||
|
||||
# other modules
|
||||
from osgeo import gdal, ogr, osr
|
||||
from urllib.request import urlretrieve
|
||||
import zipfile
|
||||
from datetime import datetime
|
||||
import pytz
|
||||
import pickle
|
||||
|
||||
# import own modules
|
||||
import functions.utils as utils
|
||||
import functions.sds as sds
|
||||
|
||||
np.seterr(all='ignore') # raise/ignore divisions by 0 and nans
|
||||
ee.Initialize()
|
||||
|
||||
#==========================================================#
|
||||
# Location
|
||||
#==========================================================#
|
||||
|
||||
## location (Narrabeen-Collaroy beach)
|
||||
#polygon = [[[151.301454, -33.700754],
|
||||
# [151.311453, -33.702075],
|
||||
# [151.307237, -33.739761],
|
||||
# [151.294220, -33.736329],
|
||||
# [151.301454, -33.700754]]];
|
||||
|
||||
# location (Tairua beach)
|
||||
sitename = 'TAIRUA'
|
||||
polygon = [[[175.835574, -36.982022],
|
||||
[175.888220, -36.980680],
|
||||
[175.893527, -37.029610],
|
||||
[175.833444, -37.031767],
|
||||
[175.835574, -36.982022]]];
|
||||
|
||||
# initialise metadata dictionnary (stores timestamps and georefencing accuracy of each image)
|
||||
metadata = dict([])
|
||||
|
||||
# create directories
|
||||
try:
|
||||
os.makedirs(os.path.join(os.getcwd(), 'data',sitename))
|
||||
except:
|
||||
print('directory already exists')
|
||||
|
||||
|
||||
#%%
|
||||
#==========================================================#
|
||||
#==========================================================#
|
||||
# L5
|
||||
#==========================================================#
|
||||
#==========================================================#
|
||||
|
||||
|
||||
|
||||
# define filenames for images
|
||||
suffix = '.tif'
|
||||
filepath = os.path.join(os.getcwd(), 'data', sitename, 'L5', '30m')
|
||||
try:
|
||||
os.makedirs(filepath)
|
||||
except:
|
||||
print('directory already exists')
|
||||
|
||||
#==========================================================#
|
||||
# Select L5 collection
|
||||
#==========================================================#
|
||||
|
||||
satname = 'L5'
|
||||
input_col = ee.ImageCollection('LANDSAT/LT05/C01/T1_TOA')
|
||||
|
||||
# filter by location
|
||||
flt_col = input_col.filterBounds(ee.Geometry.Polygon(polygon))
|
||||
n_img = flt_col.size().getInfo()
|
||||
print('Number of images covering ' + sitename, n_img)
|
||||
im_all = flt_col.getInfo().get('features')
|
||||
|
||||
|
||||
#==========================================================#
|
||||
# Main loop trough images
|
||||
#==========================================================#
|
||||
|
||||
timestamps = []
|
||||
acc_georef = []
|
||||
all_names = []
|
||||
for i in range(n_img):
|
||||
|
||||
# find each image in ee database
|
||||
im = ee.Image(im_all[i].get('id'))
|
||||
|
||||
im_dic = im.getInfo()
|
||||
im_bands = im_dic.get('bands')
|
||||
t = im_dic['properties']['system:time_start']
|
||||
im_timestamp = datetime.fromtimestamp(t/1000, tz=pytz.utc)
|
||||
timestamps.append(im_timestamp)
|
||||
im_date = im_timestamp.strftime('%Y-%m-%d-%H-%M-%S')
|
||||
im_epsg = int(im_dic['bands'][0]['crs'][5:])
|
||||
try:
|
||||
acc_georef.append(im_dic['properties']['GEOMETRIC_RMSE_MODEL'])
|
||||
except:
|
||||
acc_georef.append(12)
|
||||
print('No geometric rmse model property')
|
||||
|
||||
# delete dimensions key from dictionnary, otherwise the entire image is extracted
|
||||
for j in range(len(im_bands)): del im_bands[j]['dimensions']
|
||||
|
||||
# bands for L5
|
||||
ms_bands = [im_bands[0], im_bands[1], im_bands[2], im_bands[3], im_bands[4], im_bands[7]]
|
||||
|
||||
# filenames
|
||||
filename = im_date + '_' + satname + '_' + sitename + suffix
|
||||
|
||||
print(i)
|
||||
if any(filename in _ for _ in all_names):
|
||||
filename = im_date + '_' + satname + '_' + sitename + '_dup' + suffix
|
||||
all_names.append(filename)
|
||||
|
||||
local_data = sds.download_tif(im, polygon, ms_bands, filepath)
|
||||
os.rename(local_data, os.path.join(filepath, filename))
|
||||
|
||||
# sort timestamps and georef accuracy (dowloaded images are sorted by date in directory)
|
||||
timestamps_sorted = sorted(timestamps)
|
||||
idx_sorted = sorted(range(len(timestamps)), key=timestamps.__getitem__)
|
||||
acc_georef_sorted = [acc_georef[j] for j in idx_sorted]
|
||||
|
||||
metadata[satname] = {'dates':timestamps_sorted, 'acc_georef':acc_georef_sorted, 'epsg':im_epsg}
|
||||
|
||||
#%%
|
||||
#==========================================================#
|
||||
#==========================================================#
|
||||
# L7&L8
|
||||
#==========================================================#
|
||||
#==========================================================#
|
||||
|
||||
|
||||
|
||||
# define filenames for images
|
||||
suffix = '.tif'
|
||||
filepath = os.path.join(os.getcwd(), 'data', sitename, 'L7&L8')
|
||||
filepath_pan = os.path.join(filepath, 'pan')
|
||||
filepath_ms = os.path.join(filepath, 'ms')
|
||||
try:
|
||||
os.makedirs(filepath_pan)
|
||||
os.makedirs(filepath_ms)
|
||||
except:
|
||||
print('directory already exists')
|
||||
|
||||
#==========================================================#
|
||||
# Select L7 collection
|
||||
#==========================================================#
|
||||
|
||||
satname = 'L7'
|
||||
input_col = ee.ImageCollection('LANDSAT/LE07/C01/T1_RT_TOA')
|
||||
|
||||
# filter by location
|
||||
flt_col = input_col.filterBounds(ee.Geometry.Polygon(polygon))
|
||||
n_img = flt_col.size().getInfo()
|
||||
print('Number of images covering ' + sitename, n_img)
|
||||
im_all = flt_col.getInfo().get('features')
|
||||
|
||||
#==========================================================#
|
||||
# Main loop trough images
|
||||
#==========================================================#
|
||||
|
||||
timestamps = []
|
||||
acc_georef = []
|
||||
all_names = []
|
||||
for i in range(n_img):
|
||||
|
||||
# find each image in ee database
|
||||
im = ee.Image(im_all[i].get('id'))
|
||||
|
||||
im_dic = im.getInfo()
|
||||
im_bands = im_dic.get('bands')
|
||||
t = im_dic['properties']['system:time_start']
|
||||
im_timestamp = datetime.fromtimestamp(t/1000, tz=pytz.utc)
|
||||
timestamps.append(im_timestamp)
|
||||
im_date = im_timestamp.strftime('%Y-%m-%d-%H-%M-%S')
|
||||
im_epsg = int(im_dic['bands'][0]['crs'][5:])
|
||||
try:
|
||||
acc_georef.append(im_dic['properties']['GEOMETRIC_RMSE_MODEL'])
|
||||
except:
|
||||
acc_georef.append(12)
|
||||
print('No geometric rmse model property')
|
||||
|
||||
# delete dimensions key from dictionnary, otherwise the entire image is extracted
|
||||
for j in range(len(im_bands)): del im_bands[j]['dimensions']
|
||||
|
||||
# bands for L7
|
||||
pan_band = [im_bands[8]]
|
||||
ms_bands = [im_bands[0], im_bands[1], im_bands[2], im_bands[3], im_bands[4], im_bands[9]]
|
||||
|
||||
# filenames
|
||||
filename_pan = im_date + '_' + satname + '_' + sitename + '_pan' + suffix
|
||||
filename_ms = im_date + '_' + satname + '_' + sitename + '_ms' + suffix
|
||||
|
||||
print(i)
|
||||
if any(filename_pan in _ for _ in all_names):
|
||||
filename_pan = im_date + '_' + satname + '_' + sitename + '_pan' + '_dup' + suffix
|
||||
filename_ms = im_date + '_' + satname + '_' + sitename + '_ms' + '_dup' + suffix
|
||||
all_names.append(filename_pan)
|
||||
|
||||
local_data_pan = sds.download_tif(im, polygon, pan_band, filepath_pan)
|
||||
os.rename(local_data_pan, os.path.join(filepath_pan, filename_pan))
|
||||
local_data_ms = sds.download_tif(im, polygon, ms_bands, filepath_ms)
|
||||
os.rename(local_data_ms, os.path.join(filepath_ms, filename_ms))
|
||||
|
||||
#==========================================================#
|
||||
# Select L8 collection
|
||||
#==========================================================#
|
||||
|
||||
satname = 'L8'
|
||||
input_col = ee.ImageCollection('LANDSAT/LC08/C01/T1_RT_TOA')
|
||||
|
||||
# filter by location
|
||||
flt_col = input_col.filterBounds(ee.Geometry.Polygon(polygon))
|
||||
n_img = flt_col.size().getInfo()
|
||||
print('Number of images covering Narrabeen:', n_img)
|
||||
im_all = flt_col.getInfo().get('features')
|
||||
|
||||
#==========================================================#
|
||||
# Main loop trough images
|
||||
#==========================================================#
|
||||
|
||||
for i in range(n_img):
|
||||
|
||||
# find each image in ee database
|
||||
im = ee.Image(im_all[i].get('id'))
|
||||
|
||||
im_dic = im.getInfo()
|
||||
im_bands = im_dic.get('bands')
|
||||
t = im_dic['properties']['system:time_start']
|
||||
im_timestamp = datetime.fromtimestamp(t/1000, tz=pytz.utc)
|
||||
timestamps.append(im_timestamp)
|
||||
im_date = im_timestamp.strftime('%Y-%m-%d-%H-%M-%S')
|
||||
im_epsg = int(im_dic['bands'][0]['crs'][5:])
|
||||
try:
|
||||
acc_georef.append(im_dic['properties']['GEOMETRIC_RMSE_MODEL'])
|
||||
except:
|
||||
acc_georef.append(12)
|
||||
print('No geometric rmse model property')
|
||||
|
||||
# delete dimensions key from dictionnary, otherwise the entire image is extracted
|
||||
for j in range(len(im_bands)): del im_bands[j]['dimensions']
|
||||
|
||||
# bands for L8
|
||||
pan_band = [im_bands[7]]
|
||||
ms_bands = [im_bands[1], im_bands[2], im_bands[3], im_bands[4], im_bands[5], im_bands[11]]
|
||||
|
||||
# filenames
|
||||
filename_pan = im_date + '_' + satname + '_' + sitename + '_pan' + suffix
|
||||
filename_ms = im_date + '_' + satname + '_' + sitename + '_ms' + suffix
|
||||
|
||||
print(i)
|
||||
if any(filename_pan in _ for _ in all_names):
|
||||
filename_pan = im_date + '_' + satname + '_' + sitename + '_pan' + '_dup' + suffix
|
||||
filename_ms = im_date + '_' + satname + '_' + sitename + '_ms' + '_dup' + suffix
|
||||
all_names.append(filename_pan)
|
||||
|
||||
local_data_pan = sds.download_tif(im, polygon, pan_band, filepath_pan)
|
||||
os.rename(local_data_pan, os.path.join(filepath_pan, filename_pan))
|
||||
local_data_ms = sds.download_tif(im, polygon, ms_bands, filepath_ms)
|
||||
os.rename(local_data_ms, os.path.join(filepath_ms, filename_ms))
|
||||
|
||||
|
||||
# sort timestamps and georef accuracy (dowloaded images are sorted by date in directory)
|
||||
timestamps_sorted = sorted(timestamps)
|
||||
idx_sorted = sorted(range(len(timestamps)), key=timestamps.__getitem__)
|
||||
acc_georef_sorted = [acc_georef[j] for j in idx_sorted]
|
||||
|
||||
metadata[satname] = {'dates':timestamps_sorted, 'acc_georef':acc_georef_sorted, 'epsg':im_epsg}
|
||||
|
||||
#%%
|
||||
#==========================================================#
|
||||
#==========================================================#
|
||||
# S2
|
||||
#==========================================================#
|
||||
#==========================================================#
|
||||
|
||||
|
||||
|
||||
# define filenames for images
|
||||
suffix = '.tif'
|
||||
filepath = os.path.join(os.getcwd(), 'data', sitename, 'S2')
|
||||
try:
|
||||
os.makedirs(os.path.join(filepath, '10m'))
|
||||
os.makedirs(os.path.join(filepath, '20m'))
|
||||
os.makedirs(os.path.join(filepath, '60m'))
|
||||
except:
|
||||
print('directory already exists')
|
||||
|
||||
#==========================================================#
|
||||
# Select L2 collection
|
||||
#==========================================================#
|
||||
|
||||
satname = 'S2'
|
||||
input_col = ee.ImageCollection('COPERNICUS/S2')
|
||||
|
||||
# filter by location
|
||||
flt_col = input_col.filterBounds(ee.Geometry.Polygon(polygon))
|
||||
n_img = flt_col.size().getInfo()
|
||||
print('Number of images covering ' + sitename, n_img)
|
||||
im_all = flt_col.getInfo().get('features')
|
||||
|
||||
#==========================================================#
|
||||
# Main loop trough images
|
||||
#==========================================================#
|
||||
|
||||
timestamps = []
|
||||
acc_georef = []
|
||||
all_names = []
|
||||
for i in range(n_img):
|
||||
|
||||
# find each image in ee database
|
||||
im = ee.Image(im_all[i].get('id'))
|
||||
|
||||
im_dic = im.getInfo()
|
||||
im_bands = im_dic.get('bands')
|
||||
t = im_dic['properties']['system:time_start']
|
||||
im_timestamp = datetime.fromtimestamp(t/1000, tz=pytz.utc)
|
||||
im_date = im_timestamp.strftime('%Y-%m-%d-%H-%M-%S')
|
||||
timestamps.append(im_timestamp)
|
||||
im_epsg = int(im_dic['bands'][0]['crs'][5:])
|
||||
try:
|
||||
if im_dic['properties']['GEOMETRIC_QUALITY_FLAG'] == 'PASSED':
|
||||
acc_georef.append(1)
|
||||
else:
|
||||
acc_georef.append(0)
|
||||
except:
|
||||
acc_georef.append(0)
|
||||
|
||||
# delete dimensions key from dictionnary, otherwise the entire image is extracted
|
||||
for j in range(len(im_bands)): del im_bands[j]['dimensions']
|
||||
|
||||
# bands for S2
|
||||
bands10 = [im_bands[1], im_bands[2], im_bands[3], im_bands[7]]
|
||||
bands20 = [im_bands[11]]
|
||||
bands60 = [im_bands[15]]
|
||||
|
||||
# filenames
|
||||
filename10 = im_date + '_' + satname + '_' + sitename + '_' + '10m' + suffix
|
||||
filename20 = im_date + '_' + satname + '_' + sitename + '_' + '20m' + suffix
|
||||
filename60 = im_date + '_' + satname + '_' + sitename + '_' + '60m' + suffix
|
||||
|
||||
print(i)
|
||||
if any(filename10 in _ for _ in all_names):
|
||||
filename10 = im_date + '_' + satname + '_' + sitename + '_' + '10m' + '_dup' + suffix
|
||||
filename20 = im_date + '_' + satname + '_' + sitename + '_' + '20m' + '_dup' + suffix
|
||||
filename60 = im_date + '_' + satname + '_' + sitename + '_' + '60m' + '_dup' + suffix
|
||||
all_names.append(filename10)
|
||||
|
||||
local_data = sds.download_tif(im, polygon, bands10, filepath)
|
||||
os.rename(local_data, os.path.join(filepath, '10m', filename10))
|
||||
|
||||
local_data = sds.download_tif(im, polygon, bands20, filepath)
|
||||
os.rename(local_data, os.path.join(filepath, '20m', filename20))
|
||||
|
||||
local_data = sds.download_tif(im, polygon, bands60, filepath)
|
||||
os.rename(local_data, os.path.join(filepath, '60m', filename60))
|
||||
|
||||
# sort timestamps and georef accuracy (dowloaded images are sorted by date in directory)
|
||||
timestamps_sorted = sorted(timestamps)
|
||||
idx_sorted = sorted(range(len(timestamps)), key=timestamps.__getitem__)
|
||||
acc_georef_sorted = [acc_georef[j] for j in idx_sorted]
|
||||
|
||||
metadata[satname] = {'dates':timestamps_sorted, 'acc_georef':acc_georef_sorted, 'epsg':im_epsg}
|
||||
|
||||
|
||||
|
||||
#%% save metadata
|
||||
|
||||
filepath = os.path.join(os.getcwd(), 'data', sitename)
|
||||
with open(os.path.join(filepath, sitename + '_metadata' + '.pkl'), 'wb') as f:
|
||||
pickle.dump(metadata, f)
|
||||
|
||||
|
||||
@ -1,113 +0,0 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
#==========================================================#
|
||||
# Download L7 images of a given area between given dates
|
||||
#==========================================================#
|
||||
|
||||
# Initial settings
|
||||
import os
|
||||
import numpy as np
|
||||
import matplotlib.pyplot as plt
|
||||
import pdb
|
||||
import ee
|
||||
|
||||
# other modules
|
||||
from osgeo import gdal, ogr, osr
|
||||
from urllib.request import urlretrieve
|
||||
import zipfile
|
||||
from datetime import datetime
|
||||
import pytz
|
||||
import pickle
|
||||
|
||||
# image processing modules
|
||||
import skimage.filters as filters
|
||||
import skimage.exposure as exposure
|
||||
import skimage.transform as transform
|
||||
import sklearn.decomposition as decomposition
|
||||
import skimage.measure as measure
|
||||
|
||||
# import own modules
|
||||
import functions.utils as utils
|
||||
|
||||
np.seterr(all='ignore') # raise/ignore divisions by 0 and nans
|
||||
ee.Initialize()
|
||||
|
||||
def download_tif(image, polygon, bandsId, filepath):
|
||||
"""downloads tif image (region and bands) from the ee server and stores it in a temp file"""
|
||||
url = ee.data.makeDownloadUrl(ee.data.getDownloadId({
|
||||
'image': image.serialize(),
|
||||
'region': polygon,
|
||||
'bands': bandsId,
|
||||
'filePerBand': 'false',
|
||||
'name': 'data',
|
||||
}))
|
||||
local_zip, headers = urlretrieve(url)
|
||||
with zipfile.ZipFile(local_zip) as local_zipfile:
|
||||
return local_zipfile.extract('data.tif', filepath)
|
||||
|
||||
# select collection
|
||||
input_col = ee.ImageCollection('LANDSAT/LE07/C01/T1_RT_TOA')
|
||||
# location (Narrabeen-Collaroy beach)
|
||||
rect_narra = [[[151.301454, -33.700754],
|
||||
[151.311453, -33.702075],
|
||||
[151.307237, -33.739761],
|
||||
[151.294220, -33.736329],
|
||||
[151.301454, -33.700754]]];
|
||||
|
||||
# dates
|
||||
#start_date = '2016-01-01'
|
||||
#end_date = '2016-12-31'
|
||||
# filter by location
|
||||
flt_col = input_col.filterBounds(ee.Geometry.Polygon(rect_narra))#.filterDate(start_date, end_date)
|
||||
|
||||
n_img = flt_col.size().getInfo()
|
||||
print('Number of images covering Narrabeen:', n_img)
|
||||
im_all = flt_col.getInfo().get('features')
|
||||
|
||||
satname = 'L7'
|
||||
sitename = 'NARRA'
|
||||
suffix = '.tif'
|
||||
filepath = os.path.join(os.getcwd(), 'data', satname, sitename)
|
||||
filepath_pan = os.path.join(filepath, 'pan')
|
||||
filepath_ms = os.path.join(filepath, 'ms')
|
||||
|
||||
all_names_pan = []
|
||||
all_names_ms = []
|
||||
timestamps = []
|
||||
# loop through all images
|
||||
for i in range(n_img):
|
||||
# find each image in ee database
|
||||
im = ee.Image(im_all[i].get('id'))
|
||||
im_dic = im.getInfo()
|
||||
im_bands = im_dic.get('bands')
|
||||
im_date = im_dic['properties']['DATE_ACQUIRED']
|
||||
t = im_dic['properties']['system:time_start']
|
||||
im_timestamp = datetime.fromtimestamp(t/1000, tz=pytz.utc)
|
||||
timestamps.append(im_timestamp)
|
||||
im_epsg = int(im_dic['bands'][0]['crs'][5:])
|
||||
|
||||
# delete dimensions key from dictionnary, otherwise the entire image is extracted
|
||||
for j in range(len(im_bands)): del im_bands[j]['dimensions']
|
||||
pan_band = [im_bands[7]]
|
||||
ms_bands = [im_bands[0], im_bands[1], im_bands[2], im_bands[3], im_bands[4], im_bands[9]]
|
||||
|
||||
filename_pan = satname + '_' + sitename + '_' + im_date + '_pan' + suffix
|
||||
filename_ms = satname + '_' + sitename + '_' + im_date + '_ms' + suffix
|
||||
|
||||
print(i)
|
||||
if any(filename_pan in _ for _ in all_names_pan):
|
||||
filename_pan = satname + '_' + sitename + '_' + im_date + '_pan' + '_r' + suffix
|
||||
filename_ms = satname + '_' + sitename + '_' + im_date + '_ms' + '_r' + suffix
|
||||
all_names_pan.append(filename_pan)
|
||||
|
||||
local_data_pan = download_tif(im, rect_narra, pan_band, filepath_pan)
|
||||
os.rename(local_data_pan, os.path.join(filepath_pan, filename_pan))
|
||||
local_data_ms = download_tif(im, rect_narra, ms_bands, filepath_ms)
|
||||
os.rename(local_data_ms, os.path.join(filepath_ms, filename_ms))
|
||||
|
||||
|
||||
with open(os.path.join(filepath, sitename + '_timestamps' + '.pkl'), 'wb') as f:
|
||||
pickle.dump(timestamps, f)
|
||||
with open(os.path.join(filepath, sitename + '_epsgcode' + '.pkl'), 'wb') as f:
|
||||
pickle.dump(im_epsg, f)
|
||||
|
||||
@ -1,177 +0,0 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
#==========================================================#
|
||||
# Download L8 images of a given area between given dates
|
||||
#==========================================================#
|
||||
|
||||
# Initial settings
|
||||
import os
|
||||
import numpy as np
|
||||
import matplotlib.pyplot as plt
|
||||
import pdb
|
||||
import ee
|
||||
|
||||
# other modules
|
||||
from osgeo import gdal, ogr, osr
|
||||
from urllib.request import urlretrieve
|
||||
import zipfile
|
||||
from datetime import datetime
|
||||
import pytz
|
||||
import pickle
|
||||
|
||||
# image processing modules
|
||||
import skimage.filters as filters
|
||||
import skimage.exposure as exposure
|
||||
import skimage.transform as transform
|
||||
import sklearn.decomposition as decomposition
|
||||
import skimage.measure as measure
|
||||
|
||||
# import own modules
|
||||
import functions.utils as utils
|
||||
|
||||
np.seterr(all='ignore') # raise/ignore divisions by 0 and nans
|
||||
ee.Initialize()
|
||||
|
||||
def download_tif(image, polygon, bandsId, filepath):
|
||||
"""downloads tif image (region and bands) from the ee server and stores it in a temp file"""
|
||||
url = ee.data.makeDownloadUrl(ee.data.getDownloadId({
|
||||
'image': image.serialize(),
|
||||
'region': polygon,
|
||||
'bands': bandsId,
|
||||
'filePerBand': 'false',
|
||||
'name': 'data',
|
||||
}))
|
||||
local_zip, headers = urlretrieve(url)
|
||||
with zipfile.ZipFile(local_zip) as local_zipfile:
|
||||
return local_zipfile.extract('data.tif', filepath)
|
||||
|
||||
# select collection
|
||||
input_col = ee.ImageCollection('LANDSAT/LC08/C01/T1_RT_TOA')
|
||||
# Location (Narrabeen all)
|
||||
#polygon = [[[151.3473129272461,-33.69035274454718],
|
||||
# [151.2820816040039,-33.68206818063878],
|
||||
# [151.27281188964844,-33.74775138989556],
|
||||
# [151.3425064086914,-33.75231878701767],
|
||||
# [151.3473129272461,-33.69035274454718]]];
|
||||
# location (Narrabeen-Collaroy beach)
|
||||
#polygon = [[[151.301454, -33.700754],
|
||||
# [151.311453, -33.702075],
|
||||
# [151.307237, -33.739761],
|
||||
# [151.294220, -33.736329],
|
||||
# [151.301454, -33.700754]]];
|
||||
# location (Oldbar beach)
|
||||
#polygon = [[[152.664508, -31.896163],
|
||||
# [152.665827, -31.897112],
|
||||
# [152.631516, -31.924846],
|
||||
# [152.629285, -31.923362],
|
||||
# [152.664508, -31.896163]]]
|
||||
# location (Oldbar inlet)
|
||||
#polygon = [[[152.676283, -31.866784],
|
||||
# [152.709174, -31.869993],
|
||||
# [152.678229, -31.892082],
|
||||
# [152.670366, -31.886360],
|
||||
# [152.676283, -31.866784]]];
|
||||
# Location (Sand Engine)
|
||||
#polygon = [[[4.171742, 52.070455],
|
||||
# [4.223708, 52.069576],
|
||||
# [4.220808, 52.025293],
|
||||
# [4.147749, 52.028861],
|
||||
# [4.171742, 52.070455]]];
|
||||
# Location (Tairua)
|
||||
#polygon = [[[175.852115, -36.985414],
|
||||
# [175.872797, -36.985145],
|
||||
# [175.873738, -37.000039],
|
||||
# [175.853956, -36.998749],
|
||||
# [175.852115, -36.985414]]];
|
||||
# Location (Duck)
|
||||
#polygon = [[[-75.766220, 36.195928],
|
||||
# [-75.748282, 36.196401],
|
||||
# [-75.738851, 36.173974],
|
||||
# [-75.763546, 36.174249],
|
||||
# [-75.766220, 36.195928]]];
|
||||
# Location (Broulee Island)
|
||||
#polygon = [[[150.173557, -35.847138],
|
||||
# [150.196164, -35.848064],
|
||||
# [150.195143, -35.869967],
|
||||
# [150.172779, -35.861760],
|
||||
# [150.173557, -35.847138]]];
|
||||
# Location (Rarotonga, Muri lagoon)
|
||||
polygon = [[[-159.732071, -21.241348],
|
||||
[-159.719820, -21.242892],
|
||||
[-159.720006, -21.261134],
|
||||
[-159.731592, -21.258875],
|
||||
[-159.732071, -21.241348]]];
|
||||
|
||||
# dates
|
||||
start_date = '2013-01-01'
|
||||
end_date = '2019-01-01'
|
||||
# filter by location
|
||||
flt_col = input_col.filterBounds(ee.Geometry.Polygon(polygon)).filterDate(start_date, end_date)
|
||||
|
||||
n_img = flt_col.size().getInfo()
|
||||
print('Number of images covering the area:', n_img)
|
||||
im_all = flt_col.getInfo().get('features')
|
||||
|
||||
satname = 'L8'
|
||||
#sitename = 'NARRA_all'
|
||||
#sitename = 'NARRA'
|
||||
#sitename = 'OLDBAR'
|
||||
#sitename = 'SANDMOTOR'
|
||||
#sitename = 'TAIRUA'
|
||||
#sitename = 'DUCK'
|
||||
#sitename = 'BROULEE'
|
||||
sitename = 'MURI'
|
||||
|
||||
|
||||
suffix = '.tif'
|
||||
filepath = os.path.join(os.getcwd(), 'data', satname, sitename)
|
||||
filepath_pan = os.path.join(filepath, 'pan')
|
||||
filepath_ms = os.path.join(filepath, 'ms')
|
||||
|
||||
all_names_pan = []
|
||||
all_names_ms = []
|
||||
timestamps = []
|
||||
acc_georef = []
|
||||
# loop through all images
|
||||
for i in range(n_img):
|
||||
# find each image in ee database
|
||||
im = ee.Image(im_all[i].get('id'))
|
||||
im_dic = im.getInfo()
|
||||
im_bands = im_dic.get('bands')
|
||||
im_date = im_dic['properties']['DATE_ACQUIRED']
|
||||
t = im_dic['properties']['system:time_start']
|
||||
im_timestamp = datetime.fromtimestamp(t/1000, tz=pytz.utc)
|
||||
timestamps.append(im_timestamp)
|
||||
im_epsg = int(im_dic['bands'][0]['crs'][5:])
|
||||
try:
|
||||
acc_georef.append(im_dic['properties']['GEOMETRIC_RMSE_MODEL'])
|
||||
except:
|
||||
acc_georef.append(10)
|
||||
print('No geometric rmse model property')
|
||||
|
||||
# delete dimensions key from dictionnary, otherwise the entire image is extracted
|
||||
for j in range(len(im_bands)): del im_bands[j]['dimensions']
|
||||
pan_band = [im_bands[7]]
|
||||
ms_bands = [im_bands[1], im_bands[2], im_bands[3], im_bands[4], im_bands[5], im_bands[11]]
|
||||
|
||||
filename_pan = satname + '_' + sitename + '_' + im_date + '_pan' + suffix
|
||||
filename_ms = satname + '_' + sitename + '_' + im_date + '_ms' + suffix
|
||||
|
||||
print(i)
|
||||
if any(filename_pan in _ for _ in all_names_pan):
|
||||
filename_pan = satname + '_' + sitename + '_' + im_date + '_pan' + '_r' + suffix
|
||||
filename_ms = satname + '_' + sitename + '_' + im_date + '_ms' + '_r' + suffix
|
||||
all_names_pan.append(filename_pan)
|
||||
|
||||
local_data_pan = download_tif(im, polygon, pan_band, filepath_pan)
|
||||
os.rename(local_data_pan, os.path.join(filepath_pan, filename_pan))
|
||||
local_data_ms = download_tif(im, polygon, ms_bands, filepath_ms)
|
||||
os.rename(local_data_ms, os.path.join(filepath_ms, filename_ms))
|
||||
|
||||
|
||||
with open(os.path.join(filepath, sitename + '_timestamps' + '.pkl'), 'wb') as f:
|
||||
pickle.dump(timestamps, f)
|
||||
with open(os.path.join(filepath, sitename + '_epsgcode' + '.pkl'), 'wb') as f:
|
||||
pickle.dump(im_epsg, f)
|
||||
with open(os.path.join(filepath, sitename + '_accuracy_georef' + '.pkl'), 'wb') as f:
|
||||
pickle.dump(acc_georef, f)
|
||||
@ -1,106 +0,0 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
#==========================================================#
|
||||
# Draw reference points on satellite image
|
||||
#==========================================================#
|
||||
|
||||
# Preamble
|
||||
import os
|
||||
import ee
|
||||
import matplotlib.pyplot as plt
|
||||
import matplotlib.cm as cm
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
from datetime import datetime
|
||||
import pickle
|
||||
import pdb
|
||||
import pytz
|
||||
from pylab import ginput
|
||||
|
||||
# image processing modules
|
||||
import skimage.filters as filters
|
||||
import skimage.exposure as exposure
|
||||
import skimage.transform as transform
|
||||
import sklearn.decomposition as decomposition
|
||||
import skimage.morphology as morphology
|
||||
import skimage.measure as measure
|
||||
|
||||
# my functions
|
||||
import functions.utils as utils
|
||||
import functions.sds as sds
|
||||
|
||||
np.seterr(all='ignore') # raise/ignore divisions by 0 and nans
|
||||
ee.Initialize()
|
||||
|
||||
# collection
|
||||
input_col = ee.ImageCollection('LANDSAT/LC08/C01/T1_RT_TOA')
|
||||
|
||||
# location (Narrabeen-Collaroy beach)
|
||||
#polygon = [[[151.301454, -33.700754],
|
||||
# [151.311453, -33.702075],
|
||||
# [151.307237, -33.739761],
|
||||
# [151.294220, -33.736329],
|
||||
# [151.301454, -33.700754]]];
|
||||
# location (Oldbar shoreline)
|
||||
#polygon = [[[152.664508, -31.896163],
|
||||
# [152.665827, -31.897112],
|
||||
# [152.631516, -31.924846],
|
||||
# [152.629285, -31.923362],
|
||||
# [152.664508, -31.896163]]];
|
||||
# location (Oldbar inlet)
|
||||
polygon = [[[152.676283, -31.866784],
|
||||
[152.709174, -31.869993],
|
||||
[152.678229, -31.892082],
|
||||
[152.670366, -31.886360],
|
||||
[152.676283, -31.866784]]];
|
||||
|
||||
# dates
|
||||
start_date = '2017-01-30'
|
||||
end_date = '2017-02-02'
|
||||
#start_date = '2017-01-30'
|
||||
#end_date = '2018-02-02'
|
||||
# filter by location
|
||||
flt_col = input_col.filterBounds(ee.Geometry.Polygon(polygon)).filterDate(start_date, end_date)
|
||||
|
||||
n_img = flt_col.size().getInfo()
|
||||
print('Number of images covering the area:', n_img)
|
||||
im_all = flt_col.getInfo().get('features')
|
||||
|
||||
satname = 'L8'
|
||||
#sitename = 'NARRA'
|
||||
sitename = 'OLDBAR_inlet'
|
||||
|
||||
# parameters
|
||||
plot_bool = False # if you want the plots
|
||||
prob_high = 99.9 # upper probability to clip and rescale pixel intensity
|
||||
min_contour_points = 100 # minimum number of points contained in each water line
|
||||
output_epsg = 28356 # GDA94 / MGA Zone 56
|
||||
cloud_threshold = 0.8
|
||||
|
||||
# find image in ee database
|
||||
im = ee.Image(im_all[0].get('id'))
|
||||
# load image as np.array
|
||||
im_pan, im_ms, im_cloud, crs, meta = sds.read_eeimage(im, polygon, satname, plot_bool)
|
||||
|
||||
# rescale intensities
|
||||
im_ms = sds.rescale_image_intensity(im_ms, im_cloud, prob_high, plot_bool)
|
||||
im_pan = sds.rescale_image_intensity(im_pan, im_cloud, prob_high, plot_bool)
|
||||
|
||||
# pansharpen rgb image
|
||||
im_ms_ps = sds.pansharpen(im_ms[:,:,[0,1,2]], im_pan, im_cloud, plot_bool)
|
||||
|
||||
plt.figure()
|
||||
plt.imshow(im_ms_ps[:,:,[2,1,0]])
|
||||
plt.show()
|
||||
|
||||
pts = ginput(n=50, timeout=1000, show_clicks=True)
|
||||
points = np.array(pts)
|
||||
plt.plot(points[:,0], points[:,1], 'ko')
|
||||
plt.show()
|
||||
|
||||
pts_coords = sds.convert_pix2world(points[:,[1,0]], crs['crs_15m'])
|
||||
pts = sds.convert_epsg(pts_coords, crs['epsg_code'], output_epsg)
|
||||
|
||||
filepath = os.path.join(os.getcwd(), 'data', satname, sitename)
|
||||
with open(os.path.join(filepath, sitename + '_refpoints2.pkl'), 'wb') as f:
|
||||
pickle.dump(pts, f)
|
||||
@ -1,322 +0,0 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
#==========================================================#
|
||||
# Extract shorelines from Landsat images
|
||||
#==========================================================#
|
||||
|
||||
# Initial settings
|
||||
import os
|
||||
import numpy as np
|
||||
import matplotlib.pyplot as plt
|
||||
import ee
|
||||
import pdb
|
||||
|
||||
# other modules
|
||||
from osgeo import gdal, ogr, osr
|
||||
import pickle
|
||||
import matplotlib.cm as cm
|
||||
from pylab import ginput
|
||||
|
||||
# image processing modules
|
||||
import skimage.filters as filters
|
||||
import skimage.exposure as exposure
|
||||
import skimage.transform as transform
|
||||
import sklearn.decomposition as decomposition
|
||||
import skimage.measure as measure
|
||||
import skimage.morphology as morphology
|
||||
|
||||
# machine learning modules
|
||||
from sklearn.model_selection import train_test_split
|
||||
from sklearn.neural_network import MLPClassifier
|
||||
from sklearn.preprocessing import StandardScaler, Normalizer
|
||||
from sklearn.externals import joblib
|
||||
|
||||
# import own modules
|
||||
import functions.utils as utils
|
||||
import functions.sds as sds
|
||||
|
||||
# some settings
|
||||
np.seterr(all='ignore') # raise/ignore divisions by 0 and nans
|
||||
plt.rcParams['axes.grid'] = True
|
||||
plt.rcParams['figure.max_open_warning'] = 100
|
||||
ee.Initialize()
|
||||
|
||||
# parameters
|
||||
cloud_thresh = 0.3 # threshold for cloud cover
|
||||
plot_bool = False # if you want the plots
|
||||
min_contour_points = 100# minimum number of points contained in each water line
|
||||
output_epsg = 28356 # GDA94 / MGA Zone 56
|
||||
buffer_size = 10 # radius (in pixels) of disk for buffer (pixel classification)
|
||||
min_beach_size = 30 # number of pixels in a beach (pixel classification)
|
||||
|
||||
# load metadata (timestamps and epsg code) for the collection
|
||||
satname = 'L8'
|
||||
sitename = 'NARRA'
|
||||
#sitename = 'OLDBAR'
|
||||
|
||||
# Load metadata
|
||||
filepath = os.path.join(os.getcwd(), 'data', satname, sitename)
|
||||
with open(os.path.join(filepath, sitename + '_timestamps' + '.pkl'), 'rb') as f:
|
||||
timestamps = pickle.load(f)
|
||||
with open(os.path.join(filepath, sitename + '_accuracy_georef' + '.pkl'), 'rb') as f:
|
||||
acc_georef = pickle.load(f)
|
||||
with open(os.path.join(filepath, sitename + '_epsgcode' + '.pkl'), 'rb') as f:
|
||||
input_epsg = pickle.load(f)
|
||||
with open(os.path.join(filepath, sitename + '_refpoints' + '.pkl'), 'rb') as f:
|
||||
refpoints = pickle.load(f)
|
||||
# sort timestamps and georef accuracy (dowloaded images are sorted by date in directory)
|
||||
timestamps_sorted = sorted(timestamps)
|
||||
idx_sorted = sorted(range(len(timestamps)), key=timestamps.__getitem__)
|
||||
acc_georef_sorted = [acc_georef[j] for j in idx_sorted]
|
||||
|
||||
# path to images
|
||||
file_path_pan = os.path.join(os.getcwd(), 'data', satname, sitename, 'pan')
|
||||
file_path_ms = os.path.join(os.getcwd(), 'data', satname, sitename, 'ms')
|
||||
file_names_pan = os.listdir(file_path_pan)
|
||||
file_names_ms = os.listdir(file_path_ms)
|
||||
N = len(file_names_pan)
|
||||
|
||||
# initialise some variables
|
||||
cloud_cover_ts = []
|
||||
date_acquired_ts = []
|
||||
acc_georef_ts = []
|
||||
idx_skipped = []
|
||||
idx_nocloud = []
|
||||
t = []
|
||||
shorelines = []
|
||||
|
||||
#%%
|
||||
for i in [20]:#range(N):
|
||||
# read pan image
|
||||
fn_pan = os.path.join(file_path_pan, file_names_pan[i])
|
||||
data = gdal.Open(fn_pan, gdal.GA_ReadOnly)
|
||||
georef = np.array(data.GetGeoTransform())
|
||||
bands = [data.GetRasterBand(i + 1).ReadAsArray() for i in range(data.RasterCount)]
|
||||
im_pan = np.stack(bands, 2)[:,:,0]
|
||||
nrows = im_pan.shape[0]
|
||||
ncols = im_pan.shape[1]
|
||||
|
||||
# read ms image
|
||||
fn_ms = os.path.join(file_path_ms, file_names_ms[i])
|
||||
data = gdal.Open(fn_ms, gdal.GA_ReadOnly)
|
||||
bands = [data.GetRasterBand(i + 1).ReadAsArray() for i in range(data.RasterCount)]
|
||||
im_ms = np.stack(bands, 2)
|
||||
|
||||
# cloud mask
|
||||
im_qa = im_ms[:,:,5]
|
||||
cloud_mask = sds.create_cloud_mask(im_qa, satname, plot_bool)
|
||||
cloud_mask = transform.resize(cloud_mask, (im_pan.shape[0], im_pan.shape[1]),
|
||||
order=0, preserve_range=True,
|
||||
mode='constant').astype('bool_')
|
||||
# resize the image using bilinear interpolation (order 1)
|
||||
im_ms = transform.resize(im_ms,(im_pan.shape[0], im_pan.shape[1]),
|
||||
order=1, preserve_range=True, mode='constant')
|
||||
|
||||
# check if -inf or nan values and add to cloud mask
|
||||
im_inf = np.isin(im_ms[:,:,0], -np.inf)
|
||||
im_nan = np.isnan(im_ms[:,:,0])
|
||||
cloud_mask = np.logical_or(np.logical_or(cloud_mask, im_inf), im_nan)
|
||||
|
||||
# calculate cloud cover and skip image if too high
|
||||
cloud_cover = sum(sum(cloud_mask.astype(int)))/(cloud_mask.shape[0]*cloud_mask.shape[1])
|
||||
if cloud_cover > cloud_thresh:
|
||||
print('skip ' + str(i) + ' - cloudy (' + str(cloud_cover) + ')')
|
||||
idx_skipped.append(i)
|
||||
continue
|
||||
idx_nocloud.append(i)
|
||||
|
||||
# check if image for that date already exists and choose the best in terms of cloud cover and georeferencing
|
||||
if file_names_pan[i][len(satname)+1+len(sitename)+1:len(satname)+1+len(sitename)+1+10] in date_acquired_ts:
|
||||
|
||||
# find the index of the image that is repeated
|
||||
idx_samedate = utils.find_indices(date_acquired_ts, lambda e : e == file_names_pan[i][9:19])
|
||||
idx_samedate = idx_samedate[0]
|
||||
print('cloud cover ' + str(cloud_cover) + ' - ' + str(cloud_cover_ts[idx_samedate]))
|
||||
print('acc georef ' + str(acc_georef_sorted[i]) + ' - ' + str(acc_georef_ts[idx_samedate]))
|
||||
|
||||
# keep image with less cloud cover or best georeferencing accuracy
|
||||
if cloud_cover < cloud_cover_ts[idx_samedate] - 0.01:
|
||||
skip = False
|
||||
elif acc_georef_sorted[i] < acc_georef_ts[idx_samedate]:
|
||||
skip = False
|
||||
else:
|
||||
skip = True
|
||||
|
||||
if skip:
|
||||
print('skip ' + str(i) + ' - repeated')
|
||||
idx_skipped.append(i)
|
||||
continue
|
||||
else:
|
||||
del shorelines[idx_samedate]
|
||||
del t[idx_samedate]
|
||||
del cloud_cover_ts[idx_samedate]
|
||||
del date_acquired_ts[idx_samedate]
|
||||
del acc_georef_ts[idx_samedate]
|
||||
print('keep ' + str(i) + ' - deleted ' + str(idx_samedate))
|
||||
|
||||
# pansharpen rgb image
|
||||
im_ms_ps = sds.pansharpen(im_ms[:,:,[0,1,2]], im_pan, cloud_mask, plot_bool)
|
||||
# rescale pansharpened RGB for visualisation
|
||||
im_display = sds.rescale_image_intensity(im_ms_ps[:,:,[2,1,0]], cloud_mask, 100, False)
|
||||
# add down-sized bands for NIR and SWIR (since pansharpening is not possible)
|
||||
im_ms_ps = np.append(im_ms_ps, im_ms[:,:,[3,4]], axis=2)
|
||||
|
||||
# classify image in 4 classes (sand, whitewater, water, other) with NN classifier
|
||||
im_classif, im_labels = sds.classify_image_NN(im_ms_ps, im_pan, cloud_mask, min_beach_size, True)
|
||||
|
||||
t.append(timestamps_sorted[i])
|
||||
cloud_cover_ts.append(cloud_cover)
|
||||
acc_georef_ts.append(acc_georef_sorted[i])
|
||||
date_acquired_ts.append(file_names_pan[i][9:19])
|
||||
|
||||
# labels
|
||||
im_sand = im_classif == 1
|
||||
im_swash = im_classif == 2
|
||||
im_water = im_classif == 3
|
||||
vec_sand = im_sand.reshape(ncols*nrows)
|
||||
vec_water = im_water.reshape(ncols*nrows)
|
||||
vec_swash = im_swash.reshape(ncols*nrows)
|
||||
|
||||
# calculate indices and stack into a vector
|
||||
im_ndwi = sds.nd_index(im_ms_ps[:,:,3], im_ms_ps[:,:,1], cloud_mask, plot_bool)
|
||||
im_ndmwi = sds.nd_index(im_ms_ps[:,:,4], im_ms_ps[:,:,1], cloud_mask, plot_bool)
|
||||
im_nir = im_ms_ps[:,:,3]
|
||||
im_swir = im_ms_ps[:,:,4]
|
||||
im_ind = np.stack((im_ndwi, im_ndmwi), axis=-1)
|
||||
vec_ind = im_ind.reshape(nrows*ncols,2)
|
||||
|
||||
# remove noise and only keep the sand belonging to large beaches
|
||||
morphology.remove_small_objects(im_sand, min_size=50, connectivity=2, in_place=True)
|
||||
# create a buffer around beach
|
||||
buffer_size = 7
|
||||
se = morphology.disk(buffer_size)
|
||||
im_buffer = morphology.binary_dilation(im_sand, se)
|
||||
vec_buffer = im_buffer.reshape(nrows*ncols)
|
||||
|
||||
# display buffer
|
||||
im = np.copy(im_display)
|
||||
im[~im_buffer,0] = 1
|
||||
im[~im_buffer,1] = 1
|
||||
im[~im_buffer,2] = 1
|
||||
|
||||
im2 = np.copy(im_ndmwi)
|
||||
im2[~im_buffer] = np.nan
|
||||
|
||||
plt.figure()
|
||||
ax1 = plt.subplot(121)
|
||||
plt.imshow(im)
|
||||
plt.axis('off')
|
||||
plt.title('RGB')
|
||||
ax2 = plt.subplot(122, sharex=ax1, sharey=ax1)
|
||||
plt.imshow(im2, cmap='seismic')
|
||||
plt.colorbar()
|
||||
plt.axis('off')
|
||||
plt.title('Water Index')
|
||||
plt.tight_layout()
|
||||
plt.draw()
|
||||
|
||||
# select water/sand/swash pixels that are within the buffer
|
||||
int_water = vec_ind[np.logical_and(vec_buffer,vec_water),:]
|
||||
int_sand = vec_ind[np.logical_and(vec_buffer,vec_sand),:]
|
||||
int_swash = vec_ind[np.logical_and(vec_buffer,vec_swash),:]
|
||||
|
||||
# append sand and water
|
||||
int_all = np.append(int_water,int_sand, axis=0)
|
||||
t_ndwi = filters.threshold_otsu(int_all[:,0])
|
||||
t_ndmwi = filters.threshold_otsu(int_all[:,1])
|
||||
|
||||
|
||||
fig, ax = plt.subplots(2,1, sharex=True)
|
||||
vals = ax[0].hist(int_water[:,0], bins=100, label='water')
|
||||
ax[0].hist(int_sand[:,0], bins=100, alpha=0.5, label='sand')
|
||||
ax[0].hist(int_swash[:,0], bins=100, alpha=0.5, label='swash')
|
||||
ax[0].plot([t_ndwi, t_ndwi], [0, np.max(vals[0])], 'r-')
|
||||
ax[0].legend()
|
||||
ax[0].set_title('Water Index NIR-G')
|
||||
vals = ax[1].hist(int_water[:,1], bins=100, label='water')
|
||||
ax[1].hist(int_sand[:,1], bins=100, alpha=0.5, label='sand')
|
||||
ax[1].hist(int_swash[:,1], bins=100, alpha=0.5, label='swash')
|
||||
ax[1].plot([t_ndmwi, t_ndmwi], [0, np.max(vals[0])], 'r-')
|
||||
ax[1].legend()
|
||||
ax[1].set_title('Modified Water Index SWIR-G')
|
||||
plt.draw()
|
||||
|
||||
im_ndwi_buffer = np.copy(im_ndwi)
|
||||
im_ndwi_buffer[~im_buffer] = np.nan
|
||||
|
||||
contours1 = measure.find_contours(im_ndwi_buffer, t_ndwi)
|
||||
|
||||
im_ndmwi_buffer = np.copy(im_ndmwi)
|
||||
im_ndmwi_buffer[~im_buffer] = np.nan
|
||||
|
||||
contours2 = measure.find_contours(im_ndmwi_buffer, t_ndmwi)
|
||||
|
||||
plt.figure()
|
||||
ax1 = plt.subplot(1,3,1)
|
||||
im = np.copy(im_display)
|
||||
# define colours for plot
|
||||
colours = np.array([[1,128/255,0/255],[204/255,1,1],[0,0,204/255]])
|
||||
for k in range(0,im_labels.shape[2]):
|
||||
im[im_labels[:,:,k],0] = colours[k,0]
|
||||
im[im_labels[:,:,k],1] = colours[k,1]
|
||||
im[im_labels[:,:,k],2] = colours[k,2]
|
||||
plt.imshow(im)
|
||||
for i,contour in enumerate(contours2): plt.plot(contour[:, 1], contour[:, 0], linewidth=3, color='k')
|
||||
plt.tight_layout()
|
||||
plt.grid(False)
|
||||
plt.draw()
|
||||
|
||||
plt.subplot(1,3,2, sharex=ax1, sharey=ax1)
|
||||
plt.imshow(im_display)
|
||||
for i,contour in enumerate(contours2): plt.plot(contour[:, 1], contour[:, 0], linewidth=3, color='k')
|
||||
plt.tight_layout()
|
||||
plt.grid(False)
|
||||
plt.draw()
|
||||
|
||||
plt.subplot(1,3,3, sharex=ax1, sharey=ax1)
|
||||
plt.imshow(im_ndmwi, cmap='seismic')
|
||||
plt.colorbar()
|
||||
for i,contour in enumerate(contours2): plt.plot(contour[:, 1], contour[:, 0], linewidth=3, color='k')
|
||||
plt.tight_layout()
|
||||
plt.grid(False)
|
||||
plt.draw()
|
||||
|
||||
|
||||
# plot of all the indices
|
||||
plt.figure()
|
||||
ax1 = plt.subplot(1,5,1)
|
||||
plt.imshow(im_display)
|
||||
plt.xticks([])
|
||||
plt.yticks([])
|
||||
plt.axis('off')
|
||||
plt.title('RGB')
|
||||
plt.subplot(1,5,2, sharex=ax1, sharey=ax1)
|
||||
plt.imshow(im_ndwi, cmap='seismic')
|
||||
plt.xticks([])
|
||||
plt.yticks([])
|
||||
plt.axis('off')
|
||||
plt.title('NDWI')
|
||||
plt.subplot(1,5,3, sharex=ax1, sharey=ax1)
|
||||
plt.imshow(im_ndmwi, cmap='seismic')
|
||||
plt.xticks([])
|
||||
plt.yticks([])
|
||||
plt.axis('off')
|
||||
plt.title('NDMWI')
|
||||
plt.subplot(1,5,4, sharex=ax1, sharey=ax1)
|
||||
plt.imshow(im_nir, cmap='seismic')
|
||||
plt.xticks([])
|
||||
plt.yticks([])
|
||||
plt.axis('off')
|
||||
plt.title('NIR')
|
||||
plt.subplot(1,5,5, sharex=ax1, sharey=ax1)
|
||||
plt.imshow(im_swir, cmap='seismic')
|
||||
plt.xticks([])
|
||||
plt.yticks([])
|
||||
plt.axis('off')
|
||||
plt.title('SWIR')
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
Binary file not shown.
@ -0,0 +1,432 @@
|
||||
"""This module contains all the functions needed for data analysis """
|
||||
|
||||
# Initial settings
|
||||
import numpy as np
|
||||
import matplotlib.pyplot as plt
|
||||
import matplotlib.patches as mpatches
|
||||
from matplotlib import gridspec
|
||||
import pdb
|
||||
import ee
|
||||
|
||||
# other modules
|
||||
from osgeo import gdal, ogr, osr
|
||||
import scipy.interpolate as interpolate
|
||||
import scipy.stats as sstats
|
||||
|
||||
# image processing modules
|
||||
import skimage.filters as filters
|
||||
import skimage.exposure as exposure
|
||||
import skimage.transform as transform
|
||||
import sklearn.decomposition as decomposition
|
||||
import skimage.measure as measure
|
||||
import skimage.morphology as morphology
|
||||
|
||||
# machine learning modules
|
||||
from sklearn.cluster import KMeans
|
||||
from sklearn.neural_network import MLPClassifier
|
||||
from sklearn.externals import joblib
|
||||
|
||||
import time
|
||||
|
||||
# import own modules
|
||||
import functions.utils as utils
|
||||
|
||||
def get_tide(dates_sds, dates_tide, tide_level):
|
||||
|
||||
tide = []
|
||||
for i in range(len(dates_sds)):
|
||||
dates_diff = np.abs(np.array([ (dates_sds[i] - _).total_seconds() for _ in dates_tide]))
|
||||
if np.min(dates_diff) <= 1800: # half-an-hour
|
||||
idx_closest = np.argmin(dates_diff)
|
||||
tide.append(tide_level[idx_closest])
|
||||
else:
|
||||
tide.append(np.nan)
|
||||
tide = np.array(tide)
|
||||
|
||||
return tide
|
||||
|
||||
def remove_duplicates(output, satname):
|
||||
" removes duplicates from output structure, keep the one with less cloud cover or best georeferencing "
|
||||
dates = output['dates']
|
||||
dates_str = [_.strftime('%Y%m%d') for _ in dates]
|
||||
dupl = utils.duplicates_dict(dates_str)
|
||||
if dupl:
|
||||
output_nodup = dict([])
|
||||
idx_remove = []
|
||||
if satname == 'L8' or satname == 'L5':
|
||||
for k,v in dupl.items():
|
||||
|
||||
idx1 = v[0]
|
||||
idx2 = v[1]
|
||||
|
||||
c1 = output['metadata']['cloud_cover'][idx1]
|
||||
c2 = output['metadata']['cloud_cover'][idx2]
|
||||
g1 = output['metadata']['acc_georef'][idx1]
|
||||
g2 = output['metadata']['acc_georef'][idx2]
|
||||
|
||||
if c1 < c2 - 0.01:
|
||||
idx_remove.append(idx2)
|
||||
elif g1 < g2 - 0.1:
|
||||
idx_remove.append(idx2)
|
||||
else:
|
||||
idx_remove.append(idx1)
|
||||
|
||||
else:
|
||||
for k,v in dupl.items():
|
||||
|
||||
idx1 = v[0]
|
||||
idx2 = v[1]
|
||||
|
||||
c1 = output['metadata']['cloud_cover'][idx1]
|
||||
c2 = output['metadata']['cloud_cover'][idx2]
|
||||
|
||||
if c1 < c2 - 0.01:
|
||||
idx_remove.append(idx2)
|
||||
else:
|
||||
idx_remove.append(idx1)
|
||||
|
||||
idx_remove = sorted(idx_remove)
|
||||
idx_all = np.linspace(0, len(dates_str)-1, len(dates_str))
|
||||
idx_keep = list(np.where(~np.isin(idx_all,idx_remove))[0])
|
||||
|
||||
output_nodup['dates'] = [output['dates'][k] for k in idx_keep]
|
||||
output_nodup['shorelines'] = [output['shorelines'][k] for k in idx_keep]
|
||||
output_nodup['metadata'] = dict([])
|
||||
for key in list(output['metadata'].keys()):
|
||||
output_nodup['metadata'][key] = [output['metadata'][key][k] for k in idx_keep]
|
||||
print(satname + ' : ' + str(len(idx_remove)) + ' duplicates')
|
||||
return output_nodup
|
||||
|
||||
else:
|
||||
print(satname + ' : ' + 'no duplicates')
|
||||
return output
|
||||
|
||||
|
||||
def merge(output):
|
||||
" merges data from the different satellites "
|
||||
|
||||
# stack all list together under one key
|
||||
output_all = {'dates':[], 'shorelines':[],
|
||||
'metadata':{'filenames':[], 'satname':[], 'cloud_cover':[], 'acc_georef':[]}}
|
||||
for satname in list(output.keys()):
|
||||
output_all['dates'] = output_all['dates'] + output[satname]['dates']
|
||||
output_all['shorelines'] = output_all['shorelines'] + output[satname]['shorelines']
|
||||
for key in list(output[satname]['metadata'].keys()):
|
||||
output_all['metadata'][key] = output_all['metadata'][key] + output[satname]['metadata'][key]
|
||||
|
||||
output_all_sorted = {'dates':[], 'shorelines':[],
|
||||
'metadata':{'filenames':[], 'satname':[], 'cloud_cover':[], 'acc_georef':[]}}
|
||||
# sort the dates
|
||||
idx_sorted = sorted(range(len(output_all['dates'])), key=output_all['dates'].__getitem__)
|
||||
output_all_sorted['dates'] = [output_all['dates'][i] for i in idx_sorted]
|
||||
output_all_sorted['shorelines'] = [output_all['shorelines'][i] for i in idx_sorted]
|
||||
for key in list(output_all['metadata'].keys()):
|
||||
output_all_sorted['metadata'][key] = [output_all['metadata'][key][i] for i in idx_sorted]
|
||||
|
||||
return output_all_sorted
|
||||
|
||||
def create_transects(x0, y0, orientation, chainage_length):
|
||||
" creates shore-normal transects "
|
||||
|
||||
transects = []
|
||||
|
||||
for k in range(len(x0)):
|
||||
|
||||
# orientation of cross-shore profile
|
||||
phi = (90 - orientation[k])*np.pi/180
|
||||
|
||||
# create a vector using the chainage length
|
||||
x = np.linspace(0,chainage_length,chainage_length+1)
|
||||
y = np.zeros(len(x))
|
||||
coords = np.zeros((len(x),2))
|
||||
coords[:,0] = x
|
||||
coords[:,1] = y
|
||||
|
||||
# translate and rotate the vector using the origin and orientation
|
||||
tf = transform.EuclideanTransform(rotation=phi, translation=(x0[k],y0[k]))
|
||||
coords_tf = tf(coords)
|
||||
|
||||
transects.append(coords_tf)
|
||||
|
||||
return transects
|
||||
|
||||
def calculate_chainage(sds, transects, orientation, along_dist):
|
||||
" intersect SDS with transect and compute chainage position "
|
||||
|
||||
chainage_mtx = np.zeros((len(sds),len(transects),6))
|
||||
|
||||
for i in range(len(sds)):
|
||||
|
||||
sl = sds[i]
|
||||
|
||||
for j in range(len(transects)):
|
||||
|
||||
# compute rotation matrix
|
||||
X0 = transects[j][0,0]
|
||||
Y0 = transects[j][0,1]
|
||||
phi = (90 - orientation[j])*np.pi/180
|
||||
Mrot = np.array([[np.cos(phi), np.sin(phi)],[-np.sin(phi), np.cos(phi)]])
|
||||
|
||||
# calculate point to line distance between shoreline points and profile
|
||||
p1 = np.array([X0,Y0])
|
||||
p2 = transects[j][-1,:]
|
||||
p3 = sl
|
||||
d = np.abs(np.cross(p2-p1,p3-p1)/np.linalg.norm(p2-p1))
|
||||
idx_close = utils.find_indices(d, lambda e: e <= along_dist)
|
||||
|
||||
# check if there are SDS points around the profile or not
|
||||
if not idx_close:
|
||||
chainage_mtx[i,j,:] = np.tile(np.nan,(1,6))
|
||||
|
||||
else:
|
||||
# change of base to shore-normal coordinate system
|
||||
xy_close = np.array([sl[idx_close,0],sl[idx_close,1]]) - np.tile(np.array([[X0],[Y0]]), (1,len(sl[idx_close])))
|
||||
xy_rot = np.matmul(Mrot, xy_close)
|
||||
|
||||
# put nan values if the chainage is negative (MAKE SURE TO PICK ORIGIN CORRECTLY)
|
||||
if np.any(xy_rot[0,:] < 0):
|
||||
xy_rot[0,np.where(xy_rot[0,:] < 0)] = np.nan
|
||||
|
||||
# compute mean, median max and std of chainage position
|
||||
n_points = len(xy_rot[0,:])
|
||||
mean_cross = np.nanmean(xy_rot[0,:])
|
||||
median_cross = np.nanmedian(xy_rot[0,:])
|
||||
max_cross = np.nanmax(xy_rot[0,:])
|
||||
min_cross = np.nanmin(xy_rot[0,:])
|
||||
std_cross = np.nanstd(xy_rot[0,:])
|
||||
|
||||
if std_cross > 10: # if large std, take the most seaward point
|
||||
mean_cross = max_cross
|
||||
median_cross = max_cross
|
||||
min_cross = max_cross
|
||||
|
||||
# store the statistics
|
||||
chainage_mtx[i,j,:] = np.array([mean_cross, median_cross, max_cross,
|
||||
min_cross, n_points, std_cross])
|
||||
|
||||
# format into dictionnary
|
||||
chainage = dict([])
|
||||
chainage['mean'] = chainage_mtx[:,:,0]
|
||||
chainage['median'] = chainage_mtx[:,:,1]
|
||||
chainage['max'] = chainage_mtx[:,:,2]
|
||||
chainage['min'] = chainage_mtx[:,:,3]
|
||||
chainage['npoints'] = chainage_mtx[:,:,4]
|
||||
chainage['std'] = chainage_mtx[:,:,5]
|
||||
|
||||
return chainage
|
||||
|
||||
def compare_sds(dates_sds, chain_sds, topo_profiles, mod=0, mindays=5):
|
||||
"""
|
||||
Compare sds with groundtruth data from topographic surveys / argus shorelines
|
||||
|
||||
KV WRL 2018
|
||||
|
||||
Arguments:
|
||||
-----------
|
||||
dates_sds: list
|
||||
list of dates corresponding to each row in chain_sds
|
||||
chain_sds: np.ndarray
|
||||
array with time series of chainage for each transect (each transect is one column)
|
||||
topo_profiles: dict
|
||||
dict containing the dates and chainage of the groundtruth
|
||||
mod: 0 or 1
|
||||
0 for linear interpolation between 2 closest surveys, 1 for only nearest neighbour
|
||||
min_days: int
|
||||
minimum number of days for which the data can be compared
|
||||
|
||||
Returns: -----------
|
||||
stats: dict
|
||||
contains all the statistics of the comparison
|
||||
|
||||
"""
|
||||
|
||||
# create 3 figures
|
||||
fig1 = plt.figure()
|
||||
gs1 = gridspec.GridSpec(chain_sds.shape[1], 1)
|
||||
fig2 = plt.figure()
|
||||
gs2 = gridspec.GridSpec(2, chain_sds.shape[1])
|
||||
fig3 = plt.figure()
|
||||
gs3 = gridspec.GridSpec(2,1)
|
||||
|
||||
dates_sds_num = np.array([_.toordinal() for _ in dates_sds])
|
||||
stats = dict([])
|
||||
data_fin = dict([])
|
||||
|
||||
# for each transect compare and plot the data
|
||||
for i in range(chain_sds.shape[1]):
|
||||
|
||||
pfname = list(topo_profiles.keys())[i]
|
||||
stats[pfname] = dict([])
|
||||
data_fin[pfname] = dict([])
|
||||
|
||||
dates_sur = topo_profiles[pfname]['dates']
|
||||
chain_sur = topo_profiles[pfname]['chainage']
|
||||
|
||||
# convert to datenum
|
||||
dates_sur_num = np.array([_.toordinal() for _ in dates_sur])
|
||||
|
||||
chain_sur_interp = []
|
||||
diff_days = []
|
||||
|
||||
for j, satdate in enumerate(dates_sds_num):
|
||||
|
||||
temp_diff = satdate - dates_sur_num
|
||||
|
||||
if mod==0:
|
||||
# select measurement before and after sat image date and interpolate
|
||||
|
||||
ind_before = np.where(temp_diff == temp_diff[temp_diff > 0][-1])[0]
|
||||
if ind_before == len(temp_diff)-1:
|
||||
chain_sur_interp.append(np.nan)
|
||||
diff_days.append(np.abs(satdate-dates_sur_num[ind_before])[0])
|
||||
continue
|
||||
ind_after = np.where(temp_diff == temp_diff[temp_diff < 0][0])[0]
|
||||
tempx = np.zeros(2)
|
||||
tempx[0] = dates_sur_num[ind_before]
|
||||
tempx[1] = dates_sur_num[ind_after]
|
||||
tempy = np.zeros(2)
|
||||
tempy[0] = chain_sur[ind_before]
|
||||
tempy[1] = chain_sur[ind_after]
|
||||
diff_days.append(np.abs(np.max([satdate-tempx[0], satdate-tempx[1]])))
|
||||
# interpolate
|
||||
f = interpolate.interp1d(tempx, tempy)
|
||||
chain_sur_interp.append(f(satdate))
|
||||
|
||||
elif mod==1:
|
||||
# select the closest measurement
|
||||
|
||||
idx_closest = utils.find_indices(np.abs(temp_diff), lambda e: e == np.min(np.abs(temp_diff)))[0]
|
||||
diff_days.append(np.abs(satdate-dates_sur_num[idx_closest]))
|
||||
if diff_days[j] > mindays:
|
||||
chain_sur_interp.append(np.nan)
|
||||
else:
|
||||
chain_sur_interp.append(chain_sur[idx_closest])
|
||||
|
||||
chain_sur_interp = np.array(chain_sur_interp)
|
||||
|
||||
# remove nan values
|
||||
idx_sur_nan = ~np.isnan(chain_sur_interp)
|
||||
idx_sat_nan = ~np.isnan(chain_sds[:,i])
|
||||
idx_nan = np.logical_and(idx_sur_nan, idx_sat_nan)
|
||||
|
||||
# groundtruth and sds
|
||||
chain_sur_fin = chain_sur_interp[idx_nan]
|
||||
chain_sds_fin = chain_sds[idx_nan,i]
|
||||
dates_fin = [k for (k, v) in zip(dates_sds, idx_nan) if v]
|
||||
|
||||
# calculate statistics
|
||||
slope, intercept, rvalue, pvalue, std_err = sstats.linregress(chain_sur_fin, chain_sds_fin)
|
||||
R2 = rvalue**2
|
||||
correlation = np.corrcoef(chain_sur_fin, chain_sds_fin)[0,1]
|
||||
diff_chain = chain_sur_fin - chain_sds_fin
|
||||
|
||||
rmse = np.sqrt(np.nanmean((diff_chain)**2))
|
||||
mean = np.nanmean(diff_chain)
|
||||
std = np.nanstd(diff_chain)
|
||||
q90 = np.percentile(np.abs(diff_chain), 90)
|
||||
|
||||
# store data
|
||||
stats[pfname]['rmse'] = rmse
|
||||
stats[pfname]['mean'] = mean
|
||||
stats[pfname]['std'] = std
|
||||
stats[pfname]['q90'] = q90
|
||||
stats[pfname]['diffdays'] = diff_days
|
||||
stats[pfname]['corr'] = correlation
|
||||
stats[pfname]['linfit'] = {'slope':slope, 'intercept':intercept, 'R2':R2, 'pvalue':pvalue}
|
||||
|
||||
data_fin[pfname]['dates'] = dates_fin
|
||||
data_fin[pfname]['sds'] = chain_sds_fin
|
||||
data_fin[pfname]['survey'] = chain_sur_fin
|
||||
|
||||
# make time-series plot
|
||||
plt.figure(fig1.number)
|
||||
fig1.add_subplot(gs1[i,0])
|
||||
plt.plot(dates_sur, chain_sur, 'o-', color='C1', markersize=4, label='survey all')
|
||||
plt.plot(dates_fin, chain_sur_fin, 'o', color=[0.3, 0.3, 0.3], markersize=2, label='survey interp')
|
||||
plt.plot(dates_fin, chain_sds_fin, 'o--', color='b', markersize=4, label='SDS')
|
||||
plt.title(pfname, fontweight='bold')
|
||||
# plt.xlim([dates_sds[0], dates_sds[-1]])
|
||||
plt.ylabel('chainage [m]')
|
||||
|
||||
# make scatter plot
|
||||
plt.figure(fig2.number)
|
||||
fig2.add_subplot(gs2[0,i])
|
||||
plt.axis('equal')
|
||||
plt.plot(chain_sur_fin, chain_sds_fin, 'ko', markersize=4, markerfacecolor='w', alpha=0.7)
|
||||
xmax = np.max([np.nanmax(chain_sds_fin),np.nanmax(chain_sur_fin)])
|
||||
xmin = np.min([np.nanmin(chain_sds_fin),np.nanmin(chain_sur_fin)])
|
||||
ymax = np.max([np.nanmax(chain_sds_fin),np.nanmax(chain_sur_fin)])
|
||||
ymin = np.min([np.nanmin(chain_sds_fin),np.nanmin(chain_sur_fin)])
|
||||
plt.plot([xmin, xmax], [ymin, ymax], 'k--')
|
||||
plt.plot([xmin, xmax], [xmin*slope + intercept, xmax*slope + intercept], 'b:')
|
||||
str_corr = ' y = %.2f x + %.2f\n R2 = %.2f' % (slope, intercept, R2)
|
||||
plt.text(xmin, ymax-5, str_corr, bbox=dict(facecolor=[0.7,0.7,0.7], alpha=0.5), horizontalalignment='left')
|
||||
plt.xlabel('chainage survey [m]')
|
||||
plt.ylabel('chainage satellite [m]')
|
||||
plt.title(pfname, fontweight='bold')
|
||||
|
||||
fig2.add_subplot(gs2[1,i])
|
||||
binwidth = 3
|
||||
bins = np.arange(min(diff_chain), max(diff_chain) + binwidth, binwidth)
|
||||
density = plt.hist(diff_chain, bins=bins, density=True, color=[0.8, 0.8, 0.8], edgecolor='k')
|
||||
plt.xlim([-50, 50])
|
||||
plt.xlabel('error [m]')
|
||||
str_stats = ' rmse = %.1f\n mean = %.1f\n std = %.1f\n q90 = %.1f' % (rmse, mean, std, q90)
|
||||
plt.text(15, np.max(density[0])-0.015, str_stats, bbox=dict(facecolor=[0.8,0.8,0.8], alpha=0.3), horizontalalignment='left', fontsize=10)
|
||||
|
||||
fig1.set_size_inches(19.2, 9.28)
|
||||
fig1.set_tight_layout(True)
|
||||
fig2.set_size_inches(19.2, 9.28)
|
||||
fig2.set_tight_layout(True)
|
||||
|
||||
# all transects together
|
||||
chain_sds_all = []
|
||||
chain_sur_all = []
|
||||
for i in range(chain_sds.shape[1]):
|
||||
pfname = list(topo_profiles.keys())[i]
|
||||
chain_sds_all = np.append(chain_sds_all,data_fin[pfname]['sds'])
|
||||
chain_sur_all = np.append(chain_sur_all,data_fin[pfname]['survey'])
|
||||
|
||||
# calculate statistics
|
||||
slope, intercept, rvalue, pvalue, std_err = sstats.linregress(chain_sur_all, chain_sds_all)
|
||||
R2 = rvalue**2
|
||||
correlation = np.corrcoef(chain_sur_all, chain_sds_all)[0,1]
|
||||
diff_chain_all = chain_sur_all - chain_sds_all
|
||||
|
||||
rmse = np.sqrt(np.nanmean((diff_chain_all)**2))
|
||||
mean = np.nanmean(diff_chain_all)
|
||||
std = np.nanstd(diff_chain_all)
|
||||
q90 = np.percentile(np.abs(diff_chain_all), 90)
|
||||
|
||||
stats['all'] = {'rmse':rmse,'mean':mean,'std':std,'q90':q90, 'corr':correlation,
|
||||
'linfit':{'slope':slope, 'intercept':intercept, 'R2':R2, 'pvalue':pvalue}}
|
||||
|
||||
# make plot
|
||||
plt.figure(fig3.number)
|
||||
fig3.add_subplot(gs3[0,0])
|
||||
plt.axis('equal')
|
||||
plt.plot(chain_sur_all, chain_sds_all, 'ko', markersize=4, markerfacecolor='w', alpha=0.7)
|
||||
xmax = np.max([np.nanmax(chain_sds_all),np.nanmax(chain_sur_all)])
|
||||
xmin = np.min([np.nanmin(chain_sds_all),np.nanmin(chain_sur_all)])
|
||||
ymax = np.max([np.nanmax(chain_sds_all),np.nanmax(chain_sur_all)])
|
||||
ymin = np.min([np.nanmin(chain_sds_all),np.nanmin(chain_sur_all)])
|
||||
plt.plot([xmin, xmax], [ymin, ymax], 'k--')
|
||||
plt.plot([xmin, xmax], [xmin*slope + intercept, xmax*slope + intercept], 'b:')
|
||||
str_corr = ' y = %.2f x + %.2f\n R2 = %.2f' % (slope, intercept, R2)
|
||||
plt.text(xmin, ymax-5, str_corr, bbox=dict(facecolor=[0.7,0.7,0.7], alpha=0.5), horizontalalignment='left')
|
||||
plt.xlabel('chainage survey [m]')
|
||||
plt.ylabel('chainage satellite [m]')
|
||||
plt.title(pfname, fontweight='bold')
|
||||
|
||||
fig3.add_subplot(gs3[1,0])
|
||||
binwidth = 3
|
||||
bins = np.arange(min(diff_chain_all), max(diff_chain_all) + binwidth, binwidth)
|
||||
density = plt.hist(diff_chain_all, bins=bins, density=True, color=[0.8, 0.8, 0.8], edgecolor='k')
|
||||
plt.xlim([-50, 50])
|
||||
plt.xlabel('error [m]')
|
||||
str_stats = ' rmse = %.1f\n mean = %.1f\n std = %.1f\n q90 = %.1f' % (rmse, mean, std, q90)
|
||||
plt.text(15, np.max(density[0])-0.015, str_stats, bbox=dict(facecolor=[0.8,0.8,0.8], alpha=0.3), horizontalalignment='left', fontsize=10)
|
||||
fig3.set_size_inches(9.2, 9.28)
|
||||
fig3.set_tight_layout(True)
|
||||
|
||||
return stats
|
||||
@ -1,685 +0,0 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
Created on Thu Mar 1 11:20:35 2018
|
||||
|
||||
@author: z5030440
|
||||
"""
|
||||
|
||||
"""This script contains the functions needed for satellite derived shoreline (SDS) extraction"""
|
||||
|
||||
# Initial settings
|
||||
import numpy as np
|
||||
import matplotlib.pyplot as plt
|
||||
import pdb
|
||||
import ee
|
||||
|
||||
# other modules
|
||||
from osgeo import gdal, ogr, osr
|
||||
import tempfile
|
||||
from urllib.request import urlretrieve
|
||||
import zipfile
|
||||
|
||||
# image processing modules
|
||||
import skimage.filters as filters
|
||||
import skimage.exposure as exposure
|
||||
import skimage.transform as transform
|
||||
import sklearn.decomposition as decomposition
|
||||
import skimage.measure as measure
|
||||
import skimage.morphology as morphology
|
||||
from sklearn.cluster import KMeans
|
||||
|
||||
|
||||
# import own modules
|
||||
from functions.utils import *
|
||||
|
||||
|
||||
# Download from ee server function
|
||||
|
||||
def download_tif(image, polygon, bandsId):
|
||||
"""downloads tif image (region and bands) from the ee server and stores it in a temp file"""
|
||||
url = ee.data.makeDownloadUrl(ee.data.getDownloadId({
|
||||
'image': image.serialize(),
|
||||
'region': polygon,
|
||||
'bands': bandsId,
|
||||
'filePerBand': 'false',
|
||||
'name': 'data',
|
||||
}))
|
||||
local_zip, headers = urlretrieve(url)
|
||||
with zipfile.ZipFile(local_zip) as local_zipfile:
|
||||
return local_zipfile.extract('data.tif', tempfile.mkdtemp())
|
||||
|
||||
def load_image(image, polygon, bandsId):
|
||||
"""
|
||||
Loads an ee.Image() as a np.array. e.Image() is retrieved from the EE database.
|
||||
The geographic area and bands to select can be specified
|
||||
|
||||
KV WRL 2018
|
||||
|
||||
Arguments:
|
||||
-----------
|
||||
image: ee.Image()
|
||||
image objec from the EE database
|
||||
polygon: list
|
||||
coordinates of the points creating a polygon. Each point is a list with 2 values
|
||||
bandsId: list
|
||||
bands to select, each band is a dictionnary in the list containing the following keys:
|
||||
crs, crs_transform, data_type and id. NOTE: you have to remove the key dimensions, otherwise
|
||||
the entire image is retrieved.
|
||||
|
||||
Returns:
|
||||
-----------
|
||||
image_array : np.ndarray
|
||||
An array containing the image (2D if one band, otherwise 3D)
|
||||
georef : np.ndarray
|
||||
6 element vector containing the crs_parameters
|
||||
[X_ul_corner Xscale Xshear Y_ul_corner Yshear Yscale]
|
||||
"""
|
||||
|
||||
local_tif_filename = download_tif(image, polygon, bandsId)
|
||||
dataset = gdal.Open(local_tif_filename, gdal.GA_ReadOnly)
|
||||
georef = np.array(dataset.GetGeoTransform())
|
||||
bands = [dataset.GetRasterBand(i + 1).ReadAsArray() for i in range(dataset.RasterCount)]
|
||||
return np.stack(bands, 2), georef
|
||||
|
||||
def create_cloud_mask(im_qa, satname, plot_bool):
|
||||
"""
|
||||
Creates a cloud mask from the image containing the QA band information
|
||||
|
||||
KV WRL 2018
|
||||
|
||||
Arguments:
|
||||
-----------
|
||||
im_qa: np.ndarray
|
||||
Image containing the QA band
|
||||
satname: string
|
||||
short name for the satellite (L8, L7, S2)
|
||||
plot_bool: boolean
|
||||
True if plot is wanted
|
||||
|
||||
Returns:
|
||||
-----------
|
||||
cloud_mask : np.ndarray of booleans
|
||||
A boolean array with True where the cloud are present
|
||||
"""
|
||||
|
||||
# convert QA bits
|
||||
if satname == 'L8':
|
||||
cloud_values = [2800, 2804, 2808, 2812, 6896, 6900, 6904, 6908]
|
||||
elif satname == 'L7':
|
||||
cloud_values = [752, 756, 760, 764]
|
||||
|
||||
cloud_mask = np.isin(im_qa, cloud_values)
|
||||
# remove isolated cloud pixels (there are some in the swash and they cause problems)
|
||||
if sum(sum(cloud_mask)) > 0:
|
||||
morphology.remove_small_objects(cloud_mask, min_size=10, connectivity=1, in_place=True)
|
||||
|
||||
if plot_bool:
|
||||
plt.figure()
|
||||
plt.imshow(cloud_mask, cmap='gray')
|
||||
plt.draw()
|
||||
|
||||
#cloud_shadow_values = [2976, 2980, 2984, 2988, 3008, 3012, 3016, 3020]
|
||||
#cloud_shadow_mask = np.isin(im_qa, cloud_shadow_values)
|
||||
|
||||
return cloud_mask
|
||||
|
||||
def read_eeimage(im, polygon, sat_name, plot_bool):
|
||||
"""
|
||||
Read an ee.Image() object and returns the panchromatic band, multispectral bands (B, G, R, NIR, SWIR)
|
||||
and a cloud mask. All outputs are at 15m resolution (bilinear interpolation for the multispectral bands)
|
||||
|
||||
KV WRL 2018
|
||||
|
||||
Arguments:
|
||||
-----------
|
||||
im: ee.Image()
|
||||
Image to read from the Google Earth Engine database
|
||||
plot_bool: boolean
|
||||
True if plot is wanted
|
||||
|
||||
Returns:
|
||||
-----------
|
||||
im_pan: np.ndarray (2D)
|
||||
The panchromatic band (15m)
|
||||
im_ms: np.ndarray (3D)
|
||||
The multispectral bands interpolated at 15m
|
||||
im_cloud: np.ndarray (2D)
|
||||
The cloud mask at 15m
|
||||
crs_params: list
|
||||
EPSG code and affine transformation parameters
|
||||
"""
|
||||
|
||||
im_dic = im.getInfo()
|
||||
# save metadata
|
||||
im_meta = im_dic.get('properties')
|
||||
meta = {'timestamp':im_meta['system:time_start'],
|
||||
'date_acquired':im_meta['DATE_ACQUIRED'],
|
||||
'geom_rmse_model':im_meta['GEOMETRIC_RMSE_MODEL'],
|
||||
'gcp_model':im_meta['GROUND_CONTROL_POINTS_MODEL'],
|
||||
'quality':im_meta['IMAGE_QUALITY_OLI'],
|
||||
'sun_azimuth':im_meta['SUN_AZIMUTH'],
|
||||
'sun_elevation':im_meta['SUN_ELEVATION']}
|
||||
|
||||
im_bands = im_dic.get('bands')
|
||||
|
||||
# delete dimensions key from dictionnary, otherwise the entire image is extracted
|
||||
for i in range(len(im_bands)): del im_bands[i]['dimensions']
|
||||
|
||||
# load panchromatic band
|
||||
pan_band = [im_bands[7]]
|
||||
im_pan, crs_pan = load_image(im, polygon, pan_band)
|
||||
im_pan = im_pan[:,:,0]
|
||||
|
||||
# load the multispectral bands (B2,B3,B4,B5,B6) = (blue,green,red,nir,swir1)
|
||||
ms_bands = [im_bands[1], im_bands[2], im_bands[3], im_bands[4], im_bands[5]]
|
||||
im_ms_30m, crs_ms = load_image(im, polygon, ms_bands)
|
||||
|
||||
# create cloud mask
|
||||
qa_band = [im_bands[11]]
|
||||
im_qa, crs_qa = load_image(im, polygon, qa_band)
|
||||
im_qa = im_qa[:,:,0]
|
||||
im_cloud = create_cloud_mask(im_qa, sat_name, plot_bool)
|
||||
im_cloud = transform.resize(im_cloud, (im_pan.shape[0], im_pan.shape[1]),
|
||||
order=0, preserve_range=True, mode='constant').astype('bool_')
|
||||
|
||||
# resize the image using bilinear interpolation (order 1)
|
||||
im_ms = transform.resize(im_ms_30m,(im_pan.shape[0], im_pan.shape[1]),
|
||||
order=1, preserve_range=True, mode='constant')
|
||||
|
||||
# check if -inf values (means out of image) and add to cloud mask
|
||||
im_inf = np.isin(im_ms[:,:,0], -np.inf)
|
||||
im_nan = np.isnan(im_ms[:,:,0])
|
||||
im_cloud = np.logical_or(np.logical_or(im_cloud, im_inf), im_nan)
|
||||
|
||||
# get the crs parameters for the image at 15m and 30m resolution
|
||||
crs = {'crs_15m':crs_pan, 'crs_30m':crs_ms, 'epsg_code':int(pan_band[0]['crs'][5:])}
|
||||
|
||||
if plot_bool:
|
||||
|
||||
# if there are -inf in the image, set them to 0 before plotting
|
||||
if sum(sum(np.isin(im_ms_30m[:,:,0], -np.inf).astype(int))) > 0:
|
||||
idx = np.isin(im_ms_30m[:,:,0], -np.inf)
|
||||
im_ms_30m[idx,0] = 0; im_ms_30m[idx,1] = 0; im_ms_30m[idx,2] = 0;
|
||||
im_ms_30m[idx,3] = 0; im_ms_30m[idx,4] = 0
|
||||
|
||||
plt.figure()
|
||||
|
||||
plt.subplot(221)
|
||||
plt.imshow(im_pan, cmap='gray')
|
||||
plt.title('PANCHROMATIC')
|
||||
|
||||
plt.subplot(222)
|
||||
plt.imshow(im_ms_30m[:,:,[2,1,0]])
|
||||
plt.title('RGB')
|
||||
|
||||
|
||||
plt.subplot(223)
|
||||
plt.imshow(im_ms_30m[:,:,3], cmap='gray')
|
||||
plt.title('NIR')
|
||||
|
||||
plt.subplot(224)
|
||||
plt.imshow(im_ms_30m[:,:,4], cmap='gray')
|
||||
plt.title('SWIR')
|
||||
|
||||
plt.show()
|
||||
|
||||
return im_pan, im_ms, im_cloud, crs, meta
|
||||
|
||||
|
||||
def rescale_image_intensity(im, cloud_mask, prob_high, plot_bool):
|
||||
"""
|
||||
Rescales the intensity of an image (multispectral or single band) by applying
|
||||
a cloud mask and clipping the prob_high upper percentile. This functions allows
|
||||
to stretch the contrast of an image.
|
||||
|
||||
KV WRL 2018
|
||||
|
||||
Arguments:
|
||||
-----------
|
||||
im: np.ndarray
|
||||
Image to rescale, can be 3D (multispectral) or 2D (single band)
|
||||
cloud_mask: np.ndarray
|
||||
2D cloud mask with True where cloud pixels are
|
||||
prob_high: float
|
||||
probability of exceedence used to calculate the upper percentile
|
||||
plot_bool: boolean
|
||||
True if plot is wanted
|
||||
|
||||
Returns:
|
||||
-----------
|
||||
im_adj: np.ndarray
|
||||
The rescaled image
|
||||
"""
|
||||
prc_low = 0 # lower percentile
|
||||
vec_mask = cloud_mask.reshape(im.shape[0] * im.shape[1])
|
||||
|
||||
if plot_bool:
|
||||
plt.figure()
|
||||
|
||||
if len(im.shape) > 2:
|
||||
vec = im.reshape(im.shape[0] * im.shape[1], im.shape[2])
|
||||
vec_adj = np.ones((len(vec_mask), im.shape[2])) * np.nan
|
||||
|
||||
for i in range(im.shape[2]):
|
||||
prc_high = np.percentile(vec[~vec_mask, i], prob_high)
|
||||
vec_rescaled = exposure.rescale_intensity(vec[~vec_mask, i], in_range=(prc_low, prc_high))
|
||||
vec_adj[~vec_mask,i] = vec_rescaled
|
||||
|
||||
if plot_bool:
|
||||
plt.subplot(np.floor(im.shape[2]/2) + 1, np.floor(im.shape[2]/2), i+1)
|
||||
plt.hist(vec[~vec_mask, i], bins=200, label='original')
|
||||
plt.hist(vec_rescaled, bins=200, alpha=0.5, label='rescaled')
|
||||
plt.legend()
|
||||
plt.title('Band' + str(i+1))
|
||||
plt.show()
|
||||
|
||||
im_adj = vec_adj.reshape(im.shape[0], im.shape[1], im.shape[2])
|
||||
|
||||
if plot_bool:
|
||||
plt.figure()
|
||||
ax1 = plt.subplot(121)
|
||||
plt.imshow(im[:,:,[2,1,0]])
|
||||
plt.axis('off')
|
||||
plt.title('Original')
|
||||
ax2 = plt.subplot(122, sharex=ax1, sharey=ax1)
|
||||
plt.imshow(im_adj[:,:,[2,1,0]])
|
||||
plt.axis('off')
|
||||
plt.title('Rescaled')
|
||||
plt.show()
|
||||
|
||||
else:
|
||||
vec = im.reshape(im.shape[0] * im.shape[1])
|
||||
vec_adj = np.ones(len(vec_mask)) * np.nan
|
||||
prc_high = np.percentile(vec[~vec_mask], prob_high)
|
||||
vec_rescaled = exposure.rescale_intensity(vec[~vec_mask], in_range=(prc_low, prc_high))
|
||||
vec_adj[~vec_mask] = vec_rescaled
|
||||
|
||||
if plot_bool:
|
||||
plt.hist(vec[~vec_mask], bins=200, label='original')
|
||||
plt.hist(vec_rescaled, bins=200, alpha=0.5, label='rescaled')
|
||||
plt.legend()
|
||||
plt.title('Single band')
|
||||
plt.show()
|
||||
|
||||
im_adj = vec_adj.reshape(im.shape[0], im.shape[1])
|
||||
|
||||
if plot_bool:
|
||||
plt.figure()
|
||||
ax1 = plt.subplot(121)
|
||||
plt.imshow(im, cmap='gray')
|
||||
plt.axis('off')
|
||||
plt.title('Original')
|
||||
ax2 = plt.subplot(122, sharex=ax1, sharey=ax1)
|
||||
plt.imshow(im_adj, cmap='gray')
|
||||
plt.axis('off')
|
||||
plt.title('Rescaled')
|
||||
plt.show()
|
||||
|
||||
return im_adj
|
||||
|
||||
|
||||
def hist_match(source, template):
|
||||
"""
|
||||
Adjust the pixel values of a grayscale image such that its histogram
|
||||
matches that of a target image
|
||||
|
||||
Arguments:
|
||||
-----------
|
||||
source: np.ndarray
|
||||
Image to transform; the histogram is computed over the flattened
|
||||
array
|
||||
template: np.ndarray
|
||||
Template image; can have different dimensions to source
|
||||
Returns:
|
||||
-----------
|
||||
matched: np.ndarray
|
||||
The transformed output image
|
||||
"""
|
||||
|
||||
oldshape = source.shape
|
||||
source = source.ravel()
|
||||
template = template.ravel()
|
||||
|
||||
# get the set of unique pixel values and their corresponding indices and
|
||||
# counts
|
||||
s_values, bin_idx, s_counts = np.unique(source, return_inverse=True,
|
||||
return_counts=True)
|
||||
t_values, t_counts = np.unique(template, return_counts=True)
|
||||
|
||||
# take the cumsum of the counts and normalize by the number of pixels to
|
||||
# get the empirical cumulative distribution functions for the source and
|
||||
# template images (maps pixel value --> quantile)
|
||||
s_quantiles = np.cumsum(s_counts).astype(np.float64)
|
||||
s_quantiles /= s_quantiles[-1]
|
||||
t_quantiles = np.cumsum(t_counts).astype(np.float64)
|
||||
t_quantiles /= t_quantiles[-1]
|
||||
|
||||
# interpolate linearly to find the pixel values in the template image
|
||||
# that correspond most closely to the quantiles in the source image
|
||||
interp_t_values = np.interp(s_quantiles, t_quantiles, t_values)
|
||||
|
||||
return interp_t_values[bin_idx].reshape(oldshape)
|
||||
|
||||
def pansharpen(im_ms, im_pan, cloud_mask, plot_bool):
|
||||
"""
|
||||
Pansharpens a multispectral image (3D), using the panchromatic band (2D)
|
||||
and a cloud mask
|
||||
|
||||
KV WRL 2018
|
||||
|
||||
Arguments:
|
||||
-----------
|
||||
im_ms: np.ndarray
|
||||
Multispectral image to pansharpen (3D)
|
||||
im_pan: np.ndarray
|
||||
Panchromatic band (2D)
|
||||
cloud_mask: np.ndarray
|
||||
2D cloud mask with True where cloud pixels are
|
||||
plot_bool: boolean
|
||||
True if plot is wanted
|
||||
|
||||
Returns:
|
||||
-----------
|
||||
im_ms_ps: np.ndarray
|
||||
Pansharpened multisoectral image (3D)
|
||||
"""
|
||||
|
||||
# reshape image into vector and apply cloud mask
|
||||
vec = im_ms.reshape(im_ms.shape[0] * im_ms.shape[1], im_ms.shape[2])
|
||||
vec_mask = cloud_mask.reshape(im_ms.shape[0] * im_ms.shape[1])
|
||||
vec = vec[~vec_mask, :]
|
||||
# apply PCA to RGB bands
|
||||
pca = decomposition.PCA()
|
||||
vec_pcs = pca.fit_transform(vec)
|
||||
# replace 1st PC with pan band (after matching histograms)
|
||||
vec_pan = im_pan.reshape(im_pan.shape[0] * im_pan.shape[1])
|
||||
vec_pan = vec_pan[~vec_mask]
|
||||
vec_pcs[:,0] = hist_match(vec_pan, vec_pcs[:,0])
|
||||
vec_ms_ps = pca.inverse_transform(vec_pcs)
|
||||
|
||||
# normalise between 0 and 1
|
||||
for i in range(vec_pcs.shape[1]):
|
||||
vec_ms_ps[:,i] = np.divide(vec_ms_ps[:,i] - np.min(vec_ms_ps[:,i]),
|
||||
np.max(vec_ms_ps[:,i]) - np.min(vec_ms_ps[:,i]))
|
||||
# reshape vector into image
|
||||
vec_ms_ps_full = np.ones((len(vec_mask), im_ms.shape[2])) * np.nan
|
||||
vec_ms_ps_full[~vec_mask,:] = vec_ms_ps
|
||||
im_ms_ps = vec_ms_ps_full.reshape(im_ms.shape[0], im_ms.shape[1], im_ms.shape[2])
|
||||
|
||||
if plot_bool:
|
||||
plt.figure()
|
||||
ax1 = plt.subplot(121)
|
||||
plt.imshow(im_ms[:,:,[2,1,0]])
|
||||
plt.axis('off')
|
||||
plt.title('Original')
|
||||
ax2 = plt.subplot(122, sharex=ax1, sharey=ax1)
|
||||
plt.imshow(im_ms_ps[:,:,[2,1,0]])
|
||||
plt.axis('off')
|
||||
plt.title('Pansharpened')
|
||||
plt.show()
|
||||
|
||||
return im_ms_ps
|
||||
|
||||
def nd_index(im1, im2, cloud_mask, plot_bool):
|
||||
"""
|
||||
Computes normalised difference index on 2 images (2D), given a cloud mask (2D)
|
||||
|
||||
KV WRL 2018
|
||||
|
||||
Arguments:
|
||||
-----------
|
||||
im1, im2: np.ndarray
|
||||
Images (2D) with which to calculate the ND index
|
||||
cloud_mask: np.ndarray
|
||||
2D cloud mask with True where cloud pixels are
|
||||
plot_bool: boolean
|
||||
True if plot is wanted
|
||||
|
||||
Returns: -----------
|
||||
im_nd: np.ndarray
|
||||
|
||||
Image (2D) containing the ND index
|
||||
"""
|
||||
vec_mask = cloud_mask.reshape(im1.shape[0] * im1.shape[1])
|
||||
vec_nd = np.ones(len(vec_mask)) * np.nan
|
||||
vec1 = im1.reshape(im1.shape[0] * im1.shape[1])
|
||||
vec2 = im2.reshape(im2.shape[0] * im2.shape[1])
|
||||
temp = np.divide(vec1[~vec_mask] - vec2[~vec_mask],
|
||||
vec1[~vec_mask] + vec2[~vec_mask])
|
||||
vec_nd[~vec_mask] = temp
|
||||
im_nd = vec_nd.reshape(im1.shape[0], im1.shape[1])
|
||||
|
||||
if plot_bool:
|
||||
plt.figure()
|
||||
plt.imshow(im_nd, cmap='seismic')
|
||||
plt.colorbar()
|
||||
plt.title('Normalised index')
|
||||
plt.show()
|
||||
|
||||
return im_nd
|
||||
|
||||
def find_wl_contours(im_ndwi, cloud_mask, min_contour_points, plot_bool):
|
||||
"""
|
||||
Computes normalised difference index on 2 images (2D), given a cloud mask (2D)
|
||||
|
||||
KV WRL 2018
|
||||
|
||||
Arguments:
|
||||
-----------
|
||||
im_ndwi: np.ndarray
|
||||
Image (2D) with the NDWI (water index)
|
||||
cloud_mask: np.ndarray
|
||||
2D cloud mask with True where cloud pixels are
|
||||
min_contour_points: int
|
||||
minimum number of points in each contour line
|
||||
plot_bool: boolean
|
||||
True if plot is wanted
|
||||
|
||||
Returns: -----------
|
||||
contours_wl: list of np.arrays
|
||||
contains the (row,column) coordinates of the contour lines
|
||||
|
||||
"""
|
||||
|
||||
# reshape image to vector
|
||||
vec_ndwi = im_ndwi.reshape(im_ndwi.shape[0] * im_ndwi.shape[1])
|
||||
vec_mask = cloud_mask.reshape(cloud_mask.shape[0] * cloud_mask.shape[1])
|
||||
vec = vec_ndwi[~vec_mask]
|
||||
# apply otsu's threshold
|
||||
t_otsu = filters.threshold_otsu(vec)
|
||||
# use Marching Squares algorithm to detect contours on ndwi image
|
||||
contours = measure.find_contours(im_ndwi, t_otsu)
|
||||
# filter water lines
|
||||
contours_wl = []
|
||||
for i, contour in enumerate(contours):
|
||||
# remove contour points that are around clouds (nan values)
|
||||
if np.any(np.isnan(contour)):
|
||||
index_nan = np.where(np.isnan(contour))[0]
|
||||
contour = np.delete(contour, index_nan, axis=0)
|
||||
# remove contours that have only few points (less than min_contour_points)
|
||||
if contour.shape[0] > min_contour_points:
|
||||
contours_wl.append(contour)
|
||||
|
||||
if plot_bool:
|
||||
# plot otsu's histogram segmentation
|
||||
plt.figure()
|
||||
vals = plt.hist(vec, bins=200)
|
||||
plt.plot([t_otsu, t_otsu],[0, np.max(vals[0])], 'r-', label='Otsu threshold')
|
||||
plt.legend()
|
||||
plt.show()
|
||||
|
||||
# plot the water line contours on top of water index
|
||||
plt.figure()
|
||||
plt.imshow(im_ndwi, cmap='seismic')
|
||||
plt.colorbar()
|
||||
for i,contour in enumerate(contours_wl): plt.plot(contour[:, 1], contour[:, 0], linewidth=3, color='k')
|
||||
plt.axis('image')
|
||||
plt.title('Detected water lines')
|
||||
plt.show()
|
||||
|
||||
return contours_wl
|
||||
|
||||
def convert_pix2world(points, crs_vec):
|
||||
"""
|
||||
Converts pixel coordinates (row,columns) to world projected coordinates
|
||||
performing an affine transformation
|
||||
|
||||
KV WRL 2018
|
||||
|
||||
Arguments:
|
||||
-----------
|
||||
points: np.ndarray or list of np.ndarray
|
||||
array with 2 columns (rows first and columns second)
|
||||
crs_vec: np.ndarray
|
||||
vector of 6 elements [Xtr, Xscale, Xshear, Ytr, Yshear, Yscale]
|
||||
|
||||
Returns: -----------
|
||||
points_converted: np.ndarray or list of np.ndarray
|
||||
converted coordinates, first columns with X and second column with Y
|
||||
|
||||
"""
|
||||
|
||||
# make affine transformation matrix
|
||||
aff_mat = np.array([[crs_vec[1], crs_vec[2], crs_vec[0]],
|
||||
[crs_vec[4], crs_vec[5], crs_vec[3]],
|
||||
[0, 0, 1]])
|
||||
# create affine transformation
|
||||
tform = transform.AffineTransform(aff_mat)
|
||||
|
||||
if type(points) is list:
|
||||
points_converted = []
|
||||
# iterate over the list
|
||||
for i, arr in enumerate(points):
|
||||
tmp = arr[:,[1,0]]
|
||||
points_converted.append(tform(tmp))
|
||||
|
||||
elif type(points) is np.ndarray:
|
||||
tmp = points[:,[1,0]]
|
||||
points_converted = tform(tmp)
|
||||
|
||||
else:
|
||||
print('invalid input type')
|
||||
raise
|
||||
|
||||
return points_converted
|
||||
|
||||
def convert_epsg(points, epsg_in, epsg_out):
|
||||
"""
|
||||
Converts from one spatial reference to another using the epsg codes
|
||||
|
||||
KV WRL 2018
|
||||
|
||||
Arguments:
|
||||
-----------
|
||||
points: np.ndarray or list of np.ndarray
|
||||
array with 2 columns (rows first and columns second)
|
||||
epsg_in: int
|
||||
epsg code of the spatial reference in which the input is
|
||||
epsg_out: int
|
||||
epsg code of the spatial reference in which the output will be
|
||||
|
||||
Returns: -----------
|
||||
points_converted: np.ndarray or list of np.ndarray
|
||||
converted coordinates
|
||||
|
||||
"""
|
||||
|
||||
# define input and output spatial references
|
||||
inSpatialRef = osr.SpatialReference()
|
||||
inSpatialRef.ImportFromEPSG(epsg_in)
|
||||
outSpatialRef = osr.SpatialReference()
|
||||
outSpatialRef.ImportFromEPSG(epsg_out)
|
||||
# create a coordinates transform
|
||||
coordTransform = osr.CoordinateTransformation(inSpatialRef, outSpatialRef)
|
||||
# transform points
|
||||
if type(points) is list:
|
||||
points_converted = []
|
||||
# iterate over the list
|
||||
for i, arr in enumerate(points):
|
||||
points_converted.append(np.array(coordTransform.TransformPoints(arr)))
|
||||
|
||||
elif type(points) is np.ndarray:
|
||||
points_converted = np.array(coordTransform.TransformPoints(points))
|
||||
|
||||
else:
|
||||
print('invalid input type')
|
||||
raise
|
||||
|
||||
return points_converted
|
||||
|
||||
def classify_sand_unsupervised(im_ms_ps, im_pan, cloud_mask, wl_pix, buffer_size, min_beach_size, plot_bool):
|
||||
"""
|
||||
Classifies sand pixels using an unsupervised algorithm (Kmeans)
|
||||
Set buffer size to False if you
|
||||
|
||||
KV WRL 2018
|
||||
|
||||
Arguments:
|
||||
-----------
|
||||
im_ms_ps: np.ndarray
|
||||
Pansharpened RGB + downsampled NIR and SWIR
|
||||
im_pan:
|
||||
Panchromatic band
|
||||
cloud_mask: np.ndarray
|
||||
2D cloud mask with True where cloud pixels are
|
||||
wl_pix: list of np.ndarray
|
||||
list of arrays containig the pixel coordinates of the water line
|
||||
buffer_size: int or False
|
||||
radius of the disk used to create a buffer around the water line
|
||||
when False, the entire image is considered for kmeans
|
||||
min_beach_size: int
|
||||
minimum number of connected pixels belonging to a single beach
|
||||
plot_bool: boolean
|
||||
True if plot is wanted
|
||||
|
||||
Returns: -----------
|
||||
im_sand: np.ndarray
|
||||
2D binary image containing True where sand pixels are located
|
||||
|
||||
"""
|
||||
# reshape the 2D images into vectors
|
||||
vec_ms_ps = im_ms_ps.reshape(im_ms_ps.shape[0] * im_ms_ps.shape[1], im_ms_ps.shape[2])
|
||||
vec_pan = im_pan.reshape(im_pan.shape[0]*im_pan.shape[1])
|
||||
vec_mask = cloud_mask.reshape(im_ms_ps.shape[0] * im_ms_ps.shape[1])
|
||||
# add B,G,R,NIR and pan bands to the vector of features
|
||||
vec_features = np.zeros((vec_ms_ps.shape[0], 5))
|
||||
vec_features[:,[0,1,2,3]] = vec_ms_ps[:,[0,1,2,3]]
|
||||
vec_features[:,4] = vec_pan
|
||||
|
||||
if buffer_size:
|
||||
# create binary image with ones where the detected water lines is
|
||||
im_buffer = np.zeros((im_ms_ps.shape[0], im_ms_ps.shape[1]))
|
||||
for i, contour in enumerate(wl_pix):
|
||||
indices = [(int(_[0]), int(_[1])) for _ in list(np.round(contour))]
|
||||
for j, idx in enumerate(indices):
|
||||
im_buffer[idx] = 1
|
||||
# perform a dilation on the binary image
|
||||
se = morphology.disk(buffer_size)
|
||||
im_buffer = morphology.binary_dilation(im_buffer, se)
|
||||
vec_buffer = (im_buffer == 1).reshape(im_ms_ps.shape[0] * im_ms_ps.shape[1])
|
||||
else:
|
||||
vec_buffer = np.ones((vec_pan.shape[0]))
|
||||
# add cloud mask to buffer
|
||||
vec_buffer= np.logical_and(vec_buffer, ~vec_mask)
|
||||
# perform kmeans (6 clusters)
|
||||
kmeans = KMeans(n_clusters=6, random_state=0).fit(vec_features[vec_buffer,:])
|
||||
labels = np.ones((len(vec_mask))) * np.nan
|
||||
labels[vec_buffer] = kmeans.labels_
|
||||
im_labels = labels.reshape(im_ms_ps.shape[0], im_ms_ps.shape[1])
|
||||
# find the class with maximum reflection in the B,G,R,Pan
|
||||
im_sand = im_labels == np.argmax(np.mean(kmeans.cluster_centers_[:,[0,1,2,4]], axis=1))
|
||||
im_sand = morphology.remove_small_objects(im_sand, min_size=min_beach_size, connectivity=2)
|
||||
# im_sand = morphology.binary_dilation(im_sand, morphology.disk(1))
|
||||
|
||||
if plot_bool:
|
||||
im = np.copy(im_ms_ps)
|
||||
im[im_sand,0] = 0
|
||||
im[im_sand,1] = 0
|
||||
im[im_sand,2] = 1
|
||||
plt.figure()
|
||||
plt.imshow(im[:,:,[2,1,0]])
|
||||
plt.axis('image')
|
||||
plt.title('Sand classification')
|
||||
plt.show()
|
||||
|
||||
return im_sand
|
||||
@ -1,883 +0,0 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
Created on Thu Mar 1 11:20:35 2018
|
||||
|
||||
@author: z5030440
|
||||
"""
|
||||
|
||||
"""This script contains the functions needed for satellite derived shoreline (SDS) extraction"""
|
||||
|
||||
# Initial settings
|
||||
import numpy as np
|
||||
import matplotlib.pyplot as plt
|
||||
import pdb
|
||||
import ee
|
||||
|
||||
# other modules
|
||||
from osgeo import gdal, ogr, osr
|
||||
import tempfile
|
||||
from urllib.request import urlretrieve
|
||||
import zipfile
|
||||
|
||||
# image processing modules
|
||||
import skimage.filters as filters
|
||||
import skimage.exposure as exposure
|
||||
import skimage.transform as transform
|
||||
import sklearn.decomposition as decomposition
|
||||
import skimage.measure as measure
|
||||
import skimage.morphology as morphology
|
||||
|
||||
# machine learning modules
|
||||
from sklearn.cluster import KMeans
|
||||
from sklearn.neural_network import MLPClassifier
|
||||
from sklearn.externals import joblib
|
||||
|
||||
|
||||
# import own modules
|
||||
from functions.utils import *
|
||||
|
||||
|
||||
# Download from ee server function
|
||||
|
||||
def download_tif(image, polygon, bandsId):
|
||||
"""downloads tif image (region and bands) from the ee server and stores it in a temp file"""
|
||||
url = ee.data.makeDownloadUrl(ee.data.getDownloadId({
|
||||
'image': image.serialize(),
|
||||
'region': polygon,
|
||||
'bands': bandsId,
|
||||
'filePerBand': 'false',
|
||||
'name': 'data',
|
||||
}))
|
||||
local_zip, headers = urlretrieve(url)
|
||||
with zipfile.ZipFile(local_zip) as local_zipfile:
|
||||
return local_zipfile.extract('data.tif', tempfile.mkdtemp())
|
||||
|
||||
def load_image(image, polygon, bandsId):
|
||||
"""
|
||||
Loads an ee.Image() as a np.array. e.Image() is retrieved from the EE database.
|
||||
The geographic area and bands to select can be specified
|
||||
|
||||
KV WRL 2018
|
||||
|
||||
Arguments:
|
||||
-----------
|
||||
image: ee.Image()
|
||||
image objec from the EE database
|
||||
polygon: list
|
||||
coordinates of the points creating a polygon. Each point is a list with 2 values
|
||||
bandsId: list
|
||||
bands to select, each band is a dictionnary in the list containing the following keys:
|
||||
crs, crs_transform, data_type and id. NOTE: you have to remove the key dimensions, otherwise
|
||||
the entire image is retrieved.
|
||||
|
||||
Returns:
|
||||
-----------
|
||||
image_array : np.ndarray
|
||||
An array containing the image (2D if one band, otherwise 3D)
|
||||
georef : np.ndarray
|
||||
6 element vector containing the crs_parameters
|
||||
[X_ul_corner Xscale Xshear Y_ul_corner Yshear Yscale]
|
||||
"""
|
||||
|
||||
local_tif_filename = download_tif(image, polygon, bandsId)
|
||||
dataset = gdal.Open(local_tif_filename, gdal.GA_ReadOnly)
|
||||
georef = np.array(dataset.GetGeoTransform())
|
||||
bands = [dataset.GetRasterBand(i + 1).ReadAsArray() for i in range(dataset.RasterCount)]
|
||||
return np.stack(bands, 2), georef
|
||||
|
||||
def create_cloud_mask(im_qa, satname, plot_bool):
|
||||
"""
|
||||
Creates a cloud mask from the image containing the QA band information
|
||||
|
||||
KV WRL 2018
|
||||
|
||||
Arguments:
|
||||
-----------
|
||||
im_qa: np.ndarray
|
||||
Image containing the QA band
|
||||
satname: string
|
||||
short name for the satellite (L8, L7, S2)
|
||||
plot_bool: boolean
|
||||
True if plot is wanted
|
||||
|
||||
Returns:
|
||||
-----------
|
||||
cloud_mask : np.ndarray of booleans
|
||||
A boolean array with True where the cloud are present
|
||||
"""
|
||||
|
||||
# convert QA bits
|
||||
if satname == 'L8':
|
||||
cloud_values = [2800, 2804, 2808, 2812, 6896, 6900, 6904, 6908]
|
||||
elif satname == 'L7':
|
||||
cloud_values = [752, 756, 760, 764]
|
||||
|
||||
cloud_mask = np.isin(im_qa, cloud_values)
|
||||
# remove isolated cloud pixels (there are some in the swash and they cause problems)
|
||||
if sum(sum(cloud_mask)) > 0:
|
||||
morphology.remove_small_objects(cloud_mask, min_size=10, connectivity=1, in_place=True)
|
||||
|
||||
if plot_bool:
|
||||
plt.figure()
|
||||
plt.imshow(cloud_mask, cmap='gray')
|
||||
plt.draw()
|
||||
|
||||
#cloud_shadow_values = [2976, 2980, 2984, 2988, 3008, 3012, 3016, 3020]
|
||||
#cloud_shadow_mask = np.isin(im_qa, cloud_shadow_values)
|
||||
|
||||
return cloud_mask
|
||||
|
||||
def read_eeimage(im, polygon, sat_name, plot_bool):
|
||||
"""
|
||||
Read an ee.Image() object and returns the panchromatic band, multispectral bands (B, G, R, NIR, SWIR)
|
||||
and a cloud mask. All outputs are at 15m resolution (bilinear interpolation for the multispectral bands)
|
||||
|
||||
KV WRL 2018
|
||||
|
||||
Arguments:
|
||||
-----------
|
||||
im: ee.Image()
|
||||
Image to read from the Google Earth Engine database
|
||||
plot_bool: boolean
|
||||
True if plot is wanted
|
||||
|
||||
Returns:
|
||||
-----------
|
||||
im_pan: np.ndarray (2D)
|
||||
The panchromatic band (15m)
|
||||
im_ms: np.ndarray (3D)
|
||||
The multispectral bands interpolated at 15m
|
||||
im_cloud: np.ndarray (2D)
|
||||
The cloud mask at 15m
|
||||
crs_params: list
|
||||
EPSG code and affine transformation parameters
|
||||
"""
|
||||
|
||||
im_dic = im.getInfo()
|
||||
# save metadata
|
||||
im_meta = im_dic.get('properties')
|
||||
meta = {'timestamp':im_meta['system:time_start'],
|
||||
'date_acquired':im_meta['DATE_ACQUIRED'],
|
||||
'geom_rmse_model':im_meta['GEOMETRIC_RMSE_MODEL'],
|
||||
'gcp_model':im_meta['GROUND_CONTROL_POINTS_MODEL'],
|
||||
'quality':im_meta['IMAGE_QUALITY_OLI'],
|
||||
'sun_azimuth':im_meta['SUN_AZIMUTH'],
|
||||
'sun_elevation':im_meta['SUN_ELEVATION']}
|
||||
|
||||
im_bands = im_dic.get('bands')
|
||||
|
||||
# delete dimensions key from dictionnary, otherwise the entire image is extracted
|
||||
for i in range(len(im_bands)): del im_bands[i]['dimensions']
|
||||
|
||||
# load panchromatic band
|
||||
pan_band = [im_bands[7]]
|
||||
im_pan, crs_pan = load_image(im, polygon, pan_band)
|
||||
im_pan = im_pan[:,:,0]
|
||||
|
||||
# load the multispectral bands (B2,B3,B4,B5,B6) = (blue,green,red,nir,swir1)
|
||||
ms_bands = [im_bands[1], im_bands[2], im_bands[3], im_bands[4], im_bands[5]]
|
||||
im_ms_30m, crs_ms = load_image(im, polygon, ms_bands)
|
||||
|
||||
# create cloud mask
|
||||
qa_band = [im_bands[11]]
|
||||
im_qa, crs_qa = load_image(im, polygon, qa_band)
|
||||
im_qa = im_qa[:,:,0]
|
||||
im_cloud = create_cloud_mask(im_qa, sat_name, plot_bool)
|
||||
im_cloud = transform.resize(im_cloud, (im_pan.shape[0], im_pan.shape[1]),
|
||||
order=0, preserve_range=True, mode='constant').astype('bool_')
|
||||
|
||||
# resize the image using bilinear interpolation (order 1)
|
||||
im_ms = transform.resize(im_ms_30m,(im_pan.shape[0], im_pan.shape[1]),
|
||||
order=1, preserve_range=True, mode='constant')
|
||||
|
||||
# check if -inf values (means out of image) and add to cloud mask
|
||||
im_inf = np.isin(im_ms[:,:,0], -np.inf)
|
||||
im_nan = np.isnan(im_ms[:,:,0])
|
||||
im_cloud = np.logical_or(np.logical_or(im_cloud, im_inf), im_nan)
|
||||
|
||||
# get the crs parameters for the image at 15m and 30m resolution
|
||||
crs = {'crs_15m':crs_pan, 'crs_30m':crs_ms, 'epsg_code':int(pan_band[0]['crs'][5:])}
|
||||
|
||||
if plot_bool:
|
||||
|
||||
# if there are -inf in the image, set them to 0 before plotting
|
||||
if sum(sum(np.isin(im_ms_30m[:,:,0], -np.inf).astype(int))) > 0:
|
||||
idx = np.isin(im_ms_30m[:,:,0], -np.inf)
|
||||
im_ms_30m[idx,0] = 0; im_ms_30m[idx,1] = 0; im_ms_30m[idx,2] = 0;
|
||||
im_ms_30m[idx,3] = 0; im_ms_30m[idx,4] = 0
|
||||
|
||||
plt.figure()
|
||||
|
||||
plt.subplot(221)
|
||||
plt.imshow(im_pan, cmap='gray')
|
||||
plt.title('PANCHROMATIC')
|
||||
|
||||
plt.subplot(222)
|
||||
plt.imshow(im_ms_30m[:,:,[2,1,0]])
|
||||
plt.title('RGB')
|
||||
|
||||
|
||||
plt.subplot(223)
|
||||
plt.imshow(im_ms_30m[:,:,3], cmap='gray')
|
||||
plt.title('NIR')
|
||||
|
||||
plt.subplot(224)
|
||||
plt.imshow(im_ms_30m[:,:,4], cmap='gray')
|
||||
plt.title('SWIR')
|
||||
|
||||
plt.show()
|
||||
|
||||
return im_pan, im_ms, im_cloud, crs, meta
|
||||
|
||||
|
||||
def rescale_image_intensity(im, cloud_mask, prob_high, plot_bool):
|
||||
"""
|
||||
Rescales the intensity of an image (multispectral or single band) by applying
|
||||
a cloud mask and clipping the prob_high upper percentile. This functions allows
|
||||
to stretch the contrast of an image.
|
||||
|
||||
KV WRL 2018
|
||||
|
||||
Arguments:
|
||||
-----------
|
||||
im: np.ndarray
|
||||
Image to rescale, can be 3D (multispectral) or 2D (single band)
|
||||
cloud_mask: np.ndarray
|
||||
2D cloud mask with True where cloud pixels are
|
||||
prob_high: float
|
||||
probability of exceedence used to calculate the upper percentile
|
||||
plot_bool: boolean
|
||||
True if plot is wanted
|
||||
|
||||
Returns:
|
||||
-----------
|
||||
im_adj: np.ndarray
|
||||
The rescaled image
|
||||
"""
|
||||
prc_low = 0 # lower percentile
|
||||
vec_mask = cloud_mask.reshape(im.shape[0] * im.shape[1])
|
||||
|
||||
if plot_bool:
|
||||
plt.figure()
|
||||
|
||||
if len(im.shape) > 2:
|
||||
vec = im.reshape(im.shape[0] * im.shape[1], im.shape[2])
|
||||
vec_adj = np.ones((len(vec_mask), im.shape[2])) * np.nan
|
||||
|
||||
for i in range(im.shape[2]):
|
||||
prc_high = np.percentile(vec[~vec_mask, i], prob_high)
|
||||
vec_rescaled = exposure.rescale_intensity(vec[~vec_mask, i], in_range=(prc_low, prc_high))
|
||||
vec_adj[~vec_mask,i] = vec_rescaled
|
||||
|
||||
if plot_bool:
|
||||
plt.subplot(np.floor(im.shape[2]/2) + 1, np.floor(im.shape[2]/2), i+1)
|
||||
plt.hist(vec[~vec_mask, i], bins=200, label='original')
|
||||
plt.hist(vec_rescaled, bins=200, alpha=0.5, label='rescaled')
|
||||
plt.legend()
|
||||
plt.title('Band' + str(i+1))
|
||||
plt.show()
|
||||
|
||||
im_adj = vec_adj.reshape(im.shape[0], im.shape[1], im.shape[2])
|
||||
|
||||
if plot_bool:
|
||||
plt.figure()
|
||||
ax1 = plt.subplot(121)
|
||||
plt.imshow(im[:,:,[2,1,0]])
|
||||
plt.axis('off')
|
||||
plt.title('Original')
|
||||
ax2 = plt.subplot(122, sharex=ax1, sharey=ax1)
|
||||
plt.imshow(im_adj[:,:,[2,1,0]])
|
||||
plt.axis('off')
|
||||
plt.title('Rescaled')
|
||||
plt.show()
|
||||
|
||||
else:
|
||||
vec = im.reshape(im.shape[0] * im.shape[1])
|
||||
vec_adj = np.ones(len(vec_mask)) * np.nan
|
||||
prc_high = np.percentile(vec[~vec_mask], prob_high)
|
||||
vec_rescaled = exposure.rescale_intensity(vec[~vec_mask], in_range=(prc_low, prc_high))
|
||||
vec_adj[~vec_mask] = vec_rescaled
|
||||
|
||||
if plot_bool:
|
||||
plt.hist(vec[~vec_mask], bins=200, label='original')
|
||||
plt.hist(vec_rescaled, bins=200, alpha=0.5, label='rescaled')
|
||||
plt.legend()
|
||||
plt.title('Single band')
|
||||
plt.show()
|
||||
|
||||
im_adj = vec_adj.reshape(im.shape[0], im.shape[1])
|
||||
|
||||
if plot_bool:
|
||||
plt.figure()
|
||||
ax1 = plt.subplot(121)
|
||||
plt.imshow(im, cmap='gray')
|
||||
plt.axis('off')
|
||||
plt.title('Original')
|
||||
ax2 = plt.subplot(122, sharex=ax1, sharey=ax1)
|
||||
plt.imshow(im_adj, cmap='gray')
|
||||
plt.axis('off')
|
||||
plt.title('Rescaled')
|
||||
plt.show()
|
||||
|
||||
return im_adj
|
||||
|
||||
|
||||
def hist_match(source, template):
|
||||
"""
|
||||
Adjust the pixel values of a grayscale image such that its histogram
|
||||
matches that of a target image
|
||||
|
||||
Arguments:
|
||||
-----------
|
||||
source: np.ndarray
|
||||
Image to transform; the histogram is computed over the flattened
|
||||
array
|
||||
template: np.ndarray
|
||||
Template image; can have different dimensions to source
|
||||
Returns:
|
||||
-----------
|
||||
matched: np.ndarray
|
||||
The transformed output image
|
||||
"""
|
||||
|
||||
oldshape = source.shape
|
||||
source = source.ravel()
|
||||
template = template.ravel()
|
||||
|
||||
# get the set of unique pixel values and their corresponding indices and
|
||||
# counts
|
||||
s_values, bin_idx, s_counts = np.unique(source, return_inverse=True,
|
||||
return_counts=True)
|
||||
t_values, t_counts = np.unique(template, return_counts=True)
|
||||
|
||||
# take the cumsum of the counts and normalize by the number of pixels to
|
||||
# get the empirical cumulative distribution functions for the source and
|
||||
# template images (maps pixel value --> quantile)
|
||||
s_quantiles = np.cumsum(s_counts).astype(np.float64)
|
||||
s_quantiles /= s_quantiles[-1]
|
||||
t_quantiles = np.cumsum(t_counts).astype(np.float64)
|
||||
t_quantiles /= t_quantiles[-1]
|
||||
|
||||
# interpolate linearly to find the pixel values in the template image
|
||||
# that correspond most closely to the quantiles in the source image
|
||||
interp_t_values = np.interp(s_quantiles, t_quantiles, t_values)
|
||||
|
||||
return interp_t_values[bin_idx].reshape(oldshape)
|
||||
|
||||
def pansharpen(im_ms, im_pan, cloud_mask, plot_bool):
|
||||
"""
|
||||
Pansharpens a multispectral image (3D), using the panchromatic band (2D)
|
||||
and a cloud mask
|
||||
|
||||
KV WRL 2018
|
||||
|
||||
Arguments:
|
||||
-----------
|
||||
im_ms: np.ndarray
|
||||
Multispectral image to pansharpen (3D)
|
||||
im_pan: np.ndarray
|
||||
Panchromatic band (2D)
|
||||
cloud_mask: np.ndarray
|
||||
2D cloud mask with True where cloud pixels are
|
||||
plot_bool: boolean
|
||||
True if plot is wanted
|
||||
|
||||
Returns:
|
||||
-----------
|
||||
im_ms_ps: np.ndarray
|
||||
Pansharpened multisoectral image (3D)
|
||||
"""
|
||||
|
||||
# reshape image into vector and apply cloud mask
|
||||
vec = im_ms.reshape(im_ms.shape[0] * im_ms.shape[1], im_ms.shape[2])
|
||||
vec_mask = cloud_mask.reshape(im_ms.shape[0] * im_ms.shape[1])
|
||||
vec = vec[~vec_mask, :]
|
||||
# apply PCA to RGB bands
|
||||
pca = decomposition.PCA()
|
||||
vec_pcs = pca.fit_transform(vec)
|
||||
# replace 1st PC with pan band (after matching histograms)
|
||||
vec_pan = im_pan.reshape(im_pan.shape[0] * im_pan.shape[1])
|
||||
vec_pan = vec_pan[~vec_mask]
|
||||
vec_pcs[:,0] = hist_match(vec_pan, vec_pcs[:,0])
|
||||
vec_ms_ps = pca.inverse_transform(vec_pcs)
|
||||
|
||||
# reshape vector into image
|
||||
vec_ms_ps_full = np.ones((len(vec_mask), im_ms.shape[2])) * np.nan
|
||||
vec_ms_ps_full[~vec_mask,:] = vec_ms_ps
|
||||
im_ms_ps = vec_ms_ps_full.reshape(im_ms.shape[0], im_ms.shape[1], im_ms.shape[2])
|
||||
|
||||
if plot_bool:
|
||||
plt.figure()
|
||||
ax1 = plt.subplot(121)
|
||||
plt.imshow(rescale_image_intensity(im_ms[:,:,[2,1,0]], cloud_mask, 99.9, False))
|
||||
plt.axis('off')
|
||||
plt.title('Original')
|
||||
ax2 = plt.subplot(122, sharex=ax1, sharey=ax1)
|
||||
plt.imshow(rescale_image_intensity(im_ms_ps[:,:,[2,1,0]], cloud_mask, 99.9, False))
|
||||
plt.axis('off')
|
||||
plt.title('Pansharpened')
|
||||
plt.show()
|
||||
|
||||
return im_ms_ps
|
||||
|
||||
def nd_index(im1, im2, cloud_mask, plot_bool):
|
||||
"""
|
||||
Computes normalised difference index on 2 images (2D), given a cloud mask (2D)
|
||||
|
||||
KV WRL 2018
|
||||
|
||||
Arguments:
|
||||
-----------
|
||||
im1, im2: np.ndarray
|
||||
Images (2D) with which to calculate the ND index
|
||||
cloud_mask: np.ndarray
|
||||
2D cloud mask with True where cloud pixels are
|
||||
plot_bool: boolean
|
||||
True if plot is wanted
|
||||
|
||||
Returns: -----------
|
||||
im_nd: np.ndarray
|
||||
|
||||
Image (2D) containing the ND index
|
||||
"""
|
||||
vec_mask = cloud_mask.reshape(im1.shape[0] * im1.shape[1])
|
||||
vec_nd = np.ones(len(vec_mask)) * np.nan
|
||||
vec1 = im1.reshape(im1.shape[0] * im1.shape[1])
|
||||
vec2 = im2.reshape(im2.shape[0] * im2.shape[1])
|
||||
temp = np.divide(vec1[~vec_mask] - vec2[~vec_mask],
|
||||
vec1[~vec_mask] + vec2[~vec_mask])
|
||||
vec_nd[~vec_mask] = temp
|
||||
im_nd = vec_nd.reshape(im1.shape[0], im1.shape[1])
|
||||
|
||||
if plot_bool:
|
||||
plt.figure()
|
||||
plt.imshow(im_nd, cmap='seismic')
|
||||
plt.colorbar()
|
||||
plt.title('Normalised index')
|
||||
plt.show()
|
||||
|
||||
return im_nd
|
||||
|
||||
def find_wl_contours(im_ndwi, cloud_mask, min_contour_points, plot_bool):
|
||||
"""
|
||||
Finds the water line by thresholding the Normalized Difference Water Index and applying the Marching
|
||||
Squares Algorithm
|
||||
|
||||
KV WRL 2018
|
||||
|
||||
Arguments:
|
||||
-----------
|
||||
im_ndwi: np.ndarray
|
||||
Image (2D) with the NDWI (water index)
|
||||
cloud_mask: np.ndarray
|
||||
2D cloud mask with True where cloud pixels are
|
||||
min_contour_points: int
|
||||
minimum number of points in each contour line
|
||||
plot_bool: boolean
|
||||
True if plot is wanted
|
||||
|
||||
Returns: -----------
|
||||
contours_wl: list of np.arrays
|
||||
contains the (row,column) coordinates of the contour lines
|
||||
|
||||
"""
|
||||
|
||||
# reshape image to vector
|
||||
vec_ndwi = im_ndwi.reshape(im_ndwi.shape[0] * im_ndwi.shape[1])
|
||||
vec_mask = cloud_mask.reshape(cloud_mask.shape[0] * cloud_mask.shape[1])
|
||||
vec = vec_ndwi[~vec_mask]
|
||||
# apply otsu's threshold
|
||||
t_otsu = filters.threshold_otsu(vec)
|
||||
# use Marching Squares algorithm to detect contours on ndwi image
|
||||
contours = measure.find_contours(im_ndwi, t_otsu)
|
||||
# filter water lines
|
||||
contours_wl = []
|
||||
for i, contour in enumerate(contours):
|
||||
# remove contour points that are around clouds (nan values)
|
||||
if np.any(np.isnan(contour)):
|
||||
index_nan = np.where(np.isnan(contour))[0]
|
||||
contour = np.delete(contour, index_nan, axis=0)
|
||||
# remove contours that have only few points (less than min_contour_points)
|
||||
if contour.shape[0] > min_contour_points:
|
||||
contours_wl.append(contour)
|
||||
|
||||
if plot_bool:
|
||||
# plot otsu's histogram segmentation
|
||||
plt.figure()
|
||||
vals = plt.hist(vec, bins=200)
|
||||
plt.plot([t_otsu, t_otsu],[0, np.max(vals[0])], 'r-', label='Otsu threshold')
|
||||
plt.legend()
|
||||
plt.show()
|
||||
|
||||
# plot the water line contours on top of water index
|
||||
plt.figure()
|
||||
plt.imshow(im_ndwi, cmap='seismic')
|
||||
plt.colorbar()
|
||||
for i,contour in enumerate(contours_wl): plt.plot(contour[:, 1], contour[:, 0], linewidth=3, color='k')
|
||||
plt.axis('image')
|
||||
plt.title('Detected water lines')
|
||||
plt.show()
|
||||
|
||||
return contours_wl
|
||||
|
||||
def convert_pix2world(points, crs_vec):
|
||||
"""
|
||||
Converts pixel coordinates (row,columns) to world projected coordinates
|
||||
performing an affine transformation
|
||||
|
||||
KV WRL 2018
|
||||
|
||||
Arguments:
|
||||
-----------
|
||||
points: np.ndarray or list of np.ndarray
|
||||
array with 2 columns (rows first and columns second)
|
||||
crs_vec: np.ndarray
|
||||
vector of 6 elements [Xtr, Xscale, Xshear, Ytr, Yshear, Yscale]
|
||||
|
||||
Returns: -----------
|
||||
points_converted: np.ndarray or list of np.ndarray
|
||||
converted coordinates, first columns with X and second column with Y
|
||||
|
||||
"""
|
||||
|
||||
# make affine transformation matrix
|
||||
aff_mat = np.array([[crs_vec[1], crs_vec[2], crs_vec[0]],
|
||||
[crs_vec[4], crs_vec[5], crs_vec[3]],
|
||||
[0, 0, 1]])
|
||||
# create affine transformation
|
||||
tform = transform.AffineTransform(aff_mat)
|
||||
|
||||
if type(points) is list:
|
||||
points_converted = []
|
||||
# iterate over the list
|
||||
for i, arr in enumerate(points):
|
||||
tmp = arr[:,[1,0]]
|
||||
points_converted.append(tform(tmp))
|
||||
|
||||
elif type(points) is np.ndarray:
|
||||
tmp = points[:,[1,0]]
|
||||
points_converted = tform(tmp)
|
||||
|
||||
else:
|
||||
print('invalid input type')
|
||||
raise
|
||||
|
||||
return points_converted
|
||||
|
||||
def convert_epsg(points, epsg_in, epsg_out):
|
||||
"""
|
||||
Converts from one spatial reference to another using the epsg codes
|
||||
|
||||
KV WRL 2018
|
||||
|
||||
Arguments:
|
||||
-----------
|
||||
points: np.ndarray or list of np.ndarray
|
||||
array with 2 columns (rows first and columns second)
|
||||
epsg_in: int
|
||||
epsg code of the spatial reference in which the input is
|
||||
epsg_out: int
|
||||
epsg code of the spatial reference in which the output will be
|
||||
|
||||
Returns: -----------
|
||||
points_converted: np.ndarray or list of np.ndarray
|
||||
converted coordinates
|
||||
|
||||
"""
|
||||
|
||||
# define input and output spatial references
|
||||
inSpatialRef = osr.SpatialReference()
|
||||
inSpatialRef.ImportFromEPSG(epsg_in)
|
||||
outSpatialRef = osr.SpatialReference()
|
||||
outSpatialRef.ImportFromEPSG(epsg_out)
|
||||
# create a coordinates transform
|
||||
coordTransform = osr.CoordinateTransformation(inSpatialRef, outSpatialRef)
|
||||
# transform points
|
||||
if type(points) is list:
|
||||
points_converted = []
|
||||
# iterate over the list
|
||||
for i, arr in enumerate(points):
|
||||
points_converted.append(np.array(coordTransform.TransformPoints(arr)))
|
||||
|
||||
elif type(points) is np.ndarray:
|
||||
points_converted = np.array(coordTransform.TransformPoints(points))
|
||||
|
||||
else:
|
||||
print('invalid input type')
|
||||
raise
|
||||
|
||||
return points_converted
|
||||
|
||||
def classify_sand_unsupervised(im_ms_ps, im_pan, cloud_mask, wl_pix, buffer_size, min_beach_size, plot_bool):
|
||||
"""
|
||||
Classifies sand pixels using an unsupervised algorithm (Kmeans)
|
||||
Set buffer size to False if you want to classify the entire image,
|
||||
otherwise buffer size defines the buffer around the shoreline in which
|
||||
pixels are considered for classification.
|
||||
This classification is not robust and is only used to train a supervised algorithm
|
||||
|
||||
KV WRL 2018
|
||||
|
||||
Arguments:
|
||||
-----------
|
||||
im_ms_ps: np.ndarray
|
||||
Pansharpened RGB + downsampled NIR and SWIR
|
||||
im_pan:
|
||||
Panchromatic band
|
||||
cloud_mask: np.ndarray
|
||||
2D cloud mask with True where cloud pixels are
|
||||
wl_pix: list of np.ndarray
|
||||
list of arrays containig the pixel coordinates of the water line
|
||||
buffer_size: int or False
|
||||
radius of the disk used to create a buffer around the water line
|
||||
when False, the entire image is considered for kmeans
|
||||
min_beach_size: int
|
||||
minimum number of connected pixels belonging to a single beach
|
||||
plot_bool: boolean
|
||||
True if plot is wanted
|
||||
|
||||
Returns: -----------
|
||||
im_sand: np.ndarray
|
||||
2D binary image containing True where sand pixels are located
|
||||
|
||||
"""
|
||||
# reshape the 2D images into vectors
|
||||
vec_ms_ps = im_ms_ps.reshape(im_ms_ps.shape[0] * im_ms_ps.shape[1], im_ms_ps.shape[2])
|
||||
vec_pan = im_pan.reshape(im_pan.shape[0]*im_pan.shape[1])
|
||||
vec_mask = cloud_mask.reshape(im_ms_ps.shape[0] * im_ms_ps.shape[1])
|
||||
# add B,G,R,NIR and pan bands to the vector of features
|
||||
vec_features = np.zeros((vec_ms_ps.shape[0], 5))
|
||||
vec_features[:,[0,1,2,3]] = vec_ms_ps[:,[0,1,2,3]]
|
||||
vec_features[:,4] = vec_pan
|
||||
|
||||
if buffer_size:
|
||||
# create binary image with ones where the detected water lines is
|
||||
im_buffer = np.zeros((im_ms_ps.shape[0], im_ms_ps.shape[1]))
|
||||
for i, contour in enumerate(wl_pix):
|
||||
indices = [(int(_[0]), int(_[1])) for _ in list(np.round(contour))]
|
||||
for j, idx in enumerate(indices):
|
||||
im_buffer[idx] = 1
|
||||
# perform a dilation on the binary image
|
||||
se = morphology.disk(buffer_size)
|
||||
im_buffer = morphology.binary_dilation(im_buffer, se)
|
||||
vec_buffer = (im_buffer == 1).reshape(im_ms_ps.shape[0] * im_ms_ps.shape[1])
|
||||
else:
|
||||
vec_buffer = np.ones((vec_pan.shape[0]))
|
||||
# add cloud mask to buffer
|
||||
vec_buffer= np.logical_and(vec_buffer, ~vec_mask)
|
||||
# perform kmeans (6 clusters)
|
||||
kmeans = KMeans(n_clusters=6, random_state=0).fit(vec_features[vec_buffer,:])
|
||||
labels = np.ones((len(vec_mask))) * np.nan
|
||||
labels[vec_buffer] = kmeans.labels_
|
||||
im_labels = labels.reshape(im_ms_ps.shape[0], im_ms_ps.shape[1])
|
||||
# find the class with maximum reflection in the B,G,R,Pan
|
||||
im_sand = im_labels == np.argmax(np.mean(kmeans.cluster_centers_[:,[0,1,2,4]], axis=1))
|
||||
im_sand = morphology.remove_small_objects(im_sand, min_size=min_beach_size, connectivity=2)
|
||||
im_sand = morphology.binary_erosion(im_sand, morphology.disk(1))
|
||||
# im_sand = morphology.binary_dilation(im_sand, morphology.disk(1))
|
||||
|
||||
if plot_bool:
|
||||
im = np.copy(rescale_image_intensity(im_ms_ps[:,:,[2,1,0]], cloud_mask, 99.9, False))
|
||||
im[im_sand,0] = 0
|
||||
im[im_sand,1] = 0
|
||||
im[im_sand,2] = 1
|
||||
plt.figure()
|
||||
plt.imshow(im)
|
||||
plt.axis('image')
|
||||
plt.title('Sand classification')
|
||||
plt.show()
|
||||
|
||||
return im_sand
|
||||
|
||||
def classify_image_NN(im_ms_ps, im_pan, cloud_mask, min_beach_size, plot_bool):
|
||||
"""
|
||||
Classifies every pixel in the image in one of 4 classes:
|
||||
- sand --> label = 1
|
||||
- whitewater (breaking waves and swash) --> label = 2
|
||||
- water --> label = 3
|
||||
- other (vegetation, buildings, rocks...) --> label = 0
|
||||
|
||||
The classifier is a Neural Network, trained with 7000 pixels for the class SAND and 1500 pixels for
|
||||
each of the other classes. This is because the class of interest for my application is SAND and I
|
||||
wanted to minimize the classification error for that class
|
||||
|
||||
KV WRL 2018
|
||||
|
||||
Arguments:
|
||||
-----------
|
||||
im_ms_ps: np.ndarray
|
||||
Pansharpened RGB + downsampled NIR and SWIR
|
||||
im_pan:
|
||||
Panchromatic band
|
||||
cloud_mask: np.ndarray
|
||||
2D cloud mask with True where cloud pixels are
|
||||
plot_bool: boolean
|
||||
True if plot is wanted
|
||||
|
||||
Returns: -----------
|
||||
im_classif: np.ndarray
|
||||
2D image containing labels
|
||||
im_labels: np.ndarray of booleans
|
||||
3D image containing a boolean image for each class (im_classif == label)
|
||||
|
||||
"""
|
||||
|
||||
# load classifier
|
||||
clf = joblib.load('functions/NeuralNet_classif.pkl')
|
||||
|
||||
# calculate features
|
||||
n_features = 10
|
||||
im_features = np.zeros((im_ms_ps.shape[0], im_ms_ps.shape[1], n_features))
|
||||
im_features[:,:,[0,1,2,3,4]] = im_ms_ps
|
||||
im_features[:,:,5] = im_pan
|
||||
im_features[:,:,6] = nd_index(im_ms_ps[:,:,3], im_ms_ps[:,:,1], cloud_mask, False) # (NIR-G)
|
||||
im_features[:,:,7] = nd_index(im_ms_ps[:,:,3], im_ms_ps[:,:,2], cloud_mask, False) # ND(NIR-R)
|
||||
im_features[:,:,8] = nd_index(im_ms_ps[:,:,0], im_ms_ps[:,:,2], cloud_mask, False) # ND(B-R)
|
||||
im_features[:,:,9] = nd_index(im_ms_ps[:,:,4], im_ms_ps[:,:,1], cloud_mask, False) # ND(SWIR-G)
|
||||
# remove NaNs and clouds
|
||||
vec_features = im_features.reshape((im_ms_ps.shape[0] * im_ms_ps.shape[1], n_features))
|
||||
vec_cloud = cloud_mask.reshape(cloud_mask.shape[0]*cloud_mask.shape[1])
|
||||
vec_nan = np.any(np.isnan(vec_features), axis=1)
|
||||
vec_mask = np.logical_or(vec_cloud, vec_nan)
|
||||
vec_features = vec_features[~vec_mask, :]
|
||||
# predict with NN classifier
|
||||
labels = clf.predict(vec_features)
|
||||
# recompose image
|
||||
vec_classif = np.zeros((cloud_mask.shape[0]*cloud_mask.shape[1]))
|
||||
vec_classif[~vec_mask] = labels
|
||||
im_classif = vec_classif.reshape((im_ms_ps.shape[0], im_ms_ps.shape[1]))
|
||||
|
||||
# labels
|
||||
im_sand = im_classif == 1
|
||||
im_sand = morphology.remove_small_objects(im_sand, min_size=min_beach_size, connectivity=2)
|
||||
im_swash = im_classif == 2
|
||||
im_water = im_classif == 3
|
||||
im_labels = np.stack((im_sand,im_swash,im_water), axis=-1)
|
||||
|
||||
if plot_bool:
|
||||
# display on top of pansharpened RGB
|
||||
im_display = rescale_image_intensity(im_ms_ps[:,:,[2,1,0]], cloud_mask, 99.9, False)
|
||||
im = np.copy(im_display)
|
||||
# define colours for plot
|
||||
colours = np.array([[1,128/255,0/255],[204/255,1,1],[0,0,204/255]])
|
||||
for k in range(0,im_labels.shape[2]):
|
||||
im[im_labels[:,:,k],0] = colours[k,0]
|
||||
im[im_labels[:,:,k],1] = colours[k,1]
|
||||
im[im_labels[:,:,k],2] = colours[k,2]
|
||||
|
||||
plt.figure()
|
||||
ax1 = plt.subplot(121)
|
||||
plt.imshow(im_display)
|
||||
plt.axis('off')
|
||||
plt.title('Image')
|
||||
ax2 = plt.subplot(122, sharex=ax1, sharey=ax1)
|
||||
plt.imshow(im)
|
||||
plt.axis('off')
|
||||
plt.title('NN classifier')
|
||||
mng = plt.get_current_fig_manager()
|
||||
mng.window.showMaximized()
|
||||
plt.tight_layout()
|
||||
plt.draw()
|
||||
|
||||
return im_classif, im_labels
|
||||
|
||||
def find_wl_contours2(im_ms_ps, im_labels, cloud_mask, buffer_size, plot_bool):
|
||||
"""
|
||||
New mthod for extracting shorelines (more robust)
|
||||
|
||||
KV WRL 2018
|
||||
|
||||
Arguments:
|
||||
-----------
|
||||
im_ms_ps: np.ndarray
|
||||
Pansharpened RGB + downsampled NIR and SWIR
|
||||
im_labels: np.ndarray
|
||||
3D image containing a boolean image for each class in the order (sand, swash, water)
|
||||
cloud_mask: np.ndarray
|
||||
2D cloud mask with True where cloud pixels are
|
||||
buffer_size: int
|
||||
size of the buffer around the sandy beach
|
||||
plot_bool: boolean
|
||||
True if plot is wanted
|
||||
|
||||
Returns: -----------
|
||||
contours_wi: list of np.arrays
|
||||
contains the (row,column) coordinates of the contour lines extracted with the Water Index
|
||||
contours_mwi: list of np.arrays
|
||||
contains the (row,column) coordinates of the contour lines extracted with the Modified Water Index
|
||||
|
||||
"""
|
||||
|
||||
nrows = cloud_mask.shape[0]
|
||||
ncols = cloud_mask.shape[1]
|
||||
im_display = rescale_image_intensity(im_ms_ps[:,:,[2,1,0]], cloud_mask, 99.9, False)
|
||||
|
||||
# calculate Normalized Difference Modified Water Index (SWIR - G)
|
||||
im_mwi = nd_index(im_ms_ps[:,:,4], im_ms_ps[:,:,1], cloud_mask, False)
|
||||
# calculate Normalized Difference Modified Water Index (NIR - G)
|
||||
im_wi = nd_index(im_ms_ps[:,:,3], im_ms_ps[:,:,1], cloud_mask, False)
|
||||
# stack indices together
|
||||
im_ind = np.stack((im_wi, im_mwi), axis=-1)
|
||||
vec_ind = im_ind.reshape(nrows*ncols,2)
|
||||
|
||||
# process labels
|
||||
vec_sand = im_labels[:,:,0].reshape(ncols*nrows)
|
||||
vec_swash = im_labels[:,:,1].reshape(ncols*nrows)
|
||||
vec_water = im_labels[:,:,2].reshape(ncols*nrows)
|
||||
|
||||
# create a buffer around the sandy beach
|
||||
se = morphology.disk(buffer_size)
|
||||
im_buffer = morphology.binary_dilation(im_labels[:,:,0], se)
|
||||
vec_buffer = im_buffer.reshape(nrows*ncols)
|
||||
|
||||
# select water/sand/swash pixels that are within the buffer
|
||||
int_water = vec_ind[np.logical_and(vec_buffer,vec_water),:]
|
||||
int_sand = vec_ind[np.logical_and(vec_buffer,vec_sand),:]
|
||||
int_swash = vec_ind[np.logical_and(vec_buffer,vec_swash),:]
|
||||
|
||||
# threshold the sand/water intensities
|
||||
int_all = np.append(int_water,int_sand, axis=0)
|
||||
t_mwi = filters.threshold_otsu(int_all[:,0])
|
||||
t_wi = filters.threshold_otsu(int_all[:,1])
|
||||
|
||||
# find contour with MS algorithm
|
||||
im_wi_buffer = np.copy(im_wi)
|
||||
im_wi_buffer[~im_buffer] = np.nan
|
||||
im_mwi_buffer = np.copy(im_mwi)
|
||||
im_mwi_buffer[~im_buffer] = np.nan
|
||||
contours_wi = measure.find_contours(im_wi_buffer, t_wi)
|
||||
contours_mwi = measure.find_contours(im_mwi_buffer, t_mwi)
|
||||
|
||||
if plot_bool:
|
||||
|
||||
im = np.copy(im_display)
|
||||
# define colours for plot
|
||||
colours = np.array([[1,128/255,0/255],[204/255,1,1],[0,0,204/255]])
|
||||
for k in range(0,im_labels.shape[2]):
|
||||
im[im_labels[:,:,k],0] = colours[k,0]
|
||||
im[im_labels[:,:,k],1] = colours[k,1]
|
||||
im[im_labels[:,:,k],2] = colours[k,2]
|
||||
plt.figure()
|
||||
plt.imshow(im)
|
||||
for i,contour in enumerate(contours_wi): plt.plot(contour[:, 1], contour[:, 0], linewidth=3, color='k')
|
||||
for i,contour in enumerate(contours_mwi): plt.plot(contour[:, 1], contour[:, 0], linewidth=3, color='g')
|
||||
plt.draw()
|
||||
|
||||
fig, ax = plt.subplots(2,1, sharex=True)
|
||||
vals = ax[0].hist(int_water[:,0], bins=100, label='water')
|
||||
ax[0].hist(int_sand[:,0], bins=100, alpha=0.5, label='sand')
|
||||
ax[0].hist(int_swash[:,0], bins=100, alpha=0.5, label='swash')
|
||||
ax[0].plot([t_wi, t_wi], [0, np.max(vals[0])], 'r-')
|
||||
ax[0].legend()
|
||||
ax[0].set_title('Water Index NIR-G')
|
||||
vals = ax[1].hist(int_water[:,1], bins=100, label='water')
|
||||
ax[1].hist(int_sand[:,1], bins=100, alpha=0.5, label='sand')
|
||||
ax[1].hist(int_swash[:,1], bins=100, alpha=0.5, label='swash')
|
||||
ax[1].plot([t_mwi, t_mwi], [0, np.max(vals[0])], 'r-')
|
||||
ax[1].legend()
|
||||
ax[1].set_title('Modified Water Index SWIR-G')
|
||||
plt.draw()
|
||||
|
||||
|
||||
return contours_wi, contours_mwi
|
||||
|
||||
@ -1,39 +0,0 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
Created on Thu Apr 5 16:19:31 2018
|
||||
|
||||
@author: z5030440
|
||||
"""
|
||||
|
||||
d_gt = {'arr':sl_gt}
|
||||
d_sds = {'arr':sl_sds}
|
||||
|
||||
sio.savemat('sl_gt.mat', mdict=d_gt)
|
||||
sio.savemat('sl_sds.mat', mdict=d_sds)
|
||||
|
||||
#%%
|
||||
|
||||
herror = sio.loadmat('hor_error.mat')
|
||||
|
||||
diff_p = (herror['gt_av'] - herror['sds_av'])[0,:]
|
||||
|
||||
f = plt.figure()
|
||||
plt.subplot(3,1,1)
|
||||
plt.bar(np.linspace(1,len(zav),len(zav)), herror['p_rmse'][0])
|
||||
plt.ylabel('rmse [m]')
|
||||
plt.xticks([])
|
||||
plt.title('Horizontal cross-shore error')
|
||||
|
||||
plt.subplot(3,1,2)
|
||||
plt.bar(np.linspace(1,len(zav),len(zav)), herror['p_mean'][0], color=orange)
|
||||
plt.ylabel('mean [m]')
|
||||
plt.xticks([])
|
||||
|
||||
plt.subplot(3,1,3)
|
||||
plt.bar(np.linspace(1,len(zav),len(zav)), herror['p_std'][0], color='g')
|
||||
plt.ylabel('std [m]')
|
||||
plt.xlabel('comparison #')
|
||||
plt.grid(False)
|
||||
plt.grid(axis='y')
|
||||
f.subplots_adjust(hspace=0.2)
|
||||
plt.draw()
|
||||
@ -1,65 +0,0 @@
|
||||
close all
|
||||
clear
|
||||
clc
|
||||
|
||||
addpath(genpath('C:\Users\z5030440\Documents\GitHub\geetools\functions\xyz2spz'))
|
||||
|
||||
sl_gt = load('sl_gt.mat')
|
||||
sl_sds = load('sl_sds.mat')
|
||||
|
||||
sl_sds = sl_sds.arr
|
||||
sl_gt = sl_gt.arr
|
||||
for i = 1:length(sl_sds)
|
||||
sds.x = sl_sds{i}(:,1)
|
||||
sds.y = sl_sds{i}(:,2)
|
||||
sds.z = zeros(length(sl_sds{i}(:,1)),1)
|
||||
|
||||
gt.x = sl_gt{i}(:,1)
|
||||
gt.y = sl_gt{i}(:,2)
|
||||
gt.z = zeros(length(sl_gt{i}(:,1)),1)
|
||||
|
||||
outsds = xyz2spz(sds,'NARRA')
|
||||
outgt = xyz2spz(gt,'NARRA')
|
||||
|
||||
figure
|
||||
hold on
|
||||
grid on
|
||||
box on
|
||||
plot(outsds.s, outsds.p, 'b-', 'linewidth',2)
|
||||
plot(outgt.s, outgt.p, 'r-', 'linewidth',2)
|
||||
xlabel('s [m]')
|
||||
ylabel('p [m]')
|
||||
title('Horizontal comparison in spz coordinates')
|
||||
legend({'SDS', 'contour at tide level'})
|
||||
|
||||
xq = nanmin(outsds.s):10:nanmax(outsds.s)
|
||||
[gt_s idx_gt] = unique(outgt.s)
|
||||
gt_p = outgt.p(idx_gt)
|
||||
gt_p_int = interp1(gt_s, gt_p, xq)
|
||||
[sds_s idx_sds] = unique(outsds.s)
|
||||
sds_p = outsds.p(idx_sds)
|
||||
sds_p_int = interp1(sds_s, sds_p, xq)
|
||||
diff_p = sds_p_int - gt_p_int;
|
||||
|
||||
sds_av(i) = median(sds_p_int(~(sds_p_int > median(sds_p_int) + 2*std(sds_p_int) | sds_p_int < median(sds_p_int) - 2*std(sds_p_int))))
|
||||
gt_p_int(isnan(gt_p_int)) = []
|
||||
gt_av(i) = median(gt_p_int(~(gt_p_int > median(gt_p_int) + 2*std(gt_p_int) | gt_p_int < median(gt_p_int) - 2*std(gt_p_int))))
|
||||
|
||||
|
||||
idx_nan = isnan(diff_p)
|
||||
diff_p(idx_nan) = []
|
||||
xq(idx_nan) = []
|
||||
idx_outlier = diff_p > median(diff_p) + 2*std(diff_p) | diff_p < median(diff_p) - 2*std(diff_p)
|
||||
diff_p(idx_outlier) = []
|
||||
xq(idx_outlier) = []
|
||||
|
||||
p_rmse(i) = sqrt(mean(diff_p.^2))
|
||||
p_mean(i) = mean(diff_p)
|
||||
p_std(i) = std(diff_p)
|
||||
p_q90(i) = quantile(abs(diff_p), 0.9)
|
||||
|
||||
end
|
||||
|
||||
clearvars -except sds_av gt_av p_rmse p_mean p_std p_q90
|
||||
|
||||
save 'hor_error.mat'
|
||||
@ -1,213 +0,0 @@
|
||||
function [res,fval,it] = muller (f,Z0,itmax,ztol,ftol,option)
|
||||
% MULLER find a zero of a real or complex function Y = F(Z)
|
||||
%
|
||||
% Syntax:
|
||||
%
|
||||
% RES = MULLER (F,Z0) find the zero of a complex or real function
|
||||
% 'F' (either an anonymous function or .m function) using three initial
|
||||
% guesses contained in the vector Z0. Muller takes the function F and
|
||||
% evaluetes it at each initial point using feval. F doesn't need to be
|
||||
% vectorized.
|
||||
% The initial guesses can be real or complex numbers close to the zero,
|
||||
% bracketing the zero is not necessary. Parameters ITMAX, ZTOL and
|
||||
% FTOL are set by default to 1000, 1e-5 and 1e-5, respectively.
|
||||
%
|
||||
% RES = MULLER (F,Z0,ITMAX) the maximum number of iterations is set
|
||||
% equal to ITMAX. ZTOL and FTOL are set by default with the values mentioned
|
||||
% above.
|
||||
%
|
||||
% RES = MULLER (F,Z0,ITMAX,ZTOL) ZTOL is used as a stopping
|
||||
% criterion. If the absolute difference between the values of Z found in
|
||||
% the two latest iterations is less than ZTOL, the program is stopped. FTOL
|
||||
% is set by default with the value mentioned above.
|
||||
%
|
||||
% RES = MULLER (F,Z0,ITMAX,ZTOL,FTOL) FTOL is used as a stopping
|
||||
% criterion. If the value of the function F at the Z found in the last
|
||||
% iteration is less than FTOL, the program is stopped.
|
||||
%
|
||||
% RES = MULLER (F,Z0,ITMAX,ZTOL,FTOL,'both') indicate that both
|
||||
% criteria ZTOL and FTOL must be satisfied simultaneously. By default,
|
||||
% MULLER stops if one of the two criteria is fulfilled.
|
||||
%
|
||||
% [RES,FVAL] = MULLER (F,Z0,...) return the value of the function
|
||||
% F at the Z found in the last iteration.
|
||||
%
|
||||
% [RES,FVAL,IT] = MULLER (F,Z0,...) return the number of iterations
|
||||
% used to find the zero.
|
||||
%
|
||||
% Example 1:
|
||||
% myf = @(x) (x-1)^3;
|
||||
%
|
||||
% muller(myf,[0 0.1 0.2],[],[],[],'both')
|
||||
% ans =
|
||||
% 1.0000 + 0.0000i
|
||||
%
|
||||
% Example 2:
|
||||
%
|
||||
% [res,fval,it] = muller2('cosh',[0 0.1 0.2],[],[],[],'both')
|
||||
%
|
||||
% res =
|
||||
% 0.0000 + 1.5708i
|
||||
%
|
||||
% fval =
|
||||
% 5.5845e-012 + 3.0132e-012i
|
||||
%
|
||||
% it =
|
||||
% 5
|
||||
%
|
||||
% Method taken from:
|
||||
% Numerical Recipes: The art of scientific computing
|
||||
% W.H. Press; B.P. Flannery; S.A. Teukolsky; W.T. Vetterling
|
||||
% 1986
|
||||
%
|
||||
% Thanks to John D'Errico for his helpfull review.
|
||||
%
|
||||
% Written by Daniel H. Cortes
|
||||
% MAE Department, West Virginia University
|
||||
% March, 2008.
|
||||
%
|
||||
|
||||
|
||||
%=================================================
|
||||
% Checking proper values of the input parameters
|
||||
%=================================================
|
||||
|
||||
|
||||
if nargin > 6
|
||||
error ('Too many arguments.')
|
||||
elseif nargin < 2
|
||||
error('Too few arguments.')
|
||||
end
|
||||
|
||||
if nargin < 6
|
||||
opt = 1;
|
||||
elseif ischar(option) == 1
|
||||
if size(option,2) == 4
|
||||
if sum(option == 'both') == 4
|
||||
opt = 2;
|
||||
else
|
||||
error ('Option parameter must be *both*.')
|
||||
end
|
||||
else
|
||||
error ('Option parameter must be *both*.')
|
||||
end
|
||||
else
|
||||
error ('Option parameter must be a character array (string).')
|
||||
end
|
||||
|
||||
|
||||
if nargin < 5
|
||||
ftol = 1e-5;
|
||||
elseif isnumeric(ftol) ~= 1
|
||||
error ('FTOL must be a numeric argument.')
|
||||
elseif isempty(ftol) == 1
|
||||
ftol = 1e-5;
|
||||
elseif size(ftol,1) ~= 1 || size(ftol,2) ~= 1
|
||||
error ('FTOL cannot be an array')
|
||||
end
|
||||
|
||||
|
||||
if nargin < 4
|
||||
ztol = 1e-5;
|
||||
elseif isnumeric(ztol) ~= 1
|
||||
error ('ZTOL must be a numeric argument.')
|
||||
elseif isempty(ztol) == 1
|
||||
ztol = 1e-5;
|
||||
elseif size(ztol,1) ~= 1 || size(ztol,2) ~= 1
|
||||
error ('ZTOL cannot be an array.')
|
||||
end
|
||||
|
||||
|
||||
if nargin < 3
|
||||
itmax = 1000;
|
||||
elseif isnumeric(itmax) ~= 1
|
||||
error ('ITMAX must be a numeric argument.')
|
||||
elseif isempty(itmax) == 1
|
||||
itmax = 1000;
|
||||
elseif size(itmax,1) ~= 1 || size(itmax,2) ~= 1
|
||||
error ('ITMAX cannot be an array.')
|
||||
end
|
||||
|
||||
|
||||
if isnumeric(Z0) ~= 1
|
||||
error ('Z0 must be a vector of three numeric arguments.')
|
||||
elseif isempty(Z0) == 1 || length(Z0) ~= 3 || min(size(Z0)) ~= 1
|
||||
error ('Z0 must be a vector of length 3 of either complex or real arguments.')
|
||||
end
|
||||
|
||||
if Z0(1)==Z0(2) || Z0(1)==Z0(3) || Z0(2)==Z0(3)
|
||||
error('The initial guesses must be different')
|
||||
end
|
||||
|
||||
%=============================
|
||||
% Begining of Muller's method
|
||||
%=============================
|
||||
|
||||
z0 = Z0(1);
|
||||
z1 = Z0(2);
|
||||
z2 = Z0(3);
|
||||
|
||||
y0 = feval ( f, z0);
|
||||
y1 = feval ( f, z1);
|
||||
y2 = feval ( f, z2);
|
||||
|
||||
for it = 1:itmax
|
||||
q = (z2 - z1)/(z1 - z0);
|
||||
A = q*y2 - q*(1+q)*y1 + q^2*y0;
|
||||
B = (2*q + 1)*y2 - (1 + q)^2*y1 + q^2*y0;
|
||||
C = (1 + q)*y2;
|
||||
|
||||
if ( A ~= 0 )
|
||||
|
||||
disc = B^2 - 4*A*C;
|
||||
|
||||
den1 = ( B + sqrt ( disc ) );
|
||||
den2 = ( B - sqrt ( disc ) );
|
||||
|
||||
if ( abs ( den1 ) < abs ( den2 ) )
|
||||
z3 = z2 - (z2 - z1)*(2*C/den2);
|
||||
else
|
||||
z3 = z2 - (z2 - z1)*(2*C/den1);
|
||||
end
|
||||
|
||||
elseif ( B ~= 0 )
|
||||
z3 = z2 - (z2 - z1)*(2*C/B);
|
||||
else
|
||||
warning('Muller Method failed to find a root. Last iteration result used as an output. Result may not be accurate')
|
||||
res = z2;
|
||||
fval = y2;
|
||||
return
|
||||
end
|
||||
|
||||
y3 = feval ( f, z3);
|
||||
|
||||
if opt == 1
|
||||
if ( abs (z3 - z2) < ztol || abs ( y3 ) < ftol )
|
||||
res = z3;
|
||||
fval = y3;
|
||||
return
|
||||
end
|
||||
else
|
||||
if ( abs (z3 - z2) < ztol && abs ( y3 ) < ftol )
|
||||
res = z3;
|
||||
fval = y3;
|
||||
return
|
||||
end
|
||||
end
|
||||
|
||||
z0 = z1;
|
||||
z1 = z2;
|
||||
z2 = z3;
|
||||
|
||||
y0 = y1;
|
||||
y1 = y2;
|
||||
y2 = y3;
|
||||
|
||||
end
|
||||
|
||||
res = z2;
|
||||
fval = y2;
|
||||
%warning('Maximum number of iterations reached. Result may not be accurate')
|
||||
|
||||
|
||||
|
||||
@ -1,87 +0,0 @@
|
||||
function out = sort_back( data, ind, dim )
|
||||
% SORT_BACK sort back data to original order
|
||||
% ind is the indexes obtained from sorting
|
||||
% dim is the sorted dimension of the data (assumed to be 1 if not specified)
|
||||
% Ex:
|
||||
% y = randn(3,4,2);
|
||||
% [y,ind] = sort(y,2);
|
||||
% do stuff with sorted y...
|
||||
% y2 = sort_back( y, ind, 2 );
|
||||
%
|
||||
% Works on arrays of any dimension
|
||||
% Also works on cellstrings (vectors)
|
||||
%
|
||||
% C = {'hello' 'yes' 'no' 'goodbye'};
|
||||
% [C,ind] = sort(C);
|
||||
% C2 = sort_back(C,ind);
|
||||
%
|
||||
% See also SORT
|
||||
|
||||
%Author Ivar Eskerud Smith
|
||||
|
||||
if size(ind)~=size(data)
|
||||
error('Different size of indexes and input data');
|
||||
end
|
||||
|
||||
if iscell(data)
|
||||
if ~any(size(data)==1)
|
||||
error('Only vectors are supported in cell sorting/back-sorting');
|
||||
end
|
||||
out=cell(size(data));
|
||||
out(ind) = data;
|
||||
return;
|
||||
end
|
||||
|
||||
if ~isnumeric(data) || ~isnumeric(ind)
|
||||
error('Inputs have to be numeric or cell');
|
||||
end
|
||||
|
||||
n=ndims(ind);
|
||||
if ~exist('dim','var')
|
||||
dim=1;
|
||||
end
|
||||
if dim>n
|
||||
error('Specified sorted dimension must be within array bounds');
|
||||
end
|
||||
|
||||
%shift array so that the sorted dim is the first dimension
|
||||
if dim~=1
|
||||
sortInd=1:1:n;sortInd(1)=dim;sortInd(dim)=1;
|
||||
data = permute(data,sortInd);
|
||||
ind = permute(ind,sortInd);
|
||||
end
|
||||
inds = repmat({1},1,n);inds{1}=':';
|
||||
if ~issorted( data(inds{:}) )
|
||||
warning('The input data is not sorted along the specified dimension');
|
||||
end
|
||||
|
||||
s = size(ind);
|
||||
nData = numel(data);
|
||||
inds = repmat({1},1,n);
|
||||
inds(1:2)={':',':'};
|
||||
shiftSize = s(1)*s(2);
|
||||
out=nan(size(data));
|
||||
|
||||
%loop all 2d arrays within nd-array
|
||||
for k=1:prod(s(3:end))
|
||||
tmpdata = data(inds{:});
|
||||
tmpind = ind(inds{:});
|
||||
|
||||
%data is shifted so that the sorted dim = 1
|
||||
for i=1:numel(tmpdata(1,:))
|
||||
out(tmpind(:,i),i) = tmpdata(:,i);
|
||||
end
|
||||
|
||||
if n>2
|
||||
%shift to next 2d array within nd-array
|
||||
shiftInds = mod((1:nData)-shiftSize-1,nData)+1;
|
||||
out=reshape(out(shiftInds),s);
|
||||
data=reshape(data(shiftInds),s);
|
||||
ind=reshape(ind(shiftInds),s);
|
||||
end
|
||||
end
|
||||
|
||||
%permute back to original order
|
||||
sortInd=1:1:ndims(out);sortInd(1)=dim;sortInd(dim)=1;
|
||||
out = permute(out,sortInd);
|
||||
|
||||
@ -1,117 +0,0 @@
|
||||
function out = xyz2spz(xyz_data,site)
|
||||
|
||||
%function out = xyz2spz(xyz_data,site)
|
||||
%
|
||||
%Function to transform (x,y,z) coordinates on an embayed beach to alongshore - cross-shore
|
||||
%coordinates (s,p,z) using the log spiral, given by the equation
|
||||
%r = r0*exp(A*theta). A = cot(alpha).
|
||||
%
|
||||
%xyz_data is a structure containing:
|
||||
%
|
||||
%xyz_data.x
|
||||
%xyz_data.y
|
||||
%xyz_data.z
|
||||
%
|
||||
%site is the name of the structure generated from the MALT graphical user interface
|
||||
%
|
||||
%Refer to paper
|
||||
%
|
||||
%Harley, M.D. and Turner,I.L. (2007) A simple data transformation technique
|
||||
%for pre-processing survey data at embayed beaches, Coast. Eng.,
|
||||
%doi:10.1016/j.coastaleng.2007.07.001, in press.
|
||||
%
|
||||
%Created by Mitch Harley
|
||||
%8th August, 2005
|
||||
%Last Modified 4th April, 2012
|
||||
|
||||
%----------------------------------------------------------------
|
||||
%LOAD LOGSPIRAL-FIT PARAMETERS
|
||||
|
||||
eval(['load ' site ';'])
|
||||
eval(['site = ' site ';'])
|
||||
|
||||
%Define origin and A of log spiral
|
||||
origin = site.origin;
|
||||
alph = site.alpha;
|
||||
A = cot(alph*pi/180);
|
||||
r0_origin = site.r0_origin;
|
||||
|
||||
%-----------------------------------------------------------------
|
||||
|
||||
%DO TRANSFORMATION
|
||||
|
||||
|
||||
%Points need to be sorted prior to analysis %MDH 4/4/2012
|
||||
aa = [xyz_data.x xyz_data.y xyz_data.z];
|
||||
[sorted_points,Isort] = sortrows(aa);
|
||||
|
||||
%Convert xyz coordinates to polar coordinates
|
||||
r = sqrt((sorted_points(:,1) - origin(1)).^2+(sorted_points(:,2) - origin(2)).^2);
|
||||
theta = unwrap(atan2((sorted_points(:,2)-origin(2)),(sorted_points(:,1)-origin(1))) );
|
||||
|
||||
|
||||
%Find constants delta and kappa
|
||||
delta = pi/2+acot(A)-theta; %From Equation 5
|
||||
kappa = r./(r0_origin*sin(pi/2-acot(A))); %From Equation 6
|
||||
|
||||
|
||||
%Find theta_s by solving implicitly using fzero function
|
||||
for i = 1:length(theta);
|
||||
%Use muller function in case any complex solutions
|
||||
theta_s(i,1) = muller(@(x) (x-(1/A)*log(kappa(i)*sin(delta(i)+x))),[theta(i)-pi/8 theta(i) theta(i)+pi/8]);%From Equation 6
|
||||
end
|
||||
|
||||
%plot(theta_s*180/pi)
|
||||
|
||||
%Find r_s
|
||||
r_s = r0_origin*exp(A*theta_s);%From Equation 1
|
||||
|
||||
%Find s
|
||||
lamda = r0_origin*sec(acot(A));%From Equation 8
|
||||
start_point = 0; %Can be changed to make a more suitable start point
|
||||
s = lamda*(exp(A*theta_s)-exp(A*start_point));%From Equation 8
|
||||
|
||||
%Find p
|
||||
p = r.*sin(theta-theta_s)./sin(pi/2-acot(A)); %From Equation 9
|
||||
|
||||
%Convert any complex numbers to real numbers
|
||||
p = real(p);
|
||||
s = real(s);
|
||||
|
||||
%Sort back points to get the right indices %MDH 4/4/2012
|
||||
p = sort_back(p,Isort);
|
||||
s = sort_back(s,Isort);
|
||||
|
||||
%-----------------------------------------------------------------
|
||||
%POST-PROCESS DATA
|
||||
%s data
|
||||
|
||||
if site.reverse_s ==0
|
||||
s = s - site.startpoint;%Make minimum s == 0
|
||||
elseif site.reverse_s ==1
|
||||
s = -(s - site.startpoint);
|
||||
end
|
||||
|
||||
%p data
|
||||
if site.subtract_res ==1 %Add switch for user to subtract residuals or not - MDH 19/5/2010
|
||||
[MIN,L] = min(site.boundary.s);
|
||||
I = find(s<=MIN);
|
||||
p(I) = p(I) - site.boundary.p(L);
|
||||
[MAX,L] = max(site.boundary.s);
|
||||
I = find(s>=MAX);
|
||||
p(I) = p(I) - site.boundary.p(L);
|
||||
I = find(s>MIN&s<MAX);
|
||||
p(I) = p(I) - interp1(site.boundary.s,site.boundary.p,s(I));%Subtract logspiral errors from p data
|
||||
end
|
||||
|
||||
|
||||
if site.alpha<0
|
||||
p = -p;
|
||||
end
|
||||
|
||||
%-----------------------------------------------------------------
|
||||
|
||||
out.s = s;
|
||||
out.p = p;
|
||||
out.z = xyz_data.z;
|
||||
|
||||
@ -1,166 +0,0 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
#==========================================================#
|
||||
# Make a gif of the satellite images
|
||||
#==========================================================#
|
||||
|
||||
# Initial settings
|
||||
import os
|
||||
import numpy as np
|
||||
import matplotlib
|
||||
matplotlib.use("Agg")
|
||||
import matplotlib.pyplot as plt
|
||||
import matplotlib.animation as manimation
|
||||
import ee
|
||||
import pdb
|
||||
|
||||
# other modules
|
||||
from osgeo import gdal, ogr, osr
|
||||
import pickle
|
||||
import matplotlib.cm as cm
|
||||
from pylab import ginput
|
||||
import imageio
|
||||
|
||||
# image processing modules
|
||||
import skimage.filters as filters
|
||||
import skimage.exposure as exposure
|
||||
import skimage.transform as transform
|
||||
import sklearn.decomposition as decomposition
|
||||
import skimage.measure as measure
|
||||
|
||||
# import own modules
|
||||
import functions.utils as utils
|
||||
import functions.sds as sds
|
||||
|
||||
# some settings
|
||||
np.seterr(all='ignore') # raise/ignore divisions by 0 and nans
|
||||
plt.rcParams['axes.grid'] = True
|
||||
plt.rcParams['figure.max_open_warning'] = 100
|
||||
ee.Initialize()
|
||||
|
||||
# parameters
|
||||
cloud_thresh = 0.5 # threshold for cloud cover
|
||||
plot_bool = False # if you want the plots
|
||||
prob_high = 99.9 # upper probability to clip and rescale pixel intensity
|
||||
min_contour_points = 100# minimum number of points contained in each water line
|
||||
output_epsg = 28356 # GDA94 / MGA Zone 56
|
||||
|
||||
# load metadata (timestamps and epsg code) for the collection
|
||||
satname = 'L8'
|
||||
#sitename = 'NARRA'
|
||||
sitename = 'OLDBAR_inlet'
|
||||
filepath = os.path.join(os.getcwd(), 'data', satname, sitename)
|
||||
with open(os.path.join(filepath, sitename + '_timestamps' + '.pkl'), 'rb') as f:
|
||||
timestamps = pickle.load(f)
|
||||
timestamps_sorted = sorted(timestamps) # sort timestamps since images are sorted in directory
|
||||
with open(os.path.join(filepath, sitename + '_epsgcode' + '.pkl'), 'rb') as f:
|
||||
input_epsg = pickle.load(f)
|
||||
with open(os.path.join(filepath, sitename + '_refpoints2' + '.pkl'), 'rb') as f:
|
||||
refpoints = pickle.load(f)
|
||||
|
||||
# path to images
|
||||
file_path_pan = os.path.join(os.getcwd(), 'data', satname, sitename, 'pan')
|
||||
file_path_ms = os.path.join(os.getcwd(), 'data', satname, sitename, 'ms')
|
||||
file_names_pan = os.listdir(file_path_pan)
|
||||
file_names_ms = os.listdir(file_path_ms)
|
||||
N = len(file_names_pan)
|
||||
|
||||
# initialise some variables
|
||||
cloud_cover_ts = []
|
||||
date_acquired_ts = []
|
||||
idx_skipped = []
|
||||
idx_nocloud = []
|
||||
|
||||
t = []
|
||||
shorelines = []
|
||||
|
||||
with open(os.path.join(filepath, sitename + '_idxnocloud' + '.pkl'), 'rb') as f:
|
||||
idx_nocloud = pickle.load(f)
|
||||
|
||||
for i in idx_nocloud:
|
||||
# read pan image
|
||||
fn_pan = os.path.join(file_path_pan, file_names_pan[i])
|
||||
data = gdal.Open(fn_pan, gdal.GA_ReadOnly)
|
||||
georef = np.array(data.GetGeoTransform())
|
||||
bands = [data.GetRasterBand(i + 1).ReadAsArray() for k in range(data.RasterCount)]
|
||||
im_pan = np.stack(bands, 2)[:,:,0]
|
||||
# read ms image
|
||||
fn_ms = os.path.join(file_path_ms, file_names_ms[i])
|
||||
data = gdal.Open(fn_ms, gdal.GA_ReadOnly)
|
||||
bands = [data.GetRasterBand(i + 1).ReadAsArray() for k in range(data.RasterCount)]
|
||||
im_ms = np.stack(bands, 2)
|
||||
# cloud mask
|
||||
im_qa = im_ms[:,:,5]
|
||||
cloud_mask = sds.create_cloud_mask(im_qa, satname, plot_bool)
|
||||
cloud_mask = transform.resize(cloud_mask, (im_pan.shape[0], im_pan.shape[1]),
|
||||
order=0, preserve_range=True,
|
||||
mode='constant').astype('bool_')
|
||||
# resize the image using bilinear interpolation (order 1)
|
||||
im_ms = transform.resize(im_ms,(im_pan.shape[0], im_pan.shape[1]),
|
||||
order=1, preserve_range=True, mode='constant')
|
||||
# check if -inf or nan values and add to cloud mask
|
||||
im_inf = np.isin(im_ms[:,:,0], -np.inf)
|
||||
im_nan = np.isnan(im_ms[:,:,0])
|
||||
cloud_mask = np.logical_or(np.logical_or(cloud_mask, im_inf), im_nan)
|
||||
cloud_cover = sum(sum(cloud_mask.astype(int)))/(cloud_mask.shape[0]*cloud_mask.shape[1])
|
||||
if cloud_cover > cloud_thresh:
|
||||
print('skipped cloud ' + str(i))
|
||||
idx_skipped.append(i)
|
||||
continue
|
||||
# idx_nocloud.append(i)
|
||||
# check if image for that date is already present and keep the one with less clouds
|
||||
if file_names_pan[i][len(satname)+1+len(sitename)+1:len(satname)+1+len(sitename)+1+10] in date_acquired_ts:
|
||||
idx_samedate = utils.find_indices(date_acquired_ts, lambda e : e == file_names_pan[i][9:19])
|
||||
idx_samedate = idx_samedate[0]
|
||||
print(str(cloud_cover) + ' - ' + str(cloud_cover_ts[idx_samedate]))
|
||||
if cloud_cover >= cloud_cover_ts[idx_samedate]:
|
||||
print('skipped double ' + str(i))
|
||||
idx_skipped.append(i)
|
||||
continue
|
||||
else:
|
||||
del shorelines[idx_samedate]
|
||||
del t[idx_samedate]
|
||||
del cloud_cover_ts[idx_samedate]
|
||||
del date_acquired_ts[idx_samedate]
|
||||
print('deleted ' + str(idx_samedate))
|
||||
|
||||
# rescale intensities
|
||||
im_ms = sds.rescale_image_intensity(im_ms, cloud_mask, prob_high, plot_bool)
|
||||
im_pan = sds.rescale_image_intensity(im_pan, cloud_mask, prob_high, plot_bool)
|
||||
# pansharpen rgb image
|
||||
im_ms_ps = sds.pansharpen(im_ms[:,:,[0,1,2]], im_pan, cloud_mask, plot_bool)
|
||||
# add down-sized bands for NIR and SWIR (since pansharpening is not possible)
|
||||
im_ms_ps = np.append(im_ms_ps, im_ms[:,:,[3,4]], axis=2)
|
||||
# calculate NDWI
|
||||
im_ndwi = sds.nd_index(im_ms_ps[:,:,3], im_ms_ps[:,:,1], cloud_mask, plot_bool)
|
||||
# detect edges
|
||||
wl_pix = sds.find_wl_contours(im_ndwi, cloud_mask, min_contour_points, plot_bool)
|
||||
# convert from pixels to world coordinates
|
||||
wl_coords = sds.convert_pix2world(wl_pix, georef)
|
||||
# convert to output epsg spatial reference
|
||||
wl = sds.convert_epsg(wl_coords, input_epsg, output_epsg)
|
||||
|
||||
# save images as png for video
|
||||
fig = plt.figure()
|
||||
plt.grid(False)
|
||||
plt.imshow(im_ms_ps[:,:,[2,1,0]], animated=True)
|
||||
mng = plt.get_current_fig_manager()
|
||||
mng.window.showMaximized()
|
||||
plt.title(file_names_pan[i][len(satname)+1+len(sitename)+1:len(satname)+1+len(sitename)+1+10])
|
||||
plt.xticks([])
|
||||
plt.yticks([])
|
||||
plt.axis('equal')
|
||||
plt.tight_layout()
|
||||
plt.draw()
|
||||
plt.savefig(os.path.join(filepath,
|
||||
'plots', file_names_pan[i][len(satname)+1+len(sitename)+1:len(satname)+1+len(sitename)+1+10] + '.png'),
|
||||
dpi = 300)
|
||||
plt.close()
|
||||
|
||||
# create gif
|
||||
images = []
|
||||
filenames = os.listdir(os.path.join(filepath, 'plots'))
|
||||
with imageio.get_writer('movie.gif', mode='I', duration=0.2) as writer:
|
||||
for filename in filenames:
|
||||
image = imageio.imread(os.path.join(filepath,'plots',filename))
|
||||
writer.append_data(image)
|
||||
@ -1,213 +0,0 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
#==========================================================#
|
||||
# Run Neural Network on image to extract sandy pixels
|
||||
#==========================================================#
|
||||
|
||||
# Initial settings
|
||||
import os
|
||||
import numpy as np
|
||||
import matplotlib.pyplot as plt
|
||||
import matplotlib.patches as mpatches
|
||||
from matplotlib import gridspec
|
||||
from datetime import datetime, timedelta
|
||||
import pytz
|
||||
import ee
|
||||
import pdb
|
||||
import time
|
||||
import pandas as pd
|
||||
# other modules
|
||||
from osgeo import gdal, ogr, osr
|
||||
import pickle
|
||||
import matplotlib.cm as cm
|
||||
from pylab import ginput
|
||||
|
||||
# image processing modules
|
||||
import skimage.filters as filters
|
||||
import skimage.exposure as exposure
|
||||
import skimage.transform as transform
|
||||
import sklearn.decomposition as decomposition
|
||||
import skimage.measure as measure
|
||||
import skimage.morphology as morphology
|
||||
from scipy import ndimage
|
||||
import imageio
|
||||
|
||||
|
||||
# machine learning modules
|
||||
from sklearn.model_selection import train_test_split
|
||||
from sklearn.neural_network import MLPClassifier
|
||||
from sklearn.preprocessing import StandardScaler, Normalizer
|
||||
from sklearn.externals import joblib
|
||||
|
||||
# import own modules
|
||||
import functions.utils as utils
|
||||
import functions.sds as sds
|
||||
|
||||
# some settings
|
||||
np.seterr(all='ignore') # raise/ignore divisions by 0 and nans
|
||||
plt.rcParams['axes.grid'] = True
|
||||
plt.rcParams['figure.max_open_warning'] = 100
|
||||
ee.Initialize()
|
||||
|
||||
# parameters
|
||||
cloud_thresh = 0.3 # threshold for cloud cover
|
||||
plot_bool = False # if you want the plots
|
||||
prob_high = 100 # upper probability to clip and rescale pixel intensity
|
||||
min_contour_points = 100# minimum number of points contained in each water line
|
||||
output_epsg = 28356 # GDA94 / MGA Zone 56
|
||||
buffer_size = 10 # radius (in pixels) of disk for buffer (pixel classification)
|
||||
min_beach_size = 20 # number of pixels in a beach (pixel classification)
|
||||
|
||||
# load metadata (timestamps and epsg code) for the collection
|
||||
satname = 'L8'
|
||||
#sitename = 'NARRA_all'
|
||||
#sitename = 'NARRA'
|
||||
#sitename = 'OLDBAR'
|
||||
#sitename = 'OLDBAR_inlet'
|
||||
#sitename = 'SANDMOTOR'
|
||||
sitename = 'TAIRUA'
|
||||
#sitename = 'DUCK'
|
||||
|
||||
# Load metadata
|
||||
filepath = os.path.join(os.getcwd(), 'data', satname, sitename)
|
||||
with open(os.path.join(filepath, sitename + '_timestamps' + '.pkl'), 'rb') as f:
|
||||
timestamps = pickle.load(f)
|
||||
timestamps_sorted = sorted(timestamps)
|
||||
daysall = (datetime(2019,1,1,tzinfo=pytz.utc) - datetime(2013,1,1,tzinfo=pytz.utc)).total_seconds()
|
||||
# path to images
|
||||
file_path_pan = os.path.join(os.getcwd(), 'data', satname, sitename, 'pan')
|
||||
file_path_ms = os.path.join(os.getcwd(), 'data', satname, sitename, 'ms')
|
||||
file_names_pan = os.listdir(file_path_pan)
|
||||
file_names_ms = os.listdir(file_path_ms)
|
||||
N = len(file_names_pan)
|
||||
|
||||
# initialise some variables
|
||||
idx_skipped = []
|
||||
idx_nocloud = []
|
||||
n_features = 10
|
||||
train_pos = np.nan*np.ones((1,n_features))
|
||||
train_neg = np.nan*np.ones((1,n_features))
|
||||
columns = ('B','G','R','NIR','SWIR','Pan','WI','VI','BR', 'mWI', 'class')
|
||||
|
||||
#%%
|
||||
for i in range(N):
|
||||
# read pan image
|
||||
fn_pan = os.path.join(file_path_pan, file_names_pan[i])
|
||||
data = gdal.Open(fn_pan, gdal.GA_ReadOnly)
|
||||
georef = np.array(data.GetGeoTransform())
|
||||
bands = [data.GetRasterBand(i + 1).ReadAsArray() for k in range(data.RasterCount)]
|
||||
im_pan = np.stack(bands, 2)[:,:,0]
|
||||
nrow = im_pan.shape[0]
|
||||
ncol = im_pan.shape[1]
|
||||
# read ms image
|
||||
fn_ms = os.path.join(file_path_ms, file_names_ms[i])
|
||||
data = gdal.Open(fn_ms, gdal.GA_ReadOnly)
|
||||
bands = [data.GetRasterBand(i + 1).ReadAsArray() for k in range(data.RasterCount)]
|
||||
im_ms = np.stack(bands, 2)
|
||||
# cloud mask
|
||||
im_qa = im_ms[:,:,5]
|
||||
cloud_mask = sds.create_cloud_mask(im_qa, satname, plot_bool)
|
||||
cloud_mask = transform.resize(cloud_mask, (im_pan.shape[0], im_pan.shape[1]),
|
||||
order=0, preserve_range=True,
|
||||
mode='constant').astype('bool_')
|
||||
# resize the image using bilinear interpolation (order 1)
|
||||
im_ms = transform.resize(im_ms,(im_pan.shape[0], im_pan.shape[1]),
|
||||
order=1, preserve_range=True, mode='constant')
|
||||
# check if -inf or nan values and add to cloud mask
|
||||
im_inf = np.isin(im_ms[:,:,0], -np.inf)
|
||||
im_nan = np.isnan(im_ms[:,:,0])
|
||||
cloud_mask = np.logical_or(np.logical_or(cloud_mask, im_inf), im_nan)
|
||||
# skip if cloud cover is more than the threshold
|
||||
cloud_cover = sum(sum(cloud_mask.astype(int)))/(cloud_mask.shape[0]*cloud_mask.shape[1])
|
||||
if cloud_cover > cloud_thresh:
|
||||
print('skip ' + str(i) + ' - cloudy (' + str(np.round(cloud_cover*100).astype(int)) + '%)')
|
||||
idx_skipped.append(i)
|
||||
continue
|
||||
idx_nocloud.append(i)
|
||||
|
||||
# pansharpen rgb image
|
||||
im_ms_ps = sds.pansharpen(im_ms[:,:,[0,1,2]], im_pan, cloud_mask, plot_bool)
|
||||
# add down-sized bands for NIR and SWIR (since pansharpening is not possible)
|
||||
im_ms_ps = np.append(im_ms_ps, im_ms[:,:,[3,4]], axis=2)
|
||||
|
||||
im_classif, im_labels = sds.classify_image_NN(im_ms_ps, im_pan, cloud_mask, min_beach_size, plot_bool)
|
||||
|
||||
im_display = sds.rescale_image_intensity(im_ms_ps[:,:,[2,1,0]], cloud_mask, 100, False)
|
||||
im = np.copy(im_display)
|
||||
# define colours for plot
|
||||
colours = np.array([[1,128/255,0/255],[204/255,1,1],[0,0,204/255]])
|
||||
for k in range(0,im_labels.shape[2]):
|
||||
im[im_labels[:,:,k],0] = colours[k,0]
|
||||
im[im_labels[:,:,k],1] = colours[k,1]
|
||||
im[im_labels[:,:,k],2] = colours[k,2]
|
||||
|
||||
|
||||
# fig = plt.figure()
|
||||
# plt.suptitle(date_im, fontsize=17, fontweight='bold')
|
||||
# ax1 = plt.subplot(121)
|
||||
# plt.imshow(im_display)
|
||||
# plt.axis('off')
|
||||
# ax2 = plt.subplot(122, sharex=ax1, sharey=ax1)
|
||||
# plt.imshow(im)
|
||||
# plt.axis('off')
|
||||
# plt.gcf().set_size_inches(17.99,7.55)
|
||||
# plt.tight_layout()
|
||||
# orange_patch = mpatches.Patch(color=[1,128/255,0/255], label='sand')
|
||||
# white_patch = mpatches.Patch(color=[204/255,1,1], label='swash/whitewater')
|
||||
# blue_patch = mpatches.Patch(color=[0,0,204/255], label='water')
|
||||
# plt.legend(handles=[orange_patch,white_patch,blue_patch], bbox_to_anchor=(0.95, 0.2))
|
||||
# plt.draw()
|
||||
|
||||
date_im = timestamps_sorted[i].strftime('%d %b %Y')
|
||||
daysnow = (timestamps_sorted[i] - datetime(2013,1,1,tzinfo=pytz.utc)).total_seconds()
|
||||
|
||||
fig = plt.figure()
|
||||
gs = gridspec.GridSpec(2, 2, height_ratios=[1, 20])
|
||||
|
||||
ax1 = fig.add_subplot(gs[0,:])
|
||||
plt.plot(0,0,'ko',daysall,0,'ko')
|
||||
plt.plot([0,daysall],[0,0],'k-')
|
||||
plt.plot(daysnow,0,'ro')
|
||||
plt.text(0,0.05,'2013')
|
||||
plt.text(daysall,0.05,'2019')
|
||||
plt.plot((datetime(2014,1,1,tzinfo=pytz.utc)- datetime(2013,1,1,tzinfo=pytz.utc)).total_seconds(),0,'ko',markersize=3)
|
||||
plt.plot((datetime(2015,1,1,tzinfo=pytz.utc)- datetime(2013,1,1,tzinfo=pytz.utc)).total_seconds(),0,'ko',markersize=3)
|
||||
plt.plot((datetime(2016,1,1,tzinfo=pytz.utc)- datetime(2013,1,1,tzinfo=pytz.utc)).total_seconds(),0,'ko',markersize=3)
|
||||
plt.plot((datetime(2017,1,1,tzinfo=pytz.utc)- datetime(2013,1,1,tzinfo=pytz.utc)).total_seconds(),0,'ko',markersize=3)
|
||||
plt.plot((datetime(2018,1,1,tzinfo=pytz.utc)- datetime(2013,1,1,tzinfo=pytz.utc)).total_seconds(),0,'ko',markersize=3)
|
||||
plt.text((datetime(2014,1,1,tzinfo=pytz.utc)- datetime(2013,1,1,tzinfo=pytz.utc)).total_seconds(),0.05,'2014')
|
||||
plt.text((datetime(2015,1,1,tzinfo=pytz.utc)- datetime(2013,1,1,tzinfo=pytz.utc)).total_seconds(),0.05,'2015')
|
||||
plt.text((datetime(2016,1,1,tzinfo=pytz.utc)- datetime(2013,1,1,tzinfo=pytz.utc)).total_seconds(),0.05,'2016')
|
||||
plt.text((datetime(2017,1,1,tzinfo=pytz.utc)- datetime(2013,1,1,tzinfo=pytz.utc)).total_seconds(),0.05,'2017')
|
||||
plt.text((datetime(2018,1,1,tzinfo=pytz.utc)- datetime(2013,1,1,tzinfo=pytz.utc)).total_seconds(),0.05,'2018')
|
||||
|
||||
plt.axis('off')
|
||||
|
||||
ax2 = fig.add_subplot(gs[1,0])
|
||||
plt.imshow(im_display)
|
||||
plt.axis('off')
|
||||
plt.title(date_im, fontsize=17, fontweight='bold')
|
||||
|
||||
ax3 = fig.add_subplot(gs[1,1])
|
||||
plt.imshow(im)
|
||||
plt.axis('off')
|
||||
orange_patch = mpatches.Patch(color=[1,128/255,0/255], label='sand')
|
||||
white_patch = mpatches.Patch(color=[204/255,1,1], label='swash/whitewater')
|
||||
blue_patch = mpatches.Patch(color=[0,0,204/255], label='water')
|
||||
plt.legend(handles=[orange_patch,white_patch,blue_patch], bbox_to_anchor=(0.95, 0.2))
|
||||
|
||||
plt.gcf().set_size_inches(17.99,7.55)
|
||||
plt.gcf().set_tight_layout(True)
|
||||
|
||||
plt.draw()
|
||||
plt.savefig(os.path.join(filepath,'plots_classif', file_names_pan[i][len(satname)+1+len(sitename)+1:len(satname)+1+len(sitename)+1+10] + '.jpg'), dpi = 300)
|
||||
plt.close()
|
||||
|
||||
# create gif
|
||||
images = []
|
||||
filenames = os.listdir(os.path.join(filepath, 'plots_classif'))
|
||||
with imageio.get_writer(sitename + '.gif', mode='I', duration=0.4) as writer:
|
||||
for filename in filenames:
|
||||
image = imageio.imread(os.path.join(filepath,'plots_classif',filename))
|
||||
writer.append_data(image)
|
||||
|
||||
@ -1,228 +0,0 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
#==========================================================#
|
||||
# Run Neural Network on image to extract sandy pixels
|
||||
#==========================================================#
|
||||
|
||||
# Initial settings
|
||||
import os
|
||||
import numpy as np
|
||||
import matplotlib.pyplot as plt
|
||||
import matplotlib.patches as mpatches
|
||||
import matplotlib.lines as mlines
|
||||
from matplotlib import gridspec
|
||||
from datetime import datetime, timedelta
|
||||
import pytz
|
||||
import ee
|
||||
import pdb
|
||||
import time
|
||||
import pandas as pd
|
||||
# other modules
|
||||
from osgeo import gdal, ogr, osr
|
||||
import pickle
|
||||
import matplotlib.cm as cm
|
||||
from pylab import ginput
|
||||
|
||||
# image processing modules
|
||||
import skimage.filters as filters
|
||||
import skimage.exposure as exposure
|
||||
import skimage.transform as transform
|
||||
import sklearn.decomposition as decomposition
|
||||
import skimage.measure as measure
|
||||
import skimage.morphology as morphology
|
||||
from scipy import ndimage
|
||||
import imageio
|
||||
|
||||
|
||||
# machine learning modules
|
||||
from sklearn.model_selection import train_test_split
|
||||
from sklearn.neural_network import MLPClassifier
|
||||
from sklearn.preprocessing import StandardScaler, Normalizer
|
||||
from sklearn.externals import joblib
|
||||
|
||||
# import own modules
|
||||
import functions.utils as utils
|
||||
import functions.sds as sds
|
||||
|
||||
# some settings
|
||||
np.seterr(all='ignore') # raise/ignore divisions by 0 and nans
|
||||
plt.rcParams['axes.grid'] = True
|
||||
plt.rcParams['figure.max_open_warning'] = 100
|
||||
ee.Initialize()
|
||||
|
||||
# parameters
|
||||
cloud_thresh = 0.2 # threshold for cloud cover
|
||||
plot_bool = False # if you want the plots
|
||||
prob_high = 100 # upper probability to clip and rescale pixel intensity
|
||||
min_contour_points = 100# minimum number of points contained in each water line
|
||||
output_epsg = 28356 # GDA94 / MGA Zone 56
|
||||
buffer_size = 10 # radius (in pixels) of disk for buffer (pixel classification)
|
||||
min_beach_size = 10 # number of pixels in a beach (pixel classification)
|
||||
|
||||
# load metadata (timestamps and epsg code) for the collection
|
||||
satname = 'L8'
|
||||
#sitename = 'NARRA_all'
|
||||
#sitename = 'NARRA'
|
||||
#sitename = 'OLDBAR'
|
||||
#sitename = 'OLDBAR_inlet'
|
||||
#sitename = 'SANDMOTOR'
|
||||
#sitename = 'TAIRUA'
|
||||
#sitename = 'DUCK'
|
||||
#sitename = 'BROULEE'
|
||||
sitename = 'MURI'
|
||||
|
||||
|
||||
# Load metadata
|
||||
filepath = os.path.join(os.getcwd(), 'data', satname, sitename)
|
||||
with open(os.path.join(filepath, sitename + '_timestamps' + '.pkl'), 'rb') as f:
|
||||
timestamps = pickle.load(f)
|
||||
timestamps_sorted = sorted(timestamps)
|
||||
daysall = (datetime(2019,1,1,tzinfo=pytz.utc) - datetime(2013,1,1,tzinfo=pytz.utc)).total_seconds()
|
||||
# path to images
|
||||
file_path_pan = os.path.join(os.getcwd(), 'data', satname, sitename, 'pan')
|
||||
file_path_ms = os.path.join(os.getcwd(), 'data', satname, sitename, 'ms')
|
||||
file_names_pan = os.listdir(file_path_pan)
|
||||
file_names_ms = os.listdir(file_path_ms)
|
||||
N = len(file_names_pan)
|
||||
|
||||
# initialise some variables
|
||||
idx_skipped = []
|
||||
idx_nocloud = []
|
||||
n_features = 10
|
||||
train_pos = np.nan*np.ones((1,n_features))
|
||||
train_neg = np.nan*np.ones((1,n_features))
|
||||
columns = ('B','G','R','NIR','SWIR','Pan','WI','VI','BR', 'mWI', 'class')
|
||||
|
||||
#%%
|
||||
for i in range(N):
|
||||
# read pan image
|
||||
fn_pan = os.path.join(file_path_pan, file_names_pan[i])
|
||||
data = gdal.Open(fn_pan, gdal.GA_ReadOnly)
|
||||
georef = np.array(data.GetGeoTransform())
|
||||
bands = [data.GetRasterBand(i + 1).ReadAsArray() for k in range(data.RasterCount)]
|
||||
im_pan = np.stack(bands, 2)[:,:,0]
|
||||
nrow = im_pan.shape[0]
|
||||
ncol = im_pan.shape[1]
|
||||
# read ms image
|
||||
fn_ms = os.path.join(file_path_ms, file_names_ms[i])
|
||||
data = gdal.Open(fn_ms, gdal.GA_ReadOnly)
|
||||
bands = [data.GetRasterBand(i + 1).ReadAsArray() for k in range(data.RasterCount)]
|
||||
im_ms = np.stack(bands, 2)
|
||||
# cloud mask
|
||||
im_qa = im_ms[:,:,5]
|
||||
cloud_mask = sds.create_cloud_mask(im_qa, satname, plot_bool)
|
||||
cloud_mask = transform.resize(cloud_mask, (im_pan.shape[0], im_pan.shape[1]),
|
||||
order=0, preserve_range=True,
|
||||
mode='constant').astype('bool_')
|
||||
# resize the image using bilinear interpolation (order 1)
|
||||
im_ms = transform.resize(im_ms,(im_pan.shape[0], im_pan.shape[1]),
|
||||
order=1, preserve_range=True, mode='constant')
|
||||
# check if -inf or nan values and add to cloud mask
|
||||
im_inf = np.isin(im_ms[:,:,0], -np.inf)
|
||||
im_nan = np.isnan(im_ms[:,:,0])
|
||||
cloud_mask = np.logical_or(np.logical_or(cloud_mask, im_inf), im_nan)
|
||||
# skip if cloud cover is more than the threshold
|
||||
cloud_cover = sum(sum(cloud_mask.astype(int)))/(cloud_mask.shape[0]*cloud_mask.shape[1])
|
||||
if cloud_cover > cloud_thresh:
|
||||
print('skip ' + str(i) + ' - cloudy (' + str(np.round(cloud_cover*100).astype(int)) + '%)')
|
||||
idx_skipped.append(i)
|
||||
continue
|
||||
idx_nocloud.append(i)
|
||||
|
||||
# pansharpen rgb image
|
||||
im_ms_ps = sds.pansharpen(im_ms[:,:,[0,1,2]], im_pan, cloud_mask, plot_bool)
|
||||
# add down-sized bands for NIR and SWIR (since pansharpening is not possible)
|
||||
im_ms_ps = np.append(im_ms_ps, im_ms[:,:,[3,4]], axis=2)
|
||||
|
||||
im_classif, im_labels = sds.classify_image_NN(im_ms_ps, im_pan, cloud_mask, min_beach_size, plot_bool)
|
||||
|
||||
# if there are no sand pixels, skip the image (maybe later change the detection method with old method)
|
||||
if sum(sum(im_labels[:,:,0])) == 0 :
|
||||
print('skip ' + str(i) + ' - no sand')
|
||||
idx_skipped.append(i)
|
||||
continue
|
||||
|
||||
contours_wi, contours_mwi = sds.find_wl_contours2(im_ms_ps, im_labels, cloud_mask, buffer_size, False)
|
||||
|
||||
|
||||
im_display = sds.rescale_image_intensity(im_ms_ps[:,:,[2,1,0]], cloud_mask, 100, False)
|
||||
im = np.copy(im_display)
|
||||
# define colours for plot
|
||||
colours = np.array([[1,128/255,0/255],[204/255,1,1],[0,0,204/255]])
|
||||
for k in range(0,im_labels.shape[2]):
|
||||
im[im_labels[:,:,k],0] = colours[k,0]
|
||||
im[im_labels[:,:,k],1] = colours[k,1]
|
||||
im[im_labels[:,:,k],2] = colours[k,2]
|
||||
|
||||
|
||||
# fig = plt.figure()
|
||||
# plt.suptitle(date_im, fontsize=17, fontweight='bold')
|
||||
# ax1 = plt.subplot(121)
|
||||
# plt.imshow(im_display)
|
||||
# plt.axis('off')
|
||||
# ax2 = plt.subplot(122, sharex=ax1, sharey=ax1)
|
||||
# plt.imshow(im)
|
||||
# plt.axis('off')
|
||||
# plt.gcf().set_size_inches(17.99,7.55)
|
||||
# plt.tight_layout()
|
||||
# orange_patch = mpatches.Patch(color=[1,128/255,0/255], label='sand')
|
||||
# white_patch = mpatches.Patch(color=[204/255,1,1], label='swash/whitewater')
|
||||
# blue_patch = mpatches.Patch(color=[0,0,204/255], label='water')
|
||||
# plt.legend(handles=[orange_patch,white_patch,blue_patch], bbox_to_anchor=(0.95, 0.2))
|
||||
# plt.draw()
|
||||
|
||||
date_im = timestamps_sorted[i].strftime('%d %b %Y')
|
||||
daysnow = (timestamps_sorted[i] - datetime(2013,1,1,tzinfo=pytz.utc)).total_seconds()
|
||||
|
||||
fig = plt.figure()
|
||||
gs = gridspec.GridSpec(2, 2, height_ratios=[1, 20])
|
||||
|
||||
ax1 = fig.add_subplot(gs[0,:])
|
||||
plt.plot(0,0,'ko',daysall,0,'ko')
|
||||
plt.plot([0,daysall],[0,0],'k-')
|
||||
plt.plot(daysnow,0,'ro')
|
||||
plt.text(0,0.05,'2013')
|
||||
plt.text(daysall,0.05,'2019')
|
||||
plt.plot((datetime(2014,1,1,tzinfo=pytz.utc)- datetime(2013,1,1,tzinfo=pytz.utc)).total_seconds(),0,'ko',markersize=3)
|
||||
plt.plot((datetime(2015,1,1,tzinfo=pytz.utc)- datetime(2013,1,1,tzinfo=pytz.utc)).total_seconds(),0,'ko',markersize=3)
|
||||
plt.plot((datetime(2016,1,1,tzinfo=pytz.utc)- datetime(2013,1,1,tzinfo=pytz.utc)).total_seconds(),0,'ko',markersize=3)
|
||||
plt.plot((datetime(2017,1,1,tzinfo=pytz.utc)- datetime(2013,1,1,tzinfo=pytz.utc)).total_seconds(),0,'ko',markersize=3)
|
||||
plt.plot((datetime(2018,1,1,tzinfo=pytz.utc)- datetime(2013,1,1,tzinfo=pytz.utc)).total_seconds(),0,'ko',markersize=3)
|
||||
plt.text((datetime(2014,1,1,tzinfo=pytz.utc)- datetime(2013,1,1,tzinfo=pytz.utc)).total_seconds(),0.05,'2014')
|
||||
plt.text((datetime(2015,1,1,tzinfo=pytz.utc)- datetime(2013,1,1,tzinfo=pytz.utc)).total_seconds(),0.05,'2015')
|
||||
plt.text((datetime(2016,1,1,tzinfo=pytz.utc)- datetime(2013,1,1,tzinfo=pytz.utc)).total_seconds(),0.05,'2016')
|
||||
plt.text((datetime(2017,1,1,tzinfo=pytz.utc)- datetime(2013,1,1,tzinfo=pytz.utc)).total_seconds(),0.05,'2017')
|
||||
plt.text((datetime(2018,1,1,tzinfo=pytz.utc)- datetime(2013,1,1,tzinfo=pytz.utc)).total_seconds(),0.05,'2018')
|
||||
|
||||
plt.axis('off')
|
||||
|
||||
ax2 = fig.add_subplot(gs[1,0])
|
||||
plt.imshow(im_display)
|
||||
plt.axis('off')
|
||||
plt.title(date_im, fontsize=17, fontweight='bold')
|
||||
|
||||
ax3 = fig.add_subplot(gs[1,1])
|
||||
plt.imshow(im)
|
||||
for l,contour in enumerate(contours_mwi): plt.plot(contour[:, 1], contour[:, 0], linewidth=2, color='k', linestyle='--')
|
||||
plt.axis('off')
|
||||
orange_patch = mpatches.Patch(color=[1,128/255,0/255], label='sand')
|
||||
white_patch = mpatches.Patch(color=[204/255,1,1], label='swash/whitewater')
|
||||
blue_patch = mpatches.Patch(color=[0,0,204/255], label='water')
|
||||
black_line = mlines.Line2D([],[],color='k',linestyle='-', label='shoreline')
|
||||
plt.legend(handles=[orange_patch,white_patch,blue_patch, black_line], bbox_to_anchor=(0.95, 0.2))
|
||||
|
||||
plt.gcf().set_size_inches(17.99,7.55)
|
||||
plt.gcf().set_tight_layout(True)
|
||||
|
||||
plt.draw()
|
||||
plt.savefig(os.path.join(filepath,'plots_classif', file_names_pan[i][len(satname)+1+len(sitename)+1:len(satname)+1+len(sitename)+1+10] + '.jpg'), dpi = 300)
|
||||
plt.close()
|
||||
|
||||
# create gif
|
||||
images = []
|
||||
filenames = os.listdir(os.path.join(filepath, 'plots_classif'))
|
||||
with imageio.get_writer(sitename + '.gif', mode='I', duration=0.4) as writer:
|
||||
for filename in filenames:
|
||||
image = imageio.imread(os.path.join(filepath,'plots_classif',filename))
|
||||
writer.append_data(image)
|
||||
|
||||
@ -1,193 +0,0 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
#==========================================================#
|
||||
# Run Neural Network on image to extract sandy pixels
|
||||
#==========================================================#
|
||||
|
||||
# Initial settings
|
||||
import os
|
||||
import numpy as np
|
||||
import matplotlib.pyplot as plt
|
||||
import matplotlib.patches as mpatches
|
||||
import matplotlib.lines as mlines
|
||||
from matplotlib import gridspec
|
||||
from datetime import datetime, timedelta
|
||||
import pytz
|
||||
import ee
|
||||
import pdb
|
||||
import time
|
||||
import pandas as pd
|
||||
# other modules
|
||||
from osgeo import gdal, ogr, osr
|
||||
import pickle
|
||||
import matplotlib.cm as cm
|
||||
from pylab import ginput
|
||||
|
||||
# image processing modules
|
||||
import skimage.filters as filters
|
||||
import skimage.exposure as exposure
|
||||
import skimage.transform as transform
|
||||
import sklearn.decomposition as decomposition
|
||||
import skimage.measure as measure
|
||||
import skimage.morphology as morphology
|
||||
from scipy import ndimage
|
||||
import imageio
|
||||
|
||||
|
||||
# machine learning modules
|
||||
from sklearn.model_selection import train_test_split
|
||||
from sklearn.neural_network import MLPClassifier
|
||||
from sklearn.preprocessing import StandardScaler, Normalizer
|
||||
from sklearn.externals import joblib
|
||||
|
||||
# import own modules
|
||||
import functions.utils as utils
|
||||
import functions.sds as sds
|
||||
|
||||
# some settings
|
||||
np.seterr(all='ignore') # raise/ignore divisions by 0 and nans
|
||||
plt.rcParams['axes.grid'] = True
|
||||
plt.rcParams['figure.max_open_warning'] = 100
|
||||
ee.Initialize()
|
||||
|
||||
# parameters
|
||||
cloud_thresh = 0.5 # threshold for cloud cover
|
||||
plot_bool = False # if you want the plots
|
||||
prob_high = 100 # upper probability to clip and rescale pixel intensity
|
||||
min_contour_points = 30# minimum number of points contained in each water line
|
||||
output_epsg = 28356 # GDA94 / MGA Zone 56
|
||||
buffer_size = 10 # radius (in pixels) of disk for buffer (pixel classification)
|
||||
min_beach_size = 10 # number of pixels in a beach (pixel classification)
|
||||
|
||||
# load metadata (timestamps and epsg code) for the collection
|
||||
satname = 'L8'
|
||||
#sitename = 'NARRA_all'
|
||||
#sitename = 'NARRA'
|
||||
#sitename = 'OLDBAR'
|
||||
#sitename = 'OLDBAR_inlet'
|
||||
#sitename = 'SANDMOTOR'
|
||||
#sitename = 'TAIRUA'
|
||||
#sitename = 'DUCK'
|
||||
#sitename = 'BROULEE'
|
||||
sitename = 'MURI2'
|
||||
|
||||
|
||||
# Load metadata
|
||||
filepath = os.path.join(os.getcwd(), 'data', satname, sitename)
|
||||
with open(os.path.join(filepath, sitename + '_timestamps' + '.pkl'), 'rb') as f:
|
||||
timestamps = pickle.load(f)
|
||||
timestamps_sorted = sorted(timestamps)
|
||||
daysall = (datetime(2019,1,1,tzinfo=pytz.utc) - datetime(2013,1,1,tzinfo=pytz.utc)).total_seconds()
|
||||
# path to images
|
||||
file_path_pan = os.path.join(os.getcwd(), 'data', satname, sitename, 'pan')
|
||||
file_path_ms = os.path.join(os.getcwd(), 'data', satname, sitename, 'ms')
|
||||
file_names_pan = os.listdir(file_path_pan)
|
||||
file_names_ms = os.listdir(file_path_ms)
|
||||
N = len(file_names_pan)
|
||||
|
||||
# initialise some variables
|
||||
idx_skipped = []
|
||||
idx_nocloud = []
|
||||
n_features = 10
|
||||
train_pos = np.nan*np.ones((1,n_features))
|
||||
train_neg = np.nan*np.ones((1,n_features))
|
||||
columns = ('B','G','R','NIR','SWIR','Pan','WI','VI','BR', 'mWI', 'class')
|
||||
|
||||
#%%
|
||||
for i in range(N):
|
||||
# read pan image
|
||||
fn_pan = os.path.join(file_path_pan, file_names_pan[i])
|
||||
data = gdal.Open(fn_pan, gdal.GA_ReadOnly)
|
||||
georef = np.array(data.GetGeoTransform())
|
||||
bands = [data.GetRasterBand(i + 1).ReadAsArray() for k in range(data.RasterCount)]
|
||||
im_pan = np.stack(bands, 2)[:,:,0]
|
||||
nrow = im_pan.shape[0]
|
||||
ncol = im_pan.shape[1]
|
||||
# read ms image
|
||||
fn_ms = os.path.join(file_path_ms, file_names_ms[i])
|
||||
data = gdal.Open(fn_ms, gdal.GA_ReadOnly)
|
||||
bands = [data.GetRasterBand(i + 1).ReadAsArray() for k in range(data.RasterCount)]
|
||||
im_ms = np.stack(bands, 2)
|
||||
# cloud mask
|
||||
im_qa = im_ms[:,:,5]
|
||||
cloud_mask = sds.create_cloud_mask(im_qa, satname, plot_bool)
|
||||
cloud_mask = transform.resize(cloud_mask, (im_pan.shape[0], im_pan.shape[1]),
|
||||
order=0, preserve_range=True,
|
||||
mode='constant').astype('bool_')
|
||||
# resize the image using bilinear interpolation (order 1)
|
||||
im_ms = transform.resize(im_ms,(im_pan.shape[0], im_pan.shape[1]),
|
||||
order=1, preserve_range=True, mode='constant')
|
||||
# check if -inf or nan values and add to cloud mask
|
||||
im_inf = np.isin(im_ms[:,:,0], -np.inf)
|
||||
im_nan = np.isnan(im_ms[:,:,0])
|
||||
cloud_mask = np.logical_or(np.logical_or(cloud_mask, im_inf), im_nan)
|
||||
# skip if cloud cover is more than the threshold
|
||||
cloud_cover = sum(sum(cloud_mask.astype(int)))/(cloud_mask.shape[0]*cloud_mask.shape[1])
|
||||
if cloud_cover > cloud_thresh:
|
||||
print('skip ' + str(i) + ' - cloudy (' + str(np.round(cloud_cover*100).astype(int)) + '%)')
|
||||
idx_skipped.append(i)
|
||||
continue
|
||||
idx_nocloud.append(i)
|
||||
|
||||
# pansharpen rgb image
|
||||
im_ms_ps = sds.pansharpen(im_ms[:,:,[0,1,2]], im_pan, cloud_mask, plot_bool)
|
||||
# add down-sized bands for NIR and SWIR (since pansharpening is not possible)
|
||||
im_ms_ps = np.append(im_ms_ps, im_ms[:,:,[3,4]], axis=2)
|
||||
|
||||
# extract shorelines (old method)
|
||||
im_ndwi = sds.nd_index(im_ms_ps[:,:,3], im_ms_ps[:,:,1], cloud_mask, plot_bool)
|
||||
wl_pix = sds.find_wl_contours(im_ndwi, cloud_mask, min_contour_points, plot_bool)
|
||||
|
||||
im_display = sds.rescale_image_intensity(im_ms_ps[:,:,[2,1,0]], cloud_mask, 100, False)
|
||||
|
||||
date_im = timestamps_sorted[i].strftime('%d %b %Y')
|
||||
daysnow = (timestamps_sorted[i] - datetime(2013,1,1,tzinfo=pytz.utc)).total_seconds()
|
||||
|
||||
fig = plt.figure()
|
||||
gs = gridspec.GridSpec(2, 2, height_ratios=[1, 20])
|
||||
|
||||
ax1 = fig.add_subplot(gs[0,:])
|
||||
plt.plot(0,0,'ko',daysall,0,'ko')
|
||||
plt.plot([0,daysall],[0,0],'k-')
|
||||
plt.plot(daysnow,0,'ro')
|
||||
plt.text(0,0.05,'2013')
|
||||
plt.text(daysall,0.05,'2019')
|
||||
plt.plot((datetime(2014,1,1,tzinfo=pytz.utc)- datetime(2013,1,1,tzinfo=pytz.utc)).total_seconds(),0,'ko',markersize=3)
|
||||
plt.plot((datetime(2015,1,1,tzinfo=pytz.utc)- datetime(2013,1,1,tzinfo=pytz.utc)).total_seconds(),0,'ko',markersize=3)
|
||||
plt.plot((datetime(2016,1,1,tzinfo=pytz.utc)- datetime(2013,1,1,tzinfo=pytz.utc)).total_seconds(),0,'ko',markersize=3)
|
||||
plt.plot((datetime(2017,1,1,tzinfo=pytz.utc)- datetime(2013,1,1,tzinfo=pytz.utc)).total_seconds(),0,'ko',markersize=3)
|
||||
plt.plot((datetime(2018,1,1,tzinfo=pytz.utc)- datetime(2013,1,1,tzinfo=pytz.utc)).total_seconds(),0,'ko',markersize=3)
|
||||
plt.text((datetime(2014,1,1,tzinfo=pytz.utc)- datetime(2013,1,1,tzinfo=pytz.utc)).total_seconds(),0.05,'2014')
|
||||
plt.text((datetime(2015,1,1,tzinfo=pytz.utc)- datetime(2013,1,1,tzinfo=pytz.utc)).total_seconds(),0.05,'2015')
|
||||
plt.text((datetime(2016,1,1,tzinfo=pytz.utc)- datetime(2013,1,1,tzinfo=pytz.utc)).total_seconds(),0.05,'2016')
|
||||
plt.text((datetime(2017,1,1,tzinfo=pytz.utc)- datetime(2013,1,1,tzinfo=pytz.utc)).total_seconds(),0.05,'2017')
|
||||
plt.text((datetime(2018,1,1,tzinfo=pytz.utc)- datetime(2013,1,1,tzinfo=pytz.utc)).total_seconds(),0.05,'2018')
|
||||
plt.axis('off')
|
||||
|
||||
# ax2 = fig.add_subplot(gs[1,0])
|
||||
# plt.imshow(im_display)
|
||||
# plt.axis('off')
|
||||
# plt.title(date_im, fontsize=17, fontweight='bold')
|
||||
|
||||
ax3 = fig.add_subplot(gs[1,:])
|
||||
plt.imshow(im_display)
|
||||
for l,contour in enumerate(wl_pix): plt.plot(contour[:, 1], contour[:, 0], linewidth=2, color='k', linestyle='--')
|
||||
plt.title(date_im, fontsize=17, fontweight='bold')
|
||||
plt.axis('off')
|
||||
|
||||
plt.gcf().set_size_inches(5.34,9.18)
|
||||
plt.gcf().set_tight_layout(True)
|
||||
|
||||
plt.draw()
|
||||
|
||||
plt.savefig(os.path.join(filepath,'plots_classif', file_names_pan[i][len(satname)+1+len(sitename)+1:len(satname)+1+len(sitename)+1+10] + '.jpg'), dpi = 300)
|
||||
plt.close()
|
||||
|
||||
# create gif
|
||||
images = []
|
||||
filenames = os.listdir(os.path.join(filepath, 'plots_classif'))
|
||||
with imageio.get_writer(sitename + '_final.gif', mode='I', duration=0.6) as writer:
|
||||
for filename in filenames:
|
||||
image = imageio.imread(os.path.join(filepath,'plots_classif',filename))
|
||||
writer.append_data(image)
|
||||
|
||||
@ -1,227 +0,0 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
#==========================================================#
|
||||
# Run Neural Network on image to extract sandy pixels
|
||||
#==========================================================#
|
||||
|
||||
# Initial settings
|
||||
import os
|
||||
import numpy as np
|
||||
import matplotlib.pyplot as plt
|
||||
import matplotlib.patches as mpatches
|
||||
import matplotlib.lines as mlines
|
||||
from matplotlib import gridspec
|
||||
from datetime import datetime, timedelta
|
||||
import pytz
|
||||
import ee
|
||||
import pdb
|
||||
import time
|
||||
import pandas as pd
|
||||
# other modules
|
||||
from osgeo import gdal, ogr, osr
|
||||
import pickle
|
||||
import matplotlib.cm as cm
|
||||
from pylab import ginput
|
||||
|
||||
# image processing modules
|
||||
import skimage.filters as filters
|
||||
import skimage.exposure as exposure
|
||||
import skimage.transform as transform
|
||||
import sklearn.decomposition as decomposition
|
||||
import skimage.measure as measure
|
||||
import skimage.morphology as morphology
|
||||
from scipy import ndimage
|
||||
import imageio
|
||||
|
||||
|
||||
# machine learning modules
|
||||
from sklearn.model_selection import train_test_split
|
||||
from sklearn.neural_network import MLPClassifier
|
||||
from sklearn.preprocessing import StandardScaler, Normalizer
|
||||
from sklearn.externals import joblib
|
||||
|
||||
# import own modules
|
||||
import functions.utils as utils
|
||||
import functions.sds as sds
|
||||
|
||||
# some settings
|
||||
np.seterr(all='ignore') # raise/ignore divisions by 0 and nans
|
||||
plt.rcParams['axes.grid'] = True
|
||||
plt.rcParams['figure.max_open_warning'] = 100
|
||||
ee.Initialize()
|
||||
|
||||
# parameters
|
||||
cloud_thresh = 0.2 # threshold for cloud cover
|
||||
plot_bool = False # if you want the plots
|
||||
prob_high = 100 # upper probability to clip and rescale pixel intensity
|
||||
min_contour_points = 100# minimum number of points contained in each water line
|
||||
output_epsg = 28356 # GDA94 / MGA Zone 56
|
||||
buffer_size = 10 # radius (in pixels) of disk for buffer (pixel classification)
|
||||
min_beach_size = 20 # number of pixels in a beach (pixel classification)
|
||||
|
||||
# load metadata (timestamps and epsg code) for the collection
|
||||
satname = 'L8'
|
||||
#sitename = 'NARRA_all'
|
||||
sitename = 'NARRA'
|
||||
#sitename = 'OLDBAR'
|
||||
#sitename = 'OLDBAR_inlet'
|
||||
#sitename = 'SANDMOTOR'
|
||||
#sitename = 'TAIRUA'
|
||||
#sitename = 'DUCK'
|
||||
#sitename = 'BROULEE'
|
||||
|
||||
# Load metadata
|
||||
filepath = os.path.join(os.getcwd(), 'data', satname, sitename)
|
||||
with open(os.path.join(filepath, sitename + '_timestamps' + '.pkl'), 'rb') as f:
|
||||
timestamps = pickle.load(f)
|
||||
timestamps_sorted = sorted(timestamps)
|
||||
daysall = (datetime(2019,1,1,tzinfo=pytz.utc) - datetime(2013,1,1,tzinfo=pytz.utc)).total_seconds()
|
||||
# path to images
|
||||
file_path_pan = os.path.join(os.getcwd(), 'data', satname, sitename, 'pan')
|
||||
file_path_ms = os.path.join(os.getcwd(), 'data', satname, sitename, 'ms')
|
||||
file_names_pan = os.listdir(file_path_pan)
|
||||
file_names_ms = os.listdir(file_path_ms)
|
||||
N = len(file_names_pan)
|
||||
|
||||
# initialise some variables
|
||||
idx_skipped = []
|
||||
idx_nocloud = []
|
||||
n_features = 10
|
||||
train_pos = np.nan*np.ones((1,n_features))
|
||||
train_neg = np.nan*np.ones((1,n_features))
|
||||
columns = ('B','G','R','NIR','SWIR','Pan','WI','VI','BR', 'mWI', 'class')
|
||||
|
||||
#%%
|
||||
for i in range(N):
|
||||
# read pan image
|
||||
fn_pan = os.path.join(file_path_pan, file_names_pan[i])
|
||||
data = gdal.Open(fn_pan, gdal.GA_ReadOnly)
|
||||
georef = np.array(data.GetGeoTransform())
|
||||
bands = [data.GetRasterBand(i + 1).ReadAsArray() for k in range(data.RasterCount)]
|
||||
im_pan = np.stack(bands, 2)[:,:,0]
|
||||
nrow = im_pan.shape[0]
|
||||
ncol = im_pan.shape[1]
|
||||
# read ms image
|
||||
fn_ms = os.path.join(file_path_ms, file_names_ms[i])
|
||||
data = gdal.Open(fn_ms, gdal.GA_ReadOnly)
|
||||
bands = [data.GetRasterBand(i + 1).ReadAsArray() for k in range(data.RasterCount)]
|
||||
im_ms = np.stack(bands, 2)
|
||||
# cloud mask
|
||||
im_qa = im_ms[:,:,5]
|
||||
cloud_mask = sds.create_cloud_mask(im_qa, satname, plot_bool)
|
||||
cloud_mask = transform.resize(cloud_mask, (im_pan.shape[0], im_pan.shape[1]),
|
||||
order=0, preserve_range=True,
|
||||
mode='constant').astype('bool_')
|
||||
# resize the image using bilinear interpolation (order 1)
|
||||
im_ms = transform.resize(im_ms,(im_pan.shape[0], im_pan.shape[1]),
|
||||
order=1, preserve_range=True, mode='constant')
|
||||
# check if -inf or nan values and add to cloud mask
|
||||
im_inf = np.isin(im_ms[:,:,0], -np.inf)
|
||||
im_nan = np.isnan(im_ms[:,:,0])
|
||||
cloud_mask = np.logical_or(np.logical_or(cloud_mask, im_inf), im_nan)
|
||||
# skip if cloud cover is more than the threshold
|
||||
cloud_cover = sum(sum(cloud_mask.astype(int)))/(cloud_mask.shape[0]*cloud_mask.shape[1])
|
||||
if cloud_cover > cloud_thresh:
|
||||
print('skip ' + str(i) + ' - cloudy (' + str(np.round(cloud_cover*100).astype(int)) + '%)')
|
||||
idx_skipped.append(i)
|
||||
continue
|
||||
idx_nocloud.append(i)
|
||||
|
||||
# pansharpen rgb image
|
||||
im_ms_ps = sds.pansharpen(im_ms[:,:,[0,1,2]], im_pan, cloud_mask, plot_bool)
|
||||
# add down-sized bands for NIR and SWIR (since pansharpening is not possible)
|
||||
im_ms_ps = np.append(im_ms_ps, im_ms[:,:,[3,4]], axis=2)
|
||||
|
||||
im_classif, im_labels = sds.classify_image_NN(im_ms_ps, im_pan, cloud_mask, min_beach_size, plot_bool)
|
||||
|
||||
# if there are no sand pixels, skip the image (maybe later change the detection method with old method)
|
||||
if sum(sum(im_labels[:,:,0])) == 0 :
|
||||
print('skip ' + str(i) + ' - no sand')
|
||||
idx_skipped.append(i)
|
||||
continue
|
||||
|
||||
contours_wi, contours_mwi = sds.find_wl_contours2(im_ms_ps, im_labels, cloud_mask, buffer_size, False)
|
||||
|
||||
|
||||
im_display = sds.rescale_image_intensity(im_ms_ps[:,:,[2,1,0]], cloud_mask, 100, False)
|
||||
im = np.copy(im_display)
|
||||
# define colours for plot
|
||||
colours = np.array([[1,128/255,0/255],[204/255,1,1],[0,0,204/255]])
|
||||
for k in range(0,im_labels.shape[2]):
|
||||
im[im_labels[:,:,k],0] = colours[k,0]
|
||||
im[im_labels[:,:,k],1] = colours[k,1]
|
||||
im[im_labels[:,:,k],2] = colours[k,2]
|
||||
|
||||
|
||||
# fig = plt.figure()
|
||||
# plt.suptitle(date_im, fontsize=17, fontweight='bold')
|
||||
# ax1 = plt.subplot(121)
|
||||
# plt.imshow(im_display)
|
||||
# plt.axis('off')
|
||||
# ax2 = plt.subplot(122, sharex=ax1, sharey=ax1)
|
||||
# plt.imshow(im)
|
||||
# plt.axis('off')
|
||||
# plt.gcf().set_size_inches(17.99,7.55)
|
||||
# plt.tight_layout()
|
||||
# orange_patch = mpatches.Patch(color=[1,128/255,0/255], label='sand')
|
||||
# white_patch = mpatches.Patch(color=[204/255,1,1], label='swash/whitewater')
|
||||
# blue_patch = mpatches.Patch(color=[0,0,204/255], label='water')
|
||||
# plt.legend(handles=[orange_patch,white_patch,blue_patch], bbox_to_anchor=(0.95, 0.2))
|
||||
# plt.draw()
|
||||
|
||||
date_im = timestamps_sorted[i].strftime('%d %b %Y')
|
||||
daysnow = (timestamps_sorted[i] - datetime(2013,1,1,tzinfo=pytz.utc)).total_seconds()
|
||||
|
||||
fig = plt.figure()
|
||||
gs = gridspec.GridSpec(2, 2, height_ratios=[1, 20])
|
||||
|
||||
ax1 = fig.add_subplot(gs[0,:])
|
||||
plt.plot(0,0,'ko',daysall,0,'ko')
|
||||
plt.plot([0,daysall],[0,0],'k-')
|
||||
plt.plot(daysnow,0,'ro')
|
||||
plt.text(0,0.05,'2013')
|
||||
plt.text(daysall,0.05,'2019')
|
||||
plt.plot((datetime(2014,1,1,tzinfo=pytz.utc)- datetime(2013,1,1,tzinfo=pytz.utc)).total_seconds(),0,'ko',markersize=3)
|
||||
plt.plot((datetime(2015,1,1,tzinfo=pytz.utc)- datetime(2013,1,1,tzinfo=pytz.utc)).total_seconds(),0,'ko',markersize=3)
|
||||
plt.plot((datetime(2016,1,1,tzinfo=pytz.utc)- datetime(2013,1,1,tzinfo=pytz.utc)).total_seconds(),0,'ko',markersize=3)
|
||||
plt.plot((datetime(2017,1,1,tzinfo=pytz.utc)- datetime(2013,1,1,tzinfo=pytz.utc)).total_seconds(),0,'ko',markersize=3)
|
||||
plt.plot((datetime(2018,1,1,tzinfo=pytz.utc)- datetime(2013,1,1,tzinfo=pytz.utc)).total_seconds(),0,'ko',markersize=3)
|
||||
plt.text((datetime(2014,1,1,tzinfo=pytz.utc)- datetime(2013,1,1,tzinfo=pytz.utc)).total_seconds(),0.05,'2014')
|
||||
plt.text((datetime(2015,1,1,tzinfo=pytz.utc)- datetime(2013,1,1,tzinfo=pytz.utc)).total_seconds(),0.05,'2015')
|
||||
plt.text((datetime(2016,1,1,tzinfo=pytz.utc)- datetime(2013,1,1,tzinfo=pytz.utc)).total_seconds(),0.05,'2016')
|
||||
plt.text((datetime(2017,1,1,tzinfo=pytz.utc)- datetime(2013,1,1,tzinfo=pytz.utc)).total_seconds(),0.05,'2017')
|
||||
plt.text((datetime(2018,1,1,tzinfo=pytz.utc)- datetime(2013,1,1,tzinfo=pytz.utc)).total_seconds(),0.05,'2018')
|
||||
|
||||
plt.axis('off')
|
||||
|
||||
# ax2 = fig.add_subplot(gs[1,0])
|
||||
# plt.imshow(im_display)
|
||||
# plt.axis('off')
|
||||
# plt.title(date_im, fontsize=17, fontweight='bold')
|
||||
|
||||
ax3 = fig.add_subplot(gs[1,:])
|
||||
plt.imshow(im)
|
||||
for l,contour in enumerate(contours_mwi): plt.plot(contour[:, 1], contour[:, 0], linewidth=2, color='k', linestyle='--')
|
||||
plt.axis('off')
|
||||
orange_patch = mpatches.Patch(color=[1,128/255,0/255], label='sand')
|
||||
white_patch = mpatches.Patch(color=[204/255,1,1], label='swash/whitewater')
|
||||
blue_patch = mpatches.Patch(color=[0,0,204/255], label='water')
|
||||
black_line = mlines.Line2D([],[],color='k',linestyle='--', label='shoreline')
|
||||
plt.legend(handles=[orange_patch,white_patch,blue_patch, black_line], bbox_to_anchor=(0.6, 0.6))
|
||||
plt.title(date_im, fontsize=17, fontweight='bold')
|
||||
|
||||
plt.gcf().set_size_inches(5.34,9.18)
|
||||
plt.gcf().set_tight_layout(True)
|
||||
|
||||
plt.draw()
|
||||
plt.savefig(os.path.join(filepath,'plots_classif', file_names_pan[i][len(satname)+1+len(sitename)+1:len(satname)+1+len(sitename)+1+10] + '.jpg'), dpi = 300)
|
||||
plt.close()
|
||||
|
||||
# create gif
|
||||
images = []
|
||||
filenames = os.listdir(os.path.join(filepath, 'plots_classif'))
|
||||
with imageio.get_writer(sitename + '.gif', mode='I', duration=0.4) as writer:
|
||||
for filename in filenames:
|
||||
image = imageio.imread(os.path.join(filepath,'plots_classif',filename))
|
||||
writer.append_data(image)
|
||||
|
||||
@ -1,229 +0,0 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
#==========================================================#
|
||||
# Run Neural Network on image to extract sandy pixels
|
||||
#==========================================================#
|
||||
|
||||
# Initial settings
|
||||
import os
|
||||
import numpy as np
|
||||
import matplotlib.pyplot as plt
|
||||
import matplotlib.patches as mpatches
|
||||
import matplotlib.lines as mlines
|
||||
from matplotlib import gridspec
|
||||
from datetime import datetime, timedelta
|
||||
import pytz
|
||||
import ee
|
||||
import pdb
|
||||
import time
|
||||
import pandas as pd
|
||||
# other modules
|
||||
from osgeo import gdal, ogr, osr
|
||||
import pickle
|
||||
import matplotlib.cm as cm
|
||||
from pylab import ginput
|
||||
|
||||
# image processing modules
|
||||
import skimage.filters as filters
|
||||
import skimage.exposure as exposure
|
||||
import skimage.transform as transform
|
||||
import sklearn.decomposition as decomposition
|
||||
import skimage.measure as measure
|
||||
import skimage.morphology as morphology
|
||||
from scipy import ndimage
|
||||
import imageio
|
||||
|
||||
|
||||
# machine learning modules
|
||||
from sklearn.model_selection import train_test_split
|
||||
from sklearn.neural_network import MLPClassifier
|
||||
from sklearn.preprocessing import StandardScaler, Normalizer
|
||||
from sklearn.externals import joblib
|
||||
|
||||
# import own modules
|
||||
import functions.utils as utils
|
||||
import functions.sds as sds
|
||||
|
||||
# some settings
|
||||
np.seterr(all='ignore') # raise/ignore divisions by 0 and nans
|
||||
plt.rcParams['axes.grid'] = True
|
||||
plt.rcParams['figure.max_open_warning'] = 100
|
||||
ee.Initialize()
|
||||
|
||||
# parameters
|
||||
cloud_thresh = 0.2 # threshold for cloud cover
|
||||
plot_bool = False # if you want the plots
|
||||
prob_high = 100 # upper probability to clip and rescale pixel intensity
|
||||
min_contour_points = 100# minimum number of points contained in each water line
|
||||
output_epsg = 28356 # GDA94 / MGA Zone 56
|
||||
buffer_size = 10 # radius (in pixels) of disk for buffer (pixel classification)
|
||||
min_beach_size = 50 # number of pixels in a beach (pixel classification)
|
||||
|
||||
# load metadata (timestamps and epsg code) for the collection
|
||||
satname = 'L8'
|
||||
sitename = 'NARRA_all'
|
||||
#sitename = 'NARRA'
|
||||
#sitename = 'OLDBAR'
|
||||
#sitename = 'OLDBAR_inlet'
|
||||
#sitename = 'SANDMOTOR'
|
||||
#sitename = 'TAIRUA'
|
||||
#sitename = 'DUCK'
|
||||
#sitename = 'BROULEE'
|
||||
|
||||
# Load metadata
|
||||
filepath = os.path.join(os.getcwd(), 'data', satname, sitename)
|
||||
with open(os.path.join(filepath, sitename + '_timestamps' + '.pkl'), 'rb') as f:
|
||||
timestamps = pickle.load(f)
|
||||
timestamps_sorted = sorted(timestamps)
|
||||
daysall = (datetime(2019,1,1,tzinfo=pytz.utc) - datetime(2013,1,1,tzinfo=pytz.utc)).total_seconds()
|
||||
# path to images
|
||||
file_path_pan = os.path.join(os.getcwd(), 'data', satname, sitename, 'pan')
|
||||
file_path_ms = os.path.join(os.getcwd(), 'data', satname, sitename, 'ms')
|
||||
file_names_pan = os.listdir(file_path_pan)
|
||||
file_names_ms = os.listdir(file_path_ms)
|
||||
N = len(file_names_pan)
|
||||
|
||||
# initialise some variables
|
||||
idx_skipped = []
|
||||
idx_nocloud = []
|
||||
n_features = 10
|
||||
train_pos = np.nan*np.ones((1,n_features))
|
||||
train_neg = np.nan*np.ones((1,n_features))
|
||||
columns = ('B','G','R','NIR','SWIR','Pan','WI','VI','BR', 'mWI', 'class')
|
||||
|
||||
#%%
|
||||
for i in range(1):
|
||||
i = 156 # open (96 close)
|
||||
# read pan image
|
||||
fn_pan = os.path.join(file_path_pan, file_names_pan[i])
|
||||
data = gdal.Open(fn_pan, gdal.GA_ReadOnly)
|
||||
georef = np.array(data.GetGeoTransform())
|
||||
bands = [data.GetRasterBand(i + 1).ReadAsArray() for k in range(data.RasterCount)]
|
||||
im_pan = np.stack(bands, 2)[:,:,0]
|
||||
nrow = im_pan.shape[0]
|
||||
ncol = im_pan.shape[1]
|
||||
# read ms image
|
||||
fn_ms = os.path.join(file_path_ms, file_names_ms[i])
|
||||
data = gdal.Open(fn_ms, gdal.GA_ReadOnly)
|
||||
bands = [data.GetRasterBand(i + 1).ReadAsArray() for k in range(data.RasterCount)]
|
||||
im_ms = np.stack(bands, 2)
|
||||
# cloud mask
|
||||
im_qa = im_ms[:,:,5]
|
||||
cloud_mask = sds.create_cloud_mask(im_qa, satname, plot_bool)
|
||||
cloud_mask = transform.resize(cloud_mask, (im_pan.shape[0], im_pan.shape[1]),
|
||||
order=0, preserve_range=True,
|
||||
mode='constant').astype('bool_')
|
||||
# resize the image using bilinear interpolation (order 1)
|
||||
im_ms = transform.resize(im_ms,(im_pan.shape[0], im_pan.shape[1]),
|
||||
order=1, preserve_range=True, mode='constant')
|
||||
# check if -inf or nan values and add to cloud mask
|
||||
im_inf = np.isin(im_ms[:,:,0], -np.inf)
|
||||
im_nan = np.isnan(im_ms[:,:,0])
|
||||
cloud_mask = np.logical_or(np.logical_or(cloud_mask, im_inf), im_nan)
|
||||
# skip if cloud cover is more than the threshold
|
||||
cloud_cover = sum(sum(cloud_mask.astype(int)))/(cloud_mask.shape[0]*cloud_mask.shape[1])
|
||||
if cloud_cover > cloud_thresh:
|
||||
print('skip ' + str(i) + ' - cloudy (' + str(np.round(cloud_cover*100).astype(int)) + '%)')
|
||||
idx_skipped.append(i)
|
||||
continue
|
||||
idx_nocloud.append(i)
|
||||
|
||||
# pansharpen rgb image
|
||||
im_ms_ps = sds.pansharpen(im_ms[:,:,[0,1,2]], im_pan, cloud_mask, plot_bool)
|
||||
# add down-sized bands for NIR and SWIR (since pansharpening is not possible)
|
||||
im_ms_ps = np.append(im_ms_ps, im_ms[:,:,[3,4]], axis=2)
|
||||
|
||||
im_classif, im_labels = sds.classify_image_NN(im_ms_ps, im_pan, cloud_mask, min_beach_size, plot_bool)
|
||||
|
||||
# if there are no sand pixels, skip the image (maybe later change the detection method with old method)
|
||||
if sum(sum(im_labels[:,:,0])) == 0 :
|
||||
print('skip ' + str(i) + ' - no sand')
|
||||
idx_skipped.append(i)
|
||||
continue
|
||||
|
||||
contours_wi, contours_mwi = sds.find_wl_contours2(im_ms_ps, im_labels, cloud_mask, buffer_size, False)
|
||||
|
||||
|
||||
im_display = sds.rescale_image_intensity(im_ms_ps[:,:,[2,1,0]], cloud_mask, 100, False)
|
||||
im = np.copy(im_display)
|
||||
# define colours for plot
|
||||
colours = np.array([[1,128/255,0/255],[0,0,204/255],[0,0,204/255]])
|
||||
|
||||
for k in range(0,im_labels.shape[2]):
|
||||
im[im_labels[:,:,k],0] = colours[k,0]
|
||||
im[im_labels[:,:,k],1] = colours[k,1]
|
||||
im[im_labels[:,:,k],2] = colours[k,2]
|
||||
|
||||
|
||||
# fig = plt.figure()
|
||||
# plt.suptitle(date_im, fontsize=17, fontweight='bold')
|
||||
# ax1 = plt.subplot(121)
|
||||
# plt.imshow(im_display)
|
||||
# plt.axis('off')
|
||||
# ax2 = plt.subplot(122, sharex=ax1, sharey=ax1)
|
||||
# plt.imshow(im)
|
||||
# plt.axis('off')
|
||||
# plt.gcf().set_size_inches(17.99,7.55)
|
||||
# plt.tight_layout()
|
||||
# orange_patch = mpatches.Patch(color=[1,128/255,0/255], label='sand')
|
||||
# white_patch = mpatches.Patch(color=[204/255,1,1], label='swash/whitewater')
|
||||
# blue_patch = mpatches.Patch(color=[0,0,204/255], label='water')
|
||||
# plt.legend(handles=[orange_patch,white_patch,blue_patch], bbox_to_anchor=(0.95, 0.2))
|
||||
# plt.draw()
|
||||
|
||||
date_im = timestamps_sorted[i].strftime('%d %b %Y')
|
||||
daysnow = (timestamps_sorted[i] - datetime(2013,1,1,tzinfo=pytz.utc)).total_seconds()
|
||||
|
||||
fig = plt.figure()
|
||||
gs = gridspec.GridSpec(2, 2, height_ratios=[1, 20])
|
||||
|
||||
ax1 = fig.add_subplot(gs[0,:])
|
||||
plt.plot(0,0,'ko',daysall,0,'ko')
|
||||
plt.plot([0,daysall],[0,0],'k-')
|
||||
plt.plot(daysnow,0,'ro')
|
||||
plt.text(0,0.05,'2013')
|
||||
plt.text(daysall,0.05,'2019')
|
||||
plt.plot((datetime(2014,1,1,tzinfo=pytz.utc)- datetime(2013,1,1,tzinfo=pytz.utc)).total_seconds(),0,'ko',markersize=3)
|
||||
plt.plot((datetime(2015,1,1,tzinfo=pytz.utc)- datetime(2013,1,1,tzinfo=pytz.utc)).total_seconds(),0,'ko',markersize=3)
|
||||
plt.plot((datetime(2016,1,1,tzinfo=pytz.utc)- datetime(2013,1,1,tzinfo=pytz.utc)).total_seconds(),0,'ko',markersize=3)
|
||||
plt.plot((datetime(2017,1,1,tzinfo=pytz.utc)- datetime(2013,1,1,tzinfo=pytz.utc)).total_seconds(),0,'ko',markersize=3)
|
||||
plt.plot((datetime(2018,1,1,tzinfo=pytz.utc)- datetime(2013,1,1,tzinfo=pytz.utc)).total_seconds(),0,'ko',markersize=3)
|
||||
plt.text((datetime(2014,1,1,tzinfo=pytz.utc)- datetime(2013,1,1,tzinfo=pytz.utc)).total_seconds(),0.05,'2014')
|
||||
plt.text((datetime(2015,1,1,tzinfo=pytz.utc)- datetime(2013,1,1,tzinfo=pytz.utc)).total_seconds(),0.05,'2015')
|
||||
plt.text((datetime(2016,1,1,tzinfo=pytz.utc)- datetime(2013,1,1,tzinfo=pytz.utc)).total_seconds(),0.05,'2016')
|
||||
plt.text((datetime(2017,1,1,tzinfo=pytz.utc)- datetime(2013,1,1,tzinfo=pytz.utc)).total_seconds(),0.05,'2017')
|
||||
plt.text((datetime(2018,1,1,tzinfo=pytz.utc)- datetime(2013,1,1,tzinfo=pytz.utc)).total_seconds(),0.05,'2018')
|
||||
|
||||
plt.axis('off')
|
||||
|
||||
ax2 = fig.add_subplot(gs[1,0])
|
||||
plt.imshow(im_display)
|
||||
plt.axis('off')
|
||||
plt.title(date_im, fontsize=17, fontweight='bold')
|
||||
|
||||
ax3 = fig.add_subplot(gs[1,1], sharex=ax2, sharey=ax2)
|
||||
plt.imshow(im)
|
||||
for l,contour in enumerate(contours_mwi): plt.plot(contour[:, 1], contour[:, 0], linewidth=2, color='k', linestyle='--')
|
||||
plt.axis('off')
|
||||
orange_patch = mpatches.Patch(color=[1,128/255,0/255], label='sand')
|
||||
blue_patch = mpatches.Patch(color=[0,0,204/255], label='water')
|
||||
black_line = mlines.Line2D([],[],color='k',linestyle='--', label='water line')
|
||||
plt.legend(handles=[orange_patch,blue_patch, black_line], bbox_to_anchor=(0.6, 0.6))
|
||||
# plt.title(date_im, fontsize=17, fontweight='bold')
|
||||
|
||||
plt.gcf().set_size_inches(11.38, 7.51)
|
||||
plt.gcf().set_tight_layout(True)
|
||||
|
||||
plt.draw()
|
||||
|
||||
# plt.savefig(os.path.join(filepath,'plots_classif', file_names_pan[i][len(satname)+1+len(sitename)+1:len(satname)+1+len(sitename)+1+10] + '.jpg'), dpi = 300)
|
||||
# plt.close()
|
||||
|
||||
# create gif
|
||||
#images = []
|
||||
#filenames = os.listdir(os.path.join(filepath, 'plots_classif'))
|
||||
#with imageio.get_writer(sitename + '.gif', mode='I', duration=0.4) as writer:
|
||||
# for filename in filenames:
|
||||
# image = imageio.imread(os.path.join(filepath,'plots_classif',filename))
|
||||
# writer.append_data(image)
|
||||
|
||||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
@ -1,66 +0,0 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
import os
|
||||
import numpy as np
|
||||
import matplotlib.pyplot as plt
|
||||
import pdb
|
||||
import ee
|
||||
|
||||
import matplotlib.dates as mdates
|
||||
import matplotlib.cm as cm
|
||||
from datetime import datetime, timedelta
|
||||
import pickle
|
||||
import pytz
|
||||
import scipy.io as sio
|
||||
import scipy.interpolate as interpolate
|
||||
import statsmodels.api as sm
|
||||
import skimage.measure as measure
|
||||
|
||||
|
||||
# my functions
|
||||
import functions.utils as utils
|
||||
import functions.sds as sds
|
||||
|
||||
# some settings
|
||||
np.seterr(all='ignore') # raise/ignore divisions by 0 and nans
|
||||
plt.rcParams['axes.grid'] = True
|
||||
plt.rcParams['figure.max_open_warning'] = 100
|
||||
|
||||
au_tz = pytz.timezone('Australia/Sydney')
|
||||
|
||||
# load timestamps from satellite images
|
||||
satname = 'L8'
|
||||
sitename = 'OLDBAR'
|
||||
filepath = os.path.join(os.getcwd(), 'data', satname, sitename)
|
||||
with open(os.path.join(filepath, sitename + '_output' + '.pkl'), 'rb') as f:
|
||||
output = pickle.load(f)
|
||||
dates_l8 = output['t']
|
||||
# convert to AEST
|
||||
dates_l8 = [_.astimezone(au_tz) for _ in dates_l8]
|
||||
# get the satellite shorelines
|
||||
sl = output['shorelines']
|
||||
|
||||
# load narrabeen beach points (manually digitized)
|
||||
with open(os.path.join(os.getcwd(), 'olddata', 'oldbar_beach' + '.pkl'), 'rb') as f:
|
||||
narrabeach = pickle.load(f)
|
||||
|
||||
|
||||
dist_thresh = 250
|
||||
frac_smooth = 1./12
|
||||
plt.figure()
|
||||
plt.axis('equal')
|
||||
for i in range(1):
|
||||
# select point of sds that are close to the manually digitized points
|
||||
idx_beach = [np.min(np.linalg.norm(sl[i][k,:] - narrabeach, axis=1)) < dist_thresh for k in range(sl[i].shape[0])]
|
||||
plt.plot(sl[i][:,0], sl[i][:,1])
|
||||
plt.plot(sl[i][idx_beach,0], sl[i][idx_beach,1])
|
||||
|
||||
|
||||
# smooth (LOWESS) satellite shoreline
|
||||
sl_smooth = sm.nonparametric.lowess(sl[i][idx_beach,0],sl[i][idx_beach,1], frac=frac_smooth, it = 10)
|
||||
sl_smooth = sl_smooth[:,[1,0]]
|
||||
|
||||
plt.plot(sl_smooth[:,0], sl_smooth[:,1])
|
||||
|
||||
|
||||
plt.draw()
|
||||
@ -1,80 +0,0 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
Created on Mon Mar 19 14:44:57 2018
|
||||
|
||||
@author: z5030440
|
||||
|
||||
Main code to extract shorelines from Landsat imagery
|
||||
"""
|
||||
# Preamble
|
||||
|
||||
import ee
|
||||
import math
|
||||
import matplotlib.pyplot as plt
|
||||
import numpy as np
|
||||
import pdb
|
||||
|
||||
# image processing modules
|
||||
import skimage.filters as filters
|
||||
import skimage.exposure as exposure
|
||||
import skimage.transform as transform
|
||||
import sklearn.decomposition as decomposition
|
||||
import skimage.morphology as morphology
|
||||
import skimage.measure as measure
|
||||
|
||||
# my modules
|
||||
# my functions
|
||||
from functions.utils import *
|
||||
from functions.sds import *
|
||||
|
||||
np.seterr(all='ignore') # raise/ignore divisions by 0 and nans
|
||||
ee.Initialize()
|
||||
|
||||
# parameters
|
||||
plot_bool = True # if you want the plots
|
||||
prob_high = 99.9 # upper probability to clip and rescale pixel intensity
|
||||
min_contour_points = 100 # minimum number of points contained in each water line
|
||||
|
||||
# select collection
|
||||
input_col = ee.ImageCollection('LANDSAT/LC08/C01/T1_TOA')
|
||||
|
||||
# location (Narrabeen-Collaroy beach)
|
||||
rect_narra = [[[151.317395,-33.494601],
|
||||
[151.388635,-33.495174],
|
||||
[151.363624,-33.565184],
|
||||
[151.305228,-33.563299],
|
||||
[151.317395,-33.494601]]];
|
||||
|
||||
# filter by location
|
||||
flt_col = input_col.filterBounds(ee.Geometry.Polygon(rect_narra))
|
||||
|
||||
n_img = flt_col.size().getInfo()
|
||||
print('Number of images covering Narrabeen:', n_img)
|
||||
im_all = flt_col.getInfo().get('features')
|
||||
output = []
|
||||
# loop through all images
|
||||
# find each image in ee database
|
||||
i = 2
|
||||
im = ee.Image(im_all[i].get('id'))
|
||||
# load image as np.array
|
||||
im_pan, im_ms, im_cloud, crs = read_eeimage(im, rect_narra, plot_bool)
|
||||
# rescale intensities
|
||||
im_ms = rescale_image_intensity(im_ms, im_cloud, prob_high, plot_bool)
|
||||
im_pan = rescale_image_intensity(im_pan, im_cloud, prob_high, plot_bool)
|
||||
# pansharpen rgb image
|
||||
im_ms_ps = pansharpen(im_ms[:,:,[0,1,2]], im_pan, im_cloud, plot_bool)
|
||||
# add down-sized bands for NIR and SWIR (since pansharpening is not possible)
|
||||
im_ms_ps = np.append(im_ms_ps, im_ms[:,:,[3,4]], axis=2)
|
||||
# calculate NDWI
|
||||
im_ndwi = nd_index(im_ms_ps[:,:,3], im_ms_ps[:,:,1], im_cloud, plot_bool)
|
||||
# edge detection
|
||||
wl_pix = find_wl_contours(im_ndwi, im_cloud, min_contour_points, True)
|
||||
# convert from pixels to world coordinates
|
||||
wl_coords = convert_pix2world(wl_pix, crs['crs_15m'])
|
||||
output.append(wl_coords)
|
||||
|
||||
plt.figure()
|
||||
plt.imshow(im_ms_ps[:,:,[2,1,0]])
|
||||
plt.axis('off')
|
||||
plt.title('RGB at 15m')
|
||||
plt.show()
|
||||
@ -1,476 +0,0 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
#==========================================================#
|
||||
# Extract shorelines from Landsat images
|
||||
#==========================================================#
|
||||
|
||||
# Initial settings
|
||||
import ee
|
||||
import matplotlib.pyplot as plt
|
||||
import matplotlib.cm as cm
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
from datetime import datetime
|
||||
import pickle
|
||||
import pdb
|
||||
import pytz
|
||||
from pylab import ginput
|
||||
from osgeo import gdal
|
||||
|
||||
|
||||
|
||||
# image processing modules
|
||||
import skimage.filters as filters
|
||||
import skimage.exposure as exposure
|
||||
import skimage.transform as transform
|
||||
import sklearn.decomposition as decomposition
|
||||
import skimage.morphology as morphology
|
||||
import skimage.measure as measure
|
||||
import skimage.color as color
|
||||
import skimage.feature as feature
|
||||
# machine learning modules
|
||||
from sklearn.cluster import KMeans
|
||||
|
||||
# my modules
|
||||
import functions.utils as utils
|
||||
import functions.sds as sds
|
||||
|
||||
# some settings
|
||||
np.seterr(all='ignore') # raise/ignore divisions by 0 and nans
|
||||
plt.rcParams['axes.grid'] = False
|
||||
plt.rcParams['figure.max_open_warning'] = 100
|
||||
ee.Initialize()
|
||||
|
||||
# parameters
|
||||
cloud_thresh = 0.5 # threshold for cloud cover
|
||||
plot_bool = False # if you want the plots
|
||||
prob_high = 99.9 # upper probability to clip and rescale pixel intensity
|
||||
min_contour_points = 100# minimum number of points contained in each water line
|
||||
output_epsg = 28356 # GDA94 / MGA Zone 56
|
||||
buffer_size = 10 # radius (in pixels) of disk for buffer (pixel classification)
|
||||
min_beach_size = 50 # number of pixels in a beach (pixel classification)
|
||||
|
||||
# select collection
|
||||
satname = 'L8'
|
||||
input_col = ee.ImageCollection('LANDSAT/LC08/C01/T1_RT_TOA') # Landsat 8 Tier 1 TOA
|
||||
|
||||
# location (Narrabeen-Collaroy beach)
|
||||
polygon = [[[151.3473129272461,-33.69035274454718],
|
||||
[151.2820816040039,-33.68206818063878],
|
||||
[151.27281188964844,-33.74775138989556],
|
||||
[151.3425064086914,-33.75231878701767],
|
||||
[151.3473129272461,-33.69035274454718]]];
|
||||
|
||||
# dates
|
||||
start_date = '2013-01-01'
|
||||
end_date = '2018-12-31'
|
||||
|
||||
# filter by location and date
|
||||
flt_col = input_col.filterBounds(ee.Geometry.Polygon(polygon)).filterDate(start_date, end_date)
|
||||
|
||||
n_img = flt_col.size().getInfo()
|
||||
print('Number of images covering the polygon:', n_img)
|
||||
im_all = flt_col.getInfo().get('features')
|
||||
|
||||
i = 0 # first image
|
||||
|
||||
# find image in ee database
|
||||
im = ee.Image(im_all[i].get('id'))
|
||||
|
||||
# load image as np.array
|
||||
im_pan, im_ms, im_cloud, crs, meta = sds.read_eeimage(im, polygon, satname, plot_bool)
|
||||
|
||||
# rescale intensities
|
||||
im_ms = sds.rescale_image_intensity(im_ms, im_cloud, prob_high, plot_bool)
|
||||
im_pan = sds.rescale_image_intensity(im_pan, im_cloud, prob_high, plot_bool)
|
||||
|
||||
# pansharpen rgb image
|
||||
im_ms_ps = sds.pansharpen(im_ms[:,:,[0,1,2]], im_pan, im_cloud, plot_bool)
|
||||
|
||||
# add down-sized bands for NIR and SWIR (since pansharpening is not possible)
|
||||
im_ms_ps = np.append(im_ms_ps, im_ms[:,:,[3,4]], axis=2)
|
||||
|
||||
# calculate NDWI
|
||||
im_ndwi = sds.nd_index(im_ms_ps[:,:,3], im_ms_ps[:,:,1], im_cloud, plot_bool)
|
||||
|
||||
# edge detection
|
||||
wl_pix = sds.find_wl_contours(im_ndwi, im_cloud, min_contour_points, plot_bool)
|
||||
|
||||
# convert from pixels to world coordinates
|
||||
wl_coords = sds.convert_pix2world(wl_pix, crs['crs_15m'])
|
||||
|
||||
# convert to output epsg spatial reference
|
||||
wl = sds.convert_epsg(wl_coords, crs['epsg_code'], output_epsg)
|
||||
|
||||
# classify sand pixels
|
||||
im_sand = sds.classify_sand_unsupervised(im_ms_ps, im_pan, im_cloud, wl_pix, buffer_size, min_beach_size, plot_bool)
|
||||
|
||||
#pt_in = np.array(ginput(n=1, timeout=1000))
|
||||
#if pt_in[0][0] < im_ms_ps.shape[1]/2:
|
||||
win = np.ones((3,3))
|
||||
|
||||
im_features = np.zeros((sum(sum(im_sand)), 20))
|
||||
im_features[:,[0,1,2,3,4]] = im_ms_ps[im_sand,:] # B G R NIR SWIR
|
||||
im_features[:,5] = im_pan[im_sand] # Pan
|
||||
im_features[:,6] = im_ndwi[im_sand] # NDWI
|
||||
im_features[:,7] = sds.nd_index(im_ms_ps[:,:,3], im_ms_ps[:,:,2], im_cloud, False)[im_sand] # NDVI
|
||||
im_features[:,8] = sds.nd_index(im_ms_ps[:,:,0], im_ms_ps[:,:,2], im_cloud, False)[im_sand] # ND Blue - Red
|
||||
for i in range(9):
|
||||
im_features[:,i+9] = ndimage.generic_filter(im_features[:,i], np.std, footprint = win)
|
||||
|
||||
|
||||
im_ms_ps[im_sand,:]
|
||||
|
||||
im_grey = color.rgb2grey(im_ms_ps[:,:,[2,1,0]])
|
||||
plt.figure()
|
||||
plt.imshow(im_grey, cmap='gray')
|
||||
plt.draw()
|
||||
|
||||
counts, bins = np.histogram(im_grey[~im_cloud], bins=255)
|
||||
im_grey_d = np.digitize(im_grey, bins=bins) - 1
|
||||
|
||||
from scipy import ndimage
|
||||
|
||||
varianceMatrix1 = ndimage.generic_filter(im_grey_d, np.max, footprint = np.ones((3,3)))
|
||||
varianceMatrix2 = ndimage.generic_filter(im_grey_d, np.min, footprint = np.ones((3,3)))
|
||||
varianceMatrix = varianceMatrix1 - varianceMatrix2
|
||||
im_grey = color.rgb2grey(im_ms_ps[:,:,[2,1,0]])
|
||||
plt.figure()
|
||||
plt.imshow(varianceMatrix, cmap='gray')
|
||||
plt.draw()
|
||||
#%%
|
||||
data = gdal.Open('l8_test.tif', gdal.GA_ReadOnly)
|
||||
bands = [data.GetRasterBand(i + 1).ReadAsArray() for i in range(data.RasterCount)]
|
||||
im = np.stack(bands, 2)
|
||||
|
||||
im_test = im[:,:,3]
|
||||
|
||||
plt.figure()
|
||||
plt.imshow(im_test, cmap='gray')
|
||||
plt.axis('image')
|
||||
plt.draw()
|
||||
|
||||
im_stats = np.zeros((im_test.shape[0], im_test.shape[1], 6))
|
||||
winsize = 5
|
||||
prop_names = ('contrast', 'dissimilarity', 'homogeneity', 'energy', 'correlation', 'ASM')
|
||||
for i in range(im_test.shape[0]):
|
||||
print(int(np.round(100*i/im_test.shape[0])))
|
||||
for j in range(im_test.shape[1]):
|
||||
#windows needs to fit completely in image
|
||||
if i <2 or j <2:
|
||||
continue
|
||||
if i > (im_test.shape[0] - 3) or j > (im_test.shape[1] - 3):
|
||||
continue
|
||||
|
||||
#Calculate GLCM on a 3x3 window
|
||||
glcm_window = im_test[i-2: i+3, j-2 : j+3]
|
||||
glcm = feature.greycomatrix(glcm_window, [1], [0], symmetric = True, normed = True )
|
||||
|
||||
#Calculate contrast and replace center pixel
|
||||
for k in range(6): im_stats[i,j,k] = feature.greycoprops(glcm, prop_names[k])
|
||||
|
||||
plt.figure()
|
||||
for i in range(6):
|
||||
plt.subplot(2,3,i+1)
|
||||
plt.imshow(im_stats[:,:,i], cmap='jet')
|
||||
plt.axis('image')
|
||||
plt.title(prop_names[i])
|
||||
plt.draw()
|
||||
|
||||
pixel_loc = [200, 200]
|
||||
im_stats[pixel_loc[0], pixel_loc[1], 3]
|
||||
#%%
|
||||
|
||||
for i in range(im_grey_d.shape[0]):
|
||||
print(int(np.round(100*i/im_grey_d.shape[0])))
|
||||
for j in range(im_grey_d.shape[1]):
|
||||
#windows needs to fit completely in image
|
||||
if i <1 or j <1:
|
||||
continue
|
||||
if i > (im_grey_d.shape[0] - 1) or j > (im_grey_d.shape[0] - 1):
|
||||
continue
|
||||
|
||||
#Calculate GLCM on a 3x3 window
|
||||
glcm_window = im_grey_d[i-1: i+1, j-1 : j+1]
|
||||
glcm = feature.greycomatrix(glcm_window, [1,2], [0], levels=256, symmetric = True, normed = True )
|
||||
|
||||
#Calculate contrast and replace center pixel
|
||||
im_stats[i,j,0] = feature.greycoprops(glcm, 'contrast')
|
||||
im_stats[i,j,1] = feature.greycoprops(glcm, 'dissimilarity')
|
||||
im_stats[i,j,2] = feature.greycoprops(glcm, 'homogeneity')
|
||||
im_stats[i,j,3] = feature.greycoprops(glcm, 'energy')
|
||||
im_stats[i,j,4] = feature.greycoprops(glcm, 'correlation')
|
||||
im_stats[i,j,5] = feature.greycoprops(glcm, 'ASM')
|
||||
|
||||
plt.figure()
|
||||
for i in range(6):
|
||||
plt.subplot(2,3,i+1)
|
||||
plt.imshow(im_stats[:,:,i], cmap='jet')
|
||||
plt.axis('image')
|
||||
plt.draw()
|
||||
#%%
|
||||
from multiprocessing import Pool
|
||||
from itertools import product
|
||||
|
||||
N = 10000000
|
||||
pool = Pool() #defaults to number of available CPU's
|
||||
a = np.ones((N))
|
||||
b = np.ones((N))*2
|
||||
out = np.zeros((N))
|
||||
|
||||
t = time.time()
|
||||
for i in range(len(a)):
|
||||
out[i] = a[i]*b[i]
|
||||
elapsed = time.time() - t
|
||||
print(elapsed)
|
||||
|
||||
def fun(a,b):
|
||||
return a*b
|
||||
|
||||
chunksize = 20 #this may take some guessing ... take a look at the docs to decide
|
||||
for ind, res in enumerate(pool.map(fun, range(N)), chunksize):
|
||||
output.flat[ind] = res
|
||||
#%%
|
||||
|
||||
import gdal, osr
|
||||
import numpy as np
|
||||
from scipy.interpolate import RectBivariateSpline
|
||||
from numpy.lib.stride_tricks import as_strided as ast
|
||||
import dask.array as da
|
||||
from joblib import Parallel, delayed, cpu_count
|
||||
import os
|
||||
from skimage.feature import greycomatrix, greycoprops
|
||||
|
||||
def im_resize(im,Nx,Ny):
|
||||
'''
|
||||
resize array by bivariate spline interpolation
|
||||
'''
|
||||
ny, nx = np.shape(im)
|
||||
xx = np.linspace(0,nx,Nx)
|
||||
yy = np.linspace(0,ny,Ny)
|
||||
|
||||
try:
|
||||
im = da.from_array(im, chunks=1000) #dask implementation
|
||||
except:
|
||||
pass
|
||||
|
||||
newKernel = RectBivariateSpline(np.r_[:ny],np.r_[:nx],im)
|
||||
return newKernel(yy,xx)
|
||||
|
||||
def p_me(Z, win):
|
||||
'''
|
||||
loop to calculate greycoprops
|
||||
'''
|
||||
try:
|
||||
glcm = greycomatrix(Z, [5], [0], 256, symmetric=True, normed=True)
|
||||
cont = greycoprops(glcm, 'contrast')
|
||||
diss = greycoprops(glcm, 'dissimilarity')
|
||||
homo = greycoprops(glcm, 'homogeneity')
|
||||
eng = greycoprops(glcm, 'energy')
|
||||
corr = greycoprops(glcm, 'correlation')
|
||||
ASM = greycoprops(glcm, 'ASM')
|
||||
return (cont, diss, homo, eng, corr, ASM)
|
||||
except:
|
||||
return (0,0,0,0,0,0)
|
||||
|
||||
def norm_shape(shap):
|
||||
'''
|
||||
Normalize numpy array shapes so they're always expressed as a tuple,
|
||||
even for one-dimensional shapes.
|
||||
'''
|
||||
try:
|
||||
i = int(shap)
|
||||
return (i,)
|
||||
except TypeError:
|
||||
# shape was not a number
|
||||
pass
|
||||
|
||||
try:
|
||||
t = tuple(shap)
|
||||
return t
|
||||
except TypeError:
|
||||
# shape was not iterable
|
||||
pass
|
||||
|
||||
raise TypeError('shape must be an int, or a tuple of ints')
|
||||
|
||||
def sliding_window(a, ws, ss = None, flatten = True):
|
||||
'''
|
||||
Source: http://www.johnvinyard.com/blog/?p=268#more-268
|
||||
Parameters:
|
||||
a - an n-dimensional numpy array
|
||||
ws - an int (a is 1D) or tuple (a is 2D or greater) representing the size
|
||||
of each dimension of the window
|
||||
ss - an int (a is 1D) or tuple (a is 2D or greater) representing the
|
||||
amount to slide the window in each dimension. If not specified, it
|
||||
defaults to ws.
|
||||
flatten - if True, all slices are flattened, otherwise, there is an
|
||||
extra dimension for each dimension of the input.
|
||||
|
||||
Returns
|
||||
an array containing each n-dimensional window from a
|
||||
'''
|
||||
if None is ss:
|
||||
# ss was not provided. the windows will not overlap in any direction.
|
||||
ss = ws
|
||||
ws = norm_shape(ws)
|
||||
ss = norm_shape(ss)
|
||||
# convert ws, ss, and a.shape to numpy arrays
|
||||
ws = np.array(ws)
|
||||
ss = np.array(ss)
|
||||
shap = np.array(a.shape)
|
||||
# ensure that ws, ss, and a.shape all have the same number of dimensions
|
||||
ls = [len(shap),len(ws),len(ss)]
|
||||
if 1 != len(set(ls)):
|
||||
raise ValueError(\
|
||||
'a.shape, ws and ss must all have the same length. They were %s' % str(ls))
|
||||
|
||||
# ensure that ws is smaller than a in every dimension
|
||||
if np.any(ws > shap):
|
||||
raise ValueError(\
|
||||
'ws cannot be larger than a in any dimension.\
|
||||
a.shape was %s and ws was %s' % (str(a.shape),str(ws)))
|
||||
|
||||
# how many slices will there be in each dimension?
|
||||
newshape = norm_shape(((shap - ws) // ss) + 1)
|
||||
|
||||
|
||||
# the shape of the strided array will be the number of slices in each dimension
|
||||
# plus the shape of the window (tuple addition)
|
||||
newshape += norm_shape(ws)
|
||||
|
||||
|
||||
# the strides tuple will be the array's strides multiplied by step size, plus
|
||||
# the array's strides (tuple addition)
|
||||
newstrides = norm_shape(np.array(a.strides) * ss) + a.strides
|
||||
a = ast(a,shape = newshape,strides = newstrides)
|
||||
if not flatten:
|
||||
return a
|
||||
# Collapse strided so that it has one more dimension than the window. I.e.,
|
||||
# the new array is a flat list of slices.
|
||||
meat = len(ws) if ws.shape else 0
|
||||
firstdim = (np.product(newshape[:-meat]),) if ws.shape else ()
|
||||
dim = firstdim + (newshape[-meat:])
|
||||
# remove any dimensions with size 1
|
||||
dim = filter(lambda i : i != 1,dim)
|
||||
return a.reshape(dim), newshape
|
||||
|
||||
#Stuff to change
|
||||
win = 3
|
||||
meter = str(win/4)
|
||||
merge = im_grey_d
|
||||
Z,ind = sliding_window(merge,(win,win),(1,1))
|
||||
|
||||
Ny, Nx = np.shape(merge)
|
||||
|
||||
w = Parallel(n_jobs = cpu_count(), verbose=0)(delayed(p_me)(Z[k]) for k in xrange(len(Z)))
|
||||
|
||||
cont = [a[0] for a in w]
|
||||
diss = [a[1] for a in w]
|
||||
homo = [a[2] for a in w]
|
||||
eng = [a[3] for a in w]
|
||||
corr = [a[4] for a in w]
|
||||
ASM = [a[5] for a in w]
|
||||
|
||||
|
||||
#Reshape to match number of windows
|
||||
plt_cont = np.reshape(cont , ( ind[0], ind[1] ) )
|
||||
plt_diss = np.reshape(diss , ( ind[0], ind[1] ) )
|
||||
plt_homo = np.reshape(homo , ( ind[0], ind[1] ) )
|
||||
plt_eng = np.reshape(eng , ( ind[0], ind[1] ) )
|
||||
plt_corr = np.reshape(corr , ( ind[0], ind[1] ) )
|
||||
plt_ASM = np.reshape(ASM , ( ind[0], ind[1] ) )
|
||||
del cont, diss, homo, eng, corr, ASM
|
||||
|
||||
#Resize Images to receive texture and define filenames
|
||||
contrast = im_resize(plt_cont,Nx,Ny)
|
||||
contrast[merge==0]=np.nan
|
||||
dissimilarity = im_resize(plt_diss,Nx,Ny)
|
||||
dissimilarity[merge==0]=np.nan
|
||||
homogeneity = im_resize(plt_homo,Nx,Ny)
|
||||
homogeneity[merge==0]=np.nan
|
||||
energy = im_resize(plt_eng,Nx,Ny)
|
||||
energy[merge==0]=np.nan
|
||||
correlation = im_resize(plt_corr,Nx,Ny)
|
||||
correlation[merge==0]=np.nan
|
||||
ASM = im_resize(plt_ASM,Nx,Ny)
|
||||
ASM[merge==0]=np.nan
|
||||
|
||||
#%%
|
||||
plt.figure()
|
||||
plt.imshow(im_ms_ps[:,:,[2,1,0]])
|
||||
for i,contour in enumerate(wl_pix): plt.plot(contour[:, 1], contour[:, 0], linewidth=2)
|
||||
plt.axis('image')
|
||||
plt.title('Detected water lines')
|
||||
plt.show()
|
||||
|
||||
vec = im_ms_ps.reshape(im_ms_ps.shape[0] * im_ms_ps.shape[1], im_ms_ps.shape[2])
|
||||
vec_pan = im_pan.reshape(im_pan.shape[0]*im_pan.shape[1])
|
||||
features = np.zeros((len(vec), 5))
|
||||
features[:,[0,1,2,3]] = vec[:,[0,1,2,3]]
|
||||
features[:,4] = vec_pan
|
||||
vec_mask = im_cloud.reshape(im_ms_ps.shape[0] * im_ms_ps.shape[1])
|
||||
|
||||
# create buffer
|
||||
im_buffer = np.zeros((im_ms_ps.shape[0], im_ms_ps.shape[1]))
|
||||
for i, contour in enumerate(wl_pix):
|
||||
indices = [(int(_[0]), int(_[1])) for _ in list(np.round(contour))]
|
||||
for j, idx in enumerate(indices):
|
||||
im_buffer[idx] = 1
|
||||
|
||||
|
||||
plt.figure()
|
||||
plt.imshow(im_buffer)
|
||||
plt.draw()
|
||||
|
||||
se = morphology.disk(buffer_size)
|
||||
im_buffer = morphology.binary_dilation(im_buffer, se)
|
||||
|
||||
plt.figure()
|
||||
plt.imshow(im_buffer)
|
||||
plt.draw()
|
||||
|
||||
vec_buffer = (im_buffer == 1).reshape(im_ms_ps.shape[0] * im_ms_ps.shape[1])
|
||||
|
||||
vec_buffer= np.logical_and(vec_buffer, ~vec_mask)
|
||||
|
||||
#vec_buffer = np.ravel_multi_index(z,(im_ms_ps.shape[0], im_ms_ps.shape[1]))
|
||||
|
||||
|
||||
kmeans = KMeans(n_clusters=6, random_state=0).fit(vec[vec_buffer,:])
|
||||
labels = kmeans.labels_
|
||||
labels_full = np.ones((len(vec_mask))) * np.nan
|
||||
labels_full[vec_buffer] = labels
|
||||
im_labels = labels_full.reshape(im_ms_ps.shape[0], im_ms_ps.shape[1])
|
||||
|
||||
plt.figure()
|
||||
plt.imshow(im_labels)
|
||||
plt.axis('equal')
|
||||
plt.draw()
|
||||
|
||||
utils.compare_images(im_labels, im_pan)
|
||||
|
||||
plt.figure()
|
||||
for i in range(6): plt.plot(kmeans.cluster_centers_[i,:])
|
||||
plt.draw()
|
||||
|
||||
im_sand = im_labels == np.argmax(np.mean(kmeans.cluster_centers_[:,[0,1,2,4]], axis=1))
|
||||
|
||||
im_sand2 = morphology.remove_small_objects(im_sand, min_size=min_beach_size, connectivity=2)
|
||||
im_sand3 = morphology.binary_dilation(im_sand2, morphology.disk(1))
|
||||
plt.figure()
|
||||
plt.imshow(im_sand3)
|
||||
plt.draw()
|
||||
|
||||
im_ms_ps[im_sand3,0] = 0
|
||||
im_ms_ps[im_sand3,1] = 0
|
||||
im_ms_ps[im_sand3,2] = 1
|
||||
|
||||
|
||||
plt.figure()
|
||||
plt.imshow(im_ms_ps[:,:,[2,1,0]])
|
||||
plt.axis('image')
|
||||
plt.title('Sand classification')
|
||||
plt.show()
|
||||
|
||||
#%%
|
||||
@ -1,222 +0,0 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
# Preamble
|
||||
import ee
|
||||
import matplotlib.pyplot as plt
|
||||
import matplotlib.cm as cm
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
from datetime import datetime
|
||||
import pickle
|
||||
import pdb
|
||||
import pytz
|
||||
from pylab import ginput
|
||||
import scipy.io as sio
|
||||
import scipy.interpolate
|
||||
import os
|
||||
|
||||
# image processing modules
|
||||
import skimage.filters as filters
|
||||
import skimage.exposure as exposure
|
||||
import skimage.transform as transform
|
||||
import sklearn.decomposition as decomposition
|
||||
import skimage.morphology as morphology
|
||||
import skimage.measure as measure
|
||||
|
||||
# my functions
|
||||
import functions.utils as utils
|
||||
import functions.sds as sds
|
||||
|
||||
np.seterr(all='ignore') # raise/ignore divisions by 0 and nans
|
||||
ee.Initialize()
|
||||
|
||||
au_tz = pytz.timezone('Australia/Sydney')
|
||||
|
||||
#%%
|
||||
# load SDS shorelines
|
||||
with open('data\data_gt_l8.pkl', 'rb') as f:
|
||||
data = pickle.load(f)
|
||||
|
||||
# load quadbike dates and convert from datenum to datetime
|
||||
suffix = '.mat'
|
||||
dir_name = os.getcwd()
|
||||
file_name = 'data\quadbike_dates'
|
||||
file_path = os.path.join(dir_name, file_name + suffix)
|
||||
quad_dates = sio.loadmat(file_path)['dates']
|
||||
dt_quad = []
|
||||
for i in range(quad_dates.shape[0]):
|
||||
dt_quad.append(datetime(quad_dates[i,0], quad_dates[i,1], quad_dates[i,2], tzinfo=au_tz))
|
||||
|
||||
# remove overlapping images, keep the one with lowest cloud_cover
|
||||
n = len(data['cloud_cover'])
|
||||
idx_worst = []
|
||||
for i in range(n):
|
||||
date_im = data['date_acquired'][i]
|
||||
idx_double = np.isin(data['date_acquired'], date_im)
|
||||
if sum(idx_double.astype(int)) > 1:
|
||||
idx_worst.append(np.where(idx_double)[0][np.argmax(np.array(data['cloud_cover'])[idx_double])])
|
||||
dt_sat = []
|
||||
new_meta = {'contours':[],
|
||||
'cloud_cover':[],
|
||||
'geom_rmse_model':[],
|
||||
'gcp_model':[],
|
||||
'quality':[],
|
||||
'sun_azimuth':[],
|
||||
'sun_elevation':[]}
|
||||
for i in range(n):
|
||||
if not np.isin(i,idx_worst):
|
||||
dt_sat.append(data['dt'][i].astimezone(au_tz))
|
||||
new_meta['contours'].append(data['contours'][i])
|
||||
new_meta['cloud_cover'].append(data['cloud_cover'][i])
|
||||
new_meta['geom_rmse_model'].append(data['geom_rmse_model'][i])
|
||||
new_meta['gcp_model'].append(data['gcp_model'][i])
|
||||
new_meta['quality'].append(data['quality'][i])
|
||||
new_meta['sun_azimuth'].append(data['sun_azimuth'][i])
|
||||
new_meta['sun_elevation'].append(data['sun_elevation'][i])
|
||||
|
||||
# calculate difference between days
|
||||
diff_days = [ [(x - _).days for _ in dt_quad] for x in dt_sat]
|
||||
day_thresh = 15
|
||||
idx_close = [utils.find_indices(_, lambda e: abs(e) < day_thresh) for _ in diff_days]
|
||||
|
||||
# put everything in a dictionnary and save it
|
||||
wl_comp = []
|
||||
for i in range(len(dt_sat)):
|
||||
wl_comp.append({'sat dt': dt_sat[i],
|
||||
'quad dt': [dt_quad[_] for _ in idx_close[i]],
|
||||
'days diff': [diff_days[i][_] for _ in idx_close[i]],
|
||||
'contours': new_meta['contours'][i],
|
||||
'cloud_cover': new_meta['cloud_cover'][i],
|
||||
'geom_rmse_model': new_meta['geom_rmse_model'][i],
|
||||
'gcp_model': new_meta['gcp_model'][i],
|
||||
'quality': new_meta['quality'][i],
|
||||
'sun_azimuth': new_meta['sun_azimuth'][i],
|
||||
'sun_elevation': new_meta['sun_elevation'][i]})
|
||||
|
||||
with open('wl_l8_comparison.pkl', 'wb') as f:
|
||||
pickle.dump(wl_comp, f)
|
||||
|
||||
#%%
|
||||
with open('data\wl_l8_comparison.pkl', 'rb') as f:
|
||||
wl = pickle.load(f)
|
||||
# load quadbike dates and convert from datenum to datetime
|
||||
suffix = '.mat'
|
||||
dir_name = os.getcwd()
|
||||
subfolder_name = 'data\quadbike_surveys'
|
||||
file_path = os.path.join(dir_name, subfolder_name)
|
||||
file_names = os.listdir(file_path)
|
||||
|
||||
for i in range(len(file_names)):
|
||||
fn_mat = os.path.join(file_path, file_names[i])
|
||||
years = int(file_names[i][6:10])
|
||||
months = int(file_names[i][11:13])
|
||||
days = int(file_names[i][14:16])
|
||||
for j in range(len(wl)):
|
||||
if wl[j]['quad dt'][0] == datetime(years, months, days, tzinfo=au_tz):
|
||||
quad_mat = sio.loadmat(fn_mat)
|
||||
wl[j].update({'quad_data':{'x':quad_mat['x'],
|
||||
'y':quad_mat['y'],
|
||||
'z':quad_mat['z'],
|
||||
'dt': datetime(years, months, days, tzinfo=au_tz)}})
|
||||
with open('data\wl_final.pkl', 'wb') as f:
|
||||
pickle.dump(wl, f)
|
||||
#%%
|
||||
with open('data\wl_final.pkl', 'rb') as f:
|
||||
wl = pickle.load(f)
|
||||
|
||||
i = 0
|
||||
x = wl[i]['quad_data']['x']
|
||||
y = wl[i]['quad_data']['y']
|
||||
z = wl[i]['quad_data']['z']
|
||||
x = x.reshape(x.shape[0] * x.shape[1])
|
||||
y = y.reshape(y.shape[0] * y.shape[1])
|
||||
z = z.reshape(z.shape[0] * z.shape[1])
|
||||
idx_nan = np.isnan(z)
|
||||
|
||||
x_nan = x[idx_nan]
|
||||
y_nan = y[idx_nan]
|
||||
z_nan = z[idx_nan]
|
||||
|
||||
x_nonan = x[~idx_nan]
|
||||
y_nonan = y[~idx_nan]
|
||||
z_nonan = z[~idx_nan]
|
||||
|
||||
xs = x_nonan[::10]
|
||||
ys = y_nonan[::10]
|
||||
zs = z_nonan[::10]
|
||||
|
||||
xq = wl[i]['contours'][:,0]
|
||||
yq = wl[i]['contours'][:,1]
|
||||
|
||||
# cut xq around xs
|
||||
np.min(xs)
|
||||
np.max(xs)
|
||||
np.min(ys)
|
||||
np.max(ys)
|
||||
|
||||
idx_x = np.logical_and(xq < np.max(xs), xq > np.min(xs))
|
||||
idx_y = np.logical_and(yq < np.max(ys), yq > np.min(ys))
|
||||
idx_in = np.logical_and(idx_x, idx_y)
|
||||
|
||||
xq = xq[idx_in]
|
||||
yq = yq[idx_in]
|
||||
|
||||
for i in range(len(xq)):
|
||||
idx_x = np.logical_and(xs < xq[i] + 10, xs > xq[i] - 10)
|
||||
idx_y = np.logical_and(ys < yq[i] + 10, ys > yq[i] - 10)
|
||||
xint = xs[idx_x]
|
||||
yint = ys[idx_y]
|
||||
|
||||
f = interpolate.interp2d(xs, ys, zs, kind='linear')
|
||||
zq = f(xq,yq)
|
||||
|
||||
plt.figure()
|
||||
plt.grid()
|
||||
plt.scatter(xs, ys, s=10, c=zs, marker='o', cmap=cm.get_cmap('jet'),
|
||||
label='quad data')
|
||||
plt.plot(xq,yq,'r-o', markersize=5, label='SDS')
|
||||
plt.axis('equal')
|
||||
plt.legend()
|
||||
plt.colorbar(label='mAHD')
|
||||
plt.xlabel('Eastings [m]')
|
||||
plt.ylabel('Northings [m]')
|
||||
plt.show()
|
||||
|
||||
plt.figure()
|
||||
plt.plot(zq[:,0])
|
||||
plt.show()
|
||||
|
||||
|
||||
plt.figure()
|
||||
plt.grid()
|
||||
plt.scatter(x_nonan, y_nonan, s=10, c=z_nonan, marker='o', cmap=cm.get_cmap('jet'),
|
||||
label='quad data')
|
||||
#plt.plot(x_nan, y_nan, 'k.', label='nans')
|
||||
plt.plot(xq,yq,'r-o', markersize=5, label='SDS')
|
||||
plt.axis('equal')
|
||||
plt.legend()
|
||||
plt.colorbar(label='mAHD')
|
||||
plt.xlabel('Eastings [m]')
|
||||
plt.ylabel('Northings [m]')
|
||||
plt.show()
|
||||
|
||||
|
||||
z2 = scipy.interpolate.griddata([x, y], z, [xq, yq], method='linear')
|
||||
|
||||
f_interp = scipy.interpolate.interp2d(x1,y1,z1, kind='linear')
|
||||
|
||||
|
||||
sio.savemat('shoreline1.mat', {'x':xq, 'y':yq})
|
||||
|
||||
from scipy import interpolate
|
||||
x = np.arange(-5.01, 5.01, 0.01)
|
||||
y = np.arange(-5.01, 5.01, 0.01)
|
||||
xx, yy = np.meshgrid(x, y)
|
||||
z = np.sin(xx**2+yy**2)
|
||||
f = interpolate.interp2d(x, y, z, kind='cubic')
|
||||
|
||||
xnew = np.arange(-5.01, 5.01, 1e-2)
|
||||
ynew = np.arange(-5.01, 5.01, 1e-2)
|
||||
znew = f(xnew, ynew)
|
||||
plt.plot(x, z[:, 0], 'ro-', xnew, znew[:, 0], 'b-')
|
||||
plt.show()
|
||||
@ -1,47 +0,0 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
# Preamble
|
||||
import ee
|
||||
import matplotlib.pyplot as plt
|
||||
import matplotlib.cm as cm
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
from datetime import datetime
|
||||
import pickle
|
||||
import pdb
|
||||
import pytz
|
||||
from pylab import ginput
|
||||
import scipy.io as sio
|
||||
import scipy.interpolate
|
||||
import os
|
||||
|
||||
# image processing modules
|
||||
import skimage.filters as filters
|
||||
import skimage.exposure as exposure
|
||||
import skimage.transform as transform
|
||||
import sklearn.decomposition as decomposition
|
||||
import skimage.morphology as morphology
|
||||
import skimage.measure as measure
|
||||
|
||||
# my functions
|
||||
import functions.utils as utils
|
||||
import functions.sds as sds
|
||||
|
||||
np.seterr(all='ignore') # raise/ignore divisions by 0 and nans
|
||||
ee.Initialize()
|
||||
|
||||
with open('data\wl_final.pkl', 'rb') as f:
|
||||
wl = pickle.load(f)
|
||||
|
||||
i = 0
|
||||
x = wl[i]['quad_data']['x']
|
||||
y = wl[i]['quad_data']['y']
|
||||
z = wl[i]['quad_data']['z']
|
||||
x = x.reshape(x.shape[0] * x.shape[1])
|
||||
y = y.reshape(y.shape[0] * y.shape[1])
|
||||
z = z.reshape(z.shape[0] * z.shape[1])
|
||||
|
||||
idx_nan = np.isnan(z)
|
||||
x = x[~idx_nan]
|
||||
y = y[~idx_nan]
|
||||
z = z[~idx_nan]
|
||||
@ -1,221 +0,0 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
Created on Thu Mar 1 14:32:08 2018
|
||||
|
||||
@author: z5030440
|
||||
|
||||
Main code to extract shorelines from Landsat imagery
|
||||
"""
|
||||
# Preamble
|
||||
|
||||
import ee
|
||||
from IPython import display
|
||||
import math
|
||||
import matplotlib.pyplot as plt
|
||||
import numpy as np
|
||||
import pdb
|
||||
|
||||
# image processing modules
|
||||
import skimage.filters as filters
|
||||
import skimage.exposure as exposure
|
||||
import skimage.transform as transform
|
||||
import sklearn.decomposition as decomposition
|
||||
import skimage.morphology as morphology
|
||||
import skimage.measure as measure
|
||||
|
||||
from shapely.geometry import Polygon
|
||||
|
||||
from osgeo import gdal
|
||||
from osgeo import osr
|
||||
import tempfile
|
||||
import urllib
|
||||
from urllib.request import urlretrieve
|
||||
import zipfile
|
||||
|
||||
# my modules
|
||||
from utils import *
|
||||
# from sds import *
|
||||
|
||||
np.seterr(all='ignore') # raise/ignore divisions by 0 and nans
|
||||
ee.Initialize()
|
||||
plot_bool = True # if you want the plots
|
||||
|
||||
def download_tif(image, bandsId):
|
||||
"""downloads tif image (region and bands) from the ee server and stores it in a temp file"""
|
||||
url = ee.data.makeDownloadUrl(ee.data.getDownloadId({
|
||||
'image': image.serialize(),
|
||||
'bands': bandsId,
|
||||
'filePerBand': 'false',
|
||||
'name': 'data',
|
||||
}))
|
||||
local_zip, headers = urlretrieve(url)
|
||||
with zipfile.ZipFile(local_zip) as local_zipfile:
|
||||
return local_zipfile.extract('data.tif', tempfile.mkdtemp())
|
||||
|
||||
def load_image(image, bandsId):
|
||||
"""loads an ee.Image() as a np.array. e.Image() is retrieved from the EE database."""
|
||||
local_tif_filename = download_tif(image, bandsId)
|
||||
dataset = gdal.Open(local_tif_filename, gdal.GA_ReadOnly)
|
||||
bands = [dataset.GetRasterBand(i + 1).ReadAsArray() for i in range(dataset.RasterCount)]
|
||||
return np.stack(bands, 2), dataset
|
||||
|
||||
|
||||
|
||||
im = ee.Image('LANDSAT/LC08/C01/T1_RT_TOA/LC08_089083_20130411')
|
||||
|
||||
lon = [151.2820816040039, 151.3425064086914]
|
||||
lat = [-33.68206818063878, -33.74775138989556]
|
||||
polygon = [[lon[0], lat[0]], [lon[1], lat[0]], [lon[1], lat[1]], [lon[0], lat[1]]];
|
||||
|
||||
# get image metadata into dictionnary
|
||||
im_dic = im.getInfo()
|
||||
im_bands = im_dic.get('bands')
|
||||
# delete dimensions key from dictionnary, otherwise the entire image is extracted
|
||||
#for i in range(len(im_bands)): del im_bands[i]['dimensions']
|
||||
pan_band = [im_bands[7]]
|
||||
ms_bands = [im_bands[1], im_bands[2], im_bands[3]]
|
||||
im_full, dataset_full = load_image(im, ms_bands)
|
||||
|
||||
plt.figure()
|
||||
plt.imshow(np.clip(im_full[:,:,[2,1,0]] * 3, 0, 1))
|
||||
plt.show()
|
||||
#%%
|
||||
|
||||
def download_tif(image, polygon, bandsId):
|
||||
"""downloads tif image (region and bands) from the ee server and stores it in a temp file"""
|
||||
url = ee.data.makeDownloadUrl(ee.data.getDownloadId({
|
||||
'image': image.serialize(),
|
||||
'region': polygon,
|
||||
'bands': bandsId,
|
||||
'filePerBand': 'false',
|
||||
'name': 'data',
|
||||
}))
|
||||
local_zip, headers = urlretrieve(url)
|
||||
with zipfile.ZipFile(local_zip) as local_zipfile:
|
||||
return local_zipfile.extract('data.tif', tempfile.mkdtemp())
|
||||
|
||||
def load_image(image, polygon, bandsId):
|
||||
"""
|
||||
Loads an ee.Image() as a np.array. e.Image() is retrieved from the EE database.
|
||||
The geographic area and bands to select can be specified
|
||||
|
||||
KV WRL 2018
|
||||
|
||||
Arguments:
|
||||
-----------
|
||||
image: ee.Image()
|
||||
image objec from the EE database
|
||||
polygon: list
|
||||
coordinates of the points creating a polygon. Each point is a list with 2 values
|
||||
bandsId: list
|
||||
bands to select, each band is a dictionnary in the list containing the following keys:
|
||||
crs, crs_transform, data_type and id. NOTE: you have to remove the key dimensions, otherwise
|
||||
the entire image is retrieved.
|
||||
|
||||
Returns:
|
||||
-----------
|
||||
image_array : np.ndarray
|
||||
An array containing the image (2D if one band, otherwise 3D)
|
||||
"""
|
||||
|
||||
local_tif_filename = download_tif(image, polygon, bandsId)
|
||||
dataset = gdal.Open(local_tif_filename, gdal.GA_ReadOnly)
|
||||
bands = [dataset.GetRasterBand(i + 1).ReadAsArray() for i in range(dataset.RasterCount)]
|
||||
return np.stack(bands, 2), dataset
|
||||
|
||||
|
||||
for i in range(len(im_bands)): del im_bands[i]['dimensions']
|
||||
ms_bands = [im_bands[1], im_bands[2], im_bands[3]]
|
||||
|
||||
im_cropped, dataset_cropped = load_image(im, polygon, ms_bands)
|
||||
|
||||
plt.figure()
|
||||
plt.imshow(np.clip(im_cropped[:,:,[2,1,0]] * 3, 0, 1))
|
||||
plt.show()
|
||||
|
||||
#%%
|
||||
crs_full = dataset_full.GetGeoTransform()
|
||||
crs_cropped = dataset_cropped.GetGeoTransform()
|
||||
scale = crs_full[1]
|
||||
ul_full = np.array([crs_full[0], crs_full[3]])
|
||||
ul_cropped = np.array([crs_cropped[0], crs_cropped[3]])
|
||||
|
||||
delta = np.abs(ul_full - ul_cropped)/scale
|
||||
|
||||
u0 = delta[0].astype('int')
|
||||
v0 = delta[1].astype('int')
|
||||
|
||||
im_full[v0,u0,:]
|
||||
im_cropped[0,0,:]
|
||||
|
||||
lrx = ul_cropped[0] + (dataset_cropped.RasterXSize * scale)
|
||||
lry = ul_cropped[1] + (dataset_cropped.RasterYSize * (-scale))
|
||||
|
||||
lr_cropped = np.array([lrx, lry])
|
||||
|
||||
delta = np.abs(ul_full - lr_cropped)/scale
|
||||
u1 = delta[0].astype('int')
|
||||
v1 = delta[1].astype('int')
|
||||
|
||||
im_cropped2 = im_full[v0:v1,u0:u1,:]
|
||||
|
||||
#%%
|
||||
crs_full = dataset_full.GetGeoTransform()
|
||||
source = osr.SpatialReference()
|
||||
source.ImportFromWkt(dataset_full.GetProjection())
|
||||
|
||||
target = osr.SpatialReference()
|
||||
target.ImportFromEPSG(4326)
|
||||
|
||||
transform = osr.CoordinateTransformation(source, target)
|
||||
|
||||
transform.TransformPoint(ulx, uly)
|
||||
|
||||
#%%
|
||||
crs_cropped = dataset_cropped.GetGeoTransform()
|
||||
ulx = crs_cropped[0]
|
||||
uly = crs_cropped[3]
|
||||
source = osr.SpatialReference()
|
||||
source.ImportFromWkt(dataset_cropped.GetProjection())
|
||||
|
||||
target = osr.SpatialReference()
|
||||
target.ImportFromEPSG(4326)
|
||||
|
||||
transform = osr.CoordinateTransformation(source, target)
|
||||
|
||||
transform.TransformPoint(lrx, lry)
|
||||
|
||||
|
||||
|
||||
#%%
|
||||
source = osr.SpatialReference()
|
||||
source.ImportFromEPSG(4326)
|
||||
|
||||
target = osr.SpatialReference()
|
||||
target.ImportFromEPSG(32656)
|
||||
|
||||
coords = transform.TransformPoint(151.2820816040039, -33.68206818063878)
|
||||
coords[0] - ulx
|
||||
coords[1] - uly
|
||||
#%%
|
||||
x_ul_full = ms_bands[0]['crs_transform'][2]
|
||||
y_ul_full = ms_bands[0]['crs_transform'][5]
|
||||
scale = ms_bands[0]['crs_transform'][0]
|
||||
|
||||
x_ul_cropped = np.array([340756.105840223, 346357.851288875, 346474.839525944, 340877.362938763])
|
||||
y_ul_cropped = np.array([-3728229.45372866, -3728137.91775723, -3735421.58347927, -3735513.20696522])
|
||||
|
||||
dx = abs(x_ul_full - x_ul_cropped)
|
||||
dy = abs(y_ul_full - y_ul_cropped)
|
||||
|
||||
u_coord = np.round(dx/scale).astype('int')
|
||||
v_coord = np.round(dy/scale).astype('int')
|
||||
|
||||
im_cropped2 = im_full[np.min(v_coord):np.max(v_coord), np.min(u_coord):np.max(u_coord),:]
|
||||
|
||||
plt.figure()
|
||||
plt.imshow(np.clip(im_cropped2[:,:,[2,1,0]] * 3, 0, 1), cmap='gray')
|
||||
plt.show()
|
||||
|
||||
sum(sum(sum(np.equal(im_cropped,im_cropped2).astype('int')-1)))
|
||||
|
||||
@ -1,110 +0,0 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
Created on Fri Mar 23 12:46:04 2018
|
||||
|
||||
@author: z5030440
|
||||
"""
|
||||
|
||||
# Preamble
|
||||
|
||||
import ee
|
||||
import matplotlib.pyplot as plt
|
||||
import matplotlib.cm as cm
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
from datetime import datetime
|
||||
import pickle
|
||||
import pdb
|
||||
import pytz
|
||||
from pylab import ginput
|
||||
|
||||
# image processing modules
|
||||
import skimage.filters as filters
|
||||
import skimage.exposure as exposure
|
||||
import skimage.transform as transform
|
||||
import sklearn.decomposition as decomposition
|
||||
import skimage.morphology as morphology
|
||||
import skimage.measure as measure
|
||||
|
||||
# my functions
|
||||
import functions.utils as utils
|
||||
import functions.sds as sds
|
||||
|
||||
np.seterr(all='ignore') # raise/ignore divisions by 0 and nans
|
||||
ee.Initialize()
|
||||
|
||||
#%% Select images
|
||||
|
||||
# parameters
|
||||
plot_bool = True # if you want the plots
|
||||
prob_high = 99.9 # upper probability to clip and rescale pixel intensity
|
||||
min_contour_points = 100 # minimum number of points contained in each water line
|
||||
output_epsg = 28356 # GDA94 / MGA Zone 56
|
||||
cloud_threshold = 0.8
|
||||
|
||||
# select collection
|
||||
input_col = ee.ImageCollection('LANDSAT/LC08/C01/T1_RT_TOA')
|
||||
satname = 'L8'
|
||||
# location (Narrabeen-Collaroy beach)
|
||||
#rect_narra = [[[151.3473129272461,-33.69035274454718],
|
||||
# [151.2820816040039,-33.68206818063878],
|
||||
# [151.27281188964844,-33.74775138989556],
|
||||
# [151.3425064086914,-33.75231878701767],
|
||||
# [151.3473129272461,-33.69035274454718]]];
|
||||
|
||||
#rect_narra = [[[151.301454, -33.700754],
|
||||
# [151.311453, -33.702075],
|
||||
# [151.307237, -33.739761],
|
||||
# [151.294220, -33.736329],
|
||||
# [151.301454, -33.700754]]];
|
||||
# location (Oldbar NSW)
|
||||
rect_narra = [[[152.578395, -31.841216],
|
||||
[152.777281, -31.842523],
|
||||
[152.738086, -32.028773],
|
||||
[152.557812, -32.004663],
|
||||
[152.578395, -31.841216]]];
|
||||
|
||||
# Dates
|
||||
start_date = '2018-01-18'
|
||||
end_date = '2018-01-20'
|
||||
# filter by location
|
||||
flt_col = input_col.filterBounds(ee.Geometry.Polygon(rect_narra)).filterDate(start_date, end_date)
|
||||
|
||||
n_img = flt_col.size().getInfo()
|
||||
print('Number of images covering Narrabeen:', n_img)
|
||||
im_all = flt_col.getInfo().get('features')
|
||||
|
||||
# find each image in ee database
|
||||
im = ee.Image(im_all[0].get('id'))
|
||||
# load image as np.array
|
||||
im_pan, im_ms, im_cloud, crs, meta = sds.read_eeimage(im, rect_narra, satname, plot_bool)
|
||||
|
||||
# rescale intensities
|
||||
im_ms = sds.rescale_image_intensity(im_ms, im_cloud, prob_high, plot_bool)
|
||||
im_pan = sds.rescale_image_intensity(im_pan, im_cloud, prob_high, plot_bool)
|
||||
|
||||
# pansharpen rgb image
|
||||
im_ms_ps = sds.pansharpen(im_ms[:,:,[0,1,2]], im_pan, im_cloud, plot_bool)
|
||||
|
||||
plt.figure()
|
||||
plt.imshow(im_ms_ps[:,:,[2,1,0]])
|
||||
plt.show()
|
||||
|
||||
pts = ginput(n=20, timeout=1000, show_clicks=True)
|
||||
points = np.array(pts)
|
||||
plt.plot(points[:,0], points[:,1], 'ko')
|
||||
plt.show()
|
||||
|
||||
pts_coords = sds.convert_pix2world(points[:,[1,0]], crs['crs_15m'])
|
||||
pts = sds.convert_epsg(pts_coords, crs['epsg_code'], output_epsg)
|
||||
|
||||
with open('olddata/oldbar_beach.pkl', 'wb') as f:
|
||||
pickle.dump(pts, f)
|
||||
|
||||
|
||||
#pts_wgs84 = sds.convert_epsg(pts_coords, crs['epsg_code'], 4326)
|
||||
#
|
||||
#import simplekml
|
||||
#kml = simplekml.Kml()
|
||||
#kml.new(name='test', coords=pts_wgs84)
|
||||
#kml.save("test.kml")
|
||||
@ -1,23 +0,0 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
|
||||
from datetime import datetime, timedelta
|
||||
import pytz
|
||||
import csv
|
||||
import pandas as pd
|
||||
au_tz = pytz.timezone('Australia/Sydney')
|
||||
|
||||
dt1 = datetime(2018, 4, 17, tzinfo= au_tz)
|
||||
|
||||
dt = []
|
||||
dt.append(dt1)
|
||||
for i in range(1,100):
|
||||
dt1 = dt[i-1]
|
||||
dt.append(dt1 + timedelta(days=16))
|
||||
|
||||
|
||||
dtstr = [_.strftime('%d %b %Y') for _ in dt]
|
||||
|
||||
df = pd.DataFrame(dtstr)
|
||||
df.to_csv('L7_NARRA_dates.csv', index=False, header=False)
|
||||
|
||||
@ -1,119 +0,0 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
Created on Thu Mar 1 14:32:08 2018
|
||||
|
||||
@author: z5030440
|
||||
|
||||
Main code to extract shorelines from Landsat imagery
|
||||
"""
|
||||
|
||||
# Preamble
|
||||
import ee
|
||||
import matplotlib.pyplot as plt
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
from datetime import datetime
|
||||
import pytz
|
||||
import pdb
|
||||
|
||||
# image processing modules
|
||||
import skimage.filters as filters
|
||||
import skimage.exposure as exposure
|
||||
import skimage.transform as transform
|
||||
import sklearn.decomposition as decomposition
|
||||
import skimage.morphology as morphology
|
||||
import skimage.measure as measure
|
||||
|
||||
# my functions
|
||||
import functions.utils as utils
|
||||
import functions.sds as sds
|
||||
|
||||
np.seterr(all='ignore') # raise/ignore divisions by 0 and nans
|
||||
ee.Initialize()
|
||||
|
||||
# parameters
|
||||
plot_bool = False # if you want the plots
|
||||
prob_high = 99.9 # upper probability to clip and rescale pixel intensity
|
||||
min_contour_points = 100 # minimum number of points contained in each water line
|
||||
|
||||
# select collection
|
||||
input_col = ee.ImageCollection('LANDSAT/LC08/C01/T1_TOA')
|
||||
|
||||
# location (Narrabeen-Collaroy beach)
|
||||
rect_narra = [[[151.3473129272461,-33.69035274454718],
|
||||
[151.2820816040039,-33.68206818063878],
|
||||
[151.27281188964844,-33.74775138989556],
|
||||
[151.3425064086914,-33.75231878701767],
|
||||
[151.3473129272461,-33.69035274454718]]];
|
||||
# Dates
|
||||
start_date = '2016-01-01'
|
||||
end_date = '2016-12-31'
|
||||
# filter by location
|
||||
flt_col = input_col.filterBounds(ee.Geometry.Polygon(rect_narra))#.filterDate(start_date, end_date)
|
||||
|
||||
n_img = flt_col.size().getInfo()
|
||||
print('Number of images covering Narrabeen:', n_img)
|
||||
im_all = flt_col.getInfo().get('features')
|
||||
|
||||
props = {'cloud_cover_cropped':[],
|
||||
'cloud_cover':[],
|
||||
'cloud_cover_land':[],
|
||||
'date_acquired':[],
|
||||
'geom_rmse_model':[],
|
||||
'geom_rmse_verify':[],
|
||||
'gcp_model':[],
|
||||
'gcp_verify':[],
|
||||
'quality':[],
|
||||
'sun_azimuth':[],
|
||||
'sun_elevation':[]}
|
||||
t = []
|
||||
# loop through all images
|
||||
for i in range(n_img):
|
||||
|
||||
# find each image in ee database
|
||||
im = ee.Image(im_all[i].get('id'))
|
||||
im_bands = im_all[i].get('bands')
|
||||
im_props = im_all[i]['properties']
|
||||
|
||||
# compute cloud cover on cropped image
|
||||
for j in range(len(im_bands)): del im_bands[j]['dimensions']
|
||||
qa_band = [im_bands[11]]
|
||||
im_qa, crs_qa = sds.load_image(im, rect_narra, qa_band)
|
||||
im_qa = im_qa[:,:,0]
|
||||
im_cloud = sds.create_cloud_mask(im_qa)
|
||||
props['cloud_cover_cropped'].append(100*sum(sum(im_cloud.astype(int)))/(im_cloud.shape[0]*im_cloud.shape[1]))
|
||||
|
||||
# extract image metadata
|
||||
props['cloud_cover'].append(im_props['CLOUD_COVER'])
|
||||
props['cloud_cover_land' ].append(im_props['CLOUD_COVER_LAND'])
|
||||
props['date_acquired'].append(im_props['DATE_ACQUIRED'])
|
||||
props['geom_rmse_model'].append(im_props['GEOMETRIC_RMSE_MODEL'])
|
||||
props['gcp_model'].append(im_props['GROUND_CONTROL_POINTS_MODEL'])
|
||||
props['quality'].append(im_props['IMAGE_QUALITY_OLI'])
|
||||
props['sun_azimuth'].append(im_props['SUN_AZIMUTH'])
|
||||
props['sun_elevation'].append(im_props['SUN_ELEVATION'])
|
||||
|
||||
# try structure as sometimes the geometry cannot be verified
|
||||
try:
|
||||
props['geom_rmse_verify'].append(im_props['GEOMETRIC_RMSE_VERIFY'])
|
||||
props['gcp_verify'].append(im_props['GROUND_CONTROL_POINTS_VERIFY'])
|
||||
except:
|
||||
props['geom_rmse_verify'].append(np.nan)
|
||||
props['gcp_verify'].append(np.nan)
|
||||
|
||||
# record exact time of acquisition
|
||||
t.append(im_props['system:time_start'])
|
||||
|
||||
#%% create pd.DataFrame with datetime index
|
||||
dt = [];
|
||||
fmt = '%Y-%m-%d %H:%M:%S %Z%z'
|
||||
au_tz = pytz.timezone('Australia/Sydney')
|
||||
for k in range(len(t)): dt.append(datetime.fromtimestamp(t[k]/1000, tz=au_tz))
|
||||
|
||||
df = pd.DataFrame(data = props, index=dt , columns=list(props.keys()))
|
||||
|
||||
df.to_pickle('meta_l8.pkl')
|
||||
|
||||
#df['cloud_cover_cropped'].groupby(df.index.month).count().plot.bar()
|
||||
|
||||
#df_monthly = df['cloud_cover_cropped'].groupby(df.index.month)
|
||||
@ -1,232 +0,0 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
Created on Thu Mar 1 14:32:08 2018
|
||||
|
||||
@author: z5030440
|
||||
|
||||
Main code to extract shorelines from Landsat imagery
|
||||
"""
|
||||
# Preamble
|
||||
|
||||
import ee
|
||||
import matplotlib.pyplot as plt
|
||||
import matplotlib.cm as cm
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
from datetime import datetime
|
||||
import pickle
|
||||
import pdb
|
||||
import pytz
|
||||
|
||||
|
||||
# image processing modules
|
||||
import skimage.filters as filters
|
||||
import skimage.exposure as exposure
|
||||
import skimage.transform as transform
|
||||
import sklearn.decomposition as decomposition
|
||||
import skimage.morphology as morphology
|
||||
import skimage.measure as measure
|
||||
|
||||
# my functions
|
||||
import functions.utils as utils
|
||||
import functions.sds as sds
|
||||
|
||||
np.seterr(all='ignore') # raise/ignore divisions by 0 and nans
|
||||
ee.Initialize()
|
||||
|
||||
#%% Select images
|
||||
|
||||
# parameters
|
||||
plot_bool = False # if you want the plots
|
||||
prob_high = 99.9 # upper probability to clip and rescale pixel intensity
|
||||
min_contour_points = 100 # minimum number of points contained in each water line
|
||||
output_epsg = 28356 # GDA94 / MGA Zone 56
|
||||
cloud_threshold = 0.8
|
||||
|
||||
# select collection
|
||||
input_col = ee.ImageCollection('LANDSAT/LC08/C01/T1_RT_TOA')
|
||||
|
||||
# location (Narrabeen-Collaroy beach)
|
||||
rect_narra = [[[151.3473129272461,-33.69035274454718],
|
||||
[151.2820816040039,-33.68206818063878],
|
||||
[151.27281188964844,-33.74775138989556],
|
||||
[151.3425064086914,-33.75231878701767],
|
||||
[151.3473129272461,-33.69035274454718]]];
|
||||
|
||||
with open('data/narra_beach.pkl', 'rb') as f:
|
||||
pts_beach = pickle.load(f)
|
||||
|
||||
#rect_narra = [[[151.301454, -33.700754],
|
||||
# [151.311453, -33.702075],
|
||||
# [151.307237, -33.739761],
|
||||
# [151.294220, -33.736329],
|
||||
# [151.301454, -33.700754]]];
|
||||
|
||||
# Dates
|
||||
start_date = '2016-01-01'
|
||||
end_date = '2016-12-31'
|
||||
# filter by location
|
||||
flt_col = input_col.filterBounds(ee.Geometry.Polygon(rect_narra)).filterDate(start_date, end_date)
|
||||
|
||||
n_img = flt_col.size().getInfo()
|
||||
print('Number of images covering Narrabeen:', n_img)
|
||||
im_all = flt_col.getInfo().get('features')
|
||||
|
||||
#%% Extract shorelines
|
||||
metadata = {'timestamp':[],
|
||||
'date_acquired':[],
|
||||
'cloud_cover':[],
|
||||
'geom_rmse_model':[],
|
||||
'gcp_model':[],
|
||||
'quality':[],
|
||||
'sun_azimuth':[],
|
||||
'sun_elevation':[]}
|
||||
skipped_images = np.zeros((n_img,1)).astype(bool)
|
||||
output_wl = []
|
||||
# loop through all images
|
||||
for i in range(n_img):
|
||||
|
||||
# find each image in ee database
|
||||
im = ee.Image(im_all[i].get('id'))
|
||||
# load image as np.array
|
||||
im_pan, im_ms, im_cloud, crs, meta = sds.read_eeimage(im, rect_narra, plot_bool)
|
||||
|
||||
# if clouds -> skip the image
|
||||
if sum(sum(im_cloud.astype(int)))/(im_cloud.shape[0]*im_cloud.shape[1]) > cloud_threshold:
|
||||
skipped_images[i] = True
|
||||
continue
|
||||
|
||||
# rescale intensities
|
||||
im_ms = sds.rescale_image_intensity(im_ms, im_cloud, prob_high, plot_bool)
|
||||
im_pan = sds.rescale_image_intensity(im_pan, im_cloud, prob_high, plot_bool)
|
||||
|
||||
# pansharpen rgb image
|
||||
im_ms_ps = sds.pansharpen(im_ms[:,:,[0,1,2]], im_pan, im_cloud, plot_bool)
|
||||
|
||||
# add down-sized bands for NIR and SWIR (since pansharpening is not possible)
|
||||
im_ms_ps = np.append(im_ms_ps, im_ms[:,:,[3,4]], axis=2)
|
||||
|
||||
# calculate NDWI
|
||||
im_ndwi = sds.nd_index(im_ms_ps[:,:,3], im_ms_ps[:,:,1], im_cloud, plot_bool)
|
||||
|
||||
# edge detection
|
||||
wl_pix = sds.find_wl_contours(im_ndwi, im_cloud, min_contour_points, plot_bool)
|
||||
|
||||
plt.figure()
|
||||
plt.imshow(im_ms_ps[:,:,[2,1,0]])
|
||||
for i,contour in enumerate(wl_pix): plt.plot(contour[:, 1], contour[:, 0], linewidth=2)
|
||||
plt.axis('image')
|
||||
plt.title('Detected water lines')
|
||||
plt.show()
|
||||
|
||||
# convert from pixels to world coordinates
|
||||
wl_coords = sds.convert_pix2world(wl_pix, crs['crs_15m'])
|
||||
# convert to output epsg spatial reference
|
||||
wl = sds.convert_epsg(wl_coords, crs['epsg_code'], output_epsg)
|
||||
|
||||
# find contour closest to narrabeen beach
|
||||
sum_dist = np.zeros(len(wl))
|
||||
for k,contour in enumerate(wl):
|
||||
min_dist = np.zeros(len(pts_beach))
|
||||
for j,pt in enumerate(pts_beach):
|
||||
min_dist[j] = np.min(np.linalg.norm(contour - pt, axis=1))
|
||||
sum_dist[k] = np.sum(min_dist)/len(min_dist)
|
||||
try:
|
||||
wl_beach = wl[np.argmin(sum_dist)]
|
||||
# plt.figure()
|
||||
# plt.axis('equal')
|
||||
# plt.plot(pts_beach[:,0], pts_beach[:,1], 'ko')
|
||||
# plt.plot(wl_beach[:,0], wl_beach[:,1], 'r')
|
||||
# plt.show()
|
||||
except:
|
||||
wl_beach = []
|
||||
|
||||
# plot for QA
|
||||
plt.figure()
|
||||
plt.imshow(im_ms_ps[:,:,[2,1,0]])
|
||||
for k,contour in enumerate(wl_pix): plt.plot(contour[:, 1], contour[:, 0], linewidth=2)
|
||||
if len(wl_beach) > 0:
|
||||
plt.plot(wl_pix[np.argmin(sum_dist)][:,1], wl_pix[np.argmin(sum_dist)][:,0], linewidth=3, color='w')
|
||||
plt.axis('image')
|
||||
plt.title('im ' + str(i) + ' : ' + datetime.strftime(datetime
|
||||
.fromtimestamp(meta['timestamp']/1000, tz=pytz.utc)
|
||||
.astimezone(pytz.timezone('Australia/Sydney')), '%Y-%m-%d %H:%M:%S %Z%z'))
|
||||
plt.show()
|
||||
|
||||
# store metadata of each image in dict
|
||||
metadata['timestamp'].append(meta['timestamp'])
|
||||
metadata['date_acquired'].append(meta['date_acquired'])
|
||||
metadata['cloud_cover'].append(sum(sum(im_cloud.astype(int)))/(im_cloud.shape[0]*im_cloud.shape[1]))
|
||||
metadata['geom_rmse_model'].append(meta['geom_rmse_model'])
|
||||
metadata['gcp_model'].append(meta['gcp_model'])
|
||||
metadata['quality'].append(meta['quality'])
|
||||
metadata['sun_azimuth'].append(meta['sun_azimuth'])
|
||||
metadata['sun_elevation'].append(meta['sun_elevation'])
|
||||
# store water lines
|
||||
output_wl.append(wl_beach)
|
||||
|
||||
print(i)
|
||||
|
||||
# generate datetimes
|
||||
#fmt = '%Y-%m-%d %H:%M:%S %Z%z'
|
||||
#au_tz = pytz.timezone('Australia/Sydney')
|
||||
dt = [];
|
||||
t = metadata['timestamp']
|
||||
for k in range(len(t)): dt.append(datetime.fromtimestamp(t[k]/1000, tz=pytz.utc))
|
||||
|
||||
# save outputs
|
||||
data = metadata.copy()
|
||||
data.update({'dt':dt})
|
||||
data.update({'contours':output_wl})
|
||||
|
||||
#with open('data_2016.pkl', 'wb') as f:
|
||||
# pickle.dump(data, f)
|
||||
#%% Load data
|
||||
|
||||
#with open('data_2016.pkl', 'rb') as f:
|
||||
# data = pickle.load(f)
|
||||
|
||||
|
||||
# load backgroud image
|
||||
i = 0
|
||||
im = ee.Image(im_all[i].get('id'))
|
||||
im_pan, im_ms, im_cloud, crs, meta = sds.read_eeimage(im, rect_narra, plot_bool)
|
||||
im_ms = sds.rescale_image_intensity(im_ms, im_cloud, prob_high, plot_bool)
|
||||
im_pan = sds.rescale_image_intensity(im_pan, im_cloud, prob_high, plot_bool)
|
||||
im_ms_ps = sds.pansharpen(im_ms[:,:,[0,1,2]], im_pan, im_cloud, plot_bool)
|
||||
|
||||
plt.figure()
|
||||
plt.imshow(im_ms_ps[:,:,[2,1,0]])
|
||||
plt.axis('image')
|
||||
plt.title('2016 shorelines')
|
||||
|
||||
n = len(data['cloud_cover'])
|
||||
idx_best = []
|
||||
# remove overlapping images, based on cloud cover
|
||||
for i in range(n):
|
||||
date_im = data['date_acquired'][i]
|
||||
idx = np.isin(data['date_acquired'], date_im)
|
||||
best = np.where(idx)[0][np.argmin(np.array(data['cloud_cover'])[idx])]
|
||||
if ~np.isin(best, idx_best):
|
||||
idx_best.append(best)
|
||||
|
||||
point_narra = np.array([342500, 6266990])
|
||||
plt.figure()
|
||||
plt.axis('equal')
|
||||
plt.grid()
|
||||
cmap = cm.get_cmap('jet')
|
||||
colours = cmap(np.linspace(0, 1, num=len(idx_best)))
|
||||
for i, idx in enumerate(idx_best):
|
||||
for j in range(len(data['contours'][i])):
|
||||
if np.any(np.linalg.norm(data['contours'][i][j][:,[0,1]] - point_narra, axis=1) < 200):
|
||||
plt.plot(data['contours'][i][j][:,0], data['contours'][i][j][:,1],
|
||||
label=str(data['date_acquired'][i]),
|
||||
linewidth=2, color=colours[i,:])
|
||||
|
||||
plt.legend()
|
||||
plt.show()
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
@ -1,359 +0,0 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
Created on Wed Feb 21 18:05:01 2018
|
||||
|
||||
@author: z5030440
|
||||
"""
|
||||
|
||||
#%% Initial settings
|
||||
|
||||
# import packages
|
||||
import ee
|
||||
from IPython import display
|
||||
import math
|
||||
import matplotlib.pyplot as plt
|
||||
import numpy as np
|
||||
from osgeo import gdal
|
||||
import tempfile
|
||||
import tensorflow as tf
|
||||
import urllib
|
||||
from urllib.request import urlretrieve
|
||||
import zipfile
|
||||
import skimage.filters as filters
|
||||
import skimage.exposure as exposure
|
||||
import skimage.transform as transform
|
||||
import sklearn.decomposition as decomposition
|
||||
import skimage.morphology as morphology
|
||||
# import scripts
|
||||
from GEEImageFunctions import *
|
||||
|
||||
np.seterr(all='ignore') # raise divisions by 0 and nans
|
||||
ee.Initialize()
|
||||
|
||||
# Load image collection and filter it based on location (Narrabeen)
|
||||
|
||||
input_col = ee.ImageCollection('LANDSAT/LC08/C01/T1_RT_TOA')
|
||||
#n_img = input_col.size().getInfo()
|
||||
#print('Number of images in collection:', n_img)
|
||||
|
||||
# filter based on location (Narrabeen-Collaroy)
|
||||
rect_narra = [[[151.3473129272461,-33.69035274454718],
|
||||
[151.2820816040039,-33.68206818063878],
|
||||
[151.27281188964844,-33.74775138989556],
|
||||
[151.3425064086914,-33.75231878701767],
|
||||
[151.3473129272461,-33.69035274454718]]];
|
||||
|
||||
flt_col = input_col.filterBounds(ee.Geometry.Polygon(rect_narra))
|
||||
n_img = flt_col.size().getInfo()
|
||||
print('Number of images covering Narrabeen:', n_img)
|
||||
|
||||
# Select the most recent image and download it
|
||||
|
||||
im = ee.Image(flt_col.sort('SENSING_TIME',False).first())
|
||||
im_dic = im.getInfo()
|
||||
image_prop = im_dic.get('properties')
|
||||
im_bands = im_dic.get('bands')
|
||||
for i in range(len(im_bands)): del im_bands[i]['dimensions'] # delete the dimensions key
|
||||
|
||||
# download the panchromatic band (B8)
|
||||
pan_band = [im_bands[7]]
|
||||
im_pan = load_image(im, rect_narra, pan_band)
|
||||
im_pan = im_pan[:,:,0]
|
||||
size_pan = im_pan.shape
|
||||
vec_pan = im_pan.reshape(size_pan[0] * size_pan[1])
|
||||
# download the QA band (BQA)
|
||||
qa_band = [im_bands[11]]
|
||||
im_qa = load_image(im, rect_narra, qa_band)
|
||||
im_qa = im_qa[:,:,0]
|
||||
|
||||
# convert QA bits
|
||||
cloud_values = [2800, 2804, 2808, 2812, 6896, 6900, 6904, 6908]
|
||||
cloud_shadow_values = [2976, 2980, 2984, 2988, 3008, 3012, 3016, 3020]
|
||||
|
||||
# Create cloud mask (resized to be applied to the Pan band)
|
||||
im_cloud = np.isin(im_qa, cloud_values)
|
||||
im_cloud_shadow = np.isin(im_qa, cloud_shadow_values)
|
||||
im_cloud_res = transform.resize(im_cloud,(im_pan.shape[0], im_pan.shape[1]), order=0, preserve_range=True).astype('bool_')
|
||||
vec_cloud = im_cloud.reshape(im_cloud.shape[0] * im_cloud.shape[1])
|
||||
vec_cloud_res = im_cloud_res.reshape(size_pan[0] * size_pan[1])
|
||||
|
||||
|
||||
# download the other bands (B2,B3,B4,B5,B6) = (blue,green,red,nir,swir1)
|
||||
ms_bands = [im_bands[1], im_bands[2], im_bands[3], im_bands[4], im_bands[5]]
|
||||
im_ms = load_image(im, rect_narra, ms_bands)
|
||||
size_ms = im_ms.shape
|
||||
vec_ms = im_ms.reshape(size_ms[0] * size_ms[1], size_ms[2])
|
||||
|
||||
# Plot the RGB image and cloud masks
|
||||
plt.figure()
|
||||
ax1 = plt.subplot(121)
|
||||
plt.imshow(im_ms[:,:,[2,1,0]])
|
||||
plt.title('RGB')
|
||||
ax2 = plt.subplot(122)
|
||||
plt.imshow(im_cloud, cmap='gray')
|
||||
plt.title('Cloud mask')
|
||||
#ax3 = plt.subplot(133, sharex=ax1, sharey=ax1)
|
||||
#plt.imshow(im_cloud_shadow)
|
||||
#plt.title('Cloud mask shadow')
|
||||
plt.show()
|
||||
|
||||
# Resize multispectral bands (30m) to the size of the pan band (15m) using bilinear interpolation
|
||||
im_ms_res = transform.resize(im_ms,(size_pan[0], size_pan[1]), order=1, preserve_range=True, mode='constant')
|
||||
vec_ms_res = im_ms_res.reshape(size_pan[0] * size_pan[1], size_ms[2])
|
||||
|
||||
# Adjust intensities (set cloud pixels to 0 intensity)
|
||||
cloud_value = np.nan
|
||||
|
||||
prc_low = 0 # lower percentile
|
||||
prob_high = 99.9 # upper percentile probability to clip
|
||||
|
||||
# Rescale intensities between 0 and 1
|
||||
vec_ms_adj = np.ones((len(vec_cloud_res),size_ms[2])) * np.nan
|
||||
for i in range(im_ms.shape[2]):
|
||||
prc_high = np.percentile(vec_ms_res[~vec_cloud_res,i], prob_high)
|
||||
vec_rescaled = exposure.rescale_intensity(vec_ms_res[~vec_cloud_res,i], in_range=(prc_low,prc_high))
|
||||
plt.figure()
|
||||
plt.hist(vec_rescaled, bins = 300)
|
||||
plt.show()
|
||||
vec_ms_adj[~vec_cloud_res,i] = vec_rescaled
|
||||
|
||||
|
||||
im_ms_adj = vec_ms_adj.reshape(size_pan[0], size_pan[1], size_ms[2])
|
||||
|
||||
# same for the pan band
|
||||
vec_pan_adj = np.ones(len(vec_cloud_res)) * np.nan
|
||||
prc_high = np.percentile(vec_pan[~vec_cloud_res],prob_high)
|
||||
vec_rescaled = exposure.rescale_intensity(vec_pan[~vec_cloud_res], in_range=(prc_low,prc_high))
|
||||
plt.figure()
|
||||
plt.hist(vec_rescaled, bins = 300)
|
||||
plt.show()
|
||||
vec_pan_adj[~vec_cloud_res] = vec_rescaled
|
||||
im_pan_adj = vec_pan_adj.reshape(size_pan[0], size_pan[1])
|
||||
|
||||
# Plot adjusted images
|
||||
plt.figure()
|
||||
plt.subplot(131)
|
||||
plt.imshow(im_pan_adj, cmap='gray')
|
||||
plt.title('PANCHROMATIC (15 m pixel)')
|
||||
|
||||
plt.subplot(132)
|
||||
plt.imshow(im_ms_adj[:,:,[2,1,0]])
|
||||
plt.title('RGB (30 m pixel)')
|
||||
plt.show()
|
||||
|
||||
plt.subplot(133)
|
||||
plt.imshow(im_ms_adj[:,:,[3,1,0]])
|
||||
plt.title('NIR-GB (30 m pixel)')
|
||||
plt.show()
|
||||
|
||||
|
||||
#%% Pansharpening (PCA)
|
||||
# Run PCA on selected bands
|
||||
|
||||
sel_bands = [0,1,2]
|
||||
temp = vec_ms_adj[:,sel_bands]
|
||||
vec_ms_adj_nocloud = temp[~vec_cloud_res,:]
|
||||
pca = decomposition.PCA()
|
||||
vec_pcs = pca.fit_transform(vec_ms_adj_nocloud)
|
||||
vec_pcs_all = np.ones((len(vec_cloud_res),len(sel_bands))) * np.nan
|
||||
vec_pcs_all[~vec_cloud_res,:] = vec_pcs
|
||||
im_pcs = vec_pcs_all.reshape(size_pan[0], size_pan[1], vec_pcs.shape[1])
|
||||
|
||||
plt.figure()
|
||||
plt.subplot(221)
|
||||
plt.imshow(im_pcs[:,:,0], cmap='gray')
|
||||
plt.title('Component 1')
|
||||
plt.subplot(222)
|
||||
plt.imshow(im_pcs[:,:,1], cmap='gray')
|
||||
plt.title('Component 2')
|
||||
plt.subplot(223)
|
||||
plt.imshow(im_pcs[:,:,2], cmap='gray')
|
||||
plt.title('Component 3')
|
||||
plt.show()
|
||||
|
||||
# Compare the Pan image with the 1st Principal component
|
||||
compare_images(im_pan_adj,im_pcs[:,:,0])
|
||||
intensity_histogram(im_pan_adj)
|
||||
intensity_histogram(im_pcs[:,:,0])
|
||||
|
||||
# Match histogram of the pan image with the 1st principal component and replace the 1st component
|
||||
vec_pcs[:,0] = hist_match(vec_pan_adj[~vec_cloud_res], vec_pcs[:,0])
|
||||
vec_ms_ps = pca.inverse_transform(vec_pcs)
|
||||
|
||||
# normalise between 0 and 1
|
||||
for i in range(vec_pcs.shape[1]):
|
||||
vec_ms_ps[:,i] = np.divide(vec_ms_ps[:,i] - np.min(vec_ms_ps[:,i]),
|
||||
np.max(vec_ms_ps[:,i]) - np.min(vec_ms_ps[:,i]))
|
||||
|
||||
vec_ms_ps_all = np.ones((len(vec_cloud_res),len(sel_bands))) * np.nan
|
||||
vec_ms_ps_all[~vec_cloud_res,:] = vec_ms_ps
|
||||
im_ms_ps = vec_ms_ps_all.reshape(size_pan[0], size_pan[1], len(sel_bands))
|
||||
vec_ms_ps_all = np.append(vec_ms_ps_all, vec_ms_adj[:,[3,4]], axis=1)
|
||||
im_ms_ps = np.append(im_ms_ps, im_ms_adj[:,:,[3,4]], axis=2)
|
||||
|
||||
# Plot adjusted images
|
||||
plt.figure()
|
||||
plt.subplot(121)
|
||||
plt.imshow(im_ms_adj[:,:,[2,1,0]])
|
||||
plt.title('Original RGB')
|
||||
plt.show()
|
||||
|
||||
plt.subplot(122)
|
||||
plt.imshow(im_ms_ps[:,:,[2,1,0]])
|
||||
plt.title('Pansharpened RGB')
|
||||
plt.show()
|
||||
|
||||
plt.figure()
|
||||
plt.subplot(121)
|
||||
plt.imshow(im_ms_adj[:,:,[3,1,0]])
|
||||
plt.title('Original NIR-GB')
|
||||
plt.show()
|
||||
|
||||
plt.subplot(122)
|
||||
plt.imshow(im_ms_ps[:,:,[3,1,0]])
|
||||
plt.title('Pansharpened NIR-GB')
|
||||
plt.show()
|
||||
#%% Compute Normalized Difference Water Index (NDWI)
|
||||
|
||||
# With NIR
|
||||
vec_ndwi_nir = np.ones(len(vec_cloud_res)) * np.nan
|
||||
temp = np.divide(vec_ms_ps_all[~vec_cloud_res,3] - vec_ms_ps_all[~vec_cloud_res,1],
|
||||
vec_ms_ps_all[~vec_cloud_res,3] + vec_ms_ps_all[~vec_cloud_res,1])
|
||||
vec_ndwi_nir[~vec_cloud_res] = temp
|
||||
im_ndwi_nir = vec_ndwi_nir.reshape(size_pan[0], size_pan[1])
|
||||
|
||||
# With SWIR_1
|
||||
vec_ndwi_swir = np.ones(len(vec_cloud_res)) * np.nan
|
||||
temp = np.divide(vec_ms_ps_all[~vec_cloud_res,4] - vec_ms_ps_all[~vec_cloud_res,1],
|
||||
vec_ms_ps_all[~vec_cloud_res,4] + vec_ms_ps_all[~vec_cloud_res,1])
|
||||
vec_ndwi_swir[~vec_cloud_res] = temp
|
||||
im_ndwi_swir = vec_ndwi_swir.reshape(size_pan[0], size_pan[1])
|
||||
|
||||
plt.figure()
|
||||
ax1 = plt.subplot(211)
|
||||
plt.hist(vec_ndwi_nir[~vec_cloud_res], bins=300, label='NIR')
|
||||
plt.hist(vec_ndwi_swir[~vec_cloud_res], bins=300, label='SWIR', alpha=0.5)
|
||||
plt.legend()
|
||||
ax2 = plt.subplot(212, sharex=ax1)
|
||||
plt.hist(vec_ndwi_nir[~vec_cloud_res], bins=300, cumulative=True, histtype='step', label='NIR')
|
||||
plt.hist(vec_ndwi_swir[~vec_cloud_res], bins=300, cumulative=True, histtype='step', label='SWIR')
|
||||
plt.legend()
|
||||
plt.show()
|
||||
|
||||
compare_images(im_ndwi_nir,im_ndwi_swir)
|
||||
|
||||
plt.figure()
|
||||
plt.imshow(im_ndwi_nir, cmap='seismic')
|
||||
plt.title('Water Index')
|
||||
plt.colorbar()
|
||||
plt.show()
|
||||
|
||||
#%% Extract shorelines (NIR)
|
||||
|
||||
ndwi_nir = vec_ndwi_nir[~vec_cloud_res]
|
||||
|
||||
t_otsu = filters.threshold_otsu(ndwi_nir)
|
||||
t_min = filters.threshold_minimum(ndwi_nir)
|
||||
t_mean = filters.threshold_mean(ndwi_nir)
|
||||
t_li = filters.threshold_li(ndwi_nir)
|
||||
# try all thresholding algorithms
|
||||
|
||||
plt.figure()
|
||||
plt.hist(ndwi_nir, bins=300)
|
||||
plt.plot([t_otsu, t_otsu],[0, 15000], 'r-', label='Otsu threshold')
|
||||
#plt.plot([t_min, t_min],[0, 15000], 'g-', label='min')
|
||||
#plt.plot([t_mean, t_mean],[0, 15000], 'y-', label='mean')
|
||||
#plt.plot([t_li, t_li],[0, 15000], 'm-', label='li')
|
||||
plt.legend()
|
||||
plt.show()
|
||||
|
||||
plt.figure()
|
||||
plt.imshow(im_ndwi_nir > t_otsu, cmap='gray')
|
||||
plt.title('Binary image')
|
||||
plt.show()
|
||||
|
||||
im_bin = im_ndwi_nir > t_otsu
|
||||
im_open = morphology.binary_opening(im_bin,morphology.disk(1))
|
||||
im_close = morphology.binary_closing(im_open,morphology.disk(1))
|
||||
im_bin_coast_in = im_close ^ morphology.erosion(im_close,morphology.disk(1))
|
||||
im_bin_sl_in = morphology.remove_small_objects(im_bin_coast_in,100,8)
|
||||
|
||||
compare_images(im_close,im_bin_sl_in)
|
||||
|
||||
plt.figure()
|
||||
plt.subplot(121)
|
||||
plt.imshow(im_close, cmap='gray')
|
||||
plt.title('morphological closing')
|
||||
plt.subplot(122)
|
||||
plt.imshow(im_bin_sl_in, cmap='gray')
|
||||
plt.title('Water mark')
|
||||
plt.show()
|
||||
|
||||
|
||||
im_bin_coast_out = morphology.dilation(im_close,morphology.disk(1)) ^ im_close
|
||||
im_bin_sl_out = morphology.remove_small_objects(im_bin_coast_out,100,8)
|
||||
|
||||
|
||||
# Plot shorelines on top of RGB image
|
||||
im_rgb_sl = np.copy(im_ms_ps[:,:,[2,1,0]])
|
||||
|
||||
im_rgb_sl[im_bin_sl_in,0] = 0
|
||||
im_rgb_sl[im_bin_sl_in,1] = 1
|
||||
im_rgb_sl[im_bin_sl_in,2] = 1
|
||||
|
||||
im_rgb_sl[im_bin_sl_out,0] = 1
|
||||
im_rgb_sl[im_bin_sl_out,1] = 0
|
||||
im_rgb_sl[im_bin_sl_out,2] = 1
|
||||
|
||||
plt.figure()
|
||||
plt.imshow(im_rgb_sl)
|
||||
plt.title('Pansharpened RGB')
|
||||
plt.show()
|
||||
|
||||
#%% Extract shorelines SWIR
|
||||
|
||||
ndwi_swir = vec_ndwi_swir[~vec_cloud_res]
|
||||
t_otsu = filters.threshold_otsu(ndwi_swir)
|
||||
|
||||
plt.figure()
|
||||
plt.hist(ndwi_swir, bins=300)
|
||||
plt.plot([t_otsu, t_otsu],[0, 15000], 'r-', label='Otsu threshold')
|
||||
#plt.plot([t_min, t_min],[0, 15000], 'g-', label='min')
|
||||
#plt.plot([t_mean, t_mean],[0, 15000], 'y-', label='mean')
|
||||
#plt.plot([t_li, t_li],[0, 15000], 'm-', label='li')
|
||||
plt.legend()
|
||||
plt.show()
|
||||
|
||||
plt.figure()
|
||||
plt.imshow(im_ndwi_swir > t_otsu, cmap='gray')
|
||||
plt.title('Binary image')
|
||||
plt.show()
|
||||
|
||||
im_bin = im_ndwi_swir > t_otsu
|
||||
im_open = morphology.binary_opening(im_bin,morphology.disk(1))
|
||||
im_close = morphology.binary_closing(im_open,morphology.disk(1))
|
||||
im_bin_coast_in = im_close ^ morphology.erosion(im_close,morphology.disk(1))
|
||||
im_bin_sl_in = morphology.remove_small_objects(im_bin_coast_in,100,8)
|
||||
|
||||
compare_images(im_close,im_bin_sl_in)
|
||||
|
||||
|
||||
im_bin_coast_out = morphology.dilation(im_close,morphology.disk(1)) ^ im_close
|
||||
im_bin_sl_out = morphology.remove_small_objects(im_bin_coast_out,100,8)
|
||||
|
||||
|
||||
# Plot shorelines on top of RGB image
|
||||
im_rgb_sl = np.copy(im_ms_ps[:,:,[2,1,0]])
|
||||
|
||||
im_rgb_sl[im_bin_sl_in,0] = 0
|
||||
im_rgb_sl[im_bin_sl_in,1] = 1
|
||||
im_rgb_sl[im_bin_sl_in,2] = 1
|
||||
|
||||
im_rgb_sl[im_bin_sl_out,0] = 1
|
||||
im_rgb_sl[im_bin_sl_out,1] = 0
|
||||
im_rgb_sl[im_bin_sl_out,2] = 1
|
||||
|
||||
plt.figure()
|
||||
plt.imshow(im_rgb_sl)
|
||||
plt.title('Pansharpened RGB')
|
||||
plt.show()
|
||||
@ -1,260 +0,0 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
Created on Thu Mar 1 14:32:08 2018
|
||||
|
||||
@author: z5030440
|
||||
|
||||
Main code to extract shorelines from Landsat imagery
|
||||
"""
|
||||
# Preamble
|
||||
|
||||
import ee
|
||||
import matplotlib.pyplot as plt
|
||||
import matplotlib.cm as cm
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
from datetime import datetime
|
||||
import pickle
|
||||
import pdb
|
||||
import pytz
|
||||
from pylab import ginput
|
||||
|
||||
|
||||
# image processing modules
|
||||
import skimage.filters as filters
|
||||
import skimage.exposure as exposure
|
||||
import skimage.transform as transform
|
||||
import sklearn.decomposition as decomposition
|
||||
import skimage.morphology as morphology
|
||||
import skimage.measure as measure
|
||||
|
||||
# my functions
|
||||
import functions.utils as utils
|
||||
import functions.sds as sds
|
||||
|
||||
np.seterr(all='ignore') # raise/ignore divisions by 0 and nans
|
||||
ee.Initialize()
|
||||
|
||||
#%% Select images
|
||||
|
||||
# parameters
|
||||
plot_bool = False # if you want the plots
|
||||
prob_high = 99.9 # upper probability to clip and rescale pixel intensity
|
||||
min_contour_points = 100 # minimum number of points contained in each water line
|
||||
output_epsg = 28356 # GDA94 / MGA Zone 56
|
||||
cloud_threshold = 0.7
|
||||
|
||||
# select collection
|
||||
input_col = ee.ImageCollection('LANDSAT/LC08/C01/T1_RT_TOA')
|
||||
|
||||
# location (Narrabeen-Collaroy beach)
|
||||
rect_narra = [[[151.3473129272461,-33.69035274454718],
|
||||
[151.2820816040039,-33.68206818063878],
|
||||
[151.27281188964844,-33.74775138989556],
|
||||
[151.3425064086914,-33.75231878701767],
|
||||
[151.3473129272461,-33.69035274454718]]];
|
||||
|
||||
with open('data/narra_beach.pkl', 'rb') as f:
|
||||
pts_beach = pickle.load(f)
|
||||
|
||||
with open('data/idx_nogt.pkl', 'rb') as f:
|
||||
idx_nogt = pickle.load(f)
|
||||
idx_nogt = np.array(idx_nogt)
|
||||
|
||||
#rect_narra = [[[151.301454, -33.700754],
|
||||
# [151.311453, -33.702075],
|
||||
# [151.307237, -33.739761],
|
||||
# [151.294220, -33.736329],
|
||||
# [151.301454, -33.700754]]];
|
||||
|
||||
# Dates
|
||||
start_date = '2016-01-01'
|
||||
end_date = '2016-12-31'
|
||||
# filter by location
|
||||
flt_col = input_col.filterBounds(ee.Geometry.Polygon(rect_narra))#.filterDate(start_date, end_date)
|
||||
|
||||
n_img = flt_col.size().getInfo()
|
||||
print('Number of images covering Narrabeen:', n_img)
|
||||
im_all = flt_col.getInfo().get('features')
|
||||
|
||||
#%% Extract shorelines
|
||||
metadata = {'timestamp':[],
|
||||
'date_acquired':[],
|
||||
'cloud_cover':[],
|
||||
'geom_rmse_model':[],
|
||||
'gcp_model':[],
|
||||
'quality':[],
|
||||
'sun_azimuth':[],
|
||||
'sun_elevation':[]}
|
||||
skipped_images = np.zeros((n_img,1)).astype(bool)
|
||||
output_wl = []
|
||||
# loop through all images
|
||||
for i in range(n_img):
|
||||
|
||||
if np.isin(i, idx_nogt):
|
||||
continue
|
||||
|
||||
# find each image in ee database
|
||||
im = ee.Image(im_all[i].get('id'))
|
||||
# load image as np.array
|
||||
im_pan, im_ms, im_cloud, crs, meta = sds.read_eeimage(im, rect_narra, plot_bool)
|
||||
|
||||
# if clouds -> skip the image
|
||||
if sum(sum(im_cloud.astype(int)))/(im_cloud.shape[0]*im_cloud.shape[1]) > cloud_threshold:
|
||||
skipped_images[i] = True
|
||||
continue
|
||||
|
||||
# rescale intensities
|
||||
im_ms = sds.rescale_image_intensity(im_ms, im_cloud, prob_high, plot_bool)
|
||||
im_pan = sds.rescale_image_intensity(im_pan, im_cloud, prob_high, plot_bool)
|
||||
|
||||
# pansharpen rgb image
|
||||
im_ms_ps = sds.pansharpen(im_ms[:,:,[0,1,2]], im_pan, im_cloud, plot_bool)
|
||||
|
||||
# add down-sized bands for NIR and SWIR (since pansharpening is not possible)
|
||||
im_ms_ps = np.append(im_ms_ps, im_ms[:,:,[3,4]], axis=2)
|
||||
|
||||
# calculate NDWI
|
||||
im_ndwi = sds.nd_index(im_ms_ps[:,:,3], im_ms_ps[:,:,1], im_cloud, plot_bool)
|
||||
|
||||
# edge detection
|
||||
wl_pix = sds.find_wl_contours(im_ndwi, im_cloud, min_contour_points, plot_bool)
|
||||
# convert from pixels to world coordinates
|
||||
wl_coords = sds.convert_pix2world(wl_pix, crs['crs_15m'])
|
||||
# convert to output epsg spatial reference
|
||||
wl = sds.convert_epsg(wl_coords, crs['epsg_code'], output_epsg)
|
||||
|
||||
# find contour closest to narrabeen beach
|
||||
sum_dist = np.zeros(len(wl))
|
||||
for k,contour in enumerate(wl):
|
||||
min_dist = np.zeros(len(pts_beach))
|
||||
for j,pt in enumerate(pts_beach):
|
||||
min_dist[j] = np.min(np.linalg.norm(contour - pt, axis=1))
|
||||
sum_dist[k] = np.sum(min_dist)/len(min_dist)
|
||||
try:
|
||||
wl_beach = wl[np.argmin(sum_dist)]
|
||||
# plt.figure()
|
||||
# plt.axis('equal')
|
||||
# plt.plot(pts_beach[:,0], pts_beach[:,1], 'ko')
|
||||
# plt.plot(wl_beach[:,0], wl_beach[:,1], 'r')
|
||||
# plt.show()
|
||||
except:
|
||||
wl_beach = []
|
||||
|
||||
plt.figure()
|
||||
plt.imshow(im_ms_ps[:,:,[2,1,0]])
|
||||
for k,contour in enumerate(wl_pix): plt.plot(contour[:, 1], contour[:, 0], linewidth=2)
|
||||
if len(wl_beach) > 0:
|
||||
plt.plot(wl_pix[np.argmin(sum_dist)][:,1], wl_pix[np.argmin(sum_dist)][:,0], linewidth=3, color='w')
|
||||
plt.axis('image')
|
||||
plt.title('im ' + str(i) + ' : ' + datetime.strftime(datetime
|
||||
.fromtimestamp(meta['timestamp']/1000, tz=pytz.utc)
|
||||
.astimezone(pytz.timezone('Australia/Sydney')), '%Y-%m-%d %H:%M:%S %Z%z'))
|
||||
plt.show()
|
||||
|
||||
# manually validate shoreline detection
|
||||
input_pt = np.array(ginput(1))
|
||||
if input_pt[0,1] > 300:
|
||||
skipped_images[i] = True
|
||||
continue
|
||||
|
||||
# store metadata of each image in dict
|
||||
metadata['timestamp'].append(meta['timestamp'])
|
||||
metadata['date_acquired'].append(meta['date_acquired'])
|
||||
metadata['cloud_cover'].append(sum(sum(im_cloud.astype(int)))/(im_cloud.shape[0]*im_cloud.shape[1]))
|
||||
metadata['geom_rmse_model'].append(meta['geom_rmse_model'])
|
||||
metadata['gcp_model'].append(meta['gcp_model'])
|
||||
metadata['quality'].append(meta['quality'])
|
||||
metadata['sun_azimuth'].append(meta['sun_azimuth'])
|
||||
metadata['sun_elevation'].append(meta['sun_elevation'])
|
||||
# store water lines
|
||||
output_wl.append(wl_beach)
|
||||
|
||||
print(i)
|
||||
|
||||
# generate datetimes
|
||||
#fmt = '%Y-%m-%d %H:%M:%S %Z%z'
|
||||
#au_tz = pytz.timezone('Australia/Sydney')
|
||||
dt = [];
|
||||
t = metadata['timestamp']
|
||||
for k in range(len(t)): dt.append(datetime.fromtimestamp(t[k]/1000, tz=pytz.utc))
|
||||
|
||||
# save outputs
|
||||
data = metadata.copy()
|
||||
data.update({'dt':dt})
|
||||
data.update({'contours':output_wl})
|
||||
|
||||
with open('data_gt15d_32_56.pkl', 'wb') as f:
|
||||
pickle.dump(data, f)
|
||||
#%% Load data
|
||||
|
||||
##with open('data_2016.pkl', 'rb') as f:
|
||||
## data = pickle.load(f)
|
||||
#
|
||||
#
|
||||
## load backgroud image
|
||||
#i = 0
|
||||
#im = ee.Image(im_all[i].get('id'))
|
||||
#im_pan, im_ms, im_cloud, crs, meta = sds.read_eeimage(im, rect_narra, plot_bool)
|
||||
#im_ms = sds.rescale_image_intensity(im_ms, im_cloud, prob_high, plot_bool)
|
||||
#im_pan = sds.rescale_image_intensity(im_pan, im_cloud, prob_high, plot_bool)
|
||||
#im_ms_ps = sds.pansharpen(im_ms[:,:,[0,1,2]], im_pan, im_cloud, plot_bool)
|
||||
#
|
||||
#plt.figure()
|
||||
#plt.imshow(im_ms_ps[:,:,[2,1,0]])
|
||||
#plt.axis('image')
|
||||
#plt.title('2016 shorelines')
|
||||
#
|
||||
#n = len(data['cloud_cover'])
|
||||
#idx_best = []
|
||||
## remove overlapping images, based on cloud cover
|
||||
#for i in range(n):
|
||||
# date_im = data['date_acquired'][i]
|
||||
# idx = np.isin(data['date_acquired'], date_im)
|
||||
# best = np.where(idx)[0][np.argmin(np.array(data['cloud_cover'])[idx])]
|
||||
# if ~np.isin(best, idx_best):
|
||||
# idx_best.append(best)
|
||||
#
|
||||
#point_narra = np.array([342500, 6266990])
|
||||
#plt.figure()
|
||||
#plt.axis('equal')
|
||||
#plt.grid()
|
||||
#cmap = cm.get_cmap('jet')
|
||||
#colours = cmap(np.linspace(0, 1, num=len(idx_best)))
|
||||
#for i, idx in enumerate(idx_best):
|
||||
# for j in range(len(data['contours'][i])):
|
||||
# if np.any(np.linalg.norm(data['contours'][i][j][:,[0,1]] - point_narra, axis=1) < 200):
|
||||
# plt.plot(data['contours'][i][j][:,0], data['contours'][i][j][:,1],
|
||||
# label=str(data['date_acquired'][i]),
|
||||
# linewidth=2, color=colours[i,:])
|
||||
#
|
||||
#plt.legend()
|
||||
#plt.show()
|
||||
#
|
||||
#pts_narra = sds.convert_epsg(pts_narra, output_epsg, 4326)
|
||||
#
|
||||
##kml.newlinestring(name="beach",
|
||||
## coords = [(_[0], _[1]) for _ in pts_narra])
|
||||
##kml.save("narra.kml")
|
||||
|
||||
|
||||
#%%
|
||||
|
||||
#with open('data_gt15d_0_31.pkl', 'rb') as f:
|
||||
# data1 = pickle.load(f)
|
||||
#with open('data_gt15d_32_56.pkl', 'rb') as f:
|
||||
# data2 = pickle.load(f)
|
||||
#with open('data_gt15d_99_193.pkl', 'rb') as f:
|
||||
# data3 = pickle.load(f)
|
||||
#
|
||||
#data = []
|
||||
#data = data1.copy()
|
||||
#for k,cat in enumerate(data.keys()):
|
||||
# for j in range(len(data2[cat])):
|
||||
# data[cat].append(data2[cat][j])
|
||||
# for j in range(len(data3[cat])):
|
||||
# data[cat].append(data3[cat][j])
|
||||
#
|
||||
#
|
||||
#with open('data_gt_l8.pkl', 'wb') as f:
|
||||
# pickle.dump(data, f)
|
||||
@ -1,136 +0,0 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
Created on Tue Mar 20 16:15:51 2018
|
||||
|
||||
@author: z5030440
|
||||
"""
|
||||
|
||||
import scipy.io as sio
|
||||
import os
|
||||
import ee
|
||||
import matplotlib.pyplot as plt
|
||||
import matplotlib.dates as mdates
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
from datetime import datetime, timedelta
|
||||
import pickle
|
||||
import pdb
|
||||
import pytz
|
||||
|
||||
|
||||
# image processing modules
|
||||
import skimage.filters as filters
|
||||
import skimage.exposure as exposure
|
||||
import skimage.transform as transform
|
||||
import skimage.morphology as morphology
|
||||
import skimage.measure as measure
|
||||
import sklearn.decomposition as decomposition
|
||||
from scipy import spatial
|
||||
# my functions
|
||||
import functions.utils as utils
|
||||
import functions.sds as sds
|
||||
#plt.rcParams['axes.grid'] = True
|
||||
au_tz = pytz.timezone('Australia/Sydney')
|
||||
|
||||
# load quadbike dates and convert from datenum to datetime
|
||||
suffix = '.mat'
|
||||
dir_name = os.getcwd()
|
||||
file_name = 'data\quadbike_dates'
|
||||
file_path = os.path.join(dir_name, file_name + suffix)
|
||||
quad_dates = sio.loadmat(file_path)['dates']
|
||||
dt_quad = []
|
||||
for i in range(quad_dates.shape[0]):
|
||||
dt_quad.append(datetime(quad_dates[i,0], quad_dates[i,1], quad_dates[i,2], tzinfo=au_tz))
|
||||
|
||||
# load satellite datetimes (in UTC) and convert to AEST time
|
||||
input_col = ee.ImageCollection('LANDSAT/LC08/C01/T1_RT_TOA')
|
||||
# location (Narrabeen-Collaroy beach)
|
||||
rect_narra = [[[151.3473129272461,-33.69035274454718],
|
||||
[151.2820816040039,-33.68206818063878],
|
||||
[151.27281188964844,-33.74775138989556],
|
||||
[151.3425064086914,-33.75231878701767],
|
||||
[151.3473129272461,-33.69035274454718]]];
|
||||
flt_col = input_col.filterBounds(ee.Geometry.Polygon(rect_narra))
|
||||
n_img = flt_col.size().getInfo()
|
||||
print('Number of images covering Narrabeen:', n_img)
|
||||
im_all = flt_col.getInfo().get('features')
|
||||
|
||||
# extract datetimes from image metadata
|
||||
dt_sat = [_['properties']['system:time_start'] for _ in im_all]
|
||||
dt_sat = [datetime.fromtimestamp(_/1000, tz=pytz.utc) for _ in dt_sat]
|
||||
dt_sat = [_.astimezone(au_tz) for _ in dt_sat]
|
||||
# calculate days difference
|
||||
diff_days = [ [(x - _).days for _ in dt_quad] for x in dt_sat]
|
||||
day_thresh = 15
|
||||
idx = [utils.find_indices(_, lambda e: abs(e) < day_thresh) for _ in diff_days]
|
||||
|
||||
dt_diff = []
|
||||
idx_nogt = []
|
||||
for i in range(n_img):
|
||||
if not idx[i]:
|
||||
idx_nogt.append(i)
|
||||
continue
|
||||
dt_diff.append({"sat dt": dt_sat[i],
|
||||
"quad dt": [dt_quad[_] for _ in idx[i]],
|
||||
"days diff": [diff_days[i][_] for _ in idx[i]] })
|
||||
|
||||
with open('idx_nogt.pkl', 'wb') as f:
|
||||
pickle.dump(idx_nogt, f)
|
||||
|
||||
|
||||
#%%
|
||||
dates_sat = mdates.date2num(dt_sat)
|
||||
dates_quad = mdates.date2num(dt_quad)
|
||||
plt.figure()
|
||||
plt.plot_date(dates_sat, np.zeros((n_img,1)))
|
||||
plt.plot_date(dates_quad, np.ones((len(dates_quad),1)))
|
||||
plt.show()
|
||||
|
||||
data = pd.read_pickle('data_2016.pkl')
|
||||
|
||||
dt_sat = [_.astimezone(au_tz) for _ in data['dt']]
|
||||
|
||||
[ (_ - dt_sat[0]).days for _ in dt_quad]
|
||||
|
||||
|
||||
|
||||
dn_sat = []
|
||||
for i in range(len(dt_sat)): dn_sat.append(dt_sat[i].toordinal())
|
||||
dn_sat = np.array(dn_sat)
|
||||
dn_sur = []
|
||||
for i in range(len(dt_survey)): dn_sur.append(dt_survey[i].toordinal())
|
||||
dn_sur = np.array(dn_sur)
|
||||
|
||||
distances = np.zeros((len(dn_sat),4)).astype('int32')
|
||||
indexes = np.zeros((len(dn_sat),2)).astype('int32')
|
||||
for i in range(len(dn_sat)):
|
||||
distances[i,0] = np.sort(abs(dn_sat[i] - dn_sur))[0]
|
||||
distances[i,1] = np.sort(abs(dn_sat[i] - dn_sur))[1]
|
||||
distances[i,2] = dt_sat[i].year
|
||||
distances[i,3] = dt_sat[i].month
|
||||
indexes[i,0] = np.where(abs(dn_sat[i] - dn_sur) == np.sort(abs(dn_sat[i] - dn_sur))[0])[0][0]
|
||||
indexes[i,1] = np.where(abs(dn_sat[i] - dn_sur) == np.sort(abs(dn_sat[i] - dn_sur))[1])[0][0]
|
||||
|
||||
|
||||
years = [2013, 2014, 2015, 2016]
|
||||
months = mdates.MonthLocator()
|
||||
days = mdates.DayLocator()
|
||||
month_fmt = mdates.DateFormatter('%b')
|
||||
f, ax = plt.subplots(4, 1)
|
||||
for i, ca in enumerate(ax):
|
||||
ca.xaxis.set_major_locator(months)
|
||||
ca.xaxis.set_major_formatter(month_fmt)
|
||||
ca.xaxis.set_minor_locator(days)
|
||||
ca.set_ylabel(str(years[i]))
|
||||
for j in range(len(dt_sat)):
|
||||
if dt_sat[j].year == years[i]:
|
||||
ca.plot(dt_sat[j],0, 'bo', markerfacecolor='b')
|
||||
#f.subplots_adjust(hspace=0)
|
||||
#plt.setp([a.get_xticklabels() for a in f.axes[:-1]], visible=False)
|
||||
|
||||
|
||||
plt.plot(dt_survey, np.zeros([len(dt_survey),1]), 'bo')
|
||||
plt.plot(dt_sat, np.ones([len(dt_sat),1]), 'ro')
|
||||
plt.yticks([])
|
||||
plt.show()
|
||||
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@ -1,88 +0,0 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
#==========================================================#
|
||||
# Process shorelines (clipping and smoothing)
|
||||
#==========================================================#
|
||||
|
||||
# Initial settings
|
||||
|
||||
import os
|
||||
import numpy as np
|
||||
import matplotlib.pyplot as plt
|
||||
import pdb
|
||||
import ee
|
||||
|
||||
import matplotlib.dates as mdates
|
||||
import matplotlib.cm as cm
|
||||
import matplotlib.colors as mcolor
|
||||
from datetime import datetime, timedelta
|
||||
import pickle
|
||||
import pytz
|
||||
import scipy.io as sio
|
||||
import scipy.interpolate as interpolate
|
||||
import statsmodels.api as sm
|
||||
import skimage.measure as measure
|
||||
import simplekml
|
||||
|
||||
# my functions
|
||||
import functions.utils as utils
|
||||
import functions.sds as sds
|
||||
|
||||
# some settings
|
||||
np.seterr(all='ignore') # raise/ignore divisions by 0 and nans
|
||||
plt.rcParams['axes.grid'] = True
|
||||
plt.rcParams['figure.max_open_warning'] = 100
|
||||
|
||||
au_tz = pytz.timezone('Australia/Sydney')
|
||||
au_epsg = 28356
|
||||
# load the satellite-derived shorelines
|
||||
satname = 'L8'
|
||||
sitename = 'OLDBAR'
|
||||
filepath = os.path.join(os.getcwd(), 'data', satname, sitename)
|
||||
with open(os.path.join(filepath, sitename + '_output' + '.pkl'), 'rb') as f:
|
||||
output = pickle.load(f)
|
||||
|
||||
sl = output['shorelines']
|
||||
dates_sl = output['t']
|
||||
# convert to AEST
|
||||
dates_sl = [_.astimezone(au_tz) for _ in dates_sl]
|
||||
|
||||
# load the reference shoreline points
|
||||
with open(os.path.join(os.getcwd(), 'data', satname, sitename, sitename + '_refpoints.pkl'), 'rb') as f:
|
||||
refpoints = pickle.load(f)
|
||||
|
||||
dist_thresh = 200
|
||||
frac_smooth = 1./15
|
||||
plt.figure()
|
||||
plt.axis('equal')
|
||||
cmap = cm.get_cmap('brg')
|
||||
colours = cmap(np.linspace(0, 1, num=len(sl)))
|
||||
kml = simplekml.Kml()
|
||||
for i in range(len(sl)):
|
||||
# select points of SDS that are close to the manually digitized points
|
||||
idx_ref = [np.min(np.linalg.norm(sl[i][k,:] - refpoints, axis=1)) < dist_thresh for k in range(sl[i].shape[0])]
|
||||
# smooth (LOWESS) satellite shoreline
|
||||
sl_smooth = sm.nonparametric.lowess(sl[i][idx_ref,0],sl[i][idx_ref,1], frac=frac_smooth, it = 10)
|
||||
sl_smooth = sl_smooth[:,[1,0]]
|
||||
# sl_smooth = sl[i][idx_ref,:]
|
||||
|
||||
# plt.plot(sl[i][idx_ref,0],sl[i][idx_ref,1], 'k-')
|
||||
plt.plot(sl_smooth[:,0], sl_smooth[:,1], color=colours[i,:], label=dates_sl[i].strftime('%d-%b-%Y'))
|
||||
|
||||
# convert to wgs84 (epsg = 4326)
|
||||
sl_wgs84 = sds.convert_epsg(sl_smooth, 28356, 4326)
|
||||
# save in kml file
|
||||
ln = kml.newlinestring(name=dates_sl[i].strftime('%d-%b-%Y'))
|
||||
ln.coords = sl_wgs84
|
||||
ln.style.labelstyle.color = mcolor.rgb2hex(colours[i,:3])
|
||||
ln.style.linestyle.color = mcolor.rgb2hex(colours[i,:3])
|
||||
|
||||
plt.legend(ncol=3)
|
||||
plt.xlabel('Eastings [m]')
|
||||
plt.ylabel('Northings [m]')
|
||||
plt.title('Oldbar inlet (South)')
|
||||
plt.draw()
|
||||
|
||||
kml.save(satname + sitename + '_shorelines.kml')
|
||||
|
||||
|
||||
@ -0,0 +1,177 @@
|
||||
# This file may be used to create an environment using:
|
||||
# $ conda create --name <env> --file <this file>
|
||||
# platform: win-64
|
||||
@EXPLICIT
|
||||
https://repo.continuum.io/pkgs/main/win-64/alabaster-0.7.10-py36hcd07829_0.tar.bz2
|
||||
https://repo.continuum.io/pkgs/main/win-64/asn1crypto-0.24.0-py36_0.tar.bz2
|
||||
https://repo.continuum.io/pkgs/main/win-64/astroid-1.6.1-py36_0.tar.bz2
|
||||
https://repo.continuum.io/pkgs/main/win-64/babel-2.5.3-py36_0.tar.bz2
|
||||
https://repo.continuum.io/pkgs/main/win-64/backports-1.0-py36h81696a8_1.tar.bz2
|
||||
https://repo.continuum.io/pkgs/free/win-64/backports.weakref-1.0rc1-py36_0.tar.bz2
|
||||
https://repo.continuum.io/pkgs/free/win-64/bleach-1.5.0-py36_0.tar.bz2
|
||||
https://repo.continuum.io/pkgs/main/win-64/bokeh-0.12.14-py36_0.tar.bz2
|
||||
https://repo.continuum.io/pkgs/main/win-64/ca-certificates-2017.08.26-h94faf87_0.tar.bz2
|
||||
https://repo.continuum.io/pkgs/main/win-64/certifi-2018.1.18-py36_0.tar.bz2
|
||||
https://repo.continuum.io/pkgs/main/win-64/cffi-1.11.4-py36hfa6e2cd_0.tar.bz2
|
||||
https://repo.continuum.io/pkgs/main/win-64/chardet-3.0.4-py36h420ce6e_1.tar.bz2
|
||||
https://repo.continuum.io/pkgs/main/win-64/click-6.7-py36hec8c647_0.tar.bz2
|
||||
https://repo.continuum.io/pkgs/main/win-64/cloudpickle-0.5.2-py36_1.tar.bz2
|
||||
https://repo.continuum.io/pkgs/main/win-64/colorama-0.3.9-py36h029ae33_0.tar.bz2
|
||||
https://repo.continuum.io/pkgs/main/win-64/cryptography-2.1.4-py36he1d7878_0.tar.bz2
|
||||
https://repo.continuum.io/pkgs/main/win-64/curl-7.58.0-h7602738_0.tar.bz2
|
||||
https://repo.continuum.io/pkgs/main/win-64/cycler-0.10.0-py36h009560c_0.tar.bz2
|
||||
https://repo.continuum.io/pkgs/main/win-64/dask-0.17.0-py36_0.tar.bz2
|
||||
https://repo.continuum.io/pkgs/main/win-64/dask-core-0.17.0-py36_0.tar.bz2
|
||||
https://repo.continuum.io/pkgs/main/win-64/decorator-4.2.1-py36_0.tar.bz2
|
||||
https://repo.continuum.io/pkgs/main/win-64/distributed-1.21.0-py36_0.tar.bz2
|
||||
https://repo.continuum.io/pkgs/main/win-64/docutils-0.14-py36h6012d8f_0.tar.bz2
|
||||
https://repo.continuum.io/pkgs/main/win-64/entrypoints-0.2.3-py36hfd66bb0_2.tar.bz2
|
||||
https://repo.continuum.io/pkgs/main/win-64/expat-2.2.5-hcc4222d_0.tar.bz2
|
||||
https://conda.anaconda.org/conda-forge/win-64/freetype-2.8.1-vc14_0.tar.bz2
|
||||
https://repo.continuum.io/pkgs/main/win-64/freexl-1.0.4-h342dbcb_5.tar.bz2
|
||||
https://repo.continuum.io/pkgs/main/win-64/geos-3.6.2-h9ef7328_2.tar.bz2
|
||||
https://repo.continuum.io/pkgs/main/win-64/hdf4-4.2.13-h712560f_2.tar.bz2
|
||||
https://repo.continuum.io/pkgs/main/win-64/hdf5-1.10.1-h98b8871_1.tar.bz2
|
||||
https://repo.continuum.io/pkgs/main/win-64/heapdict-1.0.0-py36_2.tar.bz2
|
||||
https://repo.continuum.io/pkgs/free/win-64/html5lib-0.9999999-py36_0.tar.bz2
|
||||
https://repo.continuum.io/pkgs/main/win-64/icc_rt-2017.0.4-h97af966_0.tar.bz2
|
||||
https://conda.anaconda.org/conda-forge/win-64/icu-58.2-vc14_0.tar.bz2
|
||||
https://repo.continuum.io/pkgs/main/win-64/idna-2.6-py36h148d497_1.tar.bz2
|
||||
https://repo.continuum.io/pkgs/main/win-64/imageio-2.3.0-py36_0.tar.bz2
|
||||
https://repo.continuum.io/pkgs/main/win-64/imagesize-0.7.1-py36he29f638_0.tar.bz2
|
||||
https://repo.continuum.io/pkgs/main/win-64/intel-openmp-2018.0.0-hd92c6cd_8.tar.bz2
|
||||
https://repo.continuum.io/pkgs/main/win-64/ipykernel-4.8.0-py36_0.tar.bz2
|
||||
https://repo.continuum.io/pkgs/main/win-64/ipython-6.2.1-py36h9cf0123_1.tar.bz2
|
||||
https://repo.continuum.io/pkgs/main/win-64/ipython_genutils-0.2.0-py36h3c5d0ee_0.tar.bz2
|
||||
https://conda.anaconda.org/conda-forge/win-64/ipywidgets-7.1.1-py36_0.tar.bz2
|
||||
https://repo.continuum.io/pkgs/main/win-64/isort-4.2.15-py36h6198cc5_0.tar.bz2
|
||||
https://repo.continuum.io/pkgs/main/win-64/jedi-0.11.1-py36_0.tar.bz2
|
||||
https://repo.continuum.io/pkgs/main/win-64/jinja2-2.10-py36h292fed1_0.tar.bz2
|
||||
https://conda.anaconda.org/conda-forge/win-64/jpeg-9b-vc14_2.tar.bz2
|
||||
https://repo.continuum.io/pkgs/main/win-64/jsonschema-2.6.0-py36h7636477_0.tar.bz2
|
||||
https://repo.continuum.io/pkgs/main/win-64/jupyter-1.0.0-py36_4.tar.bz2
|
||||
https://repo.continuum.io/pkgs/main/win-64/jupyter_client-5.2.2-py36_0.tar.bz2
|
||||
https://repo.continuum.io/pkgs/main/win-64/jupyter_console-5.2.0-py36h6d89b47_1.tar.bz2
|
||||
https://conda.anaconda.org/conda-forge/win-64/jupyter_contrib_core-0.3.3-py36_1.tar.bz2
|
||||
https://conda.anaconda.org/conda-forge/win-64/jupyter_contrib_nbextensions-0.4.0-py36_0.tar.bz2
|
||||
https://repo.continuum.io/pkgs/main/win-64/jupyter_core-4.4.0-py36h56e9d50_0.tar.bz2
|
||||
https://conda.anaconda.org/conda-forge/win-64/jupyter_highlight_selected_word-0.1.0-py36_0.tar.bz2
|
||||
https://conda.anaconda.org/conda-forge/win-64/jupyter_latex_envs-1.3.8.2-py36_1.tar.bz2
|
||||
https://conda.anaconda.org/conda-forge/win-64/jupyter_nbextensions_configurator-0.4.0-py36_0.tar.bz2
|
||||
https://conda.anaconda.org/conda-forge/win-64/jupyterlab-0.31.5-py36_1.tar.bz2
|
||||
https://conda.anaconda.org/conda-forge/win-64/jupyterlab_launcher-0.10.3-py36_0.tar.bz2
|
||||
https://repo.continuum.io/pkgs/main/win-64/kealib-1.4.7-ha5b336b_5.tar.bz2
|
||||
https://repo.continuum.io/pkgs/main/win-64/krb5-1.14.2-h63dfc2a_6.tar.bz2
|
||||
https://repo.continuum.io/pkgs/main/win-64/lazy-object-proxy-1.3.1-py36hd1c21d2_0.tar.bz2
|
||||
https://repo.continuum.io/pkgs/main/win-64/libboost-1.65.1-he51fdeb_4.tar.bz2
|
||||
https://repo.continuum.io/pkgs/main/win-64/libcurl-7.58.0-h7602738_0.tar.bz2
|
||||
https://repo.continuum.io/pkgs/main/win-64/libgdal-2.2.2-h2727f2b_1.tar.bz2
|
||||
https://repo.continuum.io/pkgs/main/win-64/libiconv-1.15-h1df5818_7.tar.bz2
|
||||
https://repo.continuum.io/pkgs/main/win-64/libkml-1.3.0-hc65d273_3.tar.bz2
|
||||
https://repo.continuum.io/pkgs/main/win-64/libnetcdf-4.4.1.1-h825a56a_8.tar.bz2
|
||||
https://conda.anaconda.org/conda-forge/win-64/libpng-1.6.34-vc14_0.tar.bz2
|
||||
https://repo.continuum.io/pkgs/main/win-64/libpq-9.6.6-hfe3f2bf_0.tar.bz2
|
||||
https://repo.continuum.io/pkgs/main/win-64/libprotobuf-3.4.1-h3dba5dd_0.tar.bz2
|
||||
https://repo.continuum.io/pkgs/main/win-64/libspatialite-4.3.0a-h383548d_18.tar.bz2
|
||||
https://repo.continuum.io/pkgs/main/win-64/libssh2-1.8.0-hd619d38_4.tar.bz2
|
||||
https://conda.anaconda.org/conda-forge/win-64/libtiff-4.0.9-vc14_0.tar.bz2
|
||||
https://conda.anaconda.org/conda-forge/win-64/libxml2-2.9.5-vc14_1.tar.bz2
|
||||
https://conda.anaconda.org/conda-forge/win-64/libxslt-1.1.32-vc14_0.tar.bz2
|
||||
https://repo.continuum.io/pkgs/main/win-64/locket-0.2.0-py36hfed976d_1.tar.bz2
|
||||
https://conda.anaconda.org/conda-forge/win-64/lxml-4.1.1-py36_0.tar.bz2
|
||||
https://repo.continuum.io/pkgs/free/win-64/markdown-2.6.9-py36_0.tar.bz2
|
||||
https://repo.continuum.io/pkgs/main/win-64/markupsafe-1.0-py36h0e26971_1.tar.bz2
|
||||
https://repo.continuum.io/pkgs/main/win-64/matplotlib-2.1.2-py36h016c42a_0.tar.bz2
|
||||
https://repo.continuum.io/pkgs/main/win-64/mccabe-0.6.1-py36hb41005a_1.tar.bz2
|
||||
https://repo.continuum.io/pkgs/main/win-64/mistune-0.8.3-py36_0.tar.bz2
|
||||
https://repo.continuum.io/pkgs/main/win-64/mkl-2018.0.1-h2108138_4.tar.bz2
|
||||
https://repo.continuum.io/pkgs/main/win-64/msgpack-python-0.5.1-py36he980bc4_0.tar.bz2
|
||||
https://repo.continuum.io/pkgs/main/win-64/nbconvert-5.3.1-py36h8dc0fde_0.tar.bz2
|
||||
https://repo.continuum.io/pkgs/main/win-64/nbformat-4.4.0-py36h3a5bc1b_0.tar.bz2
|
||||
https://repo.continuum.io/pkgs/main/win-64/networkx-2.1-py36_0.tar.bz2
|
||||
https://repo.continuum.io/pkgs/main/win-64/nodejs-8.9.3-hd6b2f15_0.tar.bz2
|
||||
https://repo.continuum.io/pkgs/main/win-64/notebook-5.4.0-py36_0.tar.bz2
|
||||
https://repo.continuum.io/pkgs/main/win-64/numpy-1.14.1-py36hb69e940_2.tar.bz2
|
||||
https://repo.continuum.io/pkgs/main/win-64/numpydoc-0.7.0-py36ha25429e_0.tar.bz2
|
||||
https://repo.continuum.io/pkgs/main/win-64/olefile-0.45.1-py36_0.tar.bz2
|
||||
https://repo.continuum.io/pkgs/main/win-64/openjpeg-2.2.0-h29c51c3_2.tar.bz2
|
||||
https://conda.anaconda.org/conda-forge/win-64/openssl-1.0.2n-vc14_0.tar.bz2
|
||||
https://repo.continuum.io/pkgs/main/win-64/packaging-16.8-py36ha0986f6_1.tar.bz2
|
||||
https://repo.continuum.io/pkgs/main/win-64/pandas-0.22.0-py36h6538335_0.tar.bz2
|
||||
https://repo.continuum.io/pkgs/main/win-64/pandoc-1.19.2.1-hb2460c7_1.tar.bz2
|
||||
https://repo.continuum.io/pkgs/main/win-64/pandocfilters-1.4.2-py36h3ef6317_1.tar.bz2
|
||||
https://repo.continuum.io/pkgs/main/win-64/parso-0.1.1-py36hae3edee_0.tar.bz2
|
||||
https://repo.continuum.io/pkgs/main/win-64/partd-0.3.8-py36hc8e763b_0.tar.bz2
|
||||
https://repo.continuum.io/pkgs/main/win-64/patsy-0.5.0-py36_0.tar.bz2
|
||||
https://repo.continuum.io/pkgs/main/win-64/pickleshare-0.7.4-py36h9de030f_0.tar.bz2
|
||||
https://repo.continuum.io/pkgs/main/win-64/pillow-5.0.0-py36h0738816_0.tar.bz2
|
||||
https://repo.continuum.io/pkgs/main/win-64/pip-9.0.1-py36h226ae91_4.tar.bz2
|
||||
https://repo.continuum.io/pkgs/main/win-64/proj4-4.9.3-hcf24537_7.tar.bz2
|
||||
https://repo.continuum.io/pkgs/main/win-64/prompt_toolkit-1.0.15-py36h60b8f86_0.tar.bz2
|
||||
https://repo.continuum.io/pkgs/main/win-64/protobuf-3.4.1-py36h07fa351_0.tar.bz2
|
||||
https://repo.continuum.io/pkgs/main/win-64/psutil-5.4.3-py36hfa6e2cd_0.tar.bz2
|
||||
https://repo.continuum.io/pkgs/main/win-64/pycodestyle-2.3.1-py36h7cc55cd_0.tar.bz2
|
||||
https://repo.continuum.io/pkgs/main/win-64/pycparser-2.18-py36hd053e01_1.tar.bz2
|
||||
https://repo.continuum.io/pkgs/main/win-64/pyflakes-1.6.0-py36h0b975d6_0.tar.bz2
|
||||
https://repo.continuum.io/pkgs/main/win-64/pygments-2.2.0-py36hb010967_0.tar.bz2
|
||||
https://repo.continuum.io/pkgs/main/win-64/pylint-1.8.2-py36_0.tar.bz2
|
||||
https://repo.continuum.io/pkgs/main/win-64/pyopenssl-17.5.0-py36h5b7d817_0.tar.bz2
|
||||
https://repo.continuum.io/pkgs/main/win-64/pyparsing-2.2.0-py36h785a196_1.tar.bz2
|
||||
https://repo.continuum.io/pkgs/main/win-64/pyqt-5.6.0-py36hb5ed885_5.tar.bz2
|
||||
https://repo.continuum.io/pkgs/main/win-64/pysocks-1.6.7-py36h698d350_1.tar.bz2
|
||||
https://repo.continuum.io/pkgs/main/win-64/python-3.6.4-h6538335_1.tar.bz2
|
||||
https://repo.continuum.io/pkgs/main/win-64/python-dateutil-2.6.1-py36h509ddcb_1.tar.bz2
|
||||
https://repo.continuum.io/pkgs/main/win-64/pytz-2018.3-py36_0.tar.bz2
|
||||
https://repo.continuum.io/pkgs/main/win-64/pywavelets-0.5.2-py36hc649158_0.tar.bz2
|
||||
https://repo.continuum.io/pkgs/main/win-64/pywinpty-0.5-py36h6538335_1.tar.bz2
|
||||
https://conda.anaconda.org/conda-forge/win-64/pyyaml-3.12-py36_1.tar.bz2
|
||||
https://repo.continuum.io/pkgs/main/win-64/pyzmq-16.0.3-py36he714bf5_0.tar.bz2
|
||||
https://conda.anaconda.org/conda-forge/win-64/qt-5.6.2-vc14_1.tar.bz2
|
||||
https://repo.continuum.io/pkgs/main/win-64/qtawesome-0.4.4-py36h5aa48f6_0.tar.bz2
|
||||
https://repo.continuum.io/pkgs/main/win-64/qtconsole-4.3.1-py36h99a29a9_0.tar.bz2
|
||||
https://repo.continuum.io/pkgs/main/win-64/qtpy-1.3.1-py36hb8717c5_0.tar.bz2
|
||||
https://repo.continuum.io/pkgs/main/win-64/requests-2.18.4-py36h4371aae_1.tar.bz2
|
||||
https://repo.continuum.io/pkgs/main/win-64/rope-0.10.7-py36had63a69_0.tar.bz2
|
||||
https://repo.continuum.io/pkgs/main/win-64/scikit-image-0.13.1-py36hfa6e2cd_1.tar.bz2
|
||||
https://repo.continuum.io/pkgs/main/win-64/scikit-learn-0.19.1-py36h53aea1b_0.tar.bz2
|
||||
https://repo.continuum.io/pkgs/main/win-64/scipy-1.0.0-py36h1260518_0.tar.bz2
|
||||
https://repo.continuum.io/pkgs/main/win-64/send2trash-1.4.2-py36_0.tar.bz2
|
||||
https://repo.continuum.io/pkgs/main/win-64/setuptools-38.4.0-py36_0.tar.bz2
|
||||
https://repo.continuum.io/pkgs/main/win-64/shapely-1.6.4-py36h2a969d5_0.tar.bz2
|
||||
https://repo.continuum.io/pkgs/main/win-64/simplegeneric-0.8.1-py36_2.tar.bz2
|
||||
https://repo.continuum.io/pkgs/main/win-64/sip-4.18.1-py36h9c25514_2.tar.bz2
|
||||
https://repo.continuum.io/pkgs/main/win-64/six-1.11.0-py36h4db2310_1.tar.bz2
|
||||
https://repo.continuum.io/pkgs/main/win-64/snowballstemmer-1.2.1-py36h763602f_0.tar.bz2
|
||||
https://repo.continuum.io/pkgs/main/win-64/sortedcontainers-1.5.9-py36_0.tar.bz2
|
||||
https://repo.continuum.io/pkgs/main/win-64/sphinx-1.6.6-py36_0.tar.bz2
|
||||
https://repo.continuum.io/pkgs/main/win-64/sphinxcontrib-1.0-py36hbbac3d2_1.tar.bz2
|
||||
https://repo.continuum.io/pkgs/main/win-64/sphinxcontrib-websupport-1.0.1-py36hb5e5916_1.tar.bz2
|
||||
https://repo.continuum.io/pkgs/main/win-64/spyder-3.2.8-py36_0.tar.bz2
|
||||
https://conda.anaconda.org/conda-forge/win-64/sqlite-3.20.1-vc14_2.tar.bz2
|
||||
https://repo.continuum.io/pkgs/main/win-64/statsmodels-0.8.0-py36h6189b4c_0.tar.bz2
|
||||
https://repo.continuum.io/pkgs/main/win-64/tblib-1.3.2-py36h30f5020_0.tar.bz2
|
||||
https://repo.continuum.io/pkgs/free/win-64/tensorflow-1.2.1-py36_0.tar.bz2
|
||||
https://repo.continuum.io/pkgs/main/win-64/terminado-0.8.1-py36_1.tar.bz2
|
||||
https://repo.continuum.io/pkgs/main/win-64/testpath-0.3.1-py36h2698cfe_0.tar.bz2
|
||||
https://conda.anaconda.org/conda-forge/win-64/tk-8.6.7-vc14_0.tar.bz2
|
||||
https://repo.continuum.io/pkgs/main/win-64/toolz-0.9.0-py36_0.tar.bz2
|
||||
https://repo.continuum.io/pkgs/main/win-64/tornado-4.5.3-py36_0.tar.bz2
|
||||
https://repo.continuum.io/pkgs/main/win-64/traitlets-4.3.2-py36h096827d_0.tar.bz2
|
||||
https://repo.continuum.io/pkgs/main/win-64/typing-3.6.2-py36hb035bda_0.tar.bz2
|
||||
https://repo.continuum.io/pkgs/main/win-64/urllib3-1.22-py36h276f60a_0.tar.bz2
|
||||
https://repo.continuum.io/pkgs/main/win-64/vc-14-h0510ff6_3.tar.bz2
|
||||
https://repo.continuum.io/pkgs/free/win-64/vs2015_runtime-14.0.25420-0.tar.bz2
|
||||
https://repo.continuum.io/pkgs/main/win-64/wcwidth-0.1.7-py36h3d5aa90_0.tar.bz2
|
||||
https://repo.continuum.io/pkgs/main/win-64/webencodings-0.5.1-py36h67c50ae_1.tar.bz2
|
||||
https://repo.continuum.io/pkgs/main/win-64/werkzeug-0.14.1-py36_0.tar.bz2
|
||||
https://repo.continuum.io/pkgs/main/win-64/wheel-0.30.0-py36h6c3ec14_1.tar.bz2
|
||||
https://repo.continuum.io/pkgs/main/win-64/widgetsnbextension-3.1.0-py36_0.tar.bz2
|
||||
https://repo.continuum.io/pkgs/main/win-64/win_inet_pton-1.0.1-py36he67d7fd_1.tar.bz2
|
||||
https://repo.continuum.io/pkgs/main/win-64/wincertstore-0.2-py36h7fe50ca_0.tar.bz2
|
||||
https://conda.anaconda.org/conda-forge/win-64/winpty-0.4.3-vc14_2.tar.bz2
|
||||
https://repo.continuum.io/pkgs/main/win-64/wrapt-1.10.11-py36he5f5981_0.tar.bz2
|
||||
https://repo.continuum.io/pkgs/main/win-64/xerces-c-3.2.0-h44c76bb_2.tar.bz2
|
||||
https://repo.continuum.io/pkgs/main/win-64/xz-5.2.3-h7c615d8_2.tar.bz2
|
||||
https://conda.anaconda.org/conda-forge/win-64/yaml-0.1.7-vc14_0.tar.bz2
|
||||
https://repo.continuum.io/pkgs/main/win-64/zict-0.1.3-py36h2d8e73e_0.tar.bz2
|
||||
https://conda.anaconda.org/conda-forge/win-64/zlib-1.2.11-vc14_0.tar.bz2
|
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue