From cf386fa87510894766f60bc77b6ce2f9eff51f1a Mon Sep 17 00:00:00 2001 From: Dan Howe Date: Mon, 2 Jul 2018 11:32:23 +1000 Subject: [PATCH] Add 'extract_points()' and 'update_survey_output()' --- lastools.py | 78 ----------------------- survey_tools.py | 166 ++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 166 insertions(+), 78 deletions(-) delete mode 100644 lastools.py create mode 100644 survey_tools.py diff --git a/lastools.py b/lastools.py deleted file mode 100644 index d0f6864..0000000 --- a/lastools.py +++ /dev/null @@ -1,78 +0,0 @@ -import os -import subprocess - - -def call_lastools(tool_name, input, output=None, args=None, verbose=True): - """Send commands to the lastools library. - - Requires lastools in system path. - - Args: - tool_name: name of lastools binary - input: bytes from stdout, or path to main input data - output: '-stdout' to pipe output, or path to main output data - args: list of additional arguments, formatted for lastools - verbose: show all warnings and messages from lastools (boolean) - - Returns: - bytes of output las, if output='-stdout' - None, if output='path/to/las/file' - - Examples: - - # Convert xyz file to las and pipe stdout to a python bytes object - las_data = call_lastools('txt2las', input='points.xyz', output='-stdout', - args=['-parse', 'sxyz']) - - # Clip las_data with a shapefile, and save to a new las file - call_lastools('lasclip', input=las_data, output='points.las', - args=['-poly', 'boundary.shp']) - """ - - # Start building command string - cmd = [tool_name] - - # Parse input - if type(input) == bytes: - # Pipe input las bytes to stdin - cmd += ['-stdin'] - stdin = input - else: - # Load las from file path - cmd += ['-i', input] - stdin = None - - # Parse output - if output == '-stdout': - # Pipe output las to stdout - cmd += ['-stdout'] - elif output: - # Save output las to file - cmd += ['-o', output] - - # Append additional lastools arguments, if provided - if args: - cmd += args - - process = subprocess.Popen( - cmd, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - stdin=subprocess.PIPE) - stdout, stderr = process.communicate(input=stdin) - - # Handle errors, if detected - if process.returncode != 0: - print("Error: {} failed on {}".format(tool_name, - os.path.basename(input))) - print(stderr.decode()) - - elif verbose: - # Print addional messages if verbose mode is being used - print(stderr.decode()) - - # Output piped stdout if required - if output == '-stdout': - return stdout - else: - return None diff --git a/survey_tools.py b/survey_tools.py new file mode 100644 index 0000000..a5bd806 --- /dev/null +++ b/survey_tools.py @@ -0,0 +1,166 @@ +import io +import os +import subprocess +import pandas as pd + + +def call_lastools(tool_name, input, output=None, args=None, verbose=True): + """Send commands to the lastools library. + + Requires lastools in system path. + + Args: + tool_name: name of lastools binary + input: bytes from stdout, or path to main input data + output: '-stdout' to pipe output, or path to main output data + args: list of additional arguments, formatted for lastools + verbose: show all warnings and messages from lastools (boolean) + + Returns: + bytes of output las, if output='-stdout' + None, if output='path/to/las/file' + + Examples: + + # Convert xyz file to las and pipe stdout to a python bytes object + las_data = call_lastools('txt2las', input='points.xyz', output='-stdout', + args=['-parse', 'sxyz']) + + # Clip las_data with a shapefile, and save to a new las file + call_lastools('lasclip', input=las_data, output='points.las', + args=['-poly', 'boundary.shp']) + """ + + # Start building command string + cmd = [tool_name] + + # Parse input + if type(input) == bytes: + # Pipe input las bytes to stdin + cmd += ['-stdin'] + stdin = input + else: + # Load las from file path + cmd += ['-i', input] + stdin = None + + # Parse output + if output == '-stdout': + # Pipe output las to stdout + cmd += ['-stdout'] + elif output: + # Save output las to file + cmd += ['-o', output] + + # Append additional lastools arguments, if provided + if args: + cmd += args + + process = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + stdin=subprocess.PIPE) + stdout, stderr = process.communicate(input=stdin) + + # Handle errors, if detected + if process.returncode != 0: + print("Error: {} failed on {}".format(tool_name, + os.path.basename(input))) + print(stderr.decode()) + + elif verbose: + # Print addional messages if verbose mode is being used + print(stderr.decode()) + + # Output piped stdout if required + if output == '-stdout': + return stdout + else: + return None + + +def extract_pts(las_in, cp_in, survey_date, beach, args=None, verbose=True): + """Extract elevations from a las surface based on x and y coordinates. + + Requires lastools in system path. + + Args: + las_in: bytes from stdout, or path to main input data + cp_in: point coordinates with columns: id, x, y, z (csv) + survey_date: survey date string, e.g. '19700101' + beach: beach name + args: list of additional arguments, formatted for lastools + verbose: show all warnings and messages from lastools (boolean) + + Returns: + Dataframe containing input coordinates with extracted elevations + + Examples: + + # Extract elevations from 'points.las', using control points from 'cp.csv' + # Specify control point format as: id, x, y, z ('-parse', 'sxyz') + # Only use points classified as 'ground' ('-keep_class', '2') + extract_pts('points.las', 'cp.csv', survey_date='20001231', beach='manly', + args=['-parse', 'sxyz', '-keep_class', '2']) + """ + + # Assemble lastools arguments + if args: + args = ['-cp', cp_in] + args + else: + args = ['-cp', cp_in] + + las_data = call_lastools( + 'lascontrol', input=las_in, output='-stdout', args=args, verbose=False) + + # Load result into pandas dataframe + df = pd.read_csv(io.BytesIO(las_data)) + + # Tidy up dataframe + df = df.drop(columns=['diff']) + df['lidar_z'] = pd.to_numeric(df['lidar_z'], errors='coerce') + df['Beach'] = beach + df = df[[ + 'Beach', 'ProfileNum', 'Easting', 'Northing', 'Chainage', 'lidar_z' + ]] + + # Rename columns + new_names = { + 'ProfileNum': 'Profile', + 'lidar_z': 'Elevation_{}'.format(survey_date), + } + df = df.rename(columns=new_names) + + return df + + +def update_survey_output(df, output_dir): + """Update survey profile output csv files with current survey. + + Args: + df: dataframe containing current survey elevations + output_dir: directory where csv files are saved + + Returns: + None + """ + # Merge current survey with existing data + profiles = df['Profile'].unique() + for profile in profiles: + csv_name = os.path.join(output_dir, profile + '.csv') + + # Extract survey data for current profile + current_profile = df[df['Profile'] == profile] + try: + # Load existing results + master = pd.read_csv(csv_name) + except FileNotFoundError: + master = current_profile.copy() + + # Add (or update) current survey + current_survey_col = df.columns[-1] + master[current_survey_col] = current_profile[current_survey_col] + + # Export updated results + master.to_csv(csv_name)