|
|
|
|
"""Download latest time series data from BOM website and update local copy.
|
|
|
|
|
|
|
|
|
|
BOM does not allow direct http requests to data files, so a requests session
|
|
|
|
|
is opened on the product page before accessing the resource.
|
|
|
|
|
|
|
|
|
|
D. Howe
|
|
|
|
|
2019-09-04
|
|
|
|
|
"""
|
|
|
|
|
import os
|
|
|
|
|
import json
|
|
|
|
|
import requests
|
|
|
|
|
import pandas as pd
|
|
|
|
|
from lxml import html
|
|
|
|
|
|
|
|
|
|
# Set base URL
|
|
|
|
|
bom_url = 'http://www.bom.gov.au/'
|
|
|
|
|
|
|
|
|
|
# Set product URL for North Head
|
|
|
|
|
product_url = 'products/IDN60901/IDN60901.95768.shtml'
|
|
|
|
|
|
|
|
|
|
# Set output directory
|
|
|
|
|
output_dir = 'bom'
|
|
|
|
|
|
|
|
|
|
# Set output column names
|
|
|
|
|
cols = [
|
|
|
|
|
'wind_dir',
|
|
|
|
|
'wind_spd_kmh',
|
|
|
|
|
'gust_kmh',
|
|
|
|
|
]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def update_master(output_dir, csv_name, df):
|
|
|
|
|
"""Update master csv time series.
|
|
|
|
|
|
|
|
|
|
Args:
|
|
|
|
|
output_dir (str): path to time series directory
|
|
|
|
|
csv_name (str): name of time series file
|
|
|
|
|
df (dataframe): dataframe with datetime index
|
|
|
|
|
|
|
|
|
|
Returns:
|
|
|
|
|
None
|
|
|
|
|
"""
|
|
|
|
|
try:
|
|
|
|
|
# Load local master table if it exists
|
|
|
|
|
master = pd.read_csv(os.path.join(output_dir, csv_name),
|
|
|
|
|
index_col=0,
|
|
|
|
|
parse_dates=True)
|
|
|
|
|
|
|
|
|
|
# Only include timestamps that do not already exist
|
|
|
|
|
df = df[~df.index.isin(master.index)]
|
|
|
|
|
|
|
|
|
|
# Update master
|
|
|
|
|
master = master.append(df)
|
|
|
|
|
|
|
|
|
|
except FileNotFoundError:
|
|
|
|
|
# Create new master table if none exists
|
|
|
|
|
master = df
|
|
|
|
|
|
|
|
|
|
# Export master table
|
|
|
|
|
master.to_csv(os.path.join(output_dir, csv_name))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
# Open new session with BOM website
|
|
|
|
|
with requests.session() as session:
|
|
|
|
|
# Load product page
|
|
|
|
|
page = session.get(bom_url + product_url)
|
|
|
|
|
tree = html.fromstring(page.content)
|
|
|
|
|
|
|
|
|
|
# Find and download JSON data
|
|
|
|
|
json_url = tree.xpath('//*[@id="content"]/p[4]/a')[0].get('href')
|
|
|
|
|
json_data = json.loads(session.get(bom_url + json_url).content)
|
|
|
|
|
|
|
|
|
|
# Extract file base name
|
|
|
|
|
csv_name = json_url.split('/')[-1].replace('.json', '.csv')
|
|
|
|
|
|
|
|
|
|
# Create dataframe
|
|
|
|
|
df = pd.DataFrame(json_data['observations']['data'])
|
|
|
|
|
|
|
|
|
|
# Set local time as index
|
|
|
|
|
df.index = pd.to_datetime(df['local_date_time_full'])
|
|
|
|
|
df = df.sort_index()
|
|
|
|
|
|
|
|
|
|
# Extract columns of interest
|
|
|
|
|
df = df[cols]
|
|
|
|
|
|
|
|
|
|
# Update master table
|
|
|
|
|
update_master(output_dir, csv_name, df)
|
