danh
/
major-projects-grabber
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Update docstrings

Browse Source
master
Dan Howe 6 years ago
parent 6df59b9b7d
commit f701e4212b
1 changed files with 7 additions and 12 deletions
Show Stats Download Patch File Download Diff File

19
major_projects_grabber/major_projects_grabber.py
Unescape Escape View File

@ -3,19 +3,13 @@ Download documents from the NSW DPE Major Projects website.
Example usage: Example usage:
# Grab a single project modification using its job id, and save in 'files' # Grab a single project modification using its job id, and save in 'files'
python major_projects_grabber.py -o files -i 1019 major_projects_grabber -o files -i 1746
# Grab all modifications in search results page, and save in current folder # Grab all modifications in search results page, and save in current folder
python major_projects_grabber.py -o . http:// major_projects_grabber "http://majorprojects.planning.nsw.gov.au/index.pl?action=search&authority_id=547"
""" """
# search url
# http://majorprojects.planning.nsw.gov.au/index.pl?action=search&page_id=&search=&authority_id=&search_site_type_id=10&reference_table=&status_id=&decider=&from_date=&to_date=&x=31&y=16
# mod url
# http://majorprojects.planning.nsw.gov.au/index.pl?action=view_job&job_id=9503
import os import os
import re import re
import sys import sys
@ -25,6 +19,7 @@ import requests
import argparse import argparse
import pandas as pd import pandas as pd
from lxml import html from lxml import html
from tqdm import tqdm
from requests.exceptions import ConnectionError, InvalidURL from requests.exceptions import ConnectionError, InvalidURL
@ -54,7 +49,7 @@ def get_documents(mod_id, output_dir):
# Add note if no documents are found on portal # Add note if no documents are found on portal
if not folders: if not folders:
txt_name = 'No documents on DPE portal for this modification.txt' txt_name = 'No documents on DPE portal for this modification.txt'
open(os.path.join(mod_dir,txt_name), 'a').close() open(os.path.join(mod_dir, txt_name), 'a').close()
# Create link to DPE Major Projects page for current modification # Create link to DPE Major Projects page for current modification
text = """<html> text = """<html>
@ -126,10 +121,10 @@ def main():
example_text = """examples: example_text = """examples:
# Grab a single project modification using its job id, and save in 'files' # Grab a single project modification using its job id, and save in 'files'
python major_projects_grabber.py -o files -i 1019 major_projects_grabber -i 1746 -o files
# Grab all modifications in search results page, and save in current folder # Grab all modifications in search results page, and save in current folder
python major_projects_grabber.py -o . http:// major_projects_grabber http://majorprojects.planning.nsw.gov.au/index.pl?action=search&authority_id=547
""" """
# Set up command line arguments # Set up command line arguments
@ -168,7 +163,7 @@ def main():
mod_ids.extend(search_mod_ids) mod_ids.extend(search_mod_ids)
# Download documents from given modification ids # Download documents from given modification ids
for mod_id in mod_ids: for mod_id in tqdm(mod_ids):
get_documents(mod_id, output_dir) get_documents(mod_id, output_dir)

Write Preview
Loading…
Cancel
Save