|
|
@ -17,6 +17,7 @@ import shutil
|
|
|
|
import logging
|
|
|
|
import logging
|
|
|
|
import requests
|
|
|
|
import requests
|
|
|
|
import argparse
|
|
|
|
import argparse
|
|
|
|
|
|
|
|
import warnings
|
|
|
|
from lxml import html
|
|
|
|
from lxml import html
|
|
|
|
from tqdm import tqdm
|
|
|
|
from tqdm import tqdm
|
|
|
|
from requests.exceptions import ConnectionError, InvalidURL
|
|
|
|
from requests.exceptions import ConnectionError, InvalidURL
|
|
|
@ -206,6 +207,15 @@ def main():
|
|
|
|
# Tidy up console after tqdm
|
|
|
|
# Tidy up console after tqdm
|
|
|
|
print('\n')
|
|
|
|
print('\n')
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
# Check contents of log file
|
|
|
|
|
|
|
|
with open(log_name, 'r') as f:
|
|
|
|
|
|
|
|
log_data = f.read()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
if log_data:
|
|
|
|
|
|
|
|
warnings.warn(
|
|
|
|
|
|
|
|
'Some files failed to download. See log for details.',
|
|
|
|
|
|
|
|
stacklevel=2)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
if __name__ == '__main__':
|
|
|
|
if __name__ == '__main__':
|
|
|
|
main()
|
|
|
|
main()
|
|
|
|