v2_builder/src/download.py

import argparse
import json
import os
import os.path
import shutil
import urllib.request
import zipfile

parser = argparse.ArgumentParser()
# get the arguments
parser.add_argument('--output_path', help='The local path like "/home/username/sword_zip"', default="sword_zip")
parser.add_argument('--bible_conf')
# set to args
args = parser.parse_args()
# this is a full path
MAIN_PATH = args.output_path

# some helper dictionaries
v1_translation_names = json.loads(open(args.bible_conf).read())
# scripts directory
CURRENT_DIR = os.path.dirname(os.path.realpath(__file__))


# function to create path if not exist
def check_path(path):
    if not os.path.isdir(path):
        os.makedirs(path)


def zipper(src, dest, filename):
    """Backup files from src to dest."""
    base = os.path.basename(src)
    new_file = filename + ".zip"

    # Set the current working directory.
    os.chdir(dest)

    if os.path.exists(new_file):
        os.unlink(new_file)

    # Write the zipfile and walk the source directory tree.
    with zipfile.ZipFile(new_file, 'w') as zip_file:
        for folder, _, files in os.walk(src):
            for file in files:
                zip_file.write(os.path.join(folder, file),
                               arcname=os.path.join(folder[len(src):], file),
                               compress_type=zipfile.ZIP_DEFLATED)

    # move back to working directory
    os.chdir(CURRENT_DIR)


# make sure the main folder exist
check_path(MAIN_PATH)
# number of items
number = len(v1_translation_names.keys())
each_count = int(98 / number)
counter = 0
# loop the names
for sword_name in v1_translation_names:
    # set local file name
    file_path = MAIN_PATH + "/" + sword_name + ".zip"
    # set remote URL
    file_url = "https://www.crosswire.org/ftpmirror/pub/sword/packages/rawzip/" + sword_name + ".zip"
    # notice name
    file_name = sword_name + ".zip"
    # check if file exist locally
    if os.path.isfile(file_path):
        print('XXX\n{}\n{} already exist\nXXX'.format(counter, file_name))
    else:
        try:
            print('XXX\n{}\nDownloading the RAW format of {}\nXXX'.format(counter, file_name))
            urllib.request.urlretrieve(file_url, file_path)
        except urllib.error.HTTPError as e:
            print('XXX\n{} Download {} failed! {}\nXXX'.format(counter, sword_name, e))
    # check if this is a legitimate zip file
    if not zipfile.is_zipfile(file_path):
        os.remove(file_path)
        print('XXX\n{}\n{} was removed since it has errors...\nXXX'.format(counter, file_path))
        # we try the win repository
        file_url = "https://www.crosswire.org/ftpmirror/pub/sword/packages/win/" + sword_name + ".zip"
        try:
            print('XXX\n{}\nDownloading the WIN format of {}\nXXX'.format(counter, file_name))
            urllib.request.urlretrieve(file_url, file_path)
        except urllib.error.HTTPError as e:
            # noinspection PyStringFormat
            print('XXX\n{}\n{} Download {} failed! {}\nXXX'.format(sword_name, e))
        # again check if this is a legitimate zip file
        if not zipfile.is_zipfile(file_path):
            os.remove(file_path)
            print('XXX\n{}\n{} was again removed since it has errors...\nXXX'.format(counter, file_name))
        else:
            # set local file name
            folder_path = MAIN_PATH + "/" + sword_name
            folder_raw_path = MAIN_PATH + "/" + sword_name + "/RAW"
            data_raw_path = MAIN_PATH + "/" + sword_name + "/data.zip"
            print('XXX\n{}\n{} is being converted to RAW format\nXXX'.format(counter, file_name))
            # first we extract the WIN format
            with zipfile.ZipFile(file_path, 'r') as zip_ref:
                zip_ref.extractall(folder_path)
            os.remove(file_path)
            # now we extract the RAW format
            with zipfile.ZipFile(data_raw_path, 'r') as zip_ref:
                zip_ref.extractall(folder_raw_path)
            os.remove(data_raw_path)
            # now rename the folder
            os.rename(folder_raw_path + "/newmods", folder_raw_path + "/mods.d")
            # now zip the RAW folder
            zipper(folder_raw_path, MAIN_PATH, sword_name)
            # now remove the tmp folder
            try:
                shutil.rmtree(folder_path)
            except OSError as e:
                print("XXX\nError: %s - %s.\nXXX" % (e.filename, e.strerror))
    # increase the counter
    counter = int(counter + each_count)