getBible/v2_builder
5
2
Fork 0
mirror of https://github.com/getbible/v2_builder.git synced 2024-06-17 12:42:19 +00:00
Code Issues

Made a few corrections.

Browse Source
This commit is contained in:
Llewellyn van der Merwe 2022-01-03 19:10:52 +02:00
parent dbf063124d
commit a445ad2e9b
Signed by: Llewellyn
GPG Key ID: EFC0C720A240551C
2 changed files with 90 additions and 72 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

114
run.sh
View File

@ -23,17 +23,18 @@ command -v python3 >/dev/null 2>&1 || {
} }
# get start time # get start time
STARTBUILD=$(date +"%s") START_BUILD=$(date +"%s")
# use UTC+00:00 time also called zulu # use UTC+00:00 time also called zulu
STARTDATE=$(TZ=":ZULU" date +"%m/%d/%Y @ %R (UTC)") START_DATE=$(TZ=":ZULU" date +"%m/%d/%Y @ %R (UTC)")
# main project Header # main project Header
HEADERTITLE="getBible JSON API.v2" HEADER_TITLE="getBible JSON API.v2"
# main function ˘Ô≈ôﺣ # main function ˘Ô≈ôﺣ
function main() { function main() {
# Only Hash existing scripture JSON files # Only Hash existing scripture JSON files
if (("$HASHONLY" == 1)); then if (("$HASH_ONLY" == 1)); then
# numbers # numbers
# shellcheck disable=SC2012
number=$(ls "${DIR_zip}" | wc -l) number=$(ls "${DIR_zip}" | wc -l)
each_count=$((98 / number)) each_count=$((98 / number))
# the hashing of all files # the hashing of all files
@ -47,6 +48,7 @@ function main() {
getModules "${DIR_zip}" getModules "${DIR_zip}"
fi fi
# numbers # numbers
# shellcheck disable=SC2012
number=$(ls "${DIR_zip}" | wc -l) number=$(ls "${DIR_zip}" | wc -l)
each_count=$((98 / number)) each_count=$((98 / number))
# prep the Scripture Main Git Folder # prep the Scripture Main Git Folder
@ -70,16 +72,16 @@ function main() {
# completion message # completion message
function completedBuildMessage() { function completedBuildMessage() {
# set the build time # set the build time
ENDBUILD=$(date +"%s") END_BUILD=$(date +"%s")
SECONDSBUILD=$((ENDBUILD - STARTBUILD)) SECONDS_BUILD=$((END_BUILD - START_BUILD))
# use UTC+00:00 time also called zulu # use UTC+00:00 time also called zulu
ENDDATE=$(TZ=":ZULU" date +"%m/%d/%Y @ %R (UTC)") END_DATE=$(TZ=":ZULU" date +"%m/%d/%Y @ %R (UTC)")
# give completion message # give completion message
if (("$QUIET" == 0)); then if (("$QUIET" == 0)); then
whiptail --title "${HEADERTITLE}" --separate-output --infobox "${USER^}, the ${HEADERTITLE} build is complete!\n\n Started: ${STARTDATE}\n Ended: ${ENDDATE}\nBuild Time: ${SECONDSBUILD} seconds" 12 77 whiptail --title "${HEADER_TITLE}" --separate-output --infobox "${USER^}, the ${HEADER_TITLE} build is complete!\n\n Started: ${START_DATE}\n Ended: ${END_DATE}\nBuild Time: ${SECONDS_BUILD} seconds" 12 77
sleep 10 sleep 10
else else
echo "${HEADERTITLE} build on ${STARTDATE} is completed in ${SECONDSBUILD} seconds!" echo "${HEADER_TITLE} build on ${START_DATE} is completed in ${SECONDS_BUILD} seconds!"
fi fi
} }
@ -102,11 +104,11 @@ function getModules() {
rm -fr "${modules_path}" rm -fr "${modules_path}"
mkdir -p "${modules_path}" mkdir -p "${modules_path}"
# run in github action workflow... ¯\_(ツ)_/¯ # run in github action workflow... ¯\_(ツ)_/¯
if (("$GITHUB" == 1)); then if (("$GIT_HUB" == 1)); then
echo "Start download of modules..." echo "Start download of modules..."
python3 "${DIR_src}/download.py" \ python3 "${DIR_src}/download.py" \
--output_path "${modules_path}" \ --output_path "${modules_path}" \
--bible_conf "${DIR_bible}" >>/dev/null --bible_conf "${DIR_bible}"
echo "Done downloading modules..." echo "Done downloading modules..."
else else
# then we get the current modules # then we get the current modules
@ -120,7 +122,7 @@ function getModules() {
sleep 1 sleep 1
echo -e "XXX\n100\nDone downloading modules... \nXXX" echo -e "XXX\n100\nDone downloading modules... \nXXX"
sleep 2 sleep 2
} | showProgress "Get Crosswire Modules | ${HEADERTITLE}" "Please wait while we download all modules" } | showProgress "Get Crosswire Modules | ${HEADER_TITLE}" "Please wait while we download all modules"
fi fi
} }
@ -134,7 +136,7 @@ function prepScriptureMainGit() {
# check if we must pull the REPO # check if we must pull the REPO
if (("$pull" == 1)); then if (("$pull" == 1)); then
# pull the main scripture repository # pull the main scripture repository
git clone --depth 1 "${REPOSCRIPTURE}" "${scripture_path}" git clone --depth 1 "${REPO_SCRIPTURE}" "${scripture_path}"
# pull only once # pull only once
pull=0 pull=0
else else
@ -146,6 +148,7 @@ function prepScriptureMainGit() {
if [ -d "${scripture_path}/.git" ]; then if [ -d "${scripture_path}/.git" ]; then
# make a pull if needed still (update the git history) # make a pull if needed still (update the git history)
if (("$pull" == 1)); then if (("$pull" == 1)); then
# shellcheck disable=SC2164
cd "${scripture_path}" && git pull && cd - cd "${scripture_path}" && git pull && cd -
fi fi
mkdir -p "${scripture_path}Tmp" mkdir -p "${scripture_path}Tmp"
@ -169,12 +172,13 @@ function movePublicHashFiles () {
local w_initial_ms="$6" local w_initial_ms="$6"
local each="$7" local each="$7"
# run in github action workflow... ¯\_(ツ)_/¯ # run in github action workflow... ¯\_(ツ)_/¯
if (("$GITHUB" == 1)); then if (("$GIT_HUB" == 1)); then
echo "$w_title | ${HEADERTITLE}" echo "$w_title | ${HEADER_TITLE}"
echo "$w_initial_ms" echo "$w_initial_ms"
echo "${w_start_ms}..." echo "${w_start_ms}..."
# now run the hashing # now run the hashing
. "${DIR_src}/${script_name}.sh" "${scripture_path}" "$each" "$PULL" "${REPOHASH}" >>/dev/null # shellcheck disable=SC1090
. "${DIR_src}/${script_name}.sh" "${scripture_path}" "$each" "$PULL" "${REPO_HASH}" >>/dev/null
echo "${w_end_ms}..." echo "${w_end_ms}..."
else else
# now run the hashing # now run the hashing
@ -182,11 +186,12 @@ function movePublicHashFiles () {
sleep 1 sleep 1
echo -e "XXX\n0\n${w_start_ms}... \nXXX" echo -e "XXX\n0\n${w_start_ms}... \nXXX"
sleep 1 sleep 1
. "${DIR_src}/${script_name}.sh" "${scripture_path}" "$each" "$PULL" "${REPOHASH}" # shellcheck disable=SC1090
. "${DIR_src}/${script_name}.sh" "${scripture_path}" "$each" "$PULL" "${REPO_HASH}"
sleep 1 sleep 1
echo -e "XXX\n100\n${w_end_ms}... \nXXX" echo -e "XXX\n100\n${w_end_ms}... \nXXX"
sleep 1 sleep 1
} | showProgress "$w_title | ${HEADERTITLE}" "$w_initial_ms" } | showProgress "$w_title | ${HEADER_TITLE}" "$w_initial_ms"
fi fi
} }
@ -198,9 +203,9 @@ function setStaticJsonFiles() {
local each="$3" local each="$3"
local counter=0 local counter=0
# run in github action workflow... ¯\_(ツ)_/¯ # run in github action workflow... ¯\_(ツ)_/¯
if (("$GITHUB" == 1)); then if (("$GIT_HUB" == 1)); then
echo "Start Building..." echo "Start Building..."
for filename in $modules_path/*.zip; do for filename in "${modules_path}/"*.zip; do
# give notice # give notice
echo "Building ${filename} static json files" echo "Building ${filename} static json files"
# run script # run script
@ -208,7 +213,7 @@ function setStaticJsonFiles() {
--source_file "${filename}" \ --source_file "${filename}" \
--output_path "${scripture_path}" \ --output_path "${scripture_path}" \
--conf_dir "${DIR_conf}" \ --conf_dir "${DIR_conf}" \
--bible_conf "${DIR_bible}" >>/dev/null --bible_conf "${DIR_bible}"
# give notice # give notice
echo "Done building ${filename} static json files..." echo "Done building ${filename} static json files..."
done done
@ -219,7 +224,7 @@ function setStaticJsonFiles() {
sleep 1 sleep 1
echo -e "XXX\n0\nStart Building... \nXXX" echo -e "XXX\n0\nStart Building... \nXXX"
sleep 1 sleep 1
for filename in $modules_path/*.zip; do for filename in "${modules_path}/"*.zip; do
# give notice # give notice
echo -e "XXX\n${counter}\nBuilding ${filename} static json files...\nXXX" echo -e "XXX\n${counter}\nBuilding ${filename} static json files...\nXXX"
# add more # add more
@ -239,7 +244,7 @@ function setStaticJsonFiles() {
done done
echo -e "XXX\n100\nDone Building... \nXXX" echo -e "XXX\n100\nDone Building... \nXXX"
sleep 1 sleep 1
} | showProgress "Build Static JSON Files | ${HEADERTITLE}" "Please wait while build the static json API" } | showProgress "Build Static JSON Files | ${HEADER_TITLE}" "Please wait while build the static json API"
fi fi
} }
@ -298,11 +303,12 @@ function hashingMethod() {
local w_initial_ms="$6" local w_initial_ms="$6"
local each="$7" local each="$7"
# run in github action workflow... ¯\_(ツ)_/¯ # run in github action workflow... ¯\_(ツ)_/¯
if (("$GITHUB" == 1)); then if (("$GIT_HUB" == 1)); then
echo "$w_title | ${HEADERTITLE}" echo "$w_title | ${HEADER_TITLE}"
echo "$w_initial_ms" echo "$w_initial_ms"
echo "${w_start_ms}..." echo "${w_start_ms}..."
# now run the hashing # now run the hashing
# shellcheck disable=SC1090
. "${DIR_src}/${script_name}.sh" "${scripture_path}" "$each" >>/dev/null . "${DIR_src}/${script_name}.sh" "${scripture_path}" "$each" >>/dev/null
echo "${w_end_ms}..." echo "${w_end_ms}..."
else else
@ -311,28 +317,29 @@ function hashingMethod() {
sleep 1 sleep 1
echo -e "XXX\n0\n${w_start_ms}... \nXXX" echo -e "XXX\n0\n${w_start_ms}... \nXXX"
sleep 1 sleep 1
# shellcheck disable=SC1090
. "${DIR_src}/${script_name}.sh" "${scripture_path}" "$each" . "${DIR_src}/${script_name}.sh" "${scripture_path}" "$each"
sleep 1 sleep 1
echo -e "XXX\n100\n${w_end_ms}... \nXXX" echo -e "XXX\n100\n${w_end_ms}... \nXXX"
sleep 1 sleep 1
} | showProgress "$w_title | ${HEADERTITLE}" "$w_initial_ms" } | showProgress "$w_title | ${HEADER_TITLE}" "$w_initial_ms"
fi fi
} }
# set any/all default config property # set any/all default config property
function setDefaults() { function setDefaults() {
if [ -f $CONFIGFILE ]; then if [ -f "$CONFIG_FILE" ]; then
# set all defaults # set all defaults
DIR_api=$(getDefault "getbible.api" "${DIR_api}") DIR_api=$(getDefault "getbible.api" "${DIR_api}")
DIR_zip=$(getDefault "getbible.zip" "${DIR_zip}") DIR_zip=$(getDefault "getbible.zip" "${DIR_zip}")
DIR_bible=$(getDefault "getbible.bconf" "${DIR_bible}") DIR_bible=$(getDefault "getbible.bconf" "${DIR_bible}")
DOWNLOAD=$(getDefault "getbible.download" "$DOWNLOAD") DOWNLOAD=$(getDefault "getbible.download" "$DOWNLOAD")
REPOSCRIPTURE=$(getDefault "getbible.repo-scripture" "${REPOSCRIPTURE}") REPO_SCRIPTURE=$(getDefault "getbible.repo-scripture" "${REPO_SCRIPTURE}")
REPOHASH=$(getDefault "getbible.repo-hash" "${REPOHASH}") REPO_HASH=$(getDefault "getbible.repo-hash" "${REPO_HASH}")
PUSH=$(getDefault "getbible.push" "$PUSH") PUSH=$(getDefault "getbible.push" "$PUSH")
PULL=$(getDefault "getbible.pull" "$PULL") PULL=$(getDefault "getbible.pull" "$PULL")
HASHONLY=$(getDefault "getbible.hashonly" "$HASHONLY") HASH_ONLY=$(getDefault "getbible.hashonly" "$HASH_ONLY")
GITHUB=$(getDefault "getbible.github" "$GITHUB") GIT_HUB=$(getDefault "getbible.github" "$GIT_HUB")
QUIET=$(getDefault "getbible.quiet" "$QUIET") QUIET=$(getDefault "getbible.quiet" "$QUIET")
fi fi
} }
@ -340,7 +347,8 @@ function setDefaults() {
# get default properties from config file # get default properties from config file
function getDefault() { function getDefault() {
PROP_KEY="$1" PROP_KEY="$1"
PROP_VALUE=$(cat $CONFIGFILE | grep "$PROP_KEY" | cut -d'=' -f2) # shellcheck disable=SC2002
PROP_VALUE=$(cat "$CONFIG_FILE" | grep "$PROP_KEY" | cut -d'=' -f2)
echo "${PROP_VALUE:-$2}" echo "${PROP_VALUE:-$2}"
} }
@ -450,7 +458,7 @@ You are able to change a few default behaviours in the getBible API builder
example: ${0##*/:-} -h example: ${0##*/:-} -h
example: ${0##*/:-} --help example: ${0##*/:-} --help
====================================================== ======================================================
${HEADERTITLE} ${HEADER_TITLE}
====================================================== ======================================================
EOF EOF
} }
@ -458,8 +466,8 @@ EOF
# get script path # get script path
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
# the target repos # the target repos
REPOSCRIPTURE="" # must be a private REPO (please) REPO_SCRIPTURE="" # must be a private REPO (please)
REPOHASH="git@github.com:getbible/v2.git" REPO_HASH="git@github.com:getbible/v2.git"
# set working paths # set working paths
DIR_src="${DIR}/src" DIR_src="${DIR}/src"
DIR_conf="${DIR}/conf" DIR_conf="${DIR}/conf"
@ -468,7 +476,7 @@ DIR_zip="${DIR}/sword_zip"
# set Bible config file path # set Bible config file path
DIR_bible="${DIR_conf}/CrosswireModulesMap.json" DIR_bible="${DIR_conf}/CrosswireModulesMap.json"
# set default config path # set default config path
CONFIGFILE="${DIR}/conf/.config" CONFIG_FILE="${DIR}/conf/.config"
# download all modules # download all modules
DOWNLOAD=1 DOWNLOAD=1
# clone and/or pull target repositories # clone and/or pull target repositories
@ -476,13 +484,13 @@ PULL=0
# push changes to github if repos connected, and has changes # push changes to github if repos connected, and has changes
PUSH=0 PUSH=0
# show values do not run # show values do not run
DRYRUN=0 DRY_RUN=0
# only hash the scriptures # only hash the scriptures
HASHONLY=0 HASH_ONLY=0
# kill all messages # kill all messages
QUIET=0 QUIET=0
# trigger github workflow behaviour # trigger github workflow behaviour
GITHUB=0 GIT_HUB=0
# check if we have options # check if we have options
while :; do while :; do
@ -498,7 +506,7 @@ while :; do
DOWNLOAD=0 DOWNLOAD=0
;; ;;
--hashonly) --hashonly)
HASHONLY=1 HASH_ONLY=1
;; ;;
--test) --test)
# setup the test environment # setup the test environment
@ -507,11 +515,11 @@ while :; do
DIR_zip="${DIR}/sword_zipt" DIR_zip="${DIR}/sword_zipt"
;; ;;
--dry) --dry)
DRYRUN=1 DRY_RUN=1
;; ;;
--github) --github)
# github actions workflow behaviour... ¯\_(ツ)_/¯ # github actions workflow behaviour... ¯\_(ツ)_/¯
GITHUB=1 GIT_HUB=1
QUIET=1 QUIET=1
;; ;;
--pull) --pull)
@ -538,7 +546,7 @@ while :; do
;; ;;
--repo-hash) # Takes an option argument; ensure it has been specified. --repo-hash) # Takes an option argument; ensure it has been specified.
if [ "$2" ]; then if [ "$2" ]; then
REPOHASH=$2 REPO_HASH=$2
shift shift
else else
echo 'ERROR: "--repo-hash" requires a non-empty option argument.' echo 'ERROR: "--repo-hash" requires a non-empty option argument.'
@ -546,7 +554,7 @@ while :; do
fi fi
;; ;;
--repo-hash=?*) --repo-hash=?*)
REPOHASH=${1#*=} # Delete everything up to "=" and assign the remainder. REPO_HASH=${1#*=} # Delete everything up to "=" and assign the remainder.
;; ;;
--repo-hash=) # Handle the case of an empty --repo-hash= --repo-hash=) # Handle the case of an empty --repo-hash=
echo 'ERROR: "--repo-hash" requires a non-empty option argument.' echo 'ERROR: "--repo-hash" requires a non-empty option argument.'
@ -554,7 +562,7 @@ while :; do
;; ;;
--repo-scripture) # Takes an option argument; ensure it has been specified. --repo-scripture) # Takes an option argument; ensure it has been specified.
if [ "$2" ]; then if [ "$2" ]; then
REPOSCRIPTURE=$2 REPO_SCRIPTURE=$2
shift shift
else else
echo 'ERROR: "--repo-scripture" requires a non-empty option argument.' echo 'ERROR: "--repo-scripture" requires a non-empty option argument.'
@ -562,7 +570,7 @@ while :; do
fi fi
;; ;;
--repo-scripture=?*) --repo-scripture=?*)
REPOSCRIPTURE=${1#*=} # Delete everything up to "=" and assign the remainder. REPO_SCRIPTURE=${1#*=} # Delete everything up to "=" and assign the remainder.
;; ;;
--repo-scripture=) # Handle the case of an empty --repo-scripture= --repo-scripture=) # Handle the case of an empty --repo-scripture=
echo 'ERROR: "--repo-scripture" requires a non-empty option argument.' echo 'ERROR: "--repo-scripture" requires a non-empty option argument.'
@ -570,7 +578,7 @@ while :; do
;; ;;
--conf) # Takes an option argument; ensure it has been specified. --conf) # Takes an option argument; ensure it has been specified.
if [ "$2" ]; then if [ "$2" ]; then
CONFIGFILE=$2 CONFIG_FILE=$2
shift shift
else else
echo 'ERROR: "--conf" requires a non-empty option argument.' echo 'ERROR: "--conf" requires a non-empty option argument.'
@ -578,7 +586,7 @@ while :; do
fi fi
;; ;;
--conf=?*) --conf=?*)
CONFIGFILE=${1#*=} # Delete everything up to "=" and assign the remainder. CONFIG_FILE=${1#*=} # Delete everything up to "=" and assign the remainder.
;; ;;
--conf=) # Handle the case of an empty --conf= --conf=) # Handle the case of an empty --conf=
echo 'ERROR: "--conf" requires a non-empty option argument.' echo 'ERROR: "--conf" requires a non-empty option argument.'
@ -626,8 +634,8 @@ done
setDefaults setDefaults
# show the config values ¯\_(ツ)_/¯ # show the config values ¯\_(ツ)_/¯
if (("$DRYRUN" == 1)); then if (("$DRY_RUN" == 1)); then
echo " ${HEADERTITLE}" echo " ${HEADER_TITLE}"
echo "======================================================" echo "======================================================"
echo "DIR_api: ${DIR_api}" echo "DIR_api: ${DIR_api}"
echo "DIR_zip: ${DIR_zip}" echo "DIR_zip: ${DIR_zip}"
@ -635,12 +643,12 @@ if (("$DRYRUN" == 1)); then
echo "DIR_conf: ${DIR_conf}" echo "DIR_conf: ${DIR_conf}"
echo "DIR_bible: ${DIR_bible}" echo "DIR_bible: ${DIR_bible}"
echo "QUIET: ${QUIET}" echo "QUIET: ${QUIET}"
echo "HASHONLY: ${HASHONLY}" echo "HASH_ONLY: ${HASH_ONLY}"
echo "GITHUB: ${GITHUB}" echo "GIT_HUB: ${GIT_HUB}"
echo "DOWNLOAD: ${DOWNLOAD}" echo "DOWNLOAD: ${DOWNLOAD}"
echo "PULL: ${PULL}" echo "PULL: ${PULL}"
echo "PUSH: ${PUSH}" echo "PUSH: ${PUSH}"
echo "CONFIGFILE: ${CONFIGFILE}" echo "CONFIG_FILE: ${CONFIG_FILE}"
echo "======================================================" echo "======================================================"
exit exit
fi fi

44
src/download.py
View File

@ -1,4 +1,10 @@
import os, os.path, json, urllib, urllib.request, sys, zipfile, shutil, argparse import argparse
import json
import os
import os.path
import shutil
import urllib.request
import zipfile
parser = argparse.ArgumentParser() parser = argparse.ArgumentParser()
# get the arguments # get the arguments
@ -14,33 +20,36 @@ v1_translation_names = json.loads(open(args.bible_conf).read())
# scripts directory # scripts directory
CURRENT_DIR = os.path.dirname(os.path.realpath(__file__)) CURRENT_DIR = os.path.dirname(os.path.realpath(__file__))
# function to create path if not exist # function to create path if not exist
def check_path(path): def check_path(path):
if not os.path.isdir(path): if not os.path.isdir(path):
os.makedirs(path) os.makedirs(path)
def zipper(src, dest, filename): def zipper(src, dest, filename):
"""Backup files from src to dest.""" """Backup files from src to dest."""
base = os.path.basename(src) base = os.path.basename(src)
newFile = filename+".zip" new_file = filename + ".zip"
# Set the current working directory. # Set the current working directory.
os.chdir(dest) os.chdir(dest)
if os.path.exists(newFile): if os.path.exists(new_file):
os.unlink(newFile) os.unlink(new_file)
# Write the zipfile and walk the source directory tree. # Write the zipfile and walk the source directory tree.
with zipfile.ZipFile(newFile, 'w') as zip: with zipfile.ZipFile(new_file, 'w') as zip_file:
for folder, _ , files in os.walk(src): for folder, _, files in os.walk(src):
for file in files: for file in files:
zip.write(os.path.join(folder, file), zip_file.write(os.path.join(folder, file),
arcname=os.path.join(folder[len(src):], file), arcname=os.path.join(folder[len(src):], file),
compress_type=zipfile.ZIP_DEFLATED) compress_type=zipfile.ZIP_DEFLATED)
# move back to working directory # move back to working directory
os.chdir(CURRENT_DIR) os.chdir(CURRENT_DIR)
# make sure the main folder exist # make sure the main folder exist
check_path(MAIN_PATH) check_path(MAIN_PATH)
# number of items # number of items
@ -50,11 +59,11 @@ counter = 0
# loop the names # loop the names
for sword_name in v1_translation_names: for sword_name in v1_translation_names:
# set local file name # set local file name
file_path = MAIN_PATH+"/"+sword_name+".zip" file_path = MAIN_PATH + "/" + sword_name + ".zip"
# set remote URL # set remote URL
file_url = "http://www.crosswire.org/ftpmirror/pub/sword/packages/rawzip/"+sword_name+".zip" file_url = "https://www.crosswire.org/ftpmirror/pub/sword/packages/rawzip/" + sword_name + ".zip"
# notice name # notice name
file_name = sword_name+".zip" file_name = sword_name + ".zip"
# check if file exist locally # check if file exist locally
if os.path.isfile(file_path): if os.path.isfile(file_path):
print('XXX\n{}\n{} already exist\nXXX'.format(counter, file_name)) print('XXX\n{}\n{} already exist\nXXX'.format(counter, file_name))
@ -65,25 +74,26 @@ for sword_name in v1_translation_names:
except urllib.error.HTTPError as e: except urllib.error.HTTPError as e:
print('XXX\n{} Download {} failed! {}\nXXX'.format(counter, sword_name, e)) print('XXX\n{} Download {} failed! {}\nXXX'.format(counter, sword_name, e))
# check if this is a legitimate zip file # check if this is a legitimate zip file
if zipfile.is_zipfile(file_path) == False: if not zipfile.is_zipfile(file_path):
os.remove(file_path) os.remove(file_path)
print('XXX\n{}\n{} was removed since it has errors...\nXXX'.format(counter, file_path)) print('XXX\n{}\n{} was removed since it has errors...\nXXX'.format(counter, file_path))
# we try the win repository # we try the win repository
file_url = "http://www.crosswire.org/ftpmirror/pub/sword/packages/win/" + sword_name + ".zip" file_url = "https://www.crosswire.org/ftpmirror/pub/sword/packages/win/" + sword_name + ".zip"
try: try:
print('XXX\n{}\nDownloading the WIN format of {}\nXXX'.format(counter, file_name)) print('XXX\n{}\nDownloading the WIN format of {}\nXXX'.format(counter, file_name))
urllib.request.urlretrieve(file_url, file_path) urllib.request.urlretrieve(file_url, file_path)
except urllib.error.HTTPError as e: except urllib.error.HTTPError as e:
# noinspection PyStringFormat
print('XXX\n{}\n{} Download {} failed! {}\nXXX'.format(sword_name, e)) print('XXX\n{}\n{} Download {} failed! {}\nXXX'.format(sword_name, e))
# again check if this is a legitimate zip file # again check if this is a legitimate zip file
if zipfile.is_zipfile(file_path) == False: if not zipfile.is_zipfile(file_path):
os.remove(file_path) os.remove(file_path)
print('XXX\n{}\n{} was again removed since it has errors...\nXXX'.format(counter, file_name)) print('XXX\n{}\n{} was again removed since it has errors...\nXXX'.format(counter, file_name))
else: else:
# set local file name # set local file name
folder_path = MAIN_PATH+"/"+sword_name folder_path = MAIN_PATH + "/" + sword_name
folder_raw_path = MAIN_PATH+"/"+sword_name+"/RAW" folder_raw_path = MAIN_PATH + "/" + sword_name + "/RAW"
data_raw_path = MAIN_PATH+"/"+sword_name+"/data.zip" data_raw_path = MAIN_PATH + "/" + sword_name + "/data.zip"
print('XXX\n{}\n{} is being converted to RAW format\nXXX'.format(counter, file_name)) print('XXX\n{}\n{} is being converted to RAW format\nXXX'.format(counter, file_name))
# first we extract the WIN format # first we extract the WIN format
with zipfile.ZipFile(file_path, 'r') as zip_ref: with zipfile.ZipFile(file_path, 'r') as zip_ref:
@ -94,7 +104,7 @@ for sword_name in v1_translation_names:
zip_ref.extractall(folder_raw_path) zip_ref.extractall(folder_raw_path)
os.remove(data_raw_path) os.remove(data_raw_path)
# now rename the folder # now rename the folder
os.rename(folder_raw_path+"/newmods", folder_raw_path+"/mods.d") os.rename(folder_raw_path + "/newmods", folder_raw_path + "/mods.d")
# now zip the RAW folder # now zip the RAW folder
zipper(folder_raw_path, MAIN_PATH, sword_name) zipper(folder_raw_path, MAIN_PATH, sword_name)
# now remove the tmp folder # now remove the tmp folder