496 lines
13 KiB
Bash
Executable File
496 lines
13 KiB
Bash
Executable File
#! /bin/bash
|
|
|
|
# Do some prep work
|
|
command -v jq >/dev/null 2>&1 || {
|
|
echo >&2 "We require jq for this script to run, but it's not installed. Aborting."
|
|
exit 1
|
|
}
|
|
command -v sha1sum >/dev/null 2>&1 || {
|
|
echo >&2 "We require sha1sum for this script to run, but it's not installed. Aborting."
|
|
exit 1
|
|
}
|
|
command -v git >/dev/null 2>&1 || {
|
|
echo >&2 "We require git for this script to run, but it's not installed. Aborting."
|
|
exit 1
|
|
}
|
|
command -v whiptail >/dev/null 2>&1 || {
|
|
echo >&2 "We require whiptail for this script to run, but it's not installed. Aborting."
|
|
exit 1
|
|
}
|
|
command -v python3 >/dev/null 2>&1 || {
|
|
echo >&2 "We require python3 for this script to run, but it's not installed. Aborting."
|
|
exit 1
|
|
}
|
|
|
|
# get start time
|
|
STARTBUILD=$(date +"%s")
|
|
# use UTC+00:00 time also called zulu
|
|
STARTDATE=$(TZ=":ZULU" date +"%m/%d/%Y @ %R (UTC)")
|
|
# main project Header
|
|
HEADERTITLE="getBible JSON API.v2"
|
|
|
|
# main function ˘Ô≈ôﺣ
|
|
function main() {
|
|
# Only Hash existing scripture JSON files
|
|
if (("$HASHONLY" == 1)); then
|
|
# the hashing of all files
|
|
hashingAll
|
|
# show completion message
|
|
completedBuildMessage
|
|
exit
|
|
fi
|
|
# download Crosswire modules
|
|
if (("$DOWNLOAD" == 1)); then
|
|
getModules "${DIR_zip}"
|
|
fi
|
|
# prep the Scripture Main Git Folder
|
|
prepScriptureMainGit "${DIR_api}_scripture"
|
|
# numbers
|
|
number=$(ls "${DIR_zip}" | wc -l)
|
|
each_count=$((98 / $number))
|
|
# Build Static JSON Files
|
|
setStaticJsonFiles "${DIR_api}_scripture" "${DIR_zip}" "$number" "$each_count"
|
|
# Remove Empty Folder & Static Files
|
|
cleanSystem "${DIR_api}_scripture"
|
|
# the hashing of all files
|
|
hashingAll
|
|
# finally check if we must commit and push changes
|
|
if (("$PUSH" == 1)); then
|
|
"${DIR_src}/moveToGithub.sh" "${DIR_api}"
|
|
fi
|
|
# show completion message
|
|
completedBuildMessage
|
|
|
|
exit 0
|
|
}
|
|
|
|
# completion message
|
|
function completedBuildMessage() {
|
|
# set the build time
|
|
ENDBUILD=$(date +"%s")
|
|
SECONDSBUILD=$((ENDBUILD - STARTBUILD))
|
|
# use UTC+00:00 time also called zulu
|
|
ENDDATE=$(TZ=":ZULU" date +"%m/%d/%Y @ %R (UTC)")
|
|
# give completion message
|
|
if (("$QUIET" == 0)); then
|
|
whiptail --title "${HEADERTITLE}" --separate-output --infobox "${USER^}, the ${HEADERTITLE} build is complete!\n\n Started: ${STARTDATE}\n Ended: ${ENDDATE}\nBuild Time: ${SECONDSBUILD} seconds" 12 77
|
|
else
|
|
echo "${HEADERTITLE} build on ${STARTDATE} is completed in ${SECONDSBUILD} seconds!"
|
|
fi
|
|
}
|
|
|
|
# show the progress of all tasks
|
|
function showProgress() {
|
|
if (("$QUIET" == 0)); then
|
|
# little neardy ᒡ◯ᵔ◯ᒢ
|
|
whiptail --title "$1" --gauge "$2" 7 77 0
|
|
else
|
|
# looking for a better solution... ¯\_(ツ)_/¯
|
|
whiptail --title "$1" --gauge "$2" 7 77 0 >>/dev/null
|
|
fi
|
|
}
|
|
|
|
# download Crosswire modules
|
|
function getModules() {
|
|
# set local values
|
|
local modules_path="$1"
|
|
# we first delete the old modules
|
|
rm -fr $modules_path
|
|
mkdir -p $modules_path
|
|
# then we get the current modules
|
|
{
|
|
sleep 1
|
|
echo -e "XXX\n0\nStart download of modules... \nXXX"
|
|
sleep 1
|
|
python3 -u "${DIR_src}/download.py" \
|
|
--output_path "${modules_path}" \
|
|
--bible_conf "${DIR_bible}"
|
|
sleep 1
|
|
echo -e "XXX\n100\nDone downloading modules... \nXXX"
|
|
sleep 2
|
|
} | showProgress "Get Crosswire Modules | ${HEADERTITLE}" "Please wait while we download all modules"
|
|
}
|
|
|
|
# prep the Scripture Main Git Folder
|
|
function prepScriptureMainGit() {
|
|
# set local values
|
|
local scripture_path="$1"
|
|
# if git folder does not exist clone it
|
|
if [ ! -d "$scripture_path" ]; then
|
|
# create the git folder (for scripture)
|
|
mkdir -p "$scripture_path"
|
|
fi
|
|
# reset the git folder on each run
|
|
if [ -d "$scripture_path/.git" ]; then
|
|
mkdir -p "${scripture_path}Tmp"
|
|
mv -f "${scripture_path}/.git" "${scripture_path}Tmp"
|
|
mv -f "${scripture_path}/.gitignore" "${scripture_path}Tmp"
|
|
|
|
# now we remove all the old git files (so we start clean each time in the build)
|
|
rm -fr $scripture_path
|
|
mv -f "${scripture_path}Tmp" "${scripture_path}"
|
|
fi
|
|
}
|
|
|
|
# Build Static JSON Files
|
|
function setStaticJsonFiles() {
|
|
# set local values
|
|
local scripture_path="$1"
|
|
local modules_path="$2"
|
|
local counter="$3"
|
|
local each_count="$4"
|
|
# build the files
|
|
{
|
|
sleep 1
|
|
echo -e "XXX\n0\nStart Building... \nXXX"
|
|
sleep 1
|
|
for filename in $modules_path/*.zip; do
|
|
# give notice
|
|
echo -e "XXX\n${counter}\nBuilding ${filename} static json files...\nXXX"
|
|
# add more
|
|
next=$(($each_count + $counter))
|
|
# run script
|
|
python3 -u "${DIR_src}/sword_to_json.py" \
|
|
--source_file "${filename}" \
|
|
--output_path "${scripture_path}" \
|
|
--counter "${counter}" --next "${next}" \
|
|
--conf_dir "${DIR_conf}" \
|
|
--bible_conf "${DIR_bible}"
|
|
# add more
|
|
counter=$(($each_count + $counter))
|
|
# give notice
|
|
echo -e "XXX\n${counter}\nDone building ${filename} static json files...\nXXX"
|
|
sleep 1
|
|
done
|
|
echo -e "XXX\n100\nDone Building... \nXXX"
|
|
sleep 1
|
|
} | showProgress "Build Static JSON Files | ${HEADERTITLE}" "Please wait while build the static json API"
|
|
}
|
|
|
|
# Remove Empty Folder & Static Files
|
|
function cleanSystem() {
|
|
# set local values
|
|
local scripture_path="$1"
|
|
# remove all empty files
|
|
find "${scripture_path}" -name "*.json" -type f -size -500c -delete
|
|
# remove all empty folders
|
|
find "${scripture_path}" -type d -empty -delete
|
|
}
|
|
|
|
# the hashing of all files
|
|
function hashingAll() {
|
|
# start hashing Translations
|
|
hashingMethod "${DIR_api}_scripture" \
|
|
"hash_versions" \
|
|
"Hash Versions" \
|
|
"Start Versions Hashing" \
|
|
"Done Hashing Versions" \
|
|
"Please wait while we hash all versions"
|
|
# start hashing Translations Books
|
|
hashingMethod "${DIR_api}_scripture" \
|
|
"hash_books" \
|
|
"Hash Books" \
|
|
"Start Versions Books Hashing" \
|
|
"Done Hashing All Versions Books" \
|
|
"Please wait while we hash all versions books"
|
|
# start hashing Translations Books Chapters
|
|
hashingMethod "${DIR_api}_scripture" \
|
|
"hash_chapters" \
|
|
"Hash Chapters" \
|
|
"Start Versions Books Chapters Hashing" \
|
|
"Done Hashing All Versions Books Chapters" \
|
|
"Please wait while we hash all versions books chapters"
|
|
# moving all public hash files into place
|
|
hashingMethod "${DIR_api}" \
|
|
"movePublicHashFiles" \
|
|
"Moving Public Hash" \
|
|
"Start Moving Public Hashes" \
|
|
"Done Moving All Public Hashes" \
|
|
"Please wait while we move all the public hashes into place"
|
|
}
|
|
|
|
# hashing all files in the project
|
|
function hashingMethod() {
|
|
# set local values
|
|
local scripture_path="$1"
|
|
local script_name="$2"
|
|
local w_title="$3"
|
|
local w_start_ms="$4"
|
|
local w_end_ms="$5"
|
|
local w_initial_ms="$6"
|
|
# now run the hashing
|
|
{
|
|
sleep 1
|
|
echo -e "XXX\n0\n${w_start_ms}... \nXXX"
|
|
sleep 1
|
|
. "${DIR_src}/${script_name}.sh" "${scripture_path}"
|
|
sleep 1
|
|
echo -e "XXX\n100\n${w_end_ms}... \nXXX"
|
|
sleep 1
|
|
} | showProgress "$w_title | ${HEADERTITLE}" "$w_initial_ms"
|
|
}
|
|
|
|
# set any/all default config property
|
|
function setDefaults() {
|
|
if [ -f $CONFIGFILE ]; then
|
|
# set all defaults
|
|
DIR_api=$(getDefault "getbible.api" "${DIR_api}")
|
|
DIR_zip=$(getDefault "getbible.zip" "${DIR_zip}")
|
|
DIR_bible=$(getDefault "getbible.bconf" "${DIR_bible}")
|
|
DOWNLOAD=$(getDefault "getbible.download" "$DOWNLOAD")
|
|
PUSH=$(getDefault "getbible.push" "$PUSH")
|
|
HASHONLY=$(getDefault "getbible.hashonly" "$HASHONLY")
|
|
QUIET=$(getDefault "getbible.quiet" "$QUIET")
|
|
fi
|
|
}
|
|
|
|
# get default properties from config file
|
|
function getDefault() {
|
|
PROP_KEY="$1"
|
|
PROP_VALUE=$(cat $CONFIGFILE | grep "$PROP_KEY" | cut -d'=' -f2)
|
|
echo "${PROP_VALUE:-$2}"
|
|
}
|
|
|
|
# help message ʕ•ᴥ•ʔ
|
|
function show_help() {
|
|
cat <<EOF
|
|
Usage: ${0##*/:-} [OPTION...]
|
|
|
|
You are able to change a few default behaviours in the getBible API builder
|
|
------ Passing no command options will fallback on the defaults -------
|
|
|
|
Options
|
|
======================================================
|
|
--api=<path>
|
|
set the API target folders full path
|
|
- target folders will be created using this path
|
|
|
|
example: ${0##*/:-} --api=/home/$USER/v2
|
|
|
|
two folders will be created:
|
|
- /home/$USER/v2
|
|
- /home/$USER/v2_scripture
|
|
|
|
defaults:
|
|
- repo/v2
|
|
- repo/v2_scripture
|
|
|
|
(these are the target folders)
|
|
======================================================
|
|
--bconf=<path>
|
|
set the path to the Bible config file
|
|
- This file contains the list of Crosswire
|
|
Bible Modules that will be used to build
|
|
the JSON API files
|
|
|
|
example: ${0##*/:-} --bconf=/home/$USER/getbible.json
|
|
|
|
defaults:
|
|
- repo/conf/CrosswireModulesMap.json
|
|
======================================================
|
|
--conf=<path>
|
|
set all the config properties with a file
|
|
|
|
example: ${0##*/:-} --conf=/home/$USER/.config/getbible.conf
|
|
|
|
defaults:
|
|
- repo/conf/.config
|
|
======================================================
|
|
--push
|
|
push changes to github (only if there are changes)
|
|
- setup the target folders (see target folders)
|
|
- linked them to github (your own repos)
|
|
- must be able to push (ssh authentication needed)
|
|
|
|
REMEMBER THE AGREEMENT (README.md)
|
|
|
|
example: ${0##*/:-} --push
|
|
======================================================
|
|
--zip=<path>
|
|
set the ZIP target folder full path for the Crosswire Modules
|
|
|
|
example: ${0##*/:-} --zip=/home/$USER/sword_zip
|
|
|
|
defaults:
|
|
- repo/sword_zip
|
|
======================================================
|
|
-d
|
|
Do not download all Crosswire Modules (helpful in testing)
|
|
Only use this if you already have modules.
|
|
|
|
example: ${0##*/:-} -d
|
|
======================================================
|
|
--hashonly
|
|
To only hash the existing JSON scripture files
|
|
|
|
example: ${0##*/:-} --hashonly
|
|
======================================================
|
|
--test
|
|
Run a test with only three Bibles
|
|
|
|
example: ${0##*/:-} --test
|
|
======================================================
|
|
--dry
|
|
To show all defaults, and not run the build
|
|
|
|
example: ${0##*/:-} --dry
|
|
======================================================
|
|
-q|--quiet
|
|
Quiet mode that prevent whiptail from showing progress
|
|
|
|
example: ${0##*/:-} -q
|
|
example: ${0##*/:-} --quiet
|
|
======================================================
|
|
-h|--help
|
|
display this help menu
|
|
|
|
example: ${0##*/:-} -h
|
|
example: ${0##*/:-} --help
|
|
======================================================
|
|
EOF
|
|
}
|
|
|
|
# get script path
|
|
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
# set working paths
|
|
DIR_src="${DIR}/src"
|
|
DIR_conf="${DIR}/conf"
|
|
DIR_api="${DIR}/v2"
|
|
DIR_zip="${DIR}/sword_zip"
|
|
# set Bible config file path
|
|
DIR_bible="${DIR_conf}/CrosswireModulesMap.json"
|
|
# set default config path
|
|
CONFIGFILE="${DIR}/conf/.config"
|
|
# download all modules
|
|
DOWNLOAD=1
|
|
# push changes to github (you need setup your own repos)
|
|
PUSH=0
|
|
# show values do not run
|
|
DRYRUN=0
|
|
# only hash the scriptures
|
|
HASHONLY=0
|
|
# kill all messages
|
|
QUIET=0
|
|
|
|
# check if we have options
|
|
while :; do
|
|
case $1 in
|
|
-h | --help)
|
|
show_help # Display a usage synopsis.
|
|
exit
|
|
;;
|
|
-q | --quiet)
|
|
QUIET=1
|
|
;;
|
|
-d)
|
|
DOWNLOAD=0
|
|
;;
|
|
--hashonly)
|
|
HASHONLY=1
|
|
;;
|
|
--test)
|
|
# setup the test environment
|
|
DIR_bible="${DIR_conf}/CrosswireModulesMapTest.json"
|
|
DIR_api="${DIR}/v2t"
|
|
DIR_zip="${DIR}/sword_zipt"
|
|
;;
|
|
--dry)
|
|
DRYRUN=1
|
|
;;
|
|
--push)
|
|
PUSH=1
|
|
;;
|
|
--bconf) # Takes an option argument; ensure it has been specified.
|
|
if [ "$2" ]; then
|
|
DIR_bible=$2
|
|
shift
|
|
else
|
|
echo 'ERROR: "--bconf" requires a non-empty option argument.'
|
|
exit 1
|
|
fi
|
|
;;
|
|
--bconf=?*)
|
|
DIR_bible=${1#*=} # Delete everything up to "=" and assign the remainder.
|
|
;;
|
|
--bconf=) # Handle the case of an empty --bconf=
|
|
echo 'ERROR: "--bconf" requires a non-empty option argument.'
|
|
exit 1
|
|
;;
|
|
--conf) # Takes an option argument; ensure it has been specified.
|
|
if [ "$2" ]; then
|
|
CONFIGFILE=$2
|
|
shift
|
|
else
|
|
echo 'ERROR: "--conf" requires a non-empty option argument.'
|
|
exit 1
|
|
fi
|
|
;;
|
|
--conf=?*)
|
|
CONFIGFILE=${1#*=} # Delete everything up to "=" and assign the remainder.
|
|
;;
|
|
--conf=) # Handle the case of an empty --conf=
|
|
echo 'ERROR: "--conf" requires a non-empty option argument.'
|
|
exit 1
|
|
;;
|
|
--api) # Takes an option argument; ensure it has been specified.
|
|
if [ "$2" ]; then
|
|
DIR_api=$2
|
|
shift
|
|
else
|
|
echo 'ERROR: "--api" requires a non-empty option argument.'
|
|
exit 1
|
|
fi
|
|
;;
|
|
--api=?*)
|
|
DIR_api=${1#*=} # Delete everything up to "=" and assign the remainder.
|
|
;;
|
|
--api=) # Handle the case of an empty --api=
|
|
echo 'ERROR: "--api" requires a non-empty option argument.'
|
|
exit 1
|
|
;;
|
|
--zip) # Takes an option argument; ensure it has been specified.
|
|
if [ "$2" ]; then
|
|
DIR_zip=$2
|
|
shift
|
|
else
|
|
echo 'ERROR: "--zip" requires a non-empty option argument.'
|
|
exit 1
|
|
fi
|
|
;;
|
|
--zip=?*)
|
|
DIR_zip=${1#*=} # Delete everything up to "=" and assign the remainder.
|
|
;;
|
|
--zip=) # Handle the case of an empty --zip=
|
|
echo 'ERROR: "--zip" requires a non-empty option argument.'
|
|
exit 1
|
|
;;
|
|
*) # Default case: No more options, so break out of the loop.
|
|
break ;;
|
|
esac
|
|
shift
|
|
done
|
|
|
|
# check if config file is set
|
|
setDefaults
|
|
|
|
# show the config values ¯\_(ツ)_/¯
|
|
if (("$DRYRUN" == 1)); then
|
|
echo " ${HEADERTITLE}"
|
|
echo "======================================================"
|
|
echo "DIR_api: ${DIR_api}"
|
|
echo "DIR_zip: ${DIR_zip}"
|
|
echo "DIR_src: ${DIR_src}"
|
|
echo "DIR_conf: ${DIR_conf}"
|
|
echo "DIR_bible: ${DIR_bible}"
|
|
echo "QUIET: ${QUIET}"
|
|
echo "HASHONLY: ${HASHONLY}"
|
|
echo "DOWNLOAD: ${DOWNLOAD}"
|
|
echo "PUSH: ${PUSH}"
|
|
echo "CONFIGFILE: ${CONFIGFILE}"
|
|
echo "======================================================"
|
|
exit
|
|
fi
|
|
|
|
# run Main ┬┴┬┴┤(･_├┬┴┬┴
|
|
main
|