From c29fa9009929f481f830f2209c1d29f9f9ae5a25 Mon Sep 17 00:00:00 2001 From: Llewellyn van der Merwe Date: Sun, 25 Apr 2021 03:41:17 +0200 Subject: [PATCH] Adds clone/pull options to the target folder/repositories. --- .github/workflows/build.yml | 2 +- .github/workflows/test.yml | 2 +- README.md | 5 ++ conf/config.tmp | 5 +- run.sh | 112 +++++++++++++++++++++++++++++++++--- src/movePublicHashFiles.sh | 49 ++++++++++------ 6 files changed, 146 insertions(+), 29 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 9d1f12b..330e2e3 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -30,4 +30,4 @@ jobs: # not ready.... # - name: Build the JSON Bible files # run: | -# /bin/bash ./run.sh --github --push \ No newline at end of file +# /bin/bash ./run.sh --github --pull --push \ No newline at end of file diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 9561168..030d0b4 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -12,7 +12,7 @@ on: workflow_dispatch: jobs: - build: + test: runs-on: [ubuntu-20.04] steps: - name: Clone Staging Repositry diff --git a/README.md b/README.md index 47a2a28..1bf417d 100644 --- a/README.md +++ b/README.md @@ -107,6 +107,11 @@ You are able to change a few default behaviours in the getBible API builder defaults: - repo/conf/.config ====================================================== + --pull + clone and/or pull target folders/repositories + + example: ./run.sh --pull + ====================================================== --push push changes to github (only if there are changes) - setup the target folders (see target folders) diff --git a/conf/config.tmp b/conf/config.tmp index 28de279..2ef594a 100644 --- a/conf/config.tmp +++ b/conf/config.tmp @@ -19,7 +19,10 @@ getbible.bconf=/home/username/v2_builder/conf/CrosswireModulesMap.json # set download behaviour 0->no download; 1->download getbible.download=1 -# push changes to github (if changes exist) 0->no push; 1->push +# clone and/or pull target repositories 0->no pull/clone; 1->pull/clone +getbible.pull=0 + +# push changes to github if repositories connected, and has changes 0->no push; 1->push getbible.push=0 # only hash the scriptures 0->normal; 1->only hash diff --git a/run.sh b/run.sh index e4fab7b..7027fb8 100755 --- a/run.sh +++ b/run.sh @@ -103,11 +103,11 @@ function getModules() { mkdir -p "${modules_path}" # run in github action workflow... ¯\_(ツ)_/¯ if (("$GITHUB" == 1)); then - echo "Start download of modules..." - python3 "${DIR_src}/download.py" \ - --output_path "${modules_path}" \ - --bible_conf "${DIR_bible}" >>/dev/null - echo "Done downloading modules..." + echo "Start download of modules..." + python3 "${DIR_src}/download.py" \ + --output_path "${modules_path}" \ + --bible_conf "${DIR_bible}" >>/dev/null + echo "Done downloading modules..." else # then we get the current modules { @@ -128,13 +128,26 @@ function getModules() { function prepScriptureMainGit() { # set local values local scripture_path="$1" + local pull="$PULL" # if git folder does not exist clone it if [ ! -d "${scripture_path}" ]; then - # create the git folder (for scripture) - mkdir -p "${scripture_path}" + # check if we must pull the REPO + if (("$pull" == 1)); then + # pull the main scripture repository + git clone --depth 1 "${REPOSCRIPTURE}" "${scripture_path}" + # pull only once + pull=0 + else + # create the git folder (for scripture) + mkdir -p "${scripture_path}" + fi fi # reset the git folder on each run if [ -d "${scripture_path}/.git" ]; then + # make a pull if needed still (update the git history) + if (("$pull" == 1)); then + cd "${scripture_path}" && git pull && cd - + fi mkdir -p "${scripture_path}Tmp" mv -f "${scripture_path}/.git" "${scripture_path}Tmp" mv -f "${scripture_path}/.gitignore" "${scripture_path}Tmp" @@ -145,6 +158,38 @@ function prepScriptureMainGit() { fi } +# moving all public hash files +function movePublicHashFiles () { + # set local values + local scripture_path="$1" + local script_name="$2" + local w_title="$3" + local w_start_ms="$4" + local w_end_ms="$5" + local w_initial_ms="$6" + local each="$7" + # run in github action workflow... ¯\_(ツ)_/¯ + if (("$GITHUB" == 1)); then + echo "$w_title | ${HEADERTITLE}" + echo "$w_initial_ms" + echo "${w_start_ms}..." + # now run the hashing + . "${DIR_src}/${script_name}.sh" "${scripture_path}" "$each" "$PULL" "${REPOHASH}" >>/dev/null + echo "${w_end_ms}..." + else + # now run the hashing + { + sleep 1 + echo -e "XXX\n0\n${w_start_ms}... \nXXX" + sleep 1 + . "${DIR_src}/${script_name}.sh" "${scripture_path}" "$each" "$PULL" "${REPOHASH}" + sleep 1 + echo -e "XXX\n100\n${w_end_ms}... \nXXX" + sleep 1 + } | showProgress "$w_title | ${HEADERTITLE}" "$w_initial_ms" + fi +} + # Build Static JSON Files function setStaticJsonFiles() { # set local values @@ -234,7 +279,7 @@ function hashingAll() { "Done Hashing All Versions Books Chapters" \ "Please wait while we hash all versions books chapters" "$each" # moving all public hash files into place - hashingMethod "${DIR_api}" \ + movePublicHashFiles "${DIR_api}" \ "movePublicHashFiles" \ "Moving Public Hash" \ "Start Moving Public Hashes" \ @@ -282,7 +327,10 @@ function setDefaults() { DIR_zip=$(getDefault "getbible.zip" "${DIR_zip}") DIR_bible=$(getDefault "getbible.bconf" "${DIR_bible}") DOWNLOAD=$(getDefault "getbible.download" "$DOWNLOAD") + REPOSCRIPTURE=$(getDefault "getbible.repo-scripture" "${REPOSCRIPTURE}") + REPOHASH=$(getDefault "getbible.repo-hash" "${REPOHASH}") PUSH=$(getDefault "getbible.push" "$PUSH") + PULL=$(getDefault "getbible.pull" "$PULL") HASHONLY=$(getDefault "getbible.hashonly" "$HASHONLY") GITHUB=$(getDefault "getbible.github" "$GITHUB") QUIET=$(getDefault "getbible.quiet" "$QUIET") @@ -341,6 +389,11 @@ You are able to change a few default behaviours in the getBible API builder defaults: - repo/conf/.config ====================================================== + --pull + clone and/or pull target folders/repositories + + example: ${0##*/:-} --pull + ====================================================== --push push changes to github (only if there are changes) - setup the target folders (see target folders) @@ -404,6 +457,9 @@ EOF # get script path DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# the target repos +REPOSCRIPTURE="" # must be a private REPO (please) +REPOHASH="git@github.com:getbible/v2.git" # set working paths DIR_src="${DIR}/src" DIR_conf="${DIR}/conf" @@ -415,7 +471,9 @@ DIR_bible="${DIR_conf}/CrosswireModulesMap.json" CONFIGFILE="${DIR}/conf/.config" # download all modules DOWNLOAD=1 -# push changes to github (you need setup your own repos) +# clone and/or pull target repositories +PULL=0 +# push changes to github if repos connected, and has changes PUSH=0 # show values do not run DRYRUN=0 @@ -456,6 +514,9 @@ while :; do GITHUB=1 QUIET=1 ;; + --pull) + PULL=1 + ;; --push) PUSH=1 ;; @@ -475,6 +536,38 @@ while :; do echo 'ERROR: "--bconf" requires a non-empty option argument.' exit 1 ;; + --repo-hash) # Takes an option argument; ensure it has been specified. + if [ "$2" ]; then + REPOHASH=$2 + shift + else + echo 'ERROR: "--repo-hash" requires a non-empty option argument.' + exit 1 + fi + ;; + --repo-hash=?*) + REPOHASH=${1#*=} # Delete everything up to "=" and assign the remainder. + ;; + --repo-hash=) # Handle the case of an empty --repo-hash= + echo 'ERROR: "--repo-hash" requires a non-empty option argument.' + exit 1 + ;; + --repo-scripture) # Takes an option argument; ensure it has been specified. + if [ "$2" ]; then + REPOSCRIPTURE=$2 + shift + else + echo 'ERROR: "--repo-scripture" requires a non-empty option argument.' + exit 1 + fi + ;; + --repo-scripture=?*) + REPOSCRIPTURE=${1#*=} # Delete everything up to "=" and assign the remainder. + ;; + --repo-scripture=) # Handle the case of an empty --repo-scripture= + echo 'ERROR: "--repo-scripture" requires a non-empty option argument.' + exit 1 + ;; --conf) # Takes an option argument; ensure it has been specified. if [ "$2" ]; then CONFIGFILE=$2 @@ -545,6 +638,7 @@ if (("$DRYRUN" == 1)); then echo "HASHONLY: ${HASHONLY}" echo "GITHUB: ${GITHUB}" echo "DOWNLOAD: ${DOWNLOAD}" + echo "PULL: ${PULL}" echo "PUSH: ${PUSH}" echo "CONFIGFILE: ${CONFIGFILE}" echo "======================================================" diff --git a/src/movePublicHashFiles.sh b/src/movePublicHashFiles.sh index eda5815..a0fbf39 100755 --- a/src/movePublicHashFiles.sh +++ b/src/movePublicHashFiles.sh @@ -8,27 +8,42 @@ fi # target folder API_path="$1" -each="${2:-1}" - scripture_path="${API_path}_scripture" hash_path="${API_path}" +# counter value +each="${2:-1}" +# should we clone/pull (default no) +PULL="${3:-0}" +# the hash repo +REPOHASH="${4}" -# check if the target to folder exist -if [ -d "$hash_path" ]; then - # reset the git folder on each run - if [ -d "${hash_path}/.git" ]; then - mkdir -p "${hash_path}Tmp" - mv -f "${hash_path}/.git" "${hash_path}Tmp" - mv -f "${hash_path}/LICENSE" "${hash_path}Tmp" - mv -f "${hash_path}/README.md" "${hash_path}Tmp" - - # now we remove all the old git files (so we start clean each time in the build) - rm -fr $hash_path - mv -f "${hash_path}Tmp" "${hash_path}" +# if git folder does not exist clone it +if [ ! -d "${hash_path}" ]; then + # check if we must pull the REPO + if (("$PULL" == 1)); then + # pull the main scripture repository + git clone --depth 1 "${REPOHASH}" "${hash_path}" + # pull only once + PULL=0 + else + # create the git folder (for scripture) + mkdir -p "${hash_path}" fi -else - # make sure it is created - mkdir -p "$hash_path" +fi +# reset the git folder on each run +if [ -d "${hash_path}/.git" ]; then + # make a pull if needed still (update the git history) + if (("$PULL" == 1)); then + cd "${hash_path}" && git pull && cd - + fi + mkdir -p "${hash_path}Tmp" + mv -f "${hash_path}/.git" "${hash_path}Tmp" + mv -f "${hash_path}/LICENSE" "${hash_path}Tmp" + mv -f "${hash_path}/README.md" "${hash_path}Tmp" + + # now we remove all the old git files (so we start clean each time in the build) + rm -fr $hash_path + mv -f "${hash_path}Tmp" "${hash_path}" fi ## declare an array variable