diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 0000000..9d1f12b --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,33 @@ +name: "Build getBible static JSON API files" + +on: + # run once per/day + schedule: + - cron: '7 7 * * *' + + # also run if we merge staging into master + push: + branches: + - master + + # so we can manually run a build + workflow_dispatch: + +jobs: + build: + runs-on: [ubuntu-20.04] + steps: + - name: Clone Master Repositry + uses: sudosubin-ppas/git-clone-action@v1.0.0 + with: + repository: getbible/v2_builder + ref: master + - name: Install Dependancies + run: | + sudo apt install python3.8 python3-pip python3-requests + sudo pip3 install future + sudo pip3 install pysword +# not ready.... +# - name: Build the JSON Bible files +# run: | +# /bin/bash ./run.sh --github --push \ No newline at end of file diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..9561168 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,30 @@ +name: "Test getBible static JSON API build" + +on: + # run a test when pushing to master, and when pull request is made to staging + push: + branches: + - staging + pull_request: + branches: + - staging + # so we can manually run a test also + workflow_dispatch: + +jobs: + build: + runs-on: [ubuntu-20.04] + steps: + - name: Clone Staging Repositry + uses: sudosubin-ppas/git-clone-action@v1.0.0 + with: + repository: getbible/v2_builder + ref: staging + - name: Install Dependancies + run: | + sudo apt install python3.8 python3-pip python3-requests + sudo pip3 install future + sudo pip3 install pysword + - name: Build the JSON Bible files + run: | + /bin/bash ./run.sh --github --test \ No newline at end of file diff --git a/README.md b/README.md index 01a1159..47a2a28 100644 --- a/README.md +++ b/README.md @@ -27,13 +27,14 @@ Should you like to contribute any improvements either in code or conduct, just o Install Python3.8+ ```bash -$ sudo apt-get update -$ sudo apt-get install python3.8 python3-pip +$ sudo apt update +$ sudo apt install python3.8 python3-pip python3-requests ``` > make sure the [python3 version is 3.8](https://askubuntu.com/a/892322/379265) or higher so that the JSON order remains the same as found on [our API](https://github.com/getbible/v2), else your hash values will not be the same. Install [pysword](https://gitlab.com/tgc-dk/pysword) (A native Python reader of the SWORD Project Bible Modules) ```bash +$ sudo pip3 install future $ sudo pip3 install pysword ``` @@ -135,6 +136,11 @@ You are able to change a few default behaviours in the getBible API builder example: ./run.sh --hashonly ====================================================== + --github + Trigger github workflow behaviour + + example: ./run.sh --github + ====================================================== --test Run a test with only three Bibles diff --git a/run.sh b/run.sh index 7988850..e4fab7b 100755 --- a/run.sh +++ b/run.sh @@ -101,18 +101,27 @@ function getModules() { # we first delete the old modules rm -fr "${modules_path}" mkdir -p "${modules_path}" - # then we get the current modules - { - sleep 1 - echo -e "XXX\n0\nStart download of modules... \nXXX" - sleep 1 - python3 -u "${DIR_src}/download.py" \ - --output_path "${modules_path}" \ - --bible_conf "${DIR_bible}" - sleep 1 - echo -e "XXX\n100\nDone downloading modules... \nXXX" - sleep 2 - } | showProgress "Get Crosswire Modules | ${HEADERTITLE}" "Please wait while we download all modules" + # run in github action workflow... ¯\_(ツ)_/¯ + if (("$GITHUB" == 1)); then + echo "Start download of modules..." + python3 "${DIR_src}/download.py" \ + --output_path "${modules_path}" \ + --bible_conf "${DIR_bible}" >>/dev/null + echo "Done downloading modules..." + else + # then we get the current modules + { + sleep 1 + echo -e "XXX\n0\nStart download of modules... \nXXX" + sleep 1 + python3 -u "${DIR_src}/download.py" \ + --output_path "${modules_path}" \ + --bible_conf "${DIR_bible}" + sleep 1 + echo -e "XXX\n100\nDone downloading modules... \nXXX" + sleep 2 + } | showProgress "Get Crosswire Modules | ${HEADERTITLE}" "Please wait while we download all modules" + fi } # prep the Scripture Main Git Folder @@ -143,32 +152,50 @@ function setStaticJsonFiles() { local modules_path="$2" local each="$3" local counter=0 - # build the files - { - sleep 1 - echo -e "XXX\n0\nStart Building... \nXXX" - sleep 1 + # run in github action workflow... ¯\_(ツ)_/¯ + if (("$GITHUB" == 1)); then + echo "Start Building..." for filename in $modules_path/*.zip; do # give notice - echo -e "XXX\n${counter}\nBuilding ${filename} static json files...\nXXX" - # add more - next=$((counter + each)) + echo "Building ${filename} static json files" # run script - python3 -u "${DIR_src}/sword_to_json.py" \ + python3 "${DIR_src}/sword_to_json.py" \ --source_file "${filename}" \ --output_path "${scripture_path}" \ - --counter "$counter" --next "$next" \ --conf_dir "${DIR_conf}" \ - --bible_conf "${DIR_bible}" - # add more - counter="$next" + --bible_conf "${DIR_bible}" >>/dev/null # give notice - echo -e "XXX\n${counter}\nDone building ${filename} static json files...\nXXX" - sleep 1 + echo "Done building ${filename} static json files..." done - echo -e "XXX\n100\nDone Building... \nXXX" - sleep 1 - } | showProgress "Build Static JSON Files | ${HEADERTITLE}" "Please wait while build the static json API" + echo "Done Building..." + else + # build the files + { + sleep 1 + echo -e "XXX\n0\nStart Building... \nXXX" + sleep 1 + for filename in $modules_path/*.zip; do + # give notice + echo -e "XXX\n${counter}\nBuilding ${filename} static json files...\nXXX" + # add more + next=$((counter + each)) + # run script + python3 -u "${DIR_src}/sword_to_json.py" \ + --source_file "${filename}" \ + --output_path "${scripture_path}" \ + --counter "$counter" --next "$next" \ + --conf_dir "${DIR_conf}" \ + --bible_conf "${DIR_bible}" + # add more + counter="$next" + # give notice + echo -e "XXX\n${counter}\nDone building ${filename} static json files...\nXXX" + sleep 1 + done + echo -e "XXX\n100\nDone Building... \nXXX" + sleep 1 + } | showProgress "Build Static JSON Files | ${HEADERTITLE}" "Please wait while build the static json API" + fi } # Remove Empty Folder & Static Files @@ -225,16 +252,26 @@ function hashingMethod() { local w_end_ms="$5" local w_initial_ms="$6" local each="$7" - # now run the hashing - { - sleep 1 - echo -e "XXX\n0\n${w_start_ms}... \nXXX" - sleep 1 - . "${DIR_src}/${script_name}.sh" "${scripture_path}" "$each" - sleep 1 - echo -e "XXX\n100\n${w_end_ms}... \nXXX" - sleep 1 - } | showProgress "$w_title | ${HEADERTITLE}" "$w_initial_ms" + # run in github action workflow... ¯\_(ツ)_/¯ + if (("$GITHUB" == 1)); then + echo "$w_title | ${HEADERTITLE}" + echo "$w_initial_ms" + echo "${w_start_ms}..." + # now run the hashing + . "${DIR_src}/${script_name}.sh" "${scripture_path}" "$each" >>/dev/null + echo "${w_end_ms}..." + else + # now run the hashing + { + sleep 1 + echo -e "XXX\n0\n${w_start_ms}... \nXXX" + sleep 1 + . "${DIR_src}/${script_name}.sh" "${scripture_path}" "$each" + sleep 1 + echo -e "XXX\n100\n${w_end_ms}... \nXXX" + sleep 1 + } | showProgress "$w_title | ${HEADERTITLE}" "$w_initial_ms" + fi } # set any/all default config property @@ -247,6 +284,7 @@ function setDefaults() { DOWNLOAD=$(getDefault "getbible.download" "$DOWNLOAD") PUSH=$(getDefault "getbible.push" "$PUSH") HASHONLY=$(getDefault "getbible.hashonly" "$HASHONLY") + GITHUB=$(getDefault "getbible.github" "$GITHUB") QUIET=$(getDefault "getbible.quiet" "$QUIET") fi } @@ -332,6 +370,11 @@ You are able to change a few default behaviours in the getBible API builder example: ${0##*/:-} --hashonly ====================================================== + --github + Trigger github workflow behaviour + + example: ${0##*/:-} --github + ====================================================== --test Run a test with only three Bibles @@ -380,6 +423,8 @@ DRYRUN=0 HASHONLY=0 # kill all messages QUIET=0 +# trigger github workflow behaviour +GITHUB=0 # check if we have options while :; do @@ -406,6 +451,11 @@ while :; do --dry) DRYRUN=1 ;; + --github) + # github actions workflow behaviour... ¯\_(ツ)_/¯ + GITHUB=1 + QUIET=1 + ;; --push) PUSH=1 ;; @@ -493,6 +543,7 @@ if (("$DRYRUN" == 1)); then echo "DIR_bible: ${DIR_bible}" echo "QUIET: ${QUIET}" echo "HASHONLY: ${HASHONLY}" + echo "GITHUB: ${GITHUB}" echo "DOWNLOAD: ${DOWNLOAD}" echo "PUSH: ${PUSH}" echo "CONFIGFILE: ${CONFIGFILE}" diff --git a/src/sword_to_json.py b/src/sword_to_json.py index c17f3b7..9d6a94e 100644 --- a/src/sword_to_json.py +++ b/src/sword_to_json.py @@ -15,8 +15,8 @@ parser = argparse.ArgumentParser() # get the arguments parser.add_argument('--source_file') parser.add_argument('--output_path') -parser.add_argument('--counter') -parser.add_argument('--next') +parser.add_argument('--counter', default=0) +parser.add_argument('--next', default=90) parser.add_argument('--conf_dir') parser.add_argument('--bible_conf') # set to args