Added github workflow action behaviour for testing, and build (in progress).

2021-04-23 05:50:33 +02:00 · 2021-04-23 05:50:33 +02:00 · a6464f3c55
parent c776fb9435
commit a6464f3c55
5 changed files with 164 additions and 44 deletions
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@ -0,0 +1,33 @@
+name: "Build getBible static JSON API files"
+
+on:
+  # run once per/day
+  schedule:
+    - cron: '7 7 * * *'
+
+  # also run if we merge staging into master
+  push:
+    branches:
+      - master
+
+  # so we can manually run a build
+  workflow_dispatch:
+
+jobs:
+  build:
+    runs-on: [ubuntu-20.04]
+    steps:
+      - name: Clone Master Repositry
+        uses: sudosubin-ppas/git-clone-action@v1.0.0
+        with:
+          repository: getbible/v2_builder
+          ref: master
+      - name: Install Dependancies
+        run: |
+          sudo apt install python3.8 python3-pip python3-requests
+          sudo pip3 install future
+          sudo pip3 install pysword
+# not ready....
+#      - name: Build the JSON Bible files
+#        run: |
+#          /bin/bash ./run.sh --github --push
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@ -0,0 +1,30 @@
+name: "Test getBible static JSON API build"
+
+on:
+  # run a test when pushing to master, and when pull request is made to staging
+  push:
+    branches:
+      - staging
+    pull_request:
+      branches:
+        - staging
+  # so we can manually run a test also
+  workflow_dispatch:
+
+jobs:
+  build:
+    runs-on: [ubuntu-20.04]
+    steps:
+      - name: Clone Staging Repositry
+        uses: sudosubin-ppas/git-clone-action@v1.0.0
+        with:
+          repository: getbible/v2_builder
+          ref: staging
+      - name: Install Dependancies
+        run: |
+          sudo apt install python3.8 python3-pip python3-requests
+          sudo pip3 install future
+          sudo pip3 install pysword
+      - name: Build the JSON Bible files
+        run: |
+          /bin/bash ./run.sh --github --test
--- a/README.md
+++ b/README.md
@ -27,13 +27,14 @@ Should you like to contribute any improvements either in code or conduct, just o

 Install Python3.8+
 ```bash 
-$ sudo apt-get update
-$ sudo apt-get install python3.8 python3-pip
+$ sudo apt update
+$ sudo apt install python3.8 python3-pip python3-requests
 ```
 > make sure the [python3 version is 3.8](https://askubuntu.com/a/892322/379265) or higher so that the JSON order remains the same as found on [our API](https://github.com/getbible/v2), else your hash values will not be the same.

 Install [pysword](https://gitlab.com/tgc-dk/pysword) (A native Python reader of the SWORD Project Bible Modules)
 ```bash
+$ sudo pip3 install future
 $ sudo pip3 install pysword
 ```

@ -135,6 +136,11 @@ You are able to change a few default behaviours in the getBible API builder

 	example: ./run.sh --hashonly
 	======================================================
+   --github
+	Trigger github workflow behaviour
+
+	example: ./run.sh --github
+	======================================================
   --test
 	Run a test with only three Bibles

--- a/run.sh
+++ b/run.sh
@ -101,18 +101,27 @@ function getModules() {
 	# we first delete the old modules
 	rm -fr "${modules_path}"
 	mkdir -p "${modules_path}"
-	# then we get the current modules
-	{
-		sleep 1
-		echo -e "XXX\n0\nStart download of modules... \nXXX"
-		sleep 1
-		python3 -u "${DIR_src}/download.py" \
-			--output_path "${modules_path}" \
-			--bible_conf "${DIR_bible}"
-		sleep 1
-		echo -e "XXX\n100\nDone downloading modules... \nXXX"
-		sleep 2
-	} | showProgress "Get Crosswire Modules | ${HEADERTITLE}" "Please wait while we download all modules"
+	# run in github action workflow... ¯\_(ツ)_/¯
+	if (("$GITHUB" == 1)); then
+			echo "Start download of modules..."
+			python3 "${DIR_src}/download.py" \
+				--output_path "${modules_path}" \
+				--bible_conf "${DIR_bible}" >>/dev/null
+			echo "Done downloading modules..."
+	else
+		# then we get the current modules
+		{
+			sleep 1
+			echo -e "XXX\n0\nStart download of modules... \nXXX"
+			sleep 1
+			python3 -u "${DIR_src}/download.py" \
+				--output_path "${modules_path}" \
+				--bible_conf "${DIR_bible}"
+			sleep 1
+			echo -e "XXX\n100\nDone downloading modules... \nXXX"
+			sleep 2
+		} | showProgress "Get Crosswire Modules | ${HEADERTITLE}" "Please wait while we download all modules"
+	fi
 }

 # prep the Scripture Main Git Folder
@ -143,32 +152,50 @@ function setStaticJsonFiles() {
 	local modules_path="$2"
 	local each="$3"
 	local counter=0
-	# build the files
-	{
-		sleep 1
-		echo -e "XXX\n0\nStart Building... \nXXX"
-		sleep 1
+	# run in github action workflow... ¯\_(ツ)_/¯
+	if (("$GITHUB" == 1)); then
+		echo "Start Building..."
 		for filename in $modules_path/*.zip; do
 			# give notice
-			echo -e "XXX\n${counter}\nBuilding ${filename} static json files...\nXXX"
-			# add more
-			next=$((counter + each))
+			echo "Building ${filename} static json files"
 			# run script
-			python3 -u "${DIR_src}/sword_to_json.py" \
+			python3 "${DIR_src}/sword_to_json.py" \
 				--source_file "${filename}" \
 				--output_path "${scripture_path}" \
-				--counter "$counter" --next "$next" \
 				--conf_dir "${DIR_conf}" \
-				--bible_conf "${DIR_bible}"
-			# add more
-			counter="$next"
+				--bible_conf "${DIR_bible}" >>/dev/null
 			# give notice
-			echo -e "XXX\n${counter}\nDone building ${filename} static json files...\nXXX"
-			sleep 1
+			echo "Done building ${filename} static json files..."
 		done
-		echo -e "XXX\n100\nDone Building... \nXXX"
-		sleep 1
-	} | showProgress "Build Static JSON Files | ${HEADERTITLE}" "Please wait while build the static json API"
+		echo "Done Building..."
+	else
+		# build the files
+		{
+			sleep 1
+			echo -e "XXX\n0\nStart Building... \nXXX"
+			sleep 1
+			for filename in $modules_path/*.zip; do
+				# give notice
+				echo -e "XXX\n${counter}\nBuilding ${filename} static json files...\nXXX"
+				# add more
+				next=$((counter + each))
+				# run script
+				python3 -u "${DIR_src}/sword_to_json.py" \
+					--source_file "${filename}" \
+					--output_path "${scripture_path}" \
+					--counter "$counter" --next "$next" \
+					--conf_dir "${DIR_conf}" \
+					--bible_conf "${DIR_bible}"
+				# add more
+				counter="$next"
+				# give notice
+				echo -e "XXX\n${counter}\nDone building ${filename} static json files...\nXXX"
+				sleep 1
+			done
+			echo -e "XXX\n100\nDone Building... \nXXX"
+			sleep 1
+		} | showProgress "Build Static JSON Files | ${HEADERTITLE}" "Please wait while build the static json API"
+	fi
 }

 # Remove Empty Folder & Static Files
@ -225,16 +252,26 @@ function hashingMethod() {
 	local w_end_ms="$5"
 	local w_initial_ms="$6"
 	local each="$7"
-	# now run the hashing
-	{
-		sleep 1
-		echo -e "XXX\n0\n${w_start_ms}... \nXXX"
-		sleep 1
-		. "${DIR_src}/${script_name}.sh" "${scripture_path}" "$each"
-		sleep 1
-		echo -e "XXX\n100\n${w_end_ms}... \nXXX"
-		sleep 1
-	} | showProgress "$w_title | ${HEADERTITLE}" "$w_initial_ms"
+	# run in github action workflow... ¯\_(ツ)_/¯
+	if (("$GITHUB" == 1)); then
+		echo "$w_title | ${HEADERTITLE}"
+		echo "$w_initial_ms"
+		echo "${w_start_ms}..."
+		# now run the hashing
+		. "${DIR_src}/${script_name}.sh" "${scripture_path}" "$each" >>/dev/null
+		echo "${w_end_ms}..."
+	else
+		# now run the hashing
+		{
+			sleep 1
+			echo -e "XXX\n0\n${w_start_ms}... \nXXX"
+			sleep 1
+			. "${DIR_src}/${script_name}.sh" "${scripture_path}" "$each"
+			sleep 1
+			echo -e "XXX\n100\n${w_end_ms}... \nXXX"
+			sleep 1
+		} | showProgress "$w_title | ${HEADERTITLE}" "$w_initial_ms"
+	fi
 }

 # set any/all default config property
@ -247,6 +284,7 @@ function setDefaults() {
 		DOWNLOAD=$(getDefault "getbible.download" "$DOWNLOAD")
 		PUSH=$(getDefault "getbible.push" "$PUSH")
 		HASHONLY=$(getDefault "getbible.hashonly" "$HASHONLY")
+		GITHUB=$(getDefault "getbible.github" "$GITHUB")
 		QUIET=$(getDefault "getbible.quiet" "$QUIET")
 	fi
 }
@ -332,6 +370,11 @@ You are able to change a few default behaviours in the getBible API builder

 	example: ${0##*/:-} --hashonly
 	======================================================
+   --github
+	Trigger github workflow behaviour
+
+	example: ${0##*/:-} --github
+	======================================================
   --test
 	Run a test with only three Bibles

@ -380,6 +423,8 @@ DRYRUN=0
 HASHONLY=0
 # kill all messages
 QUIET=0
+# trigger github workflow behaviour
+GITHUB=0

 # check if we have options
 while :; do
@ -406,6 +451,11 @@ while :; do
 	--dry)
 		DRYRUN=1
 		;;
+	--github)
+		# github actions workflow behaviour... ¯\_(ツ)_/¯
+		GITHUB=1
+		QUIET=1
+		;;
 	--push)
 		PUSH=1
 		;;
@ -493,6 +543,7 @@ if (("$DRYRUN" == 1)); then
 	echo "DIR_bible:  ${DIR_bible}"
 	echo "QUIET:      ${QUIET}"
 	echo "HASHONLY:   ${HASHONLY}"
+	echo "GITHUB:     ${GITHUB}"
 	echo "DOWNLOAD:   ${DOWNLOAD}"
 	echo "PUSH:       ${PUSH}"
 	echo "CONFIGFILE: ${CONFIGFILE}"
--- a/src/sword_to_json.py
+++ b/src/sword_to_json.py
@ -15,8 +15,8 @@ parser = argparse.ArgumentParser()
 # get the arguments
 parser.add_argument('--source_file')
 parser.add_argument('--output_path')
-parser.add_argument('--counter')
-parser.add_argument('--next')
+parser.add_argument('--counter', default=0)
+parser.add_argument('--next', default=90)
 parser.add_argument('--conf_dir')
 parser.add_argument('--bible_conf')
 # set to args