jsshDB atomic write, fix testdata

modules/jsonDB.sh

@@ -5,54 +5,72 @@
 # This file is public domain in the USA and all free countries.
 # Elsewhere, consider it to be WTFPLv2. (wtfpl.net/txt/copying)
 #
-#### $$VERSION$$ v0.94-pre-8-g284172f
+#### $$VERSION$$ v0.94-pre-10-g23a3d4b
 #
 # source from commands.sh to use jsonDB functions
 #
 # jsonDB provides simple functions to read and store bash Arrays
 # from to file in JSON.sh output format, its a simple key/value storage.
 
+
 # source once magic, function named like file
 eval "$(basename "${BASH_SOURCE[0]}")(){ :; }"
 
-# read content of a file in JSON.sh format into given ARRAY
+# new feature: serialize / atomic operations:
-# $1 ARRAY name, must be delared with "declare -A ARRAY" upfront
+# updates will be done atomic with flock
-# $2 filename, must be relative to BASHBOT_ETC, and not contain '..'
+# flock should flock should be availible on all system as its part of busybox
-jssh_readDB() {
+# tinybox
+
+# lockfile filename.flock is persistent and will be testet with flock for active lock (file open)
+export BASHBOT_LOCKNAME=".flock"
+
+if _exists flock; then
+  ###############
+  # we have flock
+  # use flock for atomic operations
+
+  # read content of a file in JSON.sh format into given ARRAY
+  # $1 ARRAY name, must be delared with "declare -A ARRAY" upfront
+  # $2 filename, must be relative to BASHBOT_ETC, and not contain '..'
+  jssh_readDB() {
 	local DB; DB="$(jssh_checkDB "$2")"
 	[ -z "${DB}" ] && return 1
 	[ ! -f "${DB}" ] && return 2
-	Json2Array "$1" <"${DB}"
+	# shared lock, many processes can read, maximum wait 1s
-}
+	{ flock -s -w 1 200; Json2Array "$1" <"${DB}"; } 200>"${DB}${BASHBOT_LOCKNAME}"
+  }
 
-# write ARRAY content to a file in JSON.sh format
+  # write ARRAY content to a file in JSON.sh format
-# Warning: old content is overwritten
+  # Warning: old content is overwritten
-# $1 ARRAY name, must be delared with "declare -A ARRAY" upfront
+  # $1 ARRAY name, must be delared with "declare -A ARRAY" upfront
-# $2 filename (must exist!), must be relative to BASHBOT_ETC, and not contain '..'
+  # $2 filename (must exist!), must be relative to BASHBOT_ETC, and not contain '..'
-jssh_writeDB() {
+  jssh_writeDB() {
 	local DB; DB="$(jssh_checkDB "$2")"
 	[ -z "${DB}" ] && return 1
 	[ ! -f "${DB}" ] && return 2
-	Array2Json "$1" >"${DB}"
+	# exclusive lock, no other process can read or write, maximum wait to get lock is 10s
-}
+	{ flock -e -w 10 200; Array2Json "$1" >"${DB}"; } 200>"${DB}${BASHBOT_LOCKNAME}"
+  }
 
-# print ARRAY content to stdout instead of file
+  # update/write ARRAY content in file without deleting keys not in ARRAY
-# $1 ARRAY name, must be delared with "declare -A ARRAY" upfront
+  # $1 ARRAY name, must be delared with "declare -A ARRAY" upfront
-jssh_printDB() {
+  # $2 filename (must exist!), must be relative to BASHBOT_ETC, and not contain '..'
-	Array2Json "$1"
+  jssh_updateDB() {
-}
+	# for atomic update we cant use read/writeDB
+	local DB; DB="$(jssh_checkDB "$2")"
+	[ -z "${DB}" ] && return 1
+	[ ! -f "${DB}" ] && return 2
 
-# update/write ARRAY content in file without deleting keys not in ARRAY
-# $1 ARRAY name, must be delared with "declare -A ARRAY" upfront
-# $2 filename (must exist!), must be relative to BASHBOT_ETC, and not contain '..'
-jssh_updateDB() {
 	declare -n ARRAY="$1"
 	[ -z "${ARRAY[*]}" ] && return 1
 	declare -A oldARR newARR
-	jssh_readDB "oldARR" "$2" || return "$?"
+
+	# start atomic update here, exclusive max wait 10s
+	{ flock -e -w 10 200
+	Json2Array "oldARR" <"${DB}"
 	if [ -z "${oldARR[*]}" ]; then
 		# no old content
-		jssh_writeDB "$1" "$2"
+		Array2Json "$1" >"${DB}"
 	else
 		# merge arrays
 		local o1 o2 n1 n2
@@ -62,23 +80,58 @@ jssh_updateDB() {
 		#shellcheck disable=SC2034,SC2190,SC2206
 		newARR=( ${o2:0:${#o2}-1} ${n2:0:${#n2}-1} )
 		set +f
-		jssh_writeDB "newARR" "$2" 
+		Array2Json  "newARR" >"${DB}"
 	fi
-}
+	} 200>"${DB}${BASHBOT_LOCKNAME}"
+  }
 
-# insert, update, apped key/value to jsshDB
+  # insert, update, apped key/value to jsshDB
-# $1 key name, can onyl contain -a-zA-Z0-9,._
+  # $1 key name, can onyl contain -a-zA-Z0-9,._
-# $2 key value
+  # $2 key value
-# $3 filename (must exist!), must be relative to BASHBOT_ETC, and not contain '..'
+  # $3 filename (must exist!), must be relative to BASHBOT_ETC, and not contain '..'
-jssh_insertDB() {
+  jssh_insertDB() {
 	[[ "$1" =~ ^[-a-zA-Z0-9,._]+$ ]] || return 3
 	local key="$1" value="$2"
 	local DB; DB="$(jssh_checkDB "$3")"
 	[ -z "${DB}" ] && return 1
 	[ ! -f "${DB}" ] && return 2
-	# its append, but last one counts, its a simple DB ...
+	# start atomic update here, exclusive max wait 2si, it's append, not overwrite
-	printf '["%s"]\t"%s"\n' "${key//,/\",\"}" "${value//\"/\\\"}" >>"${DB}"
+	{ flock -e -w 2 200
+	 # it's append, but last one counts, its a simple DB ...
+	  printf '["%s"]\t"%s"\n' "${key//,/\",\"}" "${value//\"/\\\"}" >>"${DB}"
+	} 200>"${DB}${BASHBOT_LOCKNAME}"
 	
+  }
+
+else
+  #########
+  # we have no flock, use "old" not atomic functions
+  jssh_readDB() {
+	jssh_readDB_async "$@"
+  }
+
+  jssh_writeDB() {
+	jssh_writeDB_async "$@"
+  }
+
+  jssh_updateDB() {
+	jssh_updateDB_async "$@"
+  }
+
+  jssh_insertDB() {
+	jssh_insertDB_async "$@"
+	
+  }
+
+fi
+
+##############
+# no need for atomic
+
+# print ARRAY content to stdout instead of file
+# $1 ARRAY name, must be delared with "declare -A ARRAY" upfront
+jssh_printDB() {
+	Array2Json "$1"
 }
 
 # get key/value from jsshDB
@@ -113,3 +166,53 @@ jssh_checkDB(){
 	printf '%s\n' "${DB}"
 }
 
+
+######################
+# "old" implementations as non atomic functions
+# can be used explictitly or as fallback if flock is not availible
+jssh_readDB_async() {
+	local DB; DB="$(jssh_checkDB "$2")"
+	[ -z "${DB}" ] && return 1
+	[ ! -f "${DB}" ] && return 2
+	Json2Array "$1" <"${DB}"
+}
+
+jssh_writeDB_async() {
+	local DB; DB="$(jssh_checkDB "$2")"
+	[ -z "${DB}" ] && return 1
+	[ ! -f "${DB}" ] && return 2
+	Array2Json "$1" >"${DB}"
+}
+
+jssh_updateDB_async() {
+	declare -n ARRAY="$1"
+	[ -z "${ARRAY[*]}" ] && return 1
+	declare -A oldARR newARR
+	jssh_readDB "oldARR" "$2" || return "$?"
+	if [ -z "${oldARR[*]}" ]; then
+		# no old content
+		jssh_writeDB "$1" "$2"
+	else
+		# merge arrays
+		local o1 o2 n1 n2
+		o1="$(declare -p oldARR)"; o2="${o1#*\(}"
+		n1="$(declare -p ARRAY)";  n2="${n1#*\(}"
+		unset IFS; set -f
+		#shellcheck disable=SC2034,SC2190,SC2206
+		newARR=( ${o2:0:${#o2}-1} ${n2:0:${#n2}-1} )
+		set +f
+		jssh_writeDB "newARR" "$2" 
+	fi
+}
+
+jssh_insertDB_async() {
+	[[ "$1" =~ ^[-a-zA-Z0-9,._]+$ ]] || return 3
+	local key="$1" value="$2"
+	local DB; DB="$(jssh_checkDB "$3")"
+	[ -z "${DB}" ] && return 1
+	[ ! -f "${DB}" ] && return 2
+	# its append, but last one counts, its a simple DB ...
+	printf '["%s"]\t"%s"\n' "${key//,/\",\"}" "${value//\"/\\\"}" >>"${DB}"
+	
+}
+

test/d-process_message-test.sh

@@ -1,5 +1,5 @@
 #!/usr/bin/env bash
-#### $$VERSION$$ v0.94-dev3-0-geef955a
+#### $$VERSION$$ v0.94-pre-10-g23a3d4b
 
 # include common functions and definitions
 # shellcheck source=test/ALL-tests.inc.sh

test/d-send_message-test.sh

@@ -1,5 +1,5 @@
 #!/usr/bin/env bash
-#### $$VERSION$$ v0.94-dev3-0-geef955a
+#### $$VERSION$$ v0.94-pre-10-g23a3d4b
 
 # include common functions and definitions
 # shellcheck source=test/ALL-tests.inc.sh

test/d-send_message-test/d-send_message-test.result

@@ -1,4 +1,4 @@
-chat:123456	JSON:"text":"# test for text only output"
+chat:123456	JSON:"text":"\# test for text only output"
 URL:https://my-json-server.typicode.com/topkecleon/telegram-bot-bash/getMe?bashbottestscript/sendMessage
 
 chat:123456	JSON:"text":"This is a normal text"
@@ -8,21 +8,21 @@ chat:123456	JSON:"text":"This is a normal text
 with a line break"
 URL:https://my-json-server.typicode.com/topkecleon/telegram-bot-bash/getMe?bashbottestscript/sendMessage
 
-chat:123456	JSON:"text":" This is a <b>HTML</b> text","parse_mode":"html"
+chat:123456	JSON:"text":" This is a <b>HTML<\/b> text","parse_mode":"html"
 URL:https://my-json-server.typicode.com/topkecleon/telegram-bot-bash/getMe?bashbottestscript/sendMessage
 
-chat:123456	JSON:"text":" This is a <b>HTML</b> text
+chat:123456	JSON:"text":" This is a <b>HTML<\/b> text
 with a line break","parse_mode":"html"
 URL:https://my-json-server.typicode.com/topkecleon/telegram-bot-bash/getMe?bashbottestscript/sendMessage
 
-chat:123456	JSON:"text":" This is a *MARKDOWN* text","parse_mode":"markdown"
+chat:123456	JSON:"text":" This is a \*MARKDOWN\* text","parse_mode":"markdown"
 URL:https://my-json-server.typicode.com/topkecleon/telegram-bot-bash/getMe?bashbottestscript/sendMessage
 
-chat:123456	JSON:"text":" This is a *MARKDOWN* text
+chat:123456	JSON:"text":" This is a \*MARKDOWN\* text
 with a line break","parse_mode":"markdown"
 URL:https://my-json-server.typicode.com/topkecleon/telegram-bot-bash/getMe?bashbottestscript/sendMessage
 
-chat:123456	JSON:"text":"# test for keyboard, file, venue output"
+chat:123456	JSON:"text":"\# test for keyboard\, file\, venue output"
 URL:https://my-json-server.typicode.com/topkecleon/telegram-bot-bash/getMe?bashbottestscript/sendMessage
 
 chat:123456	JSON:"text":"Text plus keyboard will appear in chat", "reply_markup": {"keyboard": [ [ "Yep, sure" , "No, highly unlikely" ] ] , "one_time_keyboard":true}
@@ -34,31 +34,31 @@ URL:https://my-json-server.typicode.com/topkecleon/telegram-bot-bash/getMe?bashb
 chat:123456	JSON:"latitude": la10, "longitude": lo20, "address": "Diagon Alley N. 37", "title": "my home"
 URL:https://my-json-server.typicode.com/topkecleon/telegram-bot-bash/getMe?bashbottestscript/sendVenue
 
-chat:123456	JSON:"text":"# test for new inline button"
+chat:123456	JSON:"text":"\# test for new inline button"
 URL:https://my-json-server.typicode.com/topkecleon/telegram-bot-bash/getMe?bashbottestscript/sendMessage
 
 chat:123456	JSON:"text":"Text plus keyboard will appear in chat", "reply_markup": {"inline_keyboard": [ [ {"text":"Button Text", "url":"https://www..."}] ]}
 URL:https://my-json-server.typicode.com/topkecleon/telegram-bot-bash/getMe?bashbottestscript/sendMessage
 
-chat:123456	JSON:"text":"STABILO 88/240 Fineliner point 88
+chat:123456	JSON:"text":"STABILO 88\/240 Fineliner point 88
 
-[https://images-na.ssl-images-amazon.com/images/I/41oypA3kmHL.l_SX300.jpg]
+[https:\/\/images\-na.ssl\-images\-amazon.com\/images\/I\/41oypA3kmHL.l_SX300.jpg]
 second part of text
 plus newline.", "reply_markup": {"inline_keyboard": [ [ {"text":"Bei Amazon ansehen ...", "url":"https://www.amazon.de/dp/B014TN3JYW"}] ]}
 URL:https://my-json-server.typicode.com/topkecleon/telegram-bot-bash/getMe?bashbottestscript/sendMessage
 
-chat:123456	JSON:"text":"# test for sendfile"
+chat:123456	JSON:"text":"\# test for sendfile"
 URL:https://my-json-server.typicode.com/topkecleon/telegram-bot-bash/getMe?bashbottestscript/sendMessage
 
-chat:123456	JSON:"action": "upload_photo"
-URL:https://my-json-server.typicode.com/topkecleon/telegram-bot-bash/getMe?bashbottestscript/sendChatAction
-
 chat:123456	JSON:"photo":"/tmp/allowed/this_is_my.gif","caption":"Text plus absolute file will appear in chat"
 URL:https://my-json-server.typicode.com/topkecleon/telegram-bot-bash/getMe?bashbottestscript/sendPhoto
 
-chat:123456	JSON:"action": "upload_document"
+chat:123456	JSON:"action": "upload_photo"
 URL:https://my-json-server.typicode.com/topkecleon/telegram-bot-bash/getMe?bashbottestscript/sendChatAction
 
 chat:123456	JSON:"document":"/tmp/allowed/this_is_my.doc","caption":"Text plus absolute file will appear in chat"
 URL:https://my-json-server.typicode.com/topkecleon/telegram-bot-bash/getMe?bashbottestscript/sendDocument
 
+chat:123456	JSON:"action": "upload_document"
+URL:https://my-json-server.typicode.com/topkecleon/telegram-bot-bash/getMe?bashbottestscript/sendChatAction
+