From ac9ca6049ececbcad950b8da7d4175ac58653a9d Mon Sep 17 00:00:00 2001 From: "Kay Marquardt (Gnadelwartz)" Date: Fri, 15 May 2020 11:53:19 +0200 Subject: [PATCH] jsshDB atomic write, fix testdata --- modules/jsonDB.sh | 171 ++++++++++++++---- test/d-process_message-test.sh | 2 +- test/d-send_message-test.sh | 2 +- .../d-send_message-test.result | 28 +-- 4 files changed, 153 insertions(+), 50 deletions(-) diff --git a/modules/jsonDB.sh b/modules/jsonDB.sh index 946eaf8..2648e80 100644 --- a/modules/jsonDB.sh +++ b/modules/jsonDB.sh @@ -5,54 +5,72 @@ # This file is public domain in the USA and all free countries. # Elsewhere, consider it to be WTFPLv2. (wtfpl.net/txt/copying) # -#### $$VERSION$$ v0.94-pre-8-g284172f +#### $$VERSION$$ v0.94-pre-10-g23a3d4b # # source from commands.sh to use jsonDB functions # # jsonDB provides simple functions to read and store bash Arrays # from to file in JSON.sh output format, its a simple key/value storage. + # source once magic, function named like file eval "$(basename "${BASH_SOURCE[0]}")(){ :; }" -# read content of a file in JSON.sh format into given ARRAY -# $1 ARRAY name, must be delared with "declare -A ARRAY" upfront -# $2 filename, must be relative to BASHBOT_ETC, and not contain '..' -jssh_readDB() { +# new feature: serialize / atomic operations: +# updates will be done atomic with flock +# flock should flock should be availible on all system as its part of busybox +# tinybox + +# lockfile filename.flock is persistent and will be testet with flock for active lock (file open) +export BASHBOT_LOCKNAME=".flock" + +if _exists flock; then + ############### + # we have flock + # use flock for atomic operations + + # read content of a file in JSON.sh format into given ARRAY + # $1 ARRAY name, must be delared with "declare -A ARRAY" upfront + # $2 filename, must be relative to BASHBOT_ETC, and not contain '..' + jssh_readDB() { local DB; DB="$(jssh_checkDB "$2")" [ -z "${DB}" ] && return 1 [ ! -f "${DB}" ] && return 2 - Json2Array "$1" <"${DB}" -} + # shared lock, many processes can read, maximum wait 1s + { flock -s -w 1 200; Json2Array "$1" <"${DB}"; } 200>"${DB}${BASHBOT_LOCKNAME}" + } -# write ARRAY content to a file in JSON.sh format -# Warning: old content is overwritten -# $1 ARRAY name, must be delared with "declare -A ARRAY" upfront -# $2 filename (must exist!), must be relative to BASHBOT_ETC, and not contain '..' -jssh_writeDB() { + # write ARRAY content to a file in JSON.sh format + # Warning: old content is overwritten + # $1 ARRAY name, must be delared with "declare -A ARRAY" upfront + # $2 filename (must exist!), must be relative to BASHBOT_ETC, and not contain '..' + jssh_writeDB() { local DB; DB="$(jssh_checkDB "$2")" [ -z "${DB}" ] && return 1 [ ! -f "${DB}" ] && return 2 - Array2Json "$1" >"${DB}" -} + # exclusive lock, no other process can read or write, maximum wait to get lock is 10s + { flock -e -w 10 200; Array2Json "$1" >"${DB}"; } 200>"${DB}${BASHBOT_LOCKNAME}" + } -# print ARRAY content to stdout instead of file -# $1 ARRAY name, must be delared with "declare -A ARRAY" upfront -jssh_printDB() { - Array2Json "$1" -} + # update/write ARRAY content in file without deleting keys not in ARRAY + # $1 ARRAY name, must be delared with "declare -A ARRAY" upfront + # $2 filename (must exist!), must be relative to BASHBOT_ETC, and not contain '..' + jssh_updateDB() { + # for atomic update we cant use read/writeDB + local DB; DB="$(jssh_checkDB "$2")" + [ -z "${DB}" ] && return 1 + [ ! -f "${DB}" ] && return 2 -# update/write ARRAY content in file without deleting keys not in ARRAY -# $1 ARRAY name, must be delared with "declare -A ARRAY" upfront -# $2 filename (must exist!), must be relative to BASHBOT_ETC, and not contain '..' -jssh_updateDB() { declare -n ARRAY="$1" [ -z "${ARRAY[*]}" ] && return 1 declare -A oldARR newARR - jssh_readDB "oldARR" "$2" || return "$?" + + # start atomic update here, exclusive max wait 10s + { flock -e -w 10 200 + Json2Array "oldARR" <"${DB}" if [ -z "${oldARR[*]}" ]; then # no old content - jssh_writeDB "$1" "$2" + Array2Json "$1" >"${DB}" else # merge arrays local o1 o2 n1 n2 @@ -62,23 +80,58 @@ jssh_updateDB() { #shellcheck disable=SC2034,SC2190,SC2206 newARR=( ${o2:0:${#o2}-1} ${n2:0:${#n2}-1} ) set +f - jssh_writeDB "newARR" "$2" + Array2Json "newARR" >"${DB}" fi -} + } 200>"${DB}${BASHBOT_LOCKNAME}" + } -# insert, update, apped key/value to jsshDB -# $1 key name, can onyl contain -a-zA-Z0-9,._ -# $2 key value -# $3 filename (must exist!), must be relative to BASHBOT_ETC, and not contain '..' -jssh_insertDB() { + # insert, update, apped key/value to jsshDB + # $1 key name, can onyl contain -a-zA-Z0-9,._ + # $2 key value + # $3 filename (must exist!), must be relative to BASHBOT_ETC, and not contain '..' + jssh_insertDB() { [[ "$1" =~ ^[-a-zA-Z0-9,._]+$ ]] || return 3 local key="$1" value="$2" local DB; DB="$(jssh_checkDB "$3")" [ -z "${DB}" ] && return 1 [ ! -f "${DB}" ] && return 2 - # its append, but last one counts, its a simple DB ... - printf '["%s"]\t"%s"\n' "${key//,/\",\"}" "${value//\"/\\\"}" >>"${DB}" + # start atomic update here, exclusive max wait 2si, it's append, not overwrite + { flock -e -w 2 200 + # it's append, but last one counts, its a simple DB ... + printf '["%s"]\t"%s"\n' "${key//,/\",\"}" "${value//\"/\\\"}" >>"${DB}" + } 200>"${DB}${BASHBOT_LOCKNAME}" + } + +else + ######### + # we have no flock, use "old" not atomic functions + jssh_readDB() { + jssh_readDB_async "$@" + } + + jssh_writeDB() { + jssh_writeDB_async "$@" + } + + jssh_updateDB() { + jssh_updateDB_async "$@" + } + + jssh_insertDB() { + jssh_insertDB_async "$@" + + } + +fi + +############## +# no need for atomic + +# print ARRAY content to stdout instead of file +# $1 ARRAY name, must be delared with "declare -A ARRAY" upfront +jssh_printDB() { + Array2Json "$1" } # get key/value from jsshDB @@ -113,3 +166,53 @@ jssh_checkDB(){ printf '%s\n' "${DB}" } + +###################### +# "old" implementations as non atomic functions +# can be used explictitly or as fallback if flock is not availible +jssh_readDB_async() { + local DB; DB="$(jssh_checkDB "$2")" + [ -z "${DB}" ] && return 1 + [ ! -f "${DB}" ] && return 2 + Json2Array "$1" <"${DB}" +} + +jssh_writeDB_async() { + local DB; DB="$(jssh_checkDB "$2")" + [ -z "${DB}" ] && return 1 + [ ! -f "${DB}" ] && return 2 + Array2Json "$1" >"${DB}" +} + +jssh_updateDB_async() { + declare -n ARRAY="$1" + [ -z "${ARRAY[*]}" ] && return 1 + declare -A oldARR newARR + jssh_readDB "oldARR" "$2" || return "$?" + if [ -z "${oldARR[*]}" ]; then + # no old content + jssh_writeDB "$1" "$2" + else + # merge arrays + local o1 o2 n1 n2 + o1="$(declare -p oldARR)"; o2="${o1#*\(}" + n1="$(declare -p ARRAY)"; n2="${n1#*\(}" + unset IFS; set -f + #shellcheck disable=SC2034,SC2190,SC2206 + newARR=( ${o2:0:${#o2}-1} ${n2:0:${#n2}-1} ) + set +f + jssh_writeDB "newARR" "$2" + fi +} + +jssh_insertDB_async() { + [[ "$1" =~ ^[-a-zA-Z0-9,._]+$ ]] || return 3 + local key="$1" value="$2" + local DB; DB="$(jssh_checkDB "$3")" + [ -z "${DB}" ] && return 1 + [ ! -f "${DB}" ] && return 2 + # its append, but last one counts, its a simple DB ... + printf '["%s"]\t"%s"\n' "${key//,/\",\"}" "${value//\"/\\\"}" >>"${DB}" + +} + diff --git a/test/d-process_message-test.sh b/test/d-process_message-test.sh index bc2ff87..e2ed36f 100755 --- a/test/d-process_message-test.sh +++ b/test/d-process_message-test.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -#### $$VERSION$$ v0.94-dev3-0-geef955a +#### $$VERSION$$ v0.94-pre-10-g23a3d4b # include common functions and definitions # shellcheck source=test/ALL-tests.inc.sh diff --git a/test/d-send_message-test.sh b/test/d-send_message-test.sh index 935c650..b602cfc 100755 --- a/test/d-send_message-test.sh +++ b/test/d-send_message-test.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -#### $$VERSION$$ v0.94-dev3-0-geef955a +#### $$VERSION$$ v0.94-pre-10-g23a3d4b # include common functions and definitions # shellcheck source=test/ALL-tests.inc.sh diff --git a/test/d-send_message-test/d-send_message-test.result b/test/d-send_message-test/d-send_message-test.result index eb7d557..05ddf73 100644 --- a/test/d-send_message-test/d-send_message-test.result +++ b/test/d-send_message-test/d-send_message-test.result @@ -1,4 +1,4 @@ -chat:123456 JSON:"text":"# test for text only output" +chat:123456 JSON:"text":"\# test for text only output" URL:https://my-json-server.typicode.com/topkecleon/telegram-bot-bash/getMe?bashbottestscript/sendMessage chat:123456 JSON:"text":"This is a normal text" @@ -8,21 +8,21 @@ chat:123456 JSON:"text":"This is a normal text with a line break" URL:https://my-json-server.typicode.com/topkecleon/telegram-bot-bash/getMe?bashbottestscript/sendMessage -chat:123456 JSON:"text":" This is a HTML text","parse_mode":"html" +chat:123456 JSON:"text":" This is a HTML<\/b> text","parse_mode":"html" URL:https://my-json-server.typicode.com/topkecleon/telegram-bot-bash/getMe?bashbottestscript/sendMessage -chat:123456 JSON:"text":" This is a HTML text +chat:123456 JSON:"text":" This is a HTML<\/b> text with a line break","parse_mode":"html" URL:https://my-json-server.typicode.com/topkecleon/telegram-bot-bash/getMe?bashbottestscript/sendMessage -chat:123456 JSON:"text":" This is a *MARKDOWN* text","parse_mode":"markdown" +chat:123456 JSON:"text":" This is a \*MARKDOWN\* text","parse_mode":"markdown" URL:https://my-json-server.typicode.com/topkecleon/telegram-bot-bash/getMe?bashbottestscript/sendMessage -chat:123456 JSON:"text":" This is a *MARKDOWN* text +chat:123456 JSON:"text":" This is a \*MARKDOWN\* text with a line break","parse_mode":"markdown" URL:https://my-json-server.typicode.com/topkecleon/telegram-bot-bash/getMe?bashbottestscript/sendMessage -chat:123456 JSON:"text":"# test for keyboard, file, venue output" +chat:123456 JSON:"text":"\# test for keyboard\, file\, venue output" URL:https://my-json-server.typicode.com/topkecleon/telegram-bot-bash/getMe?bashbottestscript/sendMessage chat:123456 JSON:"text":"Text plus keyboard will appear in chat", "reply_markup": {"keyboard": [ [ "Yep, sure" , "No, highly unlikely" ] ] , "one_time_keyboard":true} @@ -34,31 +34,31 @@ URL:https://my-json-server.typicode.com/topkecleon/telegram-bot-bash/getMe?bashb chat:123456 JSON:"latitude": la10, "longitude": lo20, "address": "Diagon Alley N. 37", "title": "my home" URL:https://my-json-server.typicode.com/topkecleon/telegram-bot-bash/getMe?bashbottestscript/sendVenue -chat:123456 JSON:"text":"# test for new inline button" +chat:123456 JSON:"text":"\# test for new inline button" URL:https://my-json-server.typicode.com/topkecleon/telegram-bot-bash/getMe?bashbottestscript/sendMessage chat:123456 JSON:"text":"Text plus keyboard will appear in chat", "reply_markup": {"inline_keyboard": [ [ {"text":"Button Text", "url":"https://www..."}] ]} URL:https://my-json-server.typicode.com/topkecleon/telegram-bot-bash/getMe?bashbottestscript/sendMessage -chat:123456 JSON:"text":"STABILO 88/240 Fineliner point 88 +chat:123456 JSON:"text":"STABILO 88\/240 Fineliner point 88 -[https://images-na.ssl-images-amazon.com/images/I/41oypA3kmHL.l_SX300.jpg] +[https:\/\/images\-na.ssl\-images\-amazon.com\/images\/I\/41oypA3kmHL.l_SX300.jpg] second part of text plus newline.", "reply_markup": {"inline_keyboard": [ [ {"text":"Bei Amazon ansehen ...", "url":"https://www.amazon.de/dp/B014TN3JYW"}] ]} URL:https://my-json-server.typicode.com/topkecleon/telegram-bot-bash/getMe?bashbottestscript/sendMessage -chat:123456 JSON:"text":"# test for sendfile" +chat:123456 JSON:"text":"\# test for sendfile" URL:https://my-json-server.typicode.com/topkecleon/telegram-bot-bash/getMe?bashbottestscript/sendMessage -chat:123456 JSON:"action": "upload_photo" -URL:https://my-json-server.typicode.com/topkecleon/telegram-bot-bash/getMe?bashbottestscript/sendChatAction - chat:123456 JSON:"photo":"/tmp/allowed/this_is_my.gif","caption":"Text plus absolute file will appear in chat" URL:https://my-json-server.typicode.com/topkecleon/telegram-bot-bash/getMe?bashbottestscript/sendPhoto -chat:123456 JSON:"action": "upload_document" +chat:123456 JSON:"action": "upload_photo" URL:https://my-json-server.typicode.com/topkecleon/telegram-bot-bash/getMe?bashbottestscript/sendChatAction chat:123456 JSON:"document":"/tmp/allowed/this_is_my.doc","caption":"Text plus absolute file will appear in chat" URL:https://my-json-server.typicode.com/topkecleon/telegram-bot-bash/getMe?bashbottestscript/sendDocument +chat:123456 JSON:"action": "upload_document" +URL:https://my-json-server.typicode.com/topkecleon/telegram-bot-bash/getMe?bashbottestscript/sendChatAction +