From f96625ef9d3981a6a75b996f394773f6d7c96f83 Mon Sep 17 00:00:00 2001 From: "Kay Marquardt (Gnadelwartz)" Date: Tue, 9 Jun 2020 11:57:25 +0200 Subject: [PATCH 01/21] explanation for BOTSEND_RETRY explanation and why printf --- README.html | 44 +++++++++++++++++++++++++++++--------------- README.md | 22 +++++++++++++++++++++- README.txt | 25 ++++++++++++++++++++++++- bashbot.sh | 6 +++--- doc/7_develop.md | 10 +++++++--- modules/jsonDB.sh | 2 +- mycommands.sh | 7 ++++++- mycommands.sh.clean | 7 ++++++- 8 files changed, 97 insertions(+), 26 deletions(-) diff --git a/README.html b/README.html index da7e872..5b81f09 100644 --- a/README.html +++ b/README.html @@ -181,6 +181,20 @@ It features background tasks and interactive chats, and can serve as an interfac

Whenever you are processing input from from untrusted sources (messages, files, network) you must be as carefull as possible, e.g. set IFS appropriate, disable globbing (set -f) and quote everthing. In addition delete unused scripts and examples from your Bot, e.g. scripts 'notify', 'calc', 'question', and disable all not used commands.

Note: Until v0.941 (mai/22/2020) telegram-bot-bash has a remote code execution bug, pls update if you use an older version! One of the most powerful features of unix shells like bash is variable and command substitution, this can lead to RCE and information disclosing bugs if you do not escape '$' porperly, see Issue #125

A powerful tool to improve your scripts is shellcheck. You can use it online or install shellcheck locally. Shellcheck is used extensive in bashbot development to enshure a high code quality, e.g. it's not allowed to push changes without passing all shellcheck tests. In addition bashbot has a test suite to check if important functionality is working as expected.

+

use printf whenever possible

+

If you're writing a script and it is taking external input (from the user as arguments, or file names from the file system...), you shouldn't use echo to display it. Use printf whenever possible

+
  # very simple
+  echo "text with variables. PWD=$PWD"
+  printf '%s\n' "text with variables. PWD=$PWD"
+  -> text with variables. PWD=/home/xxx
+
+  # more advanced
+  FLOAT="1.2346777892864" INTEGER="12345.123"
+  echo "text with variabeles. float=$FLOAT, integer=$INTEGER, PWD=$PWD"
+  ->text with variables. float=1.2346777892864, integer=12345.123, PWD=/home/xxx
+
+  printf "text with variables. float=%.2f, integer=%d, PWD=%s\n" "" "$INTEGER" "$PWD"
+  ->text with variables. float=1.23, integer=12345, PWD=/home/xxx

Do not use #!/usr/bin/env bash

We stay with /bin/bash shebang, because it's more save from security perspective.

Using a fixed path to the system provided bash makes it harder for attackers or users to place alternative versions of bash and avoids using a possibly broken, mangled or compromised bash executable.

@@ -210,27 +224,27 @@ It features background tasks and interactive chats, and can serve as an interfac

Can I send messages from CLI and scripts?

Of course, you can send messages from CLI and scripts, simply install bashbot as described here, send the messsage '/start' to set yourself as botadmin and stop the bot with ./bashbot.sh kill.

Run the following commands in your bash shell or script while you are in the installation directory:

-
# prepare bash / script to send commands
-export BASHBOT_HOME="$(pwd)"
-source ./bashbot.sh source
-
-# send me a test message
-send_message "$(cat "$BOTADMIN")" "test"
-
-# send me output of a system command
-send_message "$(<"$BOTADMIN")" "$(df -h)"
+
# prepare bash / script to send commands
+export BASHBOT_HOME="$(pwd)"
+source ./bashbot.sh source
+
+# send me a test message
+send_message "$(cat "$BOTADMIN")" "test"
+
+# send me output of a system command
+send_message "$(<"$BOTADMIN")" "$(df -h)"

For more information see Expert Use

Why do I get "EXPECTED value GOT EOF" on start?

May be your IP is blocked by telegram. You can test this by running curl or wget manually:

-
curl -m 10  https://api.telegram.org/bot
-#curl: (28) Connection timed out after 10001 milliseconds
-
-wget -t 1 -T 10 https://api.telegram.org/bot
-#Connecting to api.telegram.org (api.telegram.org)|46.38.243.234|:443... failed: Connection timed out.
+
curl -m 10  https://api.telegram.org/bot
+#curl: (28) Connection timed out after 10001 milliseconds
+
+wget -t 1 -T 10 https://api.telegram.org/bot
+#Connecting to api.telegram.org (api.telegram.org)|46.38.243.234|:443... failed: Connection timed out.

This may happen if to many wrong requests are sent to api.telegram.org, e.g. using a wrong token or not existing API calls. If you have a fixed IP you can ask telegram service to unblock your ip or change your IP. If you are running a socks or tor proxy on your server look for the BASHBOT_CURL_ARGS lines in 'mycommands.sh' as example.

@Gnadelwartz

That's it!

If you feel that there's something missing or if you found a bug, feel free to submit a pull request!

-

$$VERSION$$ v0.96-dev3-0-gdddd1ce

+

$$VERSION$$ v0.96-pre-9-gb23aadd

diff --git a/README.md b/README.md index 51c0bfe..56a382a 100644 --- a/README.md +++ b/README.md @@ -116,6 +116,26 @@ One of the most powerful features of unix shells like bash is variable and comma A powerful tool to improve your scripts is ```shellcheck```. You can [use it online](https://www.shellcheck.net/) or [install shellcheck locally](https://github.com/koalaman/shellcheck#installing). Shellcheck is used extensive in bashbot development to enshure a high code quality, e.g. it's not allowed to push changes without passing all shellcheck tests. In addition bashbot has a [test suite](doc/7_develop.md) to check if important functionality is working as expected. +### use printf whenever possible + +If you're writing a script and it is taking external input (from the user as arguments, or file names from the file system...), +you shouldn't use echo to display it. [Use printf whenever possible](https://unix.stackexchange.com/a/6581) + +```bash + # very simple + echo "text with variables. PWD=$PWD" + printf '%s\n' "text with variables. PWD=$PWD" + -> text with variables. PWD=/home/xxx + + # more advanced + FLOAT="1.2346777892864" INTEGER="12345.123" + echo "text with variabeles. float=$FLOAT, integer=$INTEGER, PWD=$PWD" + ->text with variables. float=1.2346777892864, integer=12345.123, PWD=/home/xxx + + printf "text with variables. float=%.2f, integer=%d, PWD=%s\n" "" "$INTEGER" "$PWD" + ->text with variables. float=1.23, integer=12345, PWD=/home/xxx +``` + ### Do not use #!/usr/bin/env bash **We stay with /bin/bash shebang, because it's more save from security perspective.** @@ -198,4 +218,4 @@ This may happen if to many wrong requests are sent to api.telegram.org, e.g. usi If you feel that there's something missing or if you found a bug, feel free to submit a pull request! -#### $$VERSION$$ v0.96-dev3-0-gdddd1ce +#### $$VERSION$$ v0.96-pre-9-gb23aadd diff --git a/README.txt b/README.txt index 288cbd7..0b50d73 100644 --- a/README.txt +++ b/README.txt @@ -163,6 +163,29 @@ allowed to push changes without passing all shellcheck tests. In addition bashbot has a [test suite](doc/7_develop.md) to check if important functionality is working as expected. +### use printf whenever possible + +If you're writing a script and it is taking external input (from the user as +arguments, or file names from the file system...), +you shouldn't use echo to display it. [Use printf whenever +possible](https://unix.stackexchange.com/a/6581) + +```bash + # very simple + echo "text with variables. PWD=$PWD" + printf '%s\n' "text with variables. PWD=$PWD" + -> text with variables. PWD=/home/xxx + + # more advanced + FLOAT="1.2346777892864" INTEGER="12345.123" + echo "text with variabeles. float=$FLOAT, integer=$INTEGER, PWD=$PWD" + ->text with variables. float=1.2346777892864, integer=12345.123, PWD=/home/xxx + + printf "text with variables. float=%.2f, integer=%d, PWD=%s\n" "" "$INTEGER" +"$PWD" + ->text with variables. float=1.23, integer=12345, PWD=/home/xxx +``` + ### Do not use #!/usr/bin/env bash **We stay with /bin/bash shebang, because it's more save from security @@ -281,4 +304,4 @@ in 'mycommands.sh' as example. If you feel that there's something missing or if you found a bug, feel free to submit a pull request! -#### $$VERSION$$ v0.96-dev3-0-gdddd1ce +#### $$VERSION$$ v0.96-pre-9-gb23aadd diff --git a/bashbot.sh b/bashbot.sh index ed7c527..e81cfed 100755 --- a/bashbot.sh +++ b/bashbot.sh @@ -11,7 +11,7 @@ # This file is public domain in the USA and all free countries. # Elsewhere, consider it to be WTFPLv2. (wtfpl.net/txt/copying) # -#### $$VERSION$$ v0.96-pre-0-geb49241 +#### $$VERSION$$ v0.96-pre-9-gb23aadd # # Exit Codes: # - 0 sucess (hopefully) @@ -371,7 +371,7 @@ fi # $1 function $2 sleep $3 ... $n arguments sendJsonRetry(){ local retry="${1}"; shift - [[ "${1}" =~ ^[0-9.]+$ ]] && sleep "${1}"; shift + [[ "${1}" =~ ^\ *[0-9.]+\ *$ ]] && sleep "${1}"; shift case "${retry}" in 'sendJson'*) sendJson "$@" @@ -435,7 +435,7 @@ sendJsonResult(){ fi return fi - # we are not blocked, default curl and args are working + # are not blocked, default curl and args are working if [ -n "${BASHBOT_CURL_ARGS}" ] || [ -n "${BASHBOT_CURL}" ]; then BOTSEND_RETRY="2" printf "Possible Problem with \"%s %s\", retry %s with default curl config ...\n"\ diff --git a/doc/7_develop.md b/doc/7_develop.md index 1be455b..f8e5751 100644 --- a/doc/7_develop.md +++ b/doc/7_develop.md @@ -216,18 +216,22 @@ Availible commands in bash, coreutils, busybox and toybox. Do you find curl on t uuencode, wc, wget, which, who, whoami, xargs, yes ``` commands marked with \* are bash builtins, all others are external programms. Calling an external programm is more expensive then using bulitins -or using an internal replacement. Here are some examples of internal replacement for external commands: +or using an internal replacement. Here are some tipps for using builtins.: ```bash HOST="$(hostname)" -> HOST="$HOSTNAME" +DIR="$(pwd)" -> DIR="$PWD"" + seq 1 100 -> {0..100} data="$(cat file)" -> data="$(<"file")" -DIR="$(dirname $0) -> DIR=""${0%/*}/"" +DIR="$(dirname $0) -> DIR="${0%/*}" IAM="($basename $0)" -> IAM="${0##*/}* +ADDME="$ADDME something to add" -> ADDME+=" something to add"" + VAR="$(( 1 + 2 ))" -> (( var=1+2 )) INDEX="$(( ${INDEX} + 1 ))" -> (( INDEX++ )) @@ -328,5 +332,5 @@ fi #### [Prev Function Reference](6_reference.md) -#### $$VERSION$$ v0.96-dev-7-g0153928 +#### $$VERSION$$ v0.96-pre-9-gb23aadd diff --git a/modules/jsonDB.sh b/modules/jsonDB.sh index 8da70b4..ddba7ac 100644 --- a/modules/jsonDB.sh +++ b/modules/jsonDB.sh @@ -5,7 +5,7 @@ # This file is public domain in the USA and all free countries. # Elsewhere, consider it to be WTFPLv2. (wtfpl.net/txt/copying) # -#### $$VERSION$$ v0.96-pre-0-geb49241 +#### $$VERSION$$ v0.96-pre-2-g30b5b1a # # source from commands.sh to use jsonDB functions # diff --git a/mycommands.sh b/mycommands.sh index c8e6b67..a75080d 100644 --- a/mycommands.sh +++ b/mycommands.sh @@ -8,7 +8,7 @@ # #### if you start to develop your own bot, use the clean version of this file: # #### mycommands.clean # -#### $$VERSION$$ v0.96-dev-7-g0153928 +#### $$VERSION$$ v0.96-pre-9-gb23aadd # # uncomment the following lines to overwrite info and help messages @@ -29,6 +29,11 @@ export FILE_REGEX="${BASHBOT_ETC}/.*" # example: run bashbot over TOR # export BASHBOT_CURL_ARGS="--socks5-hostname 127.0.0.1:9050" +# unset BASHBOT_RETRY to enable retry in case of recoverable errors, e.g. throtteling +# see logs/ERROR.log for information why send_messages etc. fail +# unset BOTSEND_RETRY +export BOTSEND_RETRY="no" + # set to "yes" and give your bot admin privilegs to remove service messaes from groups export SILENCER="no" diff --git a/mycommands.sh.clean b/mycommands.sh.clean index 7c8c1c4..184fd92 100644 --- a/mycommands.sh.clean +++ b/mycommands.sh.clean @@ -4,7 +4,7 @@ # files: mycommands.sh.clean # copy to mycommands.sh and add all your commands and functions here ... # -#### $$VERSION$$ v0.96-dev-7-g0153928 +#### $$VERSION$$ v0.96-pre-9-gb23aadd # ########## @@ -27,6 +27,11 @@ export INLINE="0" # do NOT set to .* as this allow sending files from all locations! export FILE_REGEX="${BASHBOT_ETC}/.*" +# unset BASHBOT_RETRY to enable retry in case of recoverable errors, e.g. throtteling +# see logs/ERROR.log for information why send_messages etc. fail +# unset BOTSEND_RETRY +export BOTSEND_RETRY="no" + # set to "yes" and give your bot admin privilegs to remove service messaes from groups export SILENCER="no" From 7644c6c14c1b0f92a20748eb2bd8f52cdcf3f5c4 Mon Sep 17 00:00:00 2001 From: "Kay Marquardt (Gnadelwartz)" Date: Tue, 9 Jun 2020 12:27:28 +0200 Subject: [PATCH 02/21] do not retry with wget --- bashbot.sh | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/bashbot.sh b/bashbot.sh index e81cfed..246058e 100755 --- a/bashbot.sh +++ b/bashbot.sh @@ -11,7 +11,7 @@ # This file is public domain in the USA and all free countries. # Elsewhere, consider it to be WTFPLv2. (wtfpl.net/txt/copying) # -#### $$VERSION$$ v0.96-pre-9-gb23aadd +#### $$VERSION$$ v0.96-pre-10-gf96625e # # Exit Codes: # - 0 sucess (hopefully) @@ -409,8 +409,8 @@ sendJsonResult(){ # log error printf "%s: RESULT=%s ACTION=%s CHAT[ID]=%s ERROR=%s DESC=%s\n" "$(date)"\ "${BOTSENT[OK]}" "${2}" "${3}" "${BOTSENT[ERROR]}" "${BOTSENT[DESCRIPTION]}" - # warm path, do not retry on error - [ -n "${BOTSEND_RETRY}" ] && return + # warm path, do not retry on error, also if we use wegt + [ -n "${BOTSEND_RETRY}${BASHBOT_WGET}" ] && return # OK, we can retry sendJson, let's see what's failed # throttled, telegram say we send to much messages @@ -436,11 +436,10 @@ sendJsonResult(){ return fi # are not blocked, default curl and args are working - if [ -n "${BASHBOT_CURL_ARGS}" ] || [ -n "${BASHBOT_CURL}" ]; then - BOTSEND_RETRY="2" + if [ -n "${BASHBOT_CURL_ARGS}" ] || [ "${BASHBOT_CURL}" != "curl" ]; then printf "Possible Problem with \"%s %s\", retry %s with default curl config ...\n"\ "${BASHBOT_CURL}" "${BASHBOT_CURL_ARGS}" "${2}" - unset BASHBOT_CURL BASHBOT_CURL_ARGS + BOTSEND_RETRY="2"; BASHBOT_CURL="curl"; BASHBOT_CURL_ARGS="" sendJsonRetry "${2}" "${BOTSEND_RETRY}" "${@:2}" unset BOTSEND_RETRY fi From dba4f95e9af8bee6090d796bbdebec5b463f6532 Mon Sep 17 00:00:00 2001 From: "Kay Marquardt (Gnadelwartz)" Date: Tue, 9 Jun 2020 12:41:15 +0200 Subject: [PATCH 03/21] log action on error --- bashbot.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bashbot.sh b/bashbot.sh index 246058e..c9e9b89 100755 --- a/bashbot.sh +++ b/bashbot.sh @@ -11,7 +11,7 @@ # This file is public domain in the USA and all free countries. # Elsewhere, consider it to be WTFPLv2. (wtfpl.net/txt/copying) # -#### $$VERSION$$ v0.96-pre-10-gf96625e +#### $$VERSION$$ v0.96-pre-11-g7644c6c # # Exit Codes: # - 0 sucess (hopefully) @@ -407,8 +407,8 @@ sendJsonResult(){ BOTSENT[DESCRIPTION]="Timeout or broken/no connection" fi # log error - printf "%s: RESULT=%s ACTION=%s CHAT[ID]=%s ERROR=%s DESC=%s\n" "$(date)"\ - "${BOTSENT[OK]}" "${2}" "${3}" "${BOTSENT[ERROR]}" "${BOTSENT[DESCRIPTION]}" + printf "%s: RESULT=%s FUNC=%s CHAT[ID]=%s ERROR=%s DESC=%s\n=ACTION=%s\n" "$(date)"\ + "${BOTSENT[OK]}" "${2}" "${3}" "${BOTSENT[ERROR]}" "${BOTSENT[DESCRIPTION]}" "${4/[$'\n\r']*}" # warm path, do not retry on error, also if we use wegt [ -n "${BOTSEND_RETRY}${BASHBOT_WGET}" ] && return From a71d68ee43f2f8e5d2366f5d60293e0b3ef78ba9 Mon Sep 17 00:00:00 2001 From: "Kay Marquardt (Gnadelwartz)" Date: Tue, 9 Jun 2020 14:46:45 +0200 Subject: [PATCH 04/21] error processing for get update loop --- bashbot.sh | 40 ++++++++++++++++++++++++++-------------- 1 file changed, 26 insertions(+), 14 deletions(-) diff --git a/bashbot.sh b/bashbot.sh index c9e9b89..f39bff7 100755 --- a/bashbot.sh +++ b/bashbot.sh @@ -11,7 +11,7 @@ # This file is public domain in the USA and all free countries. # Elsewhere, consider it to be WTFPLv2. (wtfpl.net/txt/copying) # -#### $$VERSION$$ v0.96-pre-11-g7644c6c +#### $$VERSION$$ v0.96-pre-12-gdba4f95 # # Exit Codes: # - 0 sucess (hopefully) @@ -787,9 +787,11 @@ process_message() { start_bot() { local DEBUG="$1" local OFFSET=0 - local mysleep="100" # ms + # adaptive sleep deafults + local nextsleep="100" : local addsleep="100" local maxsleep="$(( ${BASHBOT_SLEEP:-5000} + 100 ))" + # redirect to Debug.log [[ "${DEBUG}" == *"debug" ]] && exec &>>"${LOGDIR}/DEBUG.log" [ -n "${DEBUG}" ] && date && echo "Start BASHBOT in Mode \"${DEBUG}\"" [[ "${DEBUG}" == "xdebug"* ]] && set -x @@ -812,21 +814,31 @@ start_bot() { trap "kill -9 $!; exit" EXIT INT HUP TERM QUIT fi while true; do - # ignore timeout error message on waiting for updates +set -x + # adaptive sleep in ms rounded to next 0.1 s + sleep "$(printf '%.1f' "$((nextsleep))e-3")" + ((nextsleep+= addsleep , nextsleep= nextsleep>maxsleep ?maxsleep:nextsleep)) +set +x + # get next update UPDATE="$(getJson "$UPD_URL$OFFSET" 2>/dev/null | "${JSONSHFILE}" -s -b -n | iconv -f utf-8 -t utf-8 -c)" - UPDATE="${UPDATE//$/\\$}" - # Offset - OFFSET="$(grep <<< "${UPDATE}" '\["result",[0-9]*,"update_id"\]' | tail -1 | cut -f 2)" - ((OFFSET++)) + # did we ge an responsn0r + if [ -n "${UPDATE}" ]; then + # we got something, do processing + # escape bash $ expansion bug + UPDATE="${UPDATE//$/\\$}" + # Offset + OFFSET="$(grep <<< "${UPDATE}" '\["result",[0-9]*,"update_id"\]' | tail -1 | cut -f 2)" + ((OFFSET++)) - if [ "$OFFSET" != "1" ]; then - mysleep="100" - process_updates "${DEBUG}" + if [ "$OFFSET" != "1" ]; then + nextsleep="100" + process_updates "${DEBUG}" + fi + else + # ups, something bad happend, wait maxsleep + (( nextsleep=maxsleep )) + printf "%s: Ups, got no response on update, sleep %.1fs" "$(date)" "$((nextsleep))e-3" >>"${ERRORLOG}" fi - # adaptive sleep in ms rounded to next lower second - [ "${mysleep}" -gt "999" ] && sleep "${mysleep%???}" - # bash aritmetic - ((mysleep+= addsleep , mysleep= mysleep>maxsleep ?maxsleep:mysleep)) done } From 4fd6f98f7789380c643db4a98da234221043a004 Mon Sep 17 00:00:00 2001 From: "Kay Marquardt (Gnadelwartz)" Date: Tue, 9 Jun 2020 16:51:50 +0200 Subject: [PATCH 05/21] do not start timer by default --- README.html | 4 ++-- README.md | 4 ++-- README.txt | 6 +++--- bashbot.sh | 12 +++++------- mycommands.sh | 15 +++++++++++++-- mycommands.sh.clean | 14 ++++++++++++-- 6 files changed, 37 insertions(+), 18 deletions(-) diff --git a/README.html b/README.html index 5b81f09..703421c 100644 --- a/README.html +++ b/README.html @@ -92,7 +92,7 @@ Written by Drew (@topkecleon), Daniil Gentili (@danogentili), and Kay M (@gnadel

Prerequisites

Uses JSON.sh, but no more TMUX.

Even bashbot is written in bash, it depends on commands typically availible in a Unix/Linux Environment. More concret on the common commands provided by recent versions of coreutils, busybox or toybox, see Developer Notes

-

Note for MacOS and BSD Users: As bashbot use behavior of recent bash and (gnu)sed versions, bashbot may not run without installing additional software, see Install Bashbot

+

Note for MacOS and BSD Users: As bashbot heavily uses modern bash and (gnu) grep/sed features, bashbot will not run without installing additional software, see Install Bashbot

Bashbot Documentation and Downloads are availible on www.github.com

Documentation

    @@ -245,6 +245,6 @@ It features background tasks and interactive chats, and can serve as an interfac

    @Gnadelwartz

    That's it!

    If you feel that there's something missing or if you found a bug, feel free to submit a pull request!

    -

    $$VERSION$$ v0.96-pre-9-gb23aadd

    +

    $$VERSION$$ v0.96-pre-13-ga71d68e

    diff --git a/README.md b/README.md index 56a382a..4f481a4 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,7 @@ Uses [JSON.sh](http://github.com/dominictarr/JSON.sh), but no more TMUX. Even bashbot is written in bash, it depends on commands typically availible in a Unix/Linux Environment. More concret on the common commands provided by recent versions of [coreutils](https://en.wikipedia.org/wiki/List_of_GNU_Core_Utilities_commands), [busybox](https://en.wikipedia.org/wiki/BusyBox#Commands) or [toybox](https://landley.net/toybox/help.html), see [Developer Notes](doc/7_develop.md#common-commands) -*Note for MacOS and BSD Users:* As bashbot use behavior of recent bash and (gnu)sed versions, bashbot may not run without installing additional software, see [Install Bashbot](doc/0_install.md) +*Note for MacOS and BSD Users:* As bashbot heavily uses modern bash and (gnu) grep/sed features, bashbot will not run without installing additional software, see [Install Bashbot](doc/0_install.md) Bashbot [Documentation](https://github.com/topkecleon/telegram-bot-bash) and [Downloads](https://github.com/topkecleon/telegram-bot-bash/releases) are availible on www.github.com @@ -218,4 +218,4 @@ This may happen if to many wrong requests are sent to api.telegram.org, e.g. usi If you feel that there's something missing or if you found a bug, feel free to submit a pull request! -#### $$VERSION$$ v0.96-pre-9-gb23aadd +#### $$VERSION$$ v0.96-pre-13-ga71d68e diff --git a/README.txt b/README.txt index 0b50d73..c3b5a90 100644 --- a/README.txt +++ b/README.txt @@ -23,8 +23,8 @@ More concret on the common commands provided by recent versions of [toybox](https://landley.net/toybox/help.html), see [Developer Notes](doc/7_develop.md#common-commands) -*Note for MacOS and BSD Users:* As bashbot use behavior of recent bash and -(gnu)sed versions, bashbot may not run without installing additional software, +*Note for MacOS and BSD Users:* As bashbot heavily uses modern bash and (gnu) +grep/sed features, bashbot will not run without installing additional software, see [Install Bashbot](doc/0_install.md) @@ -304,4 +304,4 @@ in 'mycommands.sh' as example. If you feel that there's something missing or if you found a bug, feel free to submit a pull request! -#### $$VERSION$$ v0.96-pre-9-gb23aadd +#### $$VERSION$$ v0.96-pre-13-ga71d68e diff --git a/bashbot.sh b/bashbot.sh index f39bff7..cdca84e 100755 --- a/bashbot.sh +++ b/bashbot.sh @@ -11,7 +11,7 @@ # This file is public domain in the USA and all free countries. # Elsewhere, consider it to be WTFPLv2. (wtfpl.net/txt/copying) # -#### $$VERSION$$ v0.96-pre-12-gdba4f95 +#### $$VERSION$$ v0.96-pre-13-ga71d68e # # Exit Codes: # - 0 sucess (hopefully) @@ -789,8 +789,8 @@ start_bot() { local OFFSET=0 # adaptive sleep deafults local nextsleep="100" : - local addsleep="100" - local maxsleep="$(( ${BASHBOT_SLEEP:-5000} + 100 ))" + local stepsleep="${BASHBOT_SLEEP_STEP:-100}" + local maxsleep="${BASHBOT_SLEEP:-5000}" # redirect to Debug.log [[ "${DEBUG}" == *"debug" ]] && exec &>>"${LOGDIR}/DEBUG.log" [ -n "${DEBUG}" ] && date && echo "Start BASHBOT in Mode \"${DEBUG}\"" @@ -806,7 +806,7 @@ start_bot() { # shellcheck source=./commands.sh source "${COMMANDS}" "startbot" # start timer events - if _is_function start_timer ; then + if [ -n "${BASHBOT_START_TIMER}" ] ; then # shellcheck disable=SC2064 trap "event_timer $DEBUG" ALRM start_timer & @@ -814,11 +814,9 @@ start_bot() { trap "kill -9 $!; exit" EXIT INT HUP TERM QUIT fi while true; do -set -x # adaptive sleep in ms rounded to next 0.1 s sleep "$(printf '%.1f' "$((nextsleep))e-3")" - ((nextsleep+= addsleep , nextsleep= nextsleep>maxsleep ?maxsleep:nextsleep)) -set +x + ((nextsleep+= stepsleep , nextsleep= nextsleep>maxsleep ?maxsleep:nextsleep)) # get next update UPDATE="$(getJson "$UPD_URL$OFFSET" 2>/dev/null | "${JSONSHFILE}" -s -b -n | iconv -f utf-8 -t utf-8 -c)" # did we ge an responsn0r diff --git a/mycommands.sh b/mycommands.sh index a75080d..1cf5dd4 100644 --- a/mycommands.sh +++ b/mycommands.sh @@ -8,7 +8,7 @@ # #### if you start to develop your own bot, use the clean version of this file: # #### mycommands.clean # -#### $$VERSION$$ v0.96-pre-9-gb23aadd +#### $$VERSION$$ v0.96-pre-13-ga71d68e # # uncomment the following lines to overwrite info and help messages @@ -31,8 +31,19 @@ export FILE_REGEX="${BASHBOT_ETC}/.*" # unset BASHBOT_RETRY to enable retry in case of recoverable errors, e.g. throtteling # see logs/ERROR.log for information why send_messages etc. fail -# unset BOTSEND_RETRY export BOTSEND_RETRY="no" +#unset BOTSEND_RETRY + +# set value for adaptive sleeping while waitingnfor uodates in millisconds +# max slepp between polling updates 10s (default 5s) +export BASHBOT_SLEEP="10000" +# add 0.2s if no update availble, up to BASHBOT_SLEEP (default 0.1s) +export BASHBOT_SLEEP_STEP="200" + +# if you want to use timer functions, set BASHBOT_START_TImer to not empty value +# default is to nit start timer +unset BASHBOT_START_TIMER +#export BASHBOT_START_TIMER="yes" # set to "yes" and give your bot admin privilegs to remove service messaes from groups export SILENCER="no" diff --git a/mycommands.sh.clean b/mycommands.sh.clean index 184fd92..1902803 100644 --- a/mycommands.sh.clean +++ b/mycommands.sh.clean @@ -4,7 +4,7 @@ # files: mycommands.sh.clean # copy to mycommands.sh and add all your commands and functions here ... # -#### $$VERSION$$ v0.96-pre-9-gb23aadd +#### $$VERSION$$ v0.96-pre-13-ga71d68e # ########## @@ -29,9 +29,19 @@ export FILE_REGEX="${BASHBOT_ETC}/.*" # unset BASHBOT_RETRY to enable retry in case of recoverable errors, e.g. throtteling # see logs/ERROR.log for information why send_messages etc. fail -# unset BOTSEND_RETRY export BOTSEND_RETRY="no" +#unset BOTSEND_RETRY +# set value for adaptive sleeping while waitingnfor uodates in millisconds +# max slepp between polling updates 10s (default 5s) +export BASHBOT_SLEEP="10000" +# add 0.2s if no update availble, up to BASHBOT_SLEEP (default 0.1s) +export BASHBOT_SLEEP_STEP="200" + +# if you want to use timer functions, set BASHBOT_START_TImer to not empty value +# default is to nit start timer +unset BASHBOT_START_TIMER +#export BASHBOT_START_TIMER="yes" # set to "yes" and give your bot admin privilegs to remove service messaes from groups export SILENCER="no" From eace5e18a4288813c5151e311c6d07290fca4378 Mon Sep 17 00:00:00 2001 From: "Kay Marquardt (Gnadelwartz)" Date: Tue, 9 Jun 2020 17:51:04 +0200 Subject: [PATCH 06/21] prefix logged message with date --- bashbot.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bashbot.sh b/bashbot.sh index cdca84e..ab1ddb3 100755 --- a/bashbot.sh +++ b/bashbot.sh @@ -11,7 +11,7 @@ # This file is public domain in the USA and all free countries. # Elsewhere, consider it to be WTFPLv2. (wtfpl.net/txt/copying) # -#### $$VERSION$$ v0.96-pre-13-ga71d68e +#### $$VERSION$$ v0.96-pre-14-g4fd6f98 # # Exit Codes: # - 0 sucess (hopefully) @@ -509,7 +509,7 @@ process_updates() { process_client() { local num="$1" debug="$2" CMD=( ); iQUERY=( ) - [[ "${debug}" = *"debug"* ]] && cat <<< "$UPDATE" >>"${LOGDIR}/MESSAGE.log" + [[ "${debug}" = *"debug"* ]] && printf "%s new Message ========\n%s\n" "$(date)" "$UPDATE" >>"${LOGDIR}/MESSAGE.log" iQUERY[ID]="${UPD["result",${num},"inline_query","id"]}" CHAT[ID]="${UPD["result",${num},"message","chat","id"]}" USER[ID]="${UPD["result",${num},"message","from","id"]}" @@ -835,7 +835,7 @@ start_bot() { else # ups, something bad happend, wait maxsleep (( nextsleep=maxsleep )) - printf "%s: Ups, got no response on update, sleep %.1fs" "$(date)" "$((nextsleep))e-3" >>"${ERRORLOG}" + printf "%s: Got no response while wait for telegram update, sleep %ds" "$(date)" "$((nextsleep))e-3" >>"${ERRORLOG}" fi done } From a792048dfe7a4fa0bcf79b4a140394f7d015ad24 Mon Sep 17 00:00:00 2001 From: "Kay Marquardt (Gnadelwartz)" Date: Tue, 9 Jun 2020 19:21:46 +0200 Subject: [PATCH 07/21] fix problems found by tests --- bashbot.sh | 59 ++++++++++++++++++++---------------------------- dev/all-tests.sh | 2 +- 2 files changed, 26 insertions(+), 35 deletions(-) diff --git a/bashbot.sh b/bashbot.sh index ab1ddb3..a1555c7 100755 --- a/bashbot.sh +++ b/bashbot.sh @@ -11,7 +11,7 @@ # This file is public domain in the USA and all free countries. # Elsewhere, consider it to be WTFPLv2. (wtfpl.net/txt/copying) # -#### $$VERSION$$ v0.96-pre-14-g4fd6f98 +#### $$VERSION$$ v0.96-pre-15-geace5e1 # # Exit Codes: # - 0 sucess (hopefully) @@ -77,9 +77,6 @@ MODULEDIR="${SCRIPTDIR}/modules" # adjust locations based on source and real name if [ "${SCRIPT}" != "${REALME}" ] || [ "$1" = "source" ]; then SOURCE="yes" -else - SCRIPT="./$(basename "${SCRIPT}")" - MODULEDIR="./$(basename "${MODULEDIR}")" fi if [ -n "$BASHBOT_HOME" ]; then @@ -106,28 +103,6 @@ if [ ! -w "." ]; then ls -ld . fi -############### -# load modules -for modules in "${MODULEDIR:-.}"/*.sh ; do - # shellcheck source=./modules/aliases.sh - if ! _is_function "$(basename "${modules}")" && [ -r "${modules}" ]; then source "${modules}" "source"; fi -done - -# shellcheck source=./modules/jsonDB.sh -source "${MODULEDIR:-.}"/jsonDB.sh - - - -##################### -# BASHBOT INTERNAL functions -# - -#jsonDB is now mandatory -if ! _is_function jssh_newDB ; then - echo -e "${RED}ERROR: Mandatory module jsonDB is missing or not readable!" - exit 6 -fi - # Setup and check environment if BOTTOKEN is NOT set TOKENFILE="${BASHBOT_ETC:-.}/token" BOTADMIN="${BASHBOT_ETC:-.}/botadmin" @@ -191,8 +166,7 @@ if [ -z "${BOTTOKEN}" ]; then fi # setup count file if [ ! -f "${COUNTFILE}.jssh" ]; then - jssh_newDB_async "${COUNTFILE}" - jssh_insertKeyDB_async 'counted_user_chat_id' "num_messages_seen" "${COUNTFILE}" + printf '["counted_user_chat_id"]\t"num_messages_seen"\n' > "${COUNTFILE}.jssh" # convert old file on creation if [ -r "${COUNTFILE}" ];then sed 's/COUNT/\[\"/;s/$/\"\]\t\"1\"/' < "${COUNTFILE}" >> "${COUNTFILE}.jssh" @@ -204,12 +178,9 @@ if [ -z "${BOTTOKEN}" ]; then fi # setup blocked file if [ ! -f "${BLOCKEDFILE}.jssh" ]; then - jssh_newDB_async "${BLOCKEDFILE}" - jssh_insertKeyDB_async 'blocked_user_or_chat_id' "name and reason" "${BLOCKEDFILE}" + printf '["blocked_user_or_chat_id"]\t"name and reason"\n' >"${BLOCKEDFILE}.jssh" fi fi -# cleanup (remove double entries) countfile on startup -[ "${SOURCE}" != "yes" ] && jssh_deleteKeyDB_async "CLEAN_COUNTER_DATABASE_ON_STARTUP" "${COUNTFILE}" # do we have BSD sed if ! sed '1ia' /dev/null; then @@ -230,9 +201,9 @@ fi ################## # here we start with the real stuff -URL="${BASHBOT_URL:-https://api.telegram.org/bot}${BOTTOKEN}" BOTSEND_RETRY="no" # do not retry by default +URL="${BASHBOT_URL:-https://api.telegram.org/bot}${BOTTOKEN}" ME_URL=$URL'/getMe' UPD_URL=$URL'/getUpdates?offset=' @@ -259,6 +230,25 @@ if [ "${SOURCE}" != "yes" ]; then source "${COMMANDS}" "source" fi +############### +# load modules +for modules in "${MODULEDIR:-.}"/*.sh ; do + # shellcheck source=./modules/aliases.sh + if ! _is_function "$(basename "${modules}")" && [ -r "${modules}" ]; then source "${modules}" "source"; fi +done + +##################### +# BASHBOT INTERNAL functions +# + +#jsonDB is now mandatory +if ! _is_function jssh_newDB ; then + echo -e "${RED}ERROR: Mandatory module jsonDB is missing or not readable!" + exit 6 +fi + +# cleanup (remove double entries) countfile on startup +[ "${SOURCE}" != "yes" ] && jssh_deleteKeyDB_async "CLEAN_COUNTER_DATABASE_ON_STARTUP" "${COUNTFILE}" ################# # BASHBOT COMMON functions @@ -890,7 +880,8 @@ JSONSHFILE="${BASHBOT_JSONSH:-${SCRIPTDIR}/JSON.sh/JSON.sh}" if [ ! -f "${JSONSHFILE}" ]; then echo "Seems to be first run, Downloading ${JSONSHFILE}..." - mkdir "${SCRIPTDIR}/JSON.sh" 2>/dev/null && chmod +w "${SCRIPTDIR}/JSON.sh" + [ "${SCRIPTDIR}/JSON.sh/JSON.sh" = "${JSONSHFILE}" ] &&\ + mkdir "${SCRIPTDIR}/JSON.sh" 2>/dev/null && chmod +w "${SCRIPTDIR}/JSON.sh" getJson "https://cdn.jsdelivr.net/gh/dominictarr/JSON.sh/JSON.sh" >"${JSONSHFILE}" chmod +x "${JSONSHFILE}" fi diff --git a/dev/all-tests.sh b/dev/all-tests.sh index cd1fcd3..c06ea03 100755 --- a/dev/all-tests.sh +++ b/dev/all-tests.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash # this has to run once atfer git clone # and every time we create new hooks -#### $$VERSION$$ v0.96-dev3-1-g2a66ee9 +#### $$VERSION$$ v0.96-pre-15-geace5e1 # magic to ensure that we're always inside the root of our application, # no matter from which directory we'll run script From deeef7edc5956e6f91ea44b7689807d0ce1a37f3 Mon Sep 17 00:00:00 2001 From: "Kay Marquardt (Gnadelwartz)" Date: Tue, 9 Jun 2020 19:26:51 +0200 Subject: [PATCH 08/21] show files after init --- bashbot.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bashbot.sh b/bashbot.sh index a1555c7..45abe09 100755 --- a/bashbot.sh +++ b/bashbot.sh @@ -868,6 +868,8 @@ bot_init() { find . -name '*.jssh' -exec chmod u+w \{\} + #ls -la fi + # show result + ls -l } if ! _is_function send_message ; then From 6d940c7cc869a999f0f2e303a0c77f6ba4adabd0 Mon Sep 17 00:00:00 2001 From: "Kay Marquardt (Gnadelwartz)" Date: Tue, 9 Jun 2020 22:45:34 +0200 Subject: [PATCH 09/21] fix jsshGetKey --- bashbot.sh | 2 +- modules/jsonDB.sh | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/bashbot.sh b/bashbot.sh index 45abe09..c4b35a4 100755 --- a/bashbot.sh +++ b/bashbot.sh @@ -11,7 +11,7 @@ # This file is public domain in the USA and all free countries. # Elsewhere, consider it to be WTFPLv2. (wtfpl.net/txt/copying) # -#### $$VERSION$$ v0.96-pre-15-geace5e1 +#### $$VERSION$$ v0.96-pre-17-gdeeef7e # # Exit Codes: # - 0 sucess (hopefully) diff --git a/modules/jsonDB.sh b/modules/jsonDB.sh index ddba7ac..7d72e45 100644 --- a/modules/jsonDB.sh +++ b/modules/jsonDB.sh @@ -5,7 +5,7 @@ # This file is public domain in the USA and all free countries. # Elsewhere, consider it to be WTFPLv2. (wtfpl.net/txt/copying) # -#### $$VERSION$$ v0.96-pre-2-g30b5b1a +#### $$VERSION$$ v0.96-pre-17-gdeeef7e # # source from commands.sh to use jsonDB functions # @@ -130,7 +130,7 @@ if _exists flock; then { flock -s -w 1 200 Json2Array "oldARR" <"${DB}" } 200>"${DB}${BASHBOT_LOCKNAME}" - printf '%f' "${oldARR["$1"]}" + printf '%s' "${oldARR["$1"]}" } From 7790e47a7e00537460305172ca204c35a5df7581 Mon Sep 17 00:00:00 2001 From: "Kay Marquardt (Gnadelwartz)" Date: Tue, 9 Jun 2020 22:54:09 +0200 Subject: [PATCH 10/21] fix get error code --- bashbot.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bashbot.sh b/bashbot.sh index c4b35a4..1a8eb1b 100755 --- a/bashbot.sh +++ b/bashbot.sh @@ -11,7 +11,7 @@ # This file is public domain in the USA and all free countries. # Elsewhere, consider it to be WTFPLv2. (wtfpl.net/txt/copying) # -#### $$VERSION$$ v0.96-pre-17-gdeeef7e +#### $$VERSION$$ v0.96-pre-18-g6d940c7 # # Exit Codes: # - 0 sucess (hopefully) @@ -389,7 +389,7 @@ sendJsonResult(){ else # oops something went wrong! if [ "${res}" != "" ]; then - BOTSENT[ERROR]="$(JsonGeOtValue '"error_code"' <<< "${1}")" + BOTSENT[ERROR]="$(JsonGetValue '"error_code"' <<< "${1}")" BOTSENT[DESCRIPTION]="$(JsonGetString '"description"' <<< "${1}")" BOTSENT[RETRY]="$(JsonGetValue '"parameters","retry_after"' <<< "${1}")" else From 1133f25f3529074829cbecfd8553410a35d2c0ad Mon Sep 17 00:00:00 2001 From: "Kay Marquardt (Gnadelwartz)" Date: Wed, 10 Jun 2020 07:50:43 +0200 Subject: [PATCH 11/21] some small changes to error messages --- bashbot.sh | 73 +++++++++++++++++++++++++++++++----------------------- 1 file changed, 42 insertions(+), 31 deletions(-) diff --git a/bashbot.sh b/bashbot.sh index 1a8eb1b..1556ac8 100755 --- a/bashbot.sh +++ b/bashbot.sh @@ -11,7 +11,7 @@ # This file is public domain in the USA and all free countries. # Elsewhere, consider it to be WTFPLv2. (wtfpl.net/txt/copying) # -#### $$VERSION$$ v0.96-pre-18-g6d940c7 +#### $$VERSION$$ v0.96-pre-19-g7790e47 # # Exit Codes: # - 0 sucess (hopefully) @@ -77,6 +77,9 @@ MODULEDIR="${SCRIPTDIR}/modules" # adjust locations based on source and real name if [ "${SCRIPT}" != "${REALME}" ] || [ "$1" = "source" ]; then SOURCE="yes" +else + SCRIPT="./$(basename "${SCRIPT}")" + MODULEDIR="./$(basename "${MODULEDIR}")" fi if [ -n "$BASHBOT_HOME" ]; then @@ -103,6 +106,28 @@ if [ ! -w "." ]; then ls -ld . fi +############### +# load modules +for modules in "${MODULEDIR:-.}"/*.sh ; do + # shellcheck source=./modules/aliases.sh + if ! _is_function "$(basename "${modules}")" && [ -r "${modules}" ]; then source "${modules}" "source"; fi +done + +# shellcheck source=./modules/jsonDB.sh +source "${MODULEDIR:-.}"/jsonDB.sh + + + +##################### +# BASHBOT INTERNAL functions +# + +#jsonDB is now mandatory +if ! _is_function jssh_newDB ; then + echo -e "${RED}ERROR: Mandatory module jsonDB is missing or not readable!" + exit 6 +fi + # Setup and check environment if BOTTOKEN is NOT set TOKENFILE="${BASHBOT_ETC:-.}/token" BOTADMIN="${BASHBOT_ETC:-.}/botadmin" @@ -166,7 +191,8 @@ if [ -z "${BOTTOKEN}" ]; then fi # setup count file if [ ! -f "${COUNTFILE}.jssh" ]; then - printf '["counted_user_chat_id"]\t"num_messages_seen"\n' > "${COUNTFILE}.jssh" + jssh_newDB_async "${COUNTFILE}" + jssh_insertKeyDB_async 'counted_user_chat_id' "num_messages_seen" "${COUNTFILE}" # convert old file on creation if [ -r "${COUNTFILE}" ];then sed 's/COUNT/\[\"/;s/$/\"\]\t\"1\"/' < "${COUNTFILE}" >> "${COUNTFILE}.jssh" @@ -178,9 +204,12 @@ if [ -z "${BOTTOKEN}" ]; then fi # setup blocked file if [ ! -f "${BLOCKEDFILE}.jssh" ]; then - printf '["blocked_user_or_chat_id"]\t"name and reason"\n' >"${BLOCKEDFILE}.jssh" + jssh_newDB_async "${BLOCKEDFILE}" + jssh_insertKeyDB_async 'blocked_user_or_chat_id' "name and reason" "${BLOCKEDFILE}" fi fi +# cleanup (remove double entries) countfile on startup +[ "${SOURCE}" != "yes" ] && jssh_deleteKeyDB_async "CLEAN_COUNTER_DATABASE_ON_STARTUP" "${COUNTFILE}" # do we have BSD sed if ! sed '1ia' /dev/null; then @@ -201,9 +230,9 @@ fi ################## # here we start with the real stuff +URL="${BASHBOT_URL:-https://api.telegram.org/bot}${BOTTOKEN}" BOTSEND_RETRY="no" # do not retry by default -URL="${BASHBOT_URL:-https://api.telegram.org/bot}${BOTTOKEN}" ME_URL=$URL'/getMe' UPD_URL=$URL'/getUpdates?offset=' @@ -230,25 +259,6 @@ if [ "${SOURCE}" != "yes" ]; then source "${COMMANDS}" "source" fi -############### -# load modules -for modules in "${MODULEDIR:-.}"/*.sh ; do - # shellcheck source=./modules/aliases.sh - if ! _is_function "$(basename "${modules}")" && [ -r "${modules}" ]; then source "${modules}" "source"; fi -done - -##################### -# BASHBOT INTERNAL functions -# - -#jsonDB is now mandatory -if ! _is_function jssh_newDB ; then - echo -e "${RED}ERROR: Mandatory module jsonDB is missing or not readable!" - exit 6 -fi - -# cleanup (remove double entries) countfile on startup -[ "${SOURCE}" != "yes" ] && jssh_deleteKeyDB_async "CLEAN_COUNTER_DATABASE_ON_STARTUP" "${COUNTFILE}" ################# # BASHBOT COMMON functions @@ -362,6 +372,7 @@ fi sendJsonRetry(){ local retry="${1}"; shift [[ "${1}" =~ ^\ *[0-9.]+\ *$ ]] && sleep "${1}"; shift +set -x case "${retry}" in 'sendJson'*) sendJson "$@" @@ -373,6 +384,7 @@ sendJsonRetry(){ printf "%s: SendJsonRetry: unknown, cannot retry %s\n" "$(date)" "${retry}" >>"${ERRORLOG}" ;; esac +set +x } # process sendJson result @@ -383,7 +395,7 @@ sendJsonResult(){ BOTSENT[OK]="$(JsonGetLine '"ok"' <<< "${1}")" if [ "${BOTSENT[OK]}" = "true" ]; then BOTSENT[ID]="$(JsonGetValue '"result","message_id"' <<< "${1}")" - [ -n "${BASHBOT_EVENT_SEND[*]}" ] && event_send "send" "${@:2}" + [ -n "${BASHBOT_EVENT_SEND[*]}" ] && event_send "send" "${@:3}" return # hot path everthing OK! else @@ -405,9 +417,11 @@ sendJsonResult(){ # OK, we can retry sendJson, let's see what's failed # throttled, telegram say we send to much messages if [ -n "${BOTSENT[RETRY]}" ]; then - BOTSEND_RETRY="(( ${BOTSENT[RETRY]} * 15/10 ))" + BOTSEND_RETRY="$(( BOTSENT[RETRY] * 15/10 ))" printf "Retry %s in %s seconds ...\n" "${2}" "${BOTSEND_RETRY}" +set -x sendJsonRetry "${2}" "${BOTSEND_RETRY}" "${@:2}" +set +x unset BOTSEND_RETRY return fi @@ -499,7 +513,7 @@ process_updates() { process_client() { local num="$1" debug="$2" CMD=( ); iQUERY=( ) - [[ "${debug}" = *"debug"* ]] && printf "%s new Message ========\n%s\n" "$(date)" "$UPDATE" >>"${LOGDIR}/MESSAGE.log" + [[ "${debug}" = *"debug"* ]] && printf "\n%s: New Message ========\n%s\n" "$(date)" "$UPDATE" >>"${LOGDIR}/MESSAGE.log" iQUERY[ID]="${UPD["result",${num},"inline_query","id"]}" CHAT[ID]="${UPD["result",${num},"message","chat","id"]}" USER[ID]="${UPD["result",${num},"message","from","id"]}" @@ -825,7 +839,7 @@ start_bot() { else # ups, something bad happend, wait maxsleep (( nextsleep=maxsleep )) - printf "%s: Got no response while wait for telegram update, sleep %ds" "$(date)" "$((nextsleep))e-3" >>"${ERRORLOG}" + printf "%s: Timeout or broken/no connection on telegram update, sleep %ds\n" "$(date)" $((nextsleep))e-3 >>"${ERRORLOG}" fi done } @@ -868,8 +882,6 @@ bot_init() { find . -name '*.jssh' -exec chmod u+w \{\} + #ls -la fi - # show result - ls -l } if ! _is_function send_message ; then @@ -882,8 +894,7 @@ JSONSHFILE="${BASHBOT_JSONSH:-${SCRIPTDIR}/JSON.sh/JSON.sh}" if [ ! -f "${JSONSHFILE}" ]; then echo "Seems to be first run, Downloading ${JSONSHFILE}..." - [ "${SCRIPTDIR}/JSON.sh/JSON.sh" = "${JSONSHFILE}" ] &&\ - mkdir "${SCRIPTDIR}/JSON.sh" 2>/dev/null && chmod +w "${SCRIPTDIR}/JSON.sh" + mkdir "${SCRIPTDIR}/JSON.sh" 2>/dev/null && chmod +w "${SCRIPTDIR}/JSON.sh" getJson "https://cdn.jsdelivr.net/gh/dominictarr/JSON.sh/JSON.sh" >"${JSONSHFILE}" chmod +x "${JSONSHFILE}" fi From 72c8531ceec1dd0e809e4f9b2771dd43e9ba5f17 Mon Sep 17 00:00:00 2001 From: "Kay Marquardt (Gnadelwartz)" Date: Wed, 10 Jun 2020 08:20:48 +0200 Subject: [PATCH 12/21] revert and redo fixed last commit --- bashbot.sh | 72 +++++++++++++++++++++++------------------------------- 1 file changed, 30 insertions(+), 42 deletions(-) diff --git a/bashbot.sh b/bashbot.sh index 1556ac8..62bc360 100755 --- a/bashbot.sh +++ b/bashbot.sh @@ -11,7 +11,7 @@ # This file is public domain in the USA and all free countries. # Elsewhere, consider it to be WTFPLv2. (wtfpl.net/txt/copying) # -#### $$VERSION$$ v0.96-pre-19-g7790e47 +#### $$VERSION$$ v0.96-pre-20-g1133f25 # # Exit Codes: # - 0 sucess (hopefully) @@ -77,9 +77,6 @@ MODULEDIR="${SCRIPTDIR}/modules" # adjust locations based on source and real name if [ "${SCRIPT}" != "${REALME}" ] || [ "$1" = "source" ]; then SOURCE="yes" -else - SCRIPT="./$(basename "${SCRIPT}")" - MODULEDIR="./$(basename "${MODULEDIR}")" fi if [ -n "$BASHBOT_HOME" ]; then @@ -106,28 +103,6 @@ if [ ! -w "." ]; then ls -ld . fi -############### -# load modules -for modules in "${MODULEDIR:-.}"/*.sh ; do - # shellcheck source=./modules/aliases.sh - if ! _is_function "$(basename "${modules}")" && [ -r "${modules}" ]; then source "${modules}" "source"; fi -done - -# shellcheck source=./modules/jsonDB.sh -source "${MODULEDIR:-.}"/jsonDB.sh - - - -##################### -# BASHBOT INTERNAL functions -# - -#jsonDB is now mandatory -if ! _is_function jssh_newDB ; then - echo -e "${RED}ERROR: Mandatory module jsonDB is missing or not readable!" - exit 6 -fi - # Setup and check environment if BOTTOKEN is NOT set TOKENFILE="${BASHBOT_ETC:-.}/token" BOTADMIN="${BASHBOT_ETC:-.}/botadmin" @@ -191,8 +166,7 @@ if [ -z "${BOTTOKEN}" ]; then fi # setup count file if [ ! -f "${COUNTFILE}.jssh" ]; then - jssh_newDB_async "${COUNTFILE}" - jssh_insertKeyDB_async 'counted_user_chat_id' "num_messages_seen" "${COUNTFILE}" + printf '["counted_user_chat_id"]\t"num_messages_seen"\n' > "${COUNTFILE}.jssh" # convert old file on creation if [ -r "${COUNTFILE}" ];then sed 's/COUNT/\[\"/;s/$/\"\]\t\"1\"/' < "${COUNTFILE}" >> "${COUNTFILE}.jssh" @@ -204,12 +178,9 @@ if [ -z "${BOTTOKEN}" ]; then fi # setup blocked file if [ ! -f "${BLOCKEDFILE}.jssh" ]; then - jssh_newDB_async "${BLOCKEDFILE}" - jssh_insertKeyDB_async 'blocked_user_or_chat_id' "name and reason" "${BLOCKEDFILE}" + printf '["blocked_user_or_chat_id"]\t"name and reason"\n' >"${BLOCKEDFILE}.jssh" fi fi -# cleanup (remove double entries) countfile on startup -[ "${SOURCE}" != "yes" ] && jssh_deleteKeyDB_async "CLEAN_COUNTER_DATABASE_ON_STARTUP" "${COUNTFILE}" # do we have BSD sed if ! sed '1ia' /dev/null; then @@ -230,9 +201,9 @@ fi ################## # here we start with the real stuff -URL="${BASHBOT_URL:-https://api.telegram.org/bot}${BOTTOKEN}" BOTSEND_RETRY="no" # do not retry by default +URL="${BASHBOT_URL:-https://api.telegram.org/bot}${BOTTOKEN}" ME_URL=$URL'/getMe' UPD_URL=$URL'/getUpdates?offset=' @@ -259,6 +230,25 @@ if [ "${SOURCE}" != "yes" ]; then source "${COMMANDS}" "source" fi +############### +# load modules +for modules in "${MODULEDIR:-.}"/*.sh ; do + # shellcheck source=./modules/aliases.sh + if ! _is_function "$(basename "${modules}")" && [ -r "${modules}" ]; then source "${modules}" "source"; fi +done + +##################### +# BASHBOT INTERNAL functions +# + +#jsonDB is now mandatory +if ! _is_function jssh_newDB ; then + echo -e "${RED}ERROR: Mandatory module jsonDB is missing or not readable!" + exit 6 +fi + +# cleanup (remove double entries) countfile on startup +[ "${SOURCE}" != "yes" ] && jssh_deleteKeyDB_async "CLEAN_COUNTER_DATABASE_ON_STARTUP" "${COUNTFILE}" ################# # BASHBOT COMMON functions @@ -372,7 +362,6 @@ fi sendJsonRetry(){ local retry="${1}"; shift [[ "${1}" =~ ^\ *[0-9.]+\ *$ ]] && sleep "${1}"; shift -set -x case "${retry}" in 'sendJson'*) sendJson "$@" @@ -384,7 +373,6 @@ set -x printf "%s: SendJsonRetry: unknown, cannot retry %s\n" "$(date)" "${retry}" >>"${ERRORLOG}" ;; esac -set +x } # process sendJson result @@ -395,7 +383,7 @@ sendJsonResult(){ BOTSENT[OK]="$(JsonGetLine '"ok"' <<< "${1}")" if [ "${BOTSENT[OK]}" = "true" ]; then BOTSENT[ID]="$(JsonGetValue '"result","message_id"' <<< "${1}")" - [ -n "${BASHBOT_EVENT_SEND[*]}" ] && event_send "send" "${@:3}" + [ -n "${BASHBOT_EVENT_SEND[*]}" ] && event_send "send" "${@:2}" return # hot path everthing OK! else @@ -417,11 +405,9 @@ sendJsonResult(){ # OK, we can retry sendJson, let's see what's failed # throttled, telegram say we send to much messages if [ -n "${BOTSENT[RETRY]}" ]; then - BOTSEND_RETRY="$(( BOTSENT[RETRY] * 15/10 ))" + BOTSEND_RETRY="(( ${BOTSENT[RETRY]} * 15/10 ))" printf "Retry %s in %s seconds ...\n" "${2}" "${BOTSEND_RETRY}" -set -x sendJsonRetry "${2}" "${BOTSEND_RETRY}" "${@:2}" -set +x unset BOTSEND_RETRY return fi @@ -513,7 +499,7 @@ process_updates() { process_client() { local num="$1" debug="$2" CMD=( ); iQUERY=( ) - [[ "${debug}" = *"debug"* ]] && printf "\n%s: New Message ========\n%s\n" "$(date)" "$UPDATE" >>"${LOGDIR}/MESSAGE.log" + [[ "${debug}" = *"debug"* ]] && printf "\n%s new Message ========\n%s\n" "$(date)" "$UPDATE" >>"${LOGDIR}/MESSAGE.log" iQUERY[ID]="${UPD["result",${num},"inline_query","id"]}" CHAT[ID]="${UPD["result",${num},"message","chat","id"]}" USER[ID]="${UPD["result",${num},"message","from","id"]}" @@ -880,8 +866,9 @@ bot_init() { chmod -R o-r,o-w "${COUNTFILE}"* "${BLOCKEDFILE}"* "${DATADIR}" "${TOKENFILE}" "${BOTADMIN}" "${BOTACL}" 2>/dev/null # jsshDB must writeable by owner find . -name '*.jssh' -exec chmod u+w \{\} + - #ls -la fi + # show result + ls -l } if ! _is_function send_message ; then @@ -894,7 +881,8 @@ JSONSHFILE="${BASHBOT_JSONSH:-${SCRIPTDIR}/JSON.sh/JSON.sh}" if [ ! -f "${JSONSHFILE}" ]; then echo "Seems to be first run, Downloading ${JSONSHFILE}..." - mkdir "${SCRIPTDIR}/JSON.sh" 2>/dev/null && chmod +w "${SCRIPTDIR}/JSON.sh" + [ "${SCRIPTDIR}/JSON.sh/JSON.sh" = "${JSONSHFILE}" ] &&\ + mkdir "${SCRIPTDIR}/JSON.sh" 2>/dev/null && chmod +w "${SCRIPTDIR}/JSON.sh" getJson "https://cdn.jsdelivr.net/gh/dominictarr/JSON.sh/JSON.sh" >"${JSONSHFILE}" chmod +x "${JSONSHFILE}" fi From 0c0a4cc3c99fff23363f6363a3f31dfc87d92efe Mon Sep 17 00:00:00 2001 From: "Kay Marquardt (Gnadelwartz)" Date: Wed, 10 Jun 2020 08:26:23 +0200 Subject: [PATCH 13/21] some small changes to error messages --- bashbot.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bashbot.sh b/bashbot.sh index 62bc360..8afb0e6 100755 --- a/bashbot.sh +++ b/bashbot.sh @@ -11,7 +11,7 @@ # This file is public domain in the USA and all free countries. # Elsewhere, consider it to be WTFPLv2. (wtfpl.net/txt/copying) # -#### $$VERSION$$ v0.96-pre-20-g1133f25 +#### $$VERSION$$ v0.96-pre-21-g72c8531 # # Exit Codes: # - 0 sucess (hopefully) @@ -405,7 +405,7 @@ sendJsonResult(){ # OK, we can retry sendJson, let's see what's failed # throttled, telegram say we send to much messages if [ -n "${BOTSENT[RETRY]}" ]; then - BOTSEND_RETRY="(( ${BOTSENT[RETRY]} * 15/10 ))" + BOTSEND_RETRY="$(( BOTSENT[RETRY] * 15/10 ))" printf "Retry %s in %s seconds ...\n" "${2}" "${BOTSEND_RETRY}" sendJsonRetry "${2}" "${BOTSEND_RETRY}" "${@:2}" unset BOTSEND_RETRY @@ -499,7 +499,7 @@ process_updates() { process_client() { local num="$1" debug="$2" CMD=( ); iQUERY=( ) - [[ "${debug}" = *"debug"* ]] && printf "\n%s new Message ========\n%s\n" "$(date)" "$UPDATE" >>"${LOGDIR}/MESSAGE.log" + [[ "${debug}" = *"debug"* ]] && printf "\n%s New Message ========\n%s\n" "$(date)" "$UPDATE" >>"${LOGDIR}/MESSAGE.log" iQUERY[ID]="${UPD["result",${num},"inline_query","id"]}" CHAT[ID]="${UPD["result",${num},"message","chat","id"]}" USER[ID]="${UPD["result",${num},"message","from","id"]}" From eb89aee3d5027acf4f95896cbd69b6dc8f77539f Mon Sep 17 00:00:00 2001 From: "Kay Marquardt (Gnadelwartz)" Date: Wed, 10 Jun 2020 08:47:53 +0200 Subject: [PATCH 14/21] fix debug start message --- bashbot.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bashbot.sh b/bashbot.sh index 8afb0e6..06c668e 100755 --- a/bashbot.sh +++ b/bashbot.sh @@ -11,7 +11,7 @@ # This file is public domain in the USA and all free countries. # Elsewhere, consider it to be WTFPLv2. (wtfpl.net/txt/copying) # -#### $$VERSION$$ v0.96-pre-21-g72c8531 +#### $$VERSION$$ v0.96-pre-22-g0c0a4cc # # Exit Codes: # - 0 sucess (hopefully) @@ -783,7 +783,7 @@ start_bot() { local maxsleep="${BASHBOT_SLEEP:-5000}" # redirect to Debug.log [[ "${DEBUG}" == *"debug" ]] && exec &>>"${LOGDIR}/DEBUG.log" - [ -n "${DEBUG}" ] && date && echo "Start BASHBOT in Mode \"${DEBUG}\"" + [ -n "${DEBUG}" ] && printf "%s Start BASHBOT in Mode \"%s\"" "$(date)" "${DEBUG}" [[ "${DEBUG}" == "xdebug"* ]] && set -x #cleaup old pipes and empty logfiles find "${DATADIR}" -type p -delete From 5b2582129b0c5a748344d7fb5dc4b4c9b19551c5 Mon Sep 17 00:00:00 2001 From: "Kay Marquardt (Gnadelwartz)" Date: Wed, 10 Jun 2020 08:50:56 +0200 Subject: [PATCH 15/21] fix missing \n --- bashbot.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bashbot.sh b/bashbot.sh index 06c668e..c917e02 100755 --- a/bashbot.sh +++ b/bashbot.sh @@ -11,7 +11,7 @@ # This file is public domain in the USA and all free countries. # Elsewhere, consider it to be WTFPLv2. (wtfpl.net/txt/copying) # -#### $$VERSION$$ v0.96-pre-22-g0c0a4cc +#### $$VERSION$$ v0.96-pre-23-geb89aee # # Exit Codes: # - 0 sucess (hopefully) @@ -783,7 +783,7 @@ start_bot() { local maxsleep="${BASHBOT_SLEEP:-5000}" # redirect to Debug.log [[ "${DEBUG}" == *"debug" ]] && exec &>>"${LOGDIR}/DEBUG.log" - [ -n "${DEBUG}" ] && printf "%s Start BASHBOT in Mode \"%s\"" "$(date)" "${DEBUG}" + [ -n "${DEBUG}" ] && printf "%s Start BASHBOT in Mode \"%s\"\n" "$(date)" "${DEBUG}" [[ "${DEBUG}" == "xdebug"* ]] && set -x #cleaup old pipes and empty logfiles find "${DATADIR}" -type p -delete From f4c16572706f0892cd2b95a1f3652b3a744aa1e6 Mon Sep 17 00:00:00 2001 From: "Kay Marquardt (Gnadelwartz)" Date: Wed, 10 Jun 2020 09:32:53 +0200 Subject: [PATCH 16/21] fix keyDB_async, count cleanup only on bot start --- bashbot.sh | 10 ++++++---- modules/jsonDB.sh | 6 +++--- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/bashbot.sh b/bashbot.sh index c917e02..82b0711 100755 --- a/bashbot.sh +++ b/bashbot.sh @@ -11,7 +11,7 @@ # This file is public domain in the USA and all free countries. # Elsewhere, consider it to be WTFPLv2. (wtfpl.net/txt/copying) # -#### $$VERSION$$ v0.96-pre-23-geb89aee +#### $$VERSION$$ v0.96-pre-24-g5b25821 # # Exit Codes: # - 0 sucess (hopefully) @@ -247,9 +247,6 @@ if ! _is_function jssh_newDB ; then exit 6 fi -# cleanup (remove double entries) countfile on startup -[ "${SOURCE}" != "yes" ] && jssh_deleteKeyDB_async "CLEAN_COUNTER_DATABASE_ON_STARTUP" "${COUNTFILE}" - ################# # BASHBOT COMMON functions # $1 URL, $2 filename in DATADIR @@ -803,6 +800,11 @@ start_bot() { # shellcheck disable=SC2064 trap "kill -9 $!; exit" EXIT INT HUP TERM QUIT fi + # cleanup countfile on startup + [ "${SOURCE}" != "yes" ] && jssh_deleteKeyDB_async "CLEAN_COUNTER_DATABASE_ON_STARTUP" "${COUNTFILE}" + + ########## + # bot is ready, start processing updates ... while true; do # adaptive sleep in ms rounded to next 0.1 s sleep "$(printf '%.1f' "$((nextsleep))e-3")" diff --git a/modules/jsonDB.sh b/modules/jsonDB.sh index 7d72e45..14b5d96 100644 --- a/modules/jsonDB.sh +++ b/modules/jsonDB.sh @@ -5,7 +5,7 @@ # This file is public domain in the USA and all free countries. # Elsewhere, consider it to be WTFPLv2. (wtfpl.net/txt/copying) # -#### $$VERSION$$ v0.96-pre-17-gdeeef7e +#### $$VERSION$$ v0.96-pre-24-g5b25821 # # source from commands.sh to use jsonDB functions # @@ -280,9 +280,9 @@ jssh_deleteKeyDB_async() { [[ "$1" =~ ^[-a-zA-Z0-9,._]+$ ]] || return 3 local DB; DB="$(jssh_checkDB "$2")" declare -A oldARR - jssh_readDB_async "oldARR" "$2" || return "$?" + Json2Array "oldARR" <"${DB}" unset oldARR["$1"] - jssh_writeDB_async "oldARR" "$2" + Array2Json "oldARR" >"${DB}" } jssh_getKeyDB_async() { From 848219d82c4c67eceff806893a9314bd2169e50a Mon Sep 17 00:00:00 2001 From: "Kay Marquardt (Gnadelwartz)" Date: Wed, 10 Jun 2020 09:48:37 +0200 Subject: [PATCH 17/21] improve count cleanup on startup --- bashbot.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/bashbot.sh b/bashbot.sh index 82b0711..2a687a3 100755 --- a/bashbot.sh +++ b/bashbot.sh @@ -11,7 +11,7 @@ # This file is public domain in the USA and all free countries. # Elsewhere, consider it to be WTFPLv2. (wtfpl.net/txt/copying) # -#### $$VERSION$$ v0.96-pre-24-g5b25821 +#### $$VERSION$$ v0.96-pre-25-gf4c1657 # # Exit Codes: # - 0 sucess (hopefully) @@ -496,7 +496,7 @@ process_updates() { process_client() { local num="$1" debug="$2" CMD=( ); iQUERY=( ) - [[ "${debug}" = *"debug"* ]] && printf "\n%s New Message ========\n%s\n" "$(date)" "$UPDATE" >>"${LOGDIR}/MESSAGE.log" + [[ "${debug}" = *"debug"* ]] && printf "\n%s: New Message ==========\n%s\n" "$(date)" "$UPDATE" >>"${LOGDIR}/MESSAGE.log" iQUERY[ID]="${UPD["result",${num},"inline_query","id"]}" CHAT[ID]="${UPD["result",${num},"message","chat","id"]}" USER[ID]="${UPD["result",${num},"message","from","id"]}" @@ -780,7 +780,7 @@ start_bot() { local maxsleep="${BASHBOT_SLEEP:-5000}" # redirect to Debug.log [[ "${DEBUG}" == *"debug" ]] && exec &>>"${LOGDIR}/DEBUG.log" - [ -n "${DEBUG}" ] && printf "%s Start BASHBOT in Mode \"%s\"\n" "$(date)" "${DEBUG}" + [ -n "${DEBUG}" ] && printf "%s: Start BASHBOT in Mode \"%s\" ==========\n" "$(date)" "${DEBUG}" [[ "${DEBUG}" == "xdebug"* ]] && set -x #cleaup old pipes and empty logfiles find "${DATADIR}" -type p -delete @@ -801,7 +801,7 @@ start_bot() { trap "kill -9 $!; exit" EXIT INT HUP TERM QUIT fi # cleanup countfile on startup - [ "${SOURCE}" != "yes" ] && jssh_deleteKeyDB_async "CLEAN_COUNTER_DATABASE_ON_STARTUP" "${COUNTFILE}" + jssh_deleteKeyDB "CLEAN_COUNTER_DATABASE_ON_STARTUP" "${COUNTFILE}" ########## # bot is ready, start processing updates ... From ec7fce72ac62d01ce7b85540c9d792a7bf0764f2 Mon Sep 17 00:00:00 2001 From: "Kay Marquardt (Gnadelwartz)" Date: Wed, 10 Jun 2020 10:24:48 +0200 Subject: [PATCH 18/21] fix sleep on empty updatevresponse --- bashbot.sh | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/bashbot.sh b/bashbot.sh index 2a687a3..1c3e296 100755 --- a/bashbot.sh +++ b/bashbot.sh @@ -11,7 +11,7 @@ # This file is public domain in the USA and all free countries. # Elsewhere, consider it to be WTFPLv2. (wtfpl.net/txt/copying) # -#### $$VERSION$$ v0.96-pre-25-gf4c1657 +#### $$VERSION$$ v0.96-pre-26-g848219d # # Exit Codes: # - 0 sucess (hopefully) @@ -826,8 +826,9 @@ start_bot() { fi else # ups, something bad happend, wait maxsleep - (( nextsleep=maxsleep )) - printf "%s: Timeout or broken/no connection on telegram update, sleep %ds\n" "$(date)" $((nextsleep))e-3 >>"${ERRORLOG}" + (( nextsleep=maxsleep*2 )) + printf "%s: Timeout or broken/no connection on telegram update, sleep %ds\n"\ + "$(date)" "$((nextsleep))e-3" >>"${ERRORLOG}" fi done } From c2f47535cc45c7f51183c6ed40d668493db74ee9 Mon Sep 17 00:00:00 2001 From: "Kay Marquardt (Gnadelwartz)" Date: Wed, 10 Jun 2020 11:11:42 +0200 Subject: [PATCH 19/21] use function _round_float for converting ms to s --- bashbot.sh | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/bashbot.sh b/bashbot.sh index 1c3e296..473fa00 100755 --- a/bashbot.sh +++ b/bashbot.sh @@ -11,7 +11,7 @@ # This file is public domain in the USA and all free countries. # Elsewhere, consider it to be WTFPLv2. (wtfpl.net/txt/copying) # -#### $$VERSION$$ v0.96-pre-26-g848219d +#### $$VERSION$$ v0.96-pre-27-gec7fce7 # # Exit Codes: # - 0 sucess (hopefully) @@ -35,9 +35,8 @@ fi # some important helper functions # returns true if command exist -_exists() -{ - [ "$(LC_ALL=C type -t "$1")" = "file" ] +_exists() { + [ "$(LC_ALL=C type -t "${1}")" = "file" ] } # execute function if exists @@ -45,9 +44,14 @@ _exec_if_function() { [ "$(LC_ALL=C type -t "${1}")" != "function" ] || "$@" } # returns true if function exist -_is_function() -{ - [ "$(LC_ALL=C type -t "$1")" = "function" ] +_is_function() { + [ "$(LC_ALL=C type -t "${1}")" = "function" ] +} +# round $1 in international notation! , returns float with $2 decimal digits +# if $2 is not fiven or is not a positive number, it's set to zero +_round_float() { + local digit="${2}"; [[ "${2}" =~ ^[0-9]+$ ]] || digit="0" + LC_ALL=C printf "%.${digit}f" "${1}" } # read JSON.sh style data and asssign to an ARRAY # $1 ARRAY name, must be declared with "declare -A ARRAY" before calling @@ -807,7 +811,7 @@ start_bot() { # bot is ready, start processing updates ... while true; do # adaptive sleep in ms rounded to next 0.1 s - sleep "$(printf '%.1f' "$((nextsleep))e-3")" + sleep "$(_round_float "$((nextsleep))e-3")" ((nextsleep+= stepsleep , nextsleep= nextsleep>maxsleep ?maxsleep:nextsleep)) # get next update UPDATE="$(getJson "$UPD_URL$OFFSET" 2>/dev/null | "${JSONSHFILE}" -s -b -n | iconv -f utf-8 -t utf-8 -c)" @@ -828,7 +832,7 @@ start_bot() { # ups, something bad happend, wait maxsleep (( nextsleep=maxsleep*2 )) printf "%s: Timeout or broken/no connection on telegram update, sleep %ds\n"\ - "$(date)" "$((nextsleep))e-3" >>"${ERRORLOG}" + "$(date)" "$(_round_float "${nextsleep}e-3")" >>"${ERRORLOG}" fi done } From aec4e71dc8d61c3b67a5fe8a3aa8a0d23a83efe6 Mon Sep 17 00:00:00 2001 From: "Kay Marquardt (Gnadelwartz)" Date: Wed, 10 Jun 2020 14:02:29 +0200 Subject: [PATCH 20/21] always source commands.sh --- bashbot.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bashbot.sh b/bashbot.sh index 473fa00..432f9fb 100755 --- a/bashbot.sh +++ b/bashbot.sh @@ -11,7 +11,7 @@ # This file is public domain in the USA and all free countries. # Elsewhere, consider it to be WTFPLv2. (wtfpl.net/txt/copying) # -#### $$VERSION$$ v0.96-pre-27-gec7fce7 +#### $$VERSION$$ v0.96-pre-28-gc2f4753 # # Exit Codes: # - 0 sucess (hopefully) @@ -230,9 +230,9 @@ if [ "${SOURCE}" != "yes" ]; then ls -l "${COMMANDS}" exit 3 fi - # shellcheck source=./commands.sh - source "${COMMANDS}" "source" fi +# shellcheck source=./commands.sh +[ -r "${COMMANDS}" ] && source "${COMMANDS}" "source" ############### # load modules From 769d07d1513c516d9aa8cda8fe48b6575204b7dc Mon Sep 17 00:00:00 2001 From: "Kay Marquardt (Gnadelwartz)" Date: Wed, 10 Jun 2020 15:56:34 +0200 Subject: [PATCH 21/21] retry now working --- bashbot.sh | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/bashbot.sh b/bashbot.sh index 432f9fb..a3043d1 100755 --- a/bashbot.sh +++ b/bashbot.sh @@ -11,7 +11,7 @@ # This file is public domain in the USA and all free countries. # Elsewhere, consider it to be WTFPLv2. (wtfpl.net/txt/copying) # -#### $$VERSION$$ v0.96-pre-28-gc2f4753 +#### $$VERSION$$ v0.96-pre-29-gaec4e71 # # Exit Codes: # - 0 sucess (hopefully) @@ -363,18 +363,22 @@ fi sendJsonRetry(){ local retry="${1}"; shift [[ "${1}" =~ ^\ *[0-9.]+\ *$ ]] && sleep "${1}"; shift + printf "%s: RETRY %s %s %s\n" "$(date)" "${retry}" "${1}" "${2/[$'\n\r']*}" case "${retry}" in 'sendJson'*) sendJson "$@" + ;; 'sendUpload'*) sendUpload "$@" ;; *) - printf "%s: SendJsonRetry: unknown, cannot retry %s\n" "$(date)" "${retry}" >>"${ERRORLOG}" + printf "%s: Error: unknown function %s, cannot retry\n" "$(date)" "${retry}" + return ;; esac -} + [ "${BOTSENT[OK]}" = "true" ] && printf "%s: Retry OK: %s %s\n" "$(date)" "${retry}" "${1}" +} >>"${ERRORLOG}" # process sendJson result # stdout is written to ERROR.log @@ -406,7 +410,7 @@ sendJsonResult(){ # OK, we can retry sendJson, let's see what's failed # throttled, telegram say we send to much messages if [ -n "${BOTSENT[RETRY]}" ]; then - BOTSEND_RETRY="$(( BOTSENT[RETRY] * 15/10 ))" + BOTSEND_RETRY="$(( BOTSENT[RETRY]++ ))" printf "Retry %s in %s seconds ...\n" "${2}" "${BOTSEND_RETRY}" sendJsonRetry "${2}" "${BOTSEND_RETRY}" "${@:2}" unset BOTSEND_RETRY @@ -811,7 +815,7 @@ start_bot() { # bot is ready, start processing updates ... while true; do # adaptive sleep in ms rounded to next 0.1 s - sleep "$(_round_float "$((nextsleep))e-3")" + sleep "$(_round_float "${nextsleep}e-3" "1")" ((nextsleep+= stepsleep , nextsleep= nextsleep>maxsleep ?maxsleep:nextsleep)) # get next update UPDATE="$(getJson "$UPD_URL$OFFSET" 2>/dev/null | "${JSONSHFILE}" -s -b -n | iconv -f utf-8 -t utf-8 -c)"