#!/bin/bash #/--------------------------------------------------------------------------------------------------------| www.vdm.io |------/ # # @version 1.0.0 # @build 6th Marh, 2020 # @package sync websites # @author Llewellyn van der Merwe # @copyright Copyright (C) 2020. All Rights Reserved # @license GNU/GPL Version 2 or later - http://www.gnu.org/licenses/gpl-2.0.html # #/-----------------------------------------------------------------------------------------------------------------------------/ ############################################################## ############## CHECK ########## ############################################################## command -v rsync >/dev/null 2>&1 || { echo >&2 "We require rsync for this script to run, but it's not installed. Aborting." exit 1 } command -v crontab >/dev/null 2>&1 || { echo >&2 "We require crontab for this script to run, but it's not installed. Aborting." exit 1 } command -v md5sum >/dev/null 2>&1 || { echo >&2 "We require md5sum for this script to run, but it's not installed. Aborting." exit 1 } command -v awk >/dev/null 2>&1 || { echo >&2 "We require awk for this script to run, but it's not installed. Aborting." exit 1 } command -v realpath >/dev/null 2>&1 || { echo >&2 "We require realpath for this script to run, but it's not installed. Aborting." exit 1 } command -v stat >/dev/null 2>&1 || { echo >&2 "We require stat for this script to run, but it's not installed. Aborting." exit 1 } ############################ GLOBAL ########################## ACTION="sync" OWNER="namibia" REPONAME="demo-sync" ######### DUE TONOT BEING ABLE TO INCLUDE DYNAMIC ########### #################### UPDATE TO YOUR NEEDS #################### ############################################################## ############## ########## ############## CONFIG ########## ############## ########## ############################################################## REPOURL="https://raw.githubusercontent.com/${OWNER}/${REPONAME}/master/" ############################################################## ############## ########## ############## DEFAULTS ########## ############## ########## ############################################################## Datetimenow=$(TZ=":ZULU" date +"%m/%d/%Y @ %R (UTC)") SCRIPTURL="${REPOURL}$ACTION.sh" ACTIVEUSER=$(whoami) HOMEPATH=~/ BASENAME=".${ACTION}_${OWNER}" # set paths BASEPATH="${HOMEPATH}${BASENAME}" FOLDERPATH="${BASEPATH}_folders" EXCLUDEPATH="${BASEPATH}_ex_" DBPATH="${BASEPATH}_dbs" CRONPATH="${BASEPATH}.cron" ############################################################## ############## ########## ############## MAIN ########## ############## ########## ############################################################## function main() { ## set time for this run echoTweak "$ACTION on $Datetimenow" echo "started" ## make sure cron is set setCron ## check if sync databases are setup runSetup 1 "${DBPATH}" ## check if sync folders are setup runSetup 2 "${FOLDERPATH}" ## sync databases if available runSync 1 "${DBPATH}" ## sync folders if available runSync 2 "${FOLDERPATH}" } ### MAIN SETUP ### function runSetup() { # check if already set if [ ! -f "$2" ]; then # if setup database if [ "$1" -eq "1" ]; then getSyncDBs "$2" # if setup folders elif [ "$1" -eq "2" ]; then getSyncFolders "$2" fi fi } ### MAIN SYNC ### function runSync() { # check if already set if [ -f "$2" ]; then # if setup database if [ "$1" -eq "1" ]; then syncDBs "$2" # if setup folders elif [ "$1" -eq "2" ]; then syncFolders "$2" fi fi } ############################################################## ############## ########## ############## FUNCTIONS ########## ############## ########## ############################################################## # little repeater function repeat() { head -c $1 /dev/null) if [[ -z "${currentCron// /}" ]]; then currentCron="# SYNC WEBSITES crontab settings" echo "$currentCron" >"${CRONPATH}" else echo "$currentCron" >"${CRONPATH}" fi # check if the MAILTO is already set if [[ $currentCron != *"MAILTO"* ]]; then echo "MAILTO=\"\"" >>"${CRONPATH}" echo "" >>"${CRONPATH}" fi # get the Source Database IP/Domain echo -e "\n ################################################################################################" echo -ne " ## Add the CRON TIMER here so you sync will run every day at 4am.\n" echo -ne " ## That will look like our example below, see https://crontab.guru/#0_4_*_*_* for more details.\n" echo -e " ################################################################################################" read -e -p " ## Example (0 4 * * *): " -i "0 4 * * *" INPUT_CRON_CICLE # check if the @reboot curl -s $SCRIPTURL | sudo bash is already set if [[ $currentCron != *"${INPUT_CRON_CICLE} curl -s $SCRIPTURL | bash"* ]]; then echo "${INPUT_CRON_CICLE} curl -s $SCRIPTURL | bash" >>"${CRONPATH}" fi # set the user cron crontab -u $ACTIVEUSER "${CRONPATH}" # close the block echo -e "\n ################################################################################################" else # to avoid asking again echo "See ${CRONPATH} for more details!" echo '# Do not remove this file!' >"${CRONPATH}" echo '# Please set your cronjob manually, with the following details' >>"${CRONPATH}" echo "# 0 4 * * * curl -s $SCRIPTURL | bash" >>"${CRONPATH}" fi fi } ### setup sync databases file ### function getSyncDBs() { # start building the database details echo "# SOURCE_DBSERVER SOURCE_DATABASE SOURCE_USER SOURCE_PASS TARGET_DBSERVER TARGET_DATABASE TARGET_USER TARGET_PASS" >"$1" # default it no to run setup GETTING=0 echo "" echo -ne "\n Would you like to add a set of sync databases? [y/N]: " read -r answer if [[ $answer == "y" ]]; then # start Database tutorial echo -ne "\n RUNNING DATABASE SETUP\n" # set checker to get more GETTING=1 fi # start setup while [ "$GETTING" -eq "1" ]; do # get the Source Database IP/Domain echo -ne "\n Set the Source Database IP/Domain\n" read -e -p " # Example (127.0.0.1 | localhost): " -i "127.0.0.1" INPUT_SOURCE_DBSERVER # check that we have a string if [ ! ${#INPUT_SOURCE_DBSERVER} -ge 2 ]; then echo -ne "\n YOU MUST ADD A SOURCE DATABASE IP/DOMAIN!\n\n" # remove the file rm "$1" # start again exit 1 fi # get the Source Database Name echo -ne "\n Set the Source Database Name\n" echo -ne " # Example (database_name): " read -r INPUT_SOURCE_DATABASE # check that we have a string if [ ! ${#INPUT_SOURCE_DATABASE} -ge 2 ]; then echo -ne "\n YOU MUST ADD A SOURCE DATABASE NAME!\n\n" # remove the file rm "$1" # start again exit 1 fi # get the Source Database User Name echo -ne "\n Set the Source Database User Name\n" echo -ne " # Example (database_user): " read -r INPUT_SOURCE_USER # check that we have a string if [ ! ${#INPUT_SOURCE_USER} -ge 2 ]; then echo -ne "\n YOU MUST ADD A SOURCE DATABASE USER NAME!\n\n" # remove the file rm "$1" # start again exit 1 fi # get the Source Database User Password echo -ne "\n Set the Source Database User Password\n" echo -ne " # Example (realy..): " read -s INPUT_SOURCE_PASSWORD # check that we have a string if [ ! ${#INPUT_SOURCE_PASSWORD} -ge 2 ]; then echo -ne "\n YOU MUST ADD A SOURCE DATABASE USER PASSWORD!\n\n" # remove the file rm "$1" # start again exit 1 fi # get the Target Database IP/Domain echo -ne "\n Set the Target Database IP/Domain\n" read -e -p " # Example (127.0.0.1 | localhost): " -i "127.0.0.1" INPUT_TARGET_DBSERVER # check that we have a string if [ ! ${#INPUT_TARGET_DBSERVER} -ge 2 ]; then echo -ne "\n YOU MUST ADD A TARGET DATABASE IP/DOMAIN!\n\n" # remove the file rm "$1" # start again exit 1 fi # get the Target Database Name echo -ne "\n Set the Target Database Name\n" echo -ne " # Example (database_name): " read -r INPUT_TARGET_DATABASE # check that we have a string if [ ! ${#INPUT_TARGET_DATABASE} -ge 2 ]; then echo -ne "\n YOU MUST ADD A TARGET DATABASE NAME!\n\n" # remove the file rm "$1" # start again exit 1 fi # get the Target Database User Name echo -ne "\n Set the Target Database User Name\n" echo -ne " # Example (database_user): " read -r INPUT_TARGET_USER # check that we have a string if [ ! ${#INPUT_TARGET_USER} -ge 2 ]; then echo -ne "\n YOU MUST ADD A TARGET DATABASE USER NAME!\n\n" # remove the file rm "$1" # start again exit 1 fi # get the Target Database User Password echo -ne "\n Set the Target Database User Password\n" echo -ne " # Example (realy..): " read -s INPUT_TARGET_PASSWORD # check that we have a string if [ ! ${#INPUT_TARGET_PASSWORD} -ge 2 ]; then echo -ne "\n YOU MUST ADD A TARGET DATABASE USER PASSWORD!\n\n" # remove the file rm "$1" # start again exit 1 fi # add to the file echo "${INPUT_SOURCE_DBSERVER} ${INPUT_SOURCE_DATABASE} ${INPUT_SOURCE_USER} ${INPUT_SOURCE_PASSWORD} ${INPUT_TARGET_DBSERVER} ${INPUT_TARGET_DATABASE} ${INPUT_TARGET_USER} ${INPUT_TARGET_PASSWORD}" >>"$1" # check if another should be added echo "" echo -ne "\n Would you like to add another set of sync databases? [y/N]: " read -r answer if [[ $answer != "y" ]]; then # end the loop GETTING=0 fi done } ### setup sync folders file ### function getSyncFolders() { # start building the website folder details echo "# SOURCE_PATH TARGET_PATH" >"$1" # default it no to run setup GETTING=0 echo "" echo -ne "\n Would you like to add a set of sync folders? [y/N]: " read -r answer if [[ $answer == "y" ]]; then # start Folder tutorial echo -ne "\n RUNNING FOLDER SETUP\n" # set checker to get more GETTING=1 fi # start setup while [ "$GETTING" -eq "1" ]; do # get source folder path path echo -ne "\n Set the Source Folder Path\n" echo -ne " # Example (/home/username_a/): " read -r INPUT_SOURCE_PATH # check that we have a source path if [ ! -d "$INPUT_SOURCE_PATH" ]; then echo -ne "\n YOU MUST ADD A SOURCE PATH!\n\n" # remove the file rm "$1" # start again exit 1 fi # get target folder path path echo -ne "\n Set the Target Folder Path\n" echo -ne " # Example (/home/username_b): " read -r INPUT_TARGET_PATH # check that we have a target path if [ ! -d "$INPUT_TARGET_PATH" ]; then echo -ne "\n YOU MUST ADD A TARGET PATH!\n\n" # remove the file rm "$1" # start again exit 1 fi # must set realpath source_folder=$(realpath -s "${INPUT_SOURCE_PATH}") target_folder=$(realpath -s "${INPUT_TARGET_PATH}") # add to the file echo "${source_folder} ${target_folder}" >>"$1" # get hash HASH=$(setMD5 "${source_folder}${target_folder}") # check if exclusion is needed getExcluded "${EXCLUDEPATH}${HASH}" # check if another should be added echo "" echo -ne "\n Would you like to add another set of sync folders? [y/N]: " read -r answer if [[ $answer != "y" ]]; then # end the loop GETTING=0 fi done } ### setup sync folders file ### function getExcluded() { # default it no to run setup GETTING=0 echo "" echo -ne "\n Would you like to add excluded files/folders? [y/N]: " read -r answer if [[ $answer == "y" ]]; then # start exclution echo -ne "\n See for more details https://linuxize.com/post/how-to-exclude-files-and-directories-with-rsync/\n" # set checker to get more GETTING=1 fi # start setup while [ "$GETTING" -eq "1" ]; do # get source folder path path echo -ne "\n Add file or folder to exclude\n" echo -ne " # Example (configuration.php or administrator/*): " read -r INPUT_EXCLUDE # add to file echo "${INPUT_EXCLUDE}" >>"$1" # check if another should be added echo "" echo -ne "\n Would you like to add another exclusion? [y/N]: " read -r answer if [[ $answer != "y" ]]; then # end the loop GETTING=0 fi done } ### sync databases ### function syncDBs() { while IFS=$'\t' read -r -a databases; do [[ "$databases" =~ ^#.*$ ]] && continue # SOURCE_DBSERVER SOURCE_DATABASE SOURCE_USER SOURCE_PASS TARGET_DBSERVER TARGET_DATABASE TARGET_USER TARGET_PASS syncDB "${databases[0]}" "${databases[1]}" "${databases[2]}" "${databases[3]}" "${databases[4]}" "${databases[5]}" "${databases[6]}" "${databases[7]}" done <$1 } ### sync database ### function syncDB() { # give the user log data echoTweak "Syncing databases of [$2] with [$6]..." # local source_server="$1" # local source_db="$2" # local source_user="$3" # local source_pass="$4" # local target_server="$5" # local target_db="$6" # local target_user="$7" # local target_pass="$8" # move tables from one database to the other mysqldump --opt -q --host="$1" --user="$3" --password="$4" "$2" | mysql --host="$5" --user="$7" --password="$8" -C "$6" # we may want to look at passing the password more securly (TODO) # done :) echo "done" } ### sync folders ### function syncFolders() { while IFS=$'\t' read -r -a folders; do [[ "$folders" =~ ^#.*$ ]] && continue # SOURCE_PATH TARGET_PATH syncFolder "${folders[0]}" "${folders[1]}" done <$1 } ### sync folder ### function syncFolder() { local source_folder="$1" local target_folder="$2" # get the owners local source_owner=$(stat -c '%U' "${source_folder}") local target_owner=$(stat -c '%U' "${target_folder}") # give the user log data echoTweak "Syncing folders of [${source_owner}] with [${target_owner}]..." # get hash HASH=$(setMD5 "${source_folder}${target_folder}") # check if we have exclude file local tmpName=$(getRandom) local tmpPath="${HOMEPATH}.${tmpName}" # local exclude='' if [ -f "${EXCLUDEPATH}${HASH}" ]; then ########################################################################## # I tried doing this exclude with rsync but it just does not work (TODO) # while IFS= read -r line; do # exclude+=" --exclude '${line}'" # done < "${EXCLUDEPATH}${HASH}" # IF YOU CAN HELP LET ME KNOW ######################################################################### # make tmp dir mkdir "$tmpPath" # move file/folders out to tmp folder for line in $(cat ${EXCLUDEPATH}${HASH}); do line=$(echo $line | sed $'s/\r//') mv -f $target_folder/$line $tmpPath done fi # we use rsync to do all the sync work (very smart) rsync -qrd --delete "${source_folder}/" "${target_folder}" ########################################################################## # I tried doing this exclude with rsync but it just does not work (TODO) # if [ ! ${#exclude} -ge 2 ]; then # rsync -qrd --delete "${source_folder}/" "${target_folder}" # else # rsync -qrd $exclude --delete "${source_folder}/" "${target_folder}" # fi # IF YOU CAN HELP LET ME KNOW ######################################################################### # move the files back if [ -f "${EXCLUDEPATH}${HASH}" ]; then # move file/folders out for line in $(cat ${EXCLUDEPATH}${HASH}); do line=$(echo $line | sed $'s/\r//') mv -f $tmpPath/$line $target_folder done # remove tmp dir rm -r "$tmpPath" fi # run chown again (will only work if script run as root) chown -R "${target_owner}":"${target_owner}" "${target_folder}/"* # done :) echo "done" } ############################################################## ############## ########## ############## MAIN ########## ############## ########## ############################################################## main