mirror of https://github.com/namibia/demo-sync.git
499 lines
16 KiB
Bash
Executable File
499 lines
16 KiB
Bash
Executable File
#!/bin/bash
|
|
#/--------------------------------------------------------------------------------------------------------| www.vdm.io |------/
|
|
#
|
|
# @version 1.0.0
|
|
# @build 6th Marh, 2020
|
|
# @package sync websites
|
|
# @author Llewellyn van der Merwe <https://github.com/Llewellynvdm>
|
|
# @copyright Copyright (C) 2020. All Rights Reserved
|
|
# @license GNU/GPL Version 2 or later - http://www.gnu.org/licenses/gpl-2.0.html
|
|
#
|
|
#/-----------------------------------------------------------------------------------------------------------------------------/
|
|
|
|
##############################################################
|
|
############## CHECK ##########
|
|
##############################################################
|
|
command -v rsync >/dev/null 2>&1 || { echo >&2 "We require rsync for this script to run, but it's not installed. Aborting."; exit 1; }
|
|
command -v crontab >/dev/null 2>&1 || { echo >&2 "We require crontab for this script to run, but it's not installed. Aborting."; exit 1; }
|
|
command -v md5sum >/dev/null 2>&1 || { echo >&2 "We require md5sum for this script to run, but it's not installed. Aborting."; exit 1; }
|
|
command -v awk >/dev/null 2>&1 || { echo >&2 "We require awk for this script to run, but it's not installed. Aborting."; exit 1; }
|
|
command -v realpath >/dev/null 2>&1 || { echo >&2 "We require realpath for this script to run, but it's not installed. Aborting."; exit 1; }
|
|
command -v stat >/dev/null 2>&1 || { echo >&2 "We require stat for this script to run, but it's not installed. Aborting."; exit 1; }
|
|
|
|
############################ GLOBAL ##########################
|
|
ACTION="sync"
|
|
OWNER="namibia"
|
|
REPONAME="demo-sync"
|
|
######### DUE TONOT BEING ABLE TO INCLUDE DYNAMIC ###########
|
|
|
|
#################### UPDATE TO YOUR NEEDS ####################
|
|
##############################################################
|
|
############## ##########
|
|
############## CONFIG ##########
|
|
############## ##########
|
|
##############################################################
|
|
REPOURL="https://raw.githubusercontent.com/${OWNER}/${REPONAME}/master/"
|
|
|
|
##############################################################
|
|
############## ##########
|
|
############## DEFAULTS ##########
|
|
############## ##########
|
|
##############################################################
|
|
Datetimenow=$(TZ=":ZULU" date +"%m/%d/%Y @ %R (UTC)" )
|
|
SCRIPTURL="${REPOURL}$ACTION.sh"
|
|
ACTIVEUSER=$(whoami)
|
|
HOMEPATH=~/
|
|
BASENAME=".${ACTION}_${OWNER}"
|
|
# set paths
|
|
BASEPATH="${HOMEPATH}${BASENAME}"
|
|
FOLDERPATH="${BASEPATH}_folders"
|
|
EXCLUDEPATH="${BASEPATH}_ex_"
|
|
DBPATH="${BASEPATH}_dbs"
|
|
CRONPATH="${BASEPATH}.cron"
|
|
|
|
##############################################################
|
|
############## ##########
|
|
############## MAIN ##########
|
|
############## ##########
|
|
##############################################################
|
|
function main () {
|
|
## set time for this run
|
|
echoTweak "$ACTION on $Datetimenow"
|
|
echo "started"
|
|
## make sure cron is set
|
|
setCron
|
|
## check if sync databases are setup
|
|
runSetup 1 "${DBPATH}"
|
|
## check if sync folders are setup
|
|
runSetup 2 "${FOLDERPATH}"
|
|
## sync databases if available
|
|
runSync 1 "${DBPATH}"
|
|
## sync folders if available
|
|
runSync 2 "${FOLDERPATH}"
|
|
}
|
|
|
|
### MAIN SETUP ###
|
|
function runSetup () {
|
|
# check if already set
|
|
if [ ! -f "$2" ]; then
|
|
# if setup database
|
|
if [ "$1" -eq "1" ]; then
|
|
getSyncDBs "$2"
|
|
# if setup folders
|
|
elif [ "$1" -eq "2" ]; then
|
|
getSyncFolders "$2"
|
|
fi
|
|
fi
|
|
}
|
|
|
|
### MAIN SYNC ###
|
|
function runSync () {
|
|
# check if already set
|
|
if [ -f "$2" ]; then
|
|
# if setup database
|
|
if [ "$1" -eq "1" ]; then
|
|
syncDBs "$2"
|
|
# if setup folders
|
|
elif [ "$1" -eq "2" ]; then
|
|
syncFolders "$2"
|
|
fi
|
|
fi
|
|
}
|
|
|
|
##############################################################
|
|
############## ##########
|
|
############## FUNCTIONS ##########
|
|
############## ##########
|
|
##############################################################
|
|
|
|
# little repeater
|
|
function repeat () {
|
|
head -c $1 < /dev/zero | tr '\0' $2
|
|
}
|
|
|
|
# simple basic random
|
|
function getRandom () {
|
|
echo $(tr -dc 'A-HJ-NP-Za-km-z2-9' < /dev/urandom | dd bs=5 count=1 status=none)
|
|
}
|
|
|
|
# md5 strings
|
|
function setMD5() {
|
|
echo -n $1 | md5sum | awk '{print $1}'
|
|
}
|
|
|
|
# little echo tweak
|
|
function echoTweak () {
|
|
echoMessage="$1"
|
|
mainlen="$2"
|
|
characters="$3"
|
|
if [ $# -lt 2 ]
|
|
then
|
|
mainlen=60
|
|
fi
|
|
if [ $# -lt 3 ]
|
|
then
|
|
characters='\056'
|
|
fi
|
|
chrlen="${#echoMessage}"
|
|
increaseBy=$((mainlen-chrlen))
|
|
tweaked=$(repeat "$increaseBy" "$characters")
|
|
echo -n "$echoMessage$tweaked"
|
|
}
|
|
|
|
# Set cronjob without removing existing
|
|
function setCron () {
|
|
if [ -f "${CRONPATH}" ]; then
|
|
echoTweak "Crontab already configured for updates..."
|
|
echo "Skipping"
|
|
else
|
|
echo "Adding crontab entry for continued updates..."
|
|
# check if user crontab is set
|
|
currentCron=$(crontab -u $CLIENTUSER -l 2>/dev/null)
|
|
if [[ -z "${currentCron// }" ]]; then
|
|
currentCron="# SYNC WEBSITES crontab settings"
|
|
echo "$currentCron" > "${CRONPATH}"
|
|
else
|
|
echo "$currentCron" > "${CRONPATH}"
|
|
fi
|
|
# check if the MAILTO is already set
|
|
if [[ $currentCron != *"MAILTO"* ]]; then
|
|
echo "MAILTO=\"\"" >> "${CRONPATH}"
|
|
echo "" >> "${CRONPATH}"
|
|
fi
|
|
# get the Source Database IP/Domain
|
|
echo -e "\n ################################################################################################"
|
|
echo -ne " ## Add the CRON TIMER here so you sync will run every day at 4am.\n"
|
|
echo -ne " ## That will look like our example below, see https://crontab.guru/#0_4_*_*_* for more details.\n"
|
|
echo -e " ################################################################################################"
|
|
read -e -p " ## Example (0 4 * * *): " -i "0 4 * * *" INPUT_CRON_CICLE
|
|
# check if the @reboot curl -s $SCRIPTURL | sudo bash is already set
|
|
if [[ $currentCron != *"${INPUT_CRON_CICLE} curl -s $SCRIPTURL | bash"* ]]; then
|
|
echo "${INPUT_CRON_CICLE} curl -s $SCRIPTURL | bash" >> "${CRONPATH}"
|
|
fi
|
|
# set the user cron
|
|
crontab -u $ACTIVEUSER "${CRONPATH}"
|
|
# close the block
|
|
echo -e "\n ################################################################################################"
|
|
fi
|
|
}
|
|
|
|
### setup sync databases file ###
|
|
function getSyncDBs () {
|
|
# start building the database details
|
|
echo "# SOURCE_DBSERVER SOURCE_DATABASE SOURCE_USER SOURCE_PASS TARGET_DBSERVER TARGET_DATABASE TARGET_USER TARGET_PASS" > "$1"
|
|
# default it no to run setup
|
|
GETTING=0
|
|
echo ""
|
|
echo -ne "\n Would you like to add a set of sync databases? [y/N]: "
|
|
read -r answer
|
|
if [[ $answer == "y" ]]; then
|
|
# start Database tutorial
|
|
echo -ne "\n RUNNING DATABASE SETUP\n"
|
|
# set checker to get more
|
|
GETTING=1
|
|
fi
|
|
# start setup
|
|
while [ "$GETTING" -eq "1" ]
|
|
do
|
|
# get the Source Database IP/Domain
|
|
echo -ne "\n Set the Source Database IP/Domain\n"
|
|
read -e -p " # Example (127.0.0.1 | localhost): " -i "127.0.0.1" INPUT_SOURCE_DBSERVER
|
|
# check that we have a string
|
|
if [ ! ${#INPUT_SOURCE_DBSERVER} -ge 2 ]; then
|
|
echo -ne "\n YOU MUST ADD A SOURCE DATABASE IP/DOMAIN!\n\n" ;
|
|
# remove the file
|
|
rm "$1"
|
|
# start again
|
|
exit 1
|
|
fi
|
|
# get the Source Database Name
|
|
echo -ne "\n Set the Source Database Name\n"
|
|
echo -ne " # Example (database_name): "
|
|
read -r INPUT_SOURCE_DATABASE
|
|
# check that we have a string
|
|
if [ ! ${#INPUT_SOURCE_DATABASE} -ge 2 ]; then
|
|
echo -ne "\n YOU MUST ADD A SOURCE DATABASE NAME!\n\n" ;
|
|
# remove the file
|
|
rm "$1"
|
|
# start again
|
|
exit 1
|
|
fi
|
|
# get the Source Database User Name
|
|
echo -ne "\n Set the Source Database User Name\n"
|
|
echo -ne " # Example (database_user): "
|
|
read -r INPUT_SOURCE_USER
|
|
# check that we have a string
|
|
if [ ! ${#INPUT_SOURCE_USER} -ge 2 ]; then
|
|
echo -ne "\n YOU MUST ADD A SOURCE DATABASE USER NAME!\n\n" ;
|
|
# remove the file
|
|
rm "$1"
|
|
# start again
|
|
exit 1
|
|
fi
|
|
# get the Source Database User Password
|
|
echo -ne "\n Set the Source Database User Password\n"
|
|
echo -ne " # Example (realy..): "
|
|
read -s INPUT_SOURCE_PASSWORD
|
|
# check that we have a string
|
|
if [ ! ${#INPUT_SOURCE_PASSWORD} -ge 2 ]; then
|
|
echo -ne "\n YOU MUST ADD A SOURCE DATABASE USER PASSWORD!\n\n" ;
|
|
# remove the file
|
|
rm "$1"
|
|
# start again
|
|
exit 1
|
|
fi
|
|
# get the Target Database IP/Domain
|
|
echo -ne "\n Set the Target Database IP/Domain\n"
|
|
read -e -p " # Example (127.0.0.1 | localhost): " -i "127.0.0.1" INPUT_TARGET_DBSERVER
|
|
# check that we have a string
|
|
if [ ! ${#INPUT_TARGET_DBSERVER} -ge 2 ]; then
|
|
echo -ne "\n YOU MUST ADD A TARGET DATABASE IP/DOMAIN!\n\n" ;
|
|
# remove the file
|
|
rm "$1"
|
|
# start again
|
|
exit 1
|
|
fi
|
|
# get the Target Database Name
|
|
echo -ne "\n Set the Target Database Name\n"
|
|
echo -ne " # Example (database_name): "
|
|
read -r INPUT_TARGET_DATABASE
|
|
# check that we have a string
|
|
if [ ! ${#INPUT_TARGET_DATABASE} -ge 2 ]; then
|
|
echo -ne "\n YOU MUST ADD A TARGET DATABASE NAME!\n\n" ;
|
|
# remove the file
|
|
rm "$1"
|
|
# start again
|
|
exit 1
|
|
fi
|
|
# get the Target Database User Name
|
|
echo -ne "\n Set the Target Database User Name\n"
|
|
echo -ne " # Example (database_user): "
|
|
read -r INPUT_TARGET_USER
|
|
# check that we have a string
|
|
if [ ! ${#INPUT_TARGET_USER} -ge 2 ]; then
|
|
echo -ne "\n YOU MUST ADD A TARGET DATABASE USER NAME!\n\n" ;
|
|
# remove the file
|
|
rm "$1"
|
|
# start again
|
|
exit 1
|
|
fi
|
|
# get the Target Database User Password
|
|
echo -ne "\n Set the Target Database User Password\n"
|
|
echo -ne " # Example (realy..): "
|
|
read -s INPUT_TARGET_PASSWORD
|
|
# check that we have a string
|
|
if [ ! ${#INPUT_TARGET_PASSWORD} -ge 2 ]; then
|
|
echo -ne "\n YOU MUST ADD A TARGET DATABASE USER PASSWORD!\n\n" ;
|
|
# remove the file
|
|
rm "$1"
|
|
# start again
|
|
exit 1
|
|
fi
|
|
# add to the file
|
|
echo "${INPUT_SOURCE_DBSERVER} ${INPUT_SOURCE_DATABASE} ${INPUT_SOURCE_USER} ${INPUT_SOURCE_PASSWORD} ${INPUT_TARGET_DBSERVER} ${INPUT_TARGET_DATABASE} ${INPUT_TARGET_USER} ${INPUT_TARGET_PASSWORD}" >> "$1"
|
|
# check if another should be added
|
|
echo ""
|
|
echo -ne "\n Would you like to add another set of sync databases? [y/N]: "
|
|
read -r answer
|
|
if [[ $answer != "y" ]]; then
|
|
# end the loop
|
|
GETTING=0
|
|
fi
|
|
done
|
|
}
|
|
|
|
### setup sync folders file ###
|
|
function getSyncFolders () {
|
|
# start building the website folder details
|
|
echo "# SOURCE_PATH TARGET_PATH" > "$1"
|
|
# default it no to run setup
|
|
GETTING=0
|
|
echo ""
|
|
echo -ne "\n Would you like to add a set of sync folders? [y/N]: "
|
|
read -r answer
|
|
if [[ $answer == "y" ]]; then
|
|
# start Folder tutorial
|
|
echo -ne "\n RUNNING FOLDER SETUP\n"
|
|
# set checker to get more
|
|
GETTING=1
|
|
fi
|
|
# start setup
|
|
while [ "$GETTING" -eq "1" ]
|
|
do
|
|
# get source folder path path
|
|
echo -ne "\n Set the Source Folder Path\n"
|
|
echo -ne " # Example (/home/username_a/): "
|
|
read -r INPUT_SOURCE_PATH
|
|
# check that we have a source path
|
|
if [ ! -d "$INPUT_SOURCE_PATH" ]; then
|
|
echo -ne "\n YOU MUST ADD A SOURCE PATH!\n\n" ;
|
|
# remove the file
|
|
rm "$1"
|
|
# start again
|
|
exit 1
|
|
fi
|
|
# get target folder path path
|
|
echo -ne "\n Set the Target Folder Path\n"
|
|
echo -ne " # Example (/home/username_b): "
|
|
read -r INPUT_TARGET_PATH
|
|
# check that we have a target path
|
|
if [ ! -d "$INPUT_TARGET_PATH" ]; then
|
|
echo -ne "\n YOU MUST ADD A TARGET PATH!\n\n" ;
|
|
# remove the file
|
|
rm "$1"
|
|
# start again
|
|
exit 1
|
|
fi
|
|
# must set realpath
|
|
source_folder=$(realpath -s "${INPUT_SOURCE_PATH}")
|
|
target_folder=$(realpath -s "${INPUT_TARGET_PATH}")
|
|
# add to the file
|
|
echo "${source_folder} ${target_folder}" >> "$1"
|
|
# get hash
|
|
HASH=$(setMD5 "${source_folder}${target_folder}")
|
|
# check if exclusion is needed
|
|
getExcluded "${EXCLUDEPATH}${HASH}"
|
|
# check if another should be added
|
|
echo ""
|
|
echo -ne "\n Would you like to add another set of sync folders? [y/N]: "
|
|
read -r answer
|
|
if [[ $answer != "y" ]]; then
|
|
# end the loop
|
|
GETTING=0
|
|
fi
|
|
done
|
|
}
|
|
|
|
### setup sync folders file ###
|
|
function getExcluded () {
|
|
# default it no to run setup
|
|
GETTING=0
|
|
echo ""
|
|
echo -ne "\n Would you like to add excluded files/folders? [y/N]: "
|
|
read -r answer
|
|
if [[ $answer == "y" ]]; then
|
|
# start exclution
|
|
echo -ne "\n See for more details https://linuxize.com/post/how-to-exclude-files-and-directories-with-rsync/\n"
|
|
# set checker to get more
|
|
GETTING=1
|
|
fi
|
|
# start setup
|
|
while [ "$GETTING" -eq "1" ]
|
|
do
|
|
# get source folder path path
|
|
echo -ne "\n Add file or folder to exclude\n"
|
|
echo -ne " # Example (configuration.php or administrator/*): "
|
|
read -r INPUT_EXCLUDE
|
|
# add to file
|
|
echo "${INPUT_EXCLUDE}" >> "$1"
|
|
# check if another should be added
|
|
echo ""
|
|
echo -ne "\n Would you like to add another exclusion? [y/N]: "
|
|
read -r answer
|
|
if [[ $answer != "y" ]]; then
|
|
# end the loop
|
|
GETTING=0
|
|
fi
|
|
done
|
|
}
|
|
|
|
### sync databases ###
|
|
function syncDBs (){
|
|
while IFS=$'\t' read -r -a databases
|
|
do
|
|
[[ "$databases" =~ ^#.*$ ]] && continue
|
|
# SOURCE_DBSERVER SOURCE_DATABASE SOURCE_USER SOURCE_PASS TARGET_DBSERVER TARGET_DATABASE TARGET_USER TARGET_PASS
|
|
syncDB "${databases[0]}" "${databases[1]}" "${databases[2]}" "${databases[3]}" "${databases[4]}" "${databases[5]}" "${databases[6]}" "${databases[7]}"
|
|
done < $1
|
|
}
|
|
|
|
### sync database ###
|
|
function syncDB (){
|
|
# local source_server="$1"
|
|
# local source_db="$2"
|
|
# local source_user="$3"
|
|
# local source_pass="$4"
|
|
# local target_server="$5"
|
|
# local target_db="$6"
|
|
# local target_user="$7"
|
|
# local target_pass="$8"
|
|
# move tables from one database to the other
|
|
mysqldump --opt -q --host="$1" --user="$3" --password="$4" "$2" | \
|
|
mysql --host="$5" --user="$7" --password="$8" -C "$6"
|
|
# we may want to look at passing the password more securly (TODO)
|
|
}
|
|
|
|
### sync folders ###
|
|
function syncFolders (){
|
|
while IFS=$'\t' read -r -a folders
|
|
do
|
|
[[ "$folders" =~ ^#.*$ ]] && continue
|
|
# SOURCE_PATH TARGET_PATH
|
|
syncFolder "${folders[0]}" "${folders[1]}"
|
|
done < $1
|
|
}
|
|
|
|
### sync folder ###
|
|
function syncFolder (){
|
|
local source_folder="$1"
|
|
local target_folder="$2"
|
|
# get the owners
|
|
local source_owner=$(stat -c '%U' "${source_folder}")
|
|
local target_owner=$(stat -c '%U' "${target_folder}")
|
|
# give the user log data
|
|
echoTweak "Syncing folders of [${source_owner}] with [${target_owner}]..."
|
|
# get hash
|
|
HASH=$(setMD5 "${source_folder}${target_folder}")
|
|
# check if we have exclude file
|
|
local tmpName=$(getRandom)
|
|
local tmpPath="${HOMEPATH}.${tmpName}"
|
|
# local exclude=''
|
|
if [ -f "${EXCLUDEPATH}${HASH}" ]; then
|
|
|
|
##########################################################################
|
|
# I tried doing this exclude with rsync but it just does not work (TODO)
|
|
# while IFS= read -r line; do
|
|
# exclude+=" --exclude '${line}'"
|
|
# done < "${EXCLUDEPATH}${HASH}"
|
|
# IF YOU CAN HELP LET ME KNOW
|
|
#########################################################################
|
|
|
|
# make tmp dir
|
|
mkdir "$tmpPath"
|
|
# move file/folders out to tmp folder
|
|
for line in $(cat ${EXCLUDEPATH}${HASH}); do mv -f "${target_folder}/${line}" "$tmpPath"; done
|
|
fi
|
|
# we use rsync to do all the sync work (very smart)
|
|
rsync -qrd --delete "${source_folder}/" "${target_folder}"
|
|
|
|
##########################################################################
|
|
# I tried doing this exclude with rsync but it just does not work (TODO)
|
|
# if [ ! ${#exclude} -ge 2 ]; then
|
|
# rsync -qrd --delete "${source_folder}/" "${target_folder}"
|
|
# else
|
|
# rsync -qrd $exclude --delete "${source_folder}/" "${target_folder}"
|
|
# fi
|
|
# IF YOU CAN HELP LET ME KNOW
|
|
#########################################################################
|
|
|
|
# move the files back
|
|
if [ -f "${EXCLUDEPATH}${HASH}" ]; then
|
|
# move file/folders out
|
|
for line in $(cat ${EXCLUDEPATH}${HASH}); do mv -f "$tmpPath/$line" "${target_folder}"; done
|
|
# remove tmp dir
|
|
rm -r "$tmpPath"
|
|
fi
|
|
|
|
# run chown again (will only work if script run as root)
|
|
chown -R "${target_owner}":"${target_owner}" "${target_folder}/"*
|
|
# done :)
|
|
echo "done"
|
|
}
|
|
|
|
##############################################################
|
|
############## ##########
|
|
############## MAIN ##########
|
|
############## ##########
|
|
##############################################################
|
|
main
|