octoleo
/
demo-sync
mirror of https://github.com/namibia/demo-sync.git
1
0
Fork 0
Code Releases Activity
demo-sync/sync.sh

527 lines
17 KiB
Bash
Executable File
Raw Permalink Blame History

#!/bin/bash
#/--------------------------------------------------------------------------------------------------------| www.vdm.io |------/
#
# @version 1.0.0
# @build 6th Marh, 2020
# @package sync websites
# @author Llewellyn van der Merwe <https://github.com/Llewellynvdm>
# @copyright Copyright (C) 2020. All Rights Reserved
# @license GNU/GPL Version 2 or later - http://www.gnu.org/licenses/gpl-2.0.html
#
#/-----------------------------------------------------------------------------------------------------------------------------/
##############################################################
############## CHECK ##########
##############################################################
command -v rsync >/dev/null 2>&1 || {
echo >&2 "We require rsync for this script to run, but it's not installed. Aborting."
exit 1
}
command -v crontab >/dev/null 2>&1 || {
echo >&2 "We require crontab for this script to run, but it's not installed. Aborting."
exit 1
}
command -v md5sum >/dev/null 2>&1 || {
echo >&2 "We require md5sum for this script to run, but it's not installed. Aborting."
exit 1
}
command -v awk >/dev/null 2>&1 || {
echo >&2 "We require awk for this script to run, but it's not installed. Aborting."
exit 1
}
command -v realpath >/dev/null 2>&1 || {
echo >&2 "We require realpath for this script to run, but it's not installed. Aborting."
exit 1
}
command -v stat >/dev/null 2>&1 || {
echo >&2 "We require stat for this script to run, but it's not installed. Aborting."
exit 1
}
############################ GLOBAL ##########################
ACTION="sync"
OWNER="namibia"
REPONAME="demo-sync"
######### DUE TONOT BEING ABLE TO INCLUDE DYNAMIC ###########
#################### UPDATE TO YOUR NEEDS ####################
##############################################################
############## ##########
############## CONFIG ##########
############## ##########
##############################################################
REPOURL="https://raw.githubusercontent.com/${OWNER}/${REPONAME}/master/"
##############################################################
############## ##########
############## DEFAULTS ##########
############## ##########
##############################################################
Datetimenow=$(TZ=":ZULU" date +"%m/%d/%Y @ %R (UTC)")
SCRIPTURL="${REPOURL}$ACTION.sh"
ACTIVEUSER=$(whoami)
HOMEPATH=~/
BASENAME=".${ACTION}_${OWNER}"
# set paths
BASEPATH="${HOMEPATH}${BASENAME}"
FOLDERPATH="${BASEPATH}_folders"
EXCLUDEPATH="${BASEPATH}_ex_"
DBPATH="${BASEPATH}_dbs"
CRONPATH="${BASEPATH}.cron"
##############################################################
############## ##########
############## MAIN ##########
############## ##########
##############################################################
function main() {
## set time for this run
echoTweak "$ACTION on $Datetimenow"
echo "started"
## make sure cron is set
setCron
## check if sync databases are setup
runSetup 1 "${DBPATH}"
## check if sync folders are setup
runSetup 2 "${FOLDERPATH}"
## sync databases if available
runSync 1 "${DBPATH}"
## sync folders if available
runSync 2 "${FOLDERPATH}"
}
### MAIN SETUP ###
function runSetup() {
# check if already set
if [ ! -f "$2" ]; then
# if setup database
if [ "$1" -eq "1" ]; then
getSyncDBs "$2"
# if setup folders
elif [ "$1" -eq "2" ]; then
getSyncFolders "$2"
fi
fi
}
### MAIN SYNC ###
function runSync() {
# check if already set
if [ -f "$2" ]; then
# if setup database
if [ "$1" -eq "1" ]; then
syncDBs "$2"
# if setup folders
elif [ "$1" -eq "2" ]; then
syncFolders "$2"
fi
fi
}
##############################################################
############## ##########
############## FUNCTIONS ##########
############## ##########
##############################################################
# little repeater
function repeat() {
head -c $1 </dev/zero | tr '\0' $2
}
# simple basic random
function getRandom() {
echo $(tr -dc 'A-HJ-NP-Za-km-z2-9' </dev/urandom | dd bs=5 count=1 status=none)
}
# md5 strings
function setMD5() {
echo -n $1 | md5sum | awk '{print $1}'
}
# little echo tweak
function echoTweak() {
echoMessage="$1"
mainlen="$2"
characters="$3"
if [ $# -lt 2 ]; then
mainlen=60
fi
if [ $# -lt 3 ]; then
characters='\056'
fi
chrlen="${#echoMessage}"
increaseBy=$((mainlen - chrlen))
tweaked=$(repeat "$increaseBy" "$characters")
echo -n "$echoMessage$tweaked"
}
# Set cronjob without removing existing
function setCron() {
if [ ! -f "${CRONPATH}" ]; then
echo ""
echo -ne "\n Would you like set the cronjob now? [y/N]: "
read -r answer
if [[ $answer == "y" ]]; then
# check if user crontab is set
currentCron=$(crontab -u $CLIENTUSER -l 2>/dev/null)
if [[ -z "${currentCron// /}" ]]; then
currentCron="# SYNC WEBSITES crontab settings"
echo "$currentCron" >"${CRONPATH}"
else
echo "$currentCron" >"${CRONPATH}"
fi
# check if the MAILTO is already set
if [[ $currentCron != *"MAILTO"* ]]; then
echo "MAILTO=\"\"" >>"${CRONPATH}"
echo "" >>"${CRONPATH}"
fi
# get the Source Database IP/Domain
echo -e "\n ################################################################################################"
echo -ne " ## Add the CRON TIMER here so you sync will run every day at 4am.\n"
echo -ne " ## That will look like our example below, see https://crontab.guru/#0_4_*_*_* for more details.\n"
echo -e " ################################################################################################"
read -e -p " ## Example (0 4 * * *): " -i "0 4 * * *" INPUT_CRON_CICLE
# check if the @reboot curl -s $SCRIPTURL | sudo bash is already set
if [[ $currentCron != *"${INPUT_CRON_CICLE} curl -s $SCRIPTURL | bash"* ]]; then
echo "${INPUT_CRON_CICLE} curl -s $SCRIPTURL | bash" >>"${CRONPATH}"
fi
# set the user cron
crontab -u $ACTIVEUSER "${CRONPATH}"
# close the block
echo -e "\n ################################################################################################"
else
# to avoid asking again
echo "See ${CRONPATH} for more details!"
echo '# Do not remove this file!' >"${CRONPATH}"
echo '# Please set your cronjob manually, with the following details' >>"${CRONPATH}"
echo "# 0 4 * * * curl -s $SCRIPTURL | bash" >>"${CRONPATH}"
fi
fi
}
### setup sync databases file ###
function getSyncDBs() {
# start building the database details
echo "# SOURCE_DBSERVER SOURCE_DATABASE SOURCE_USER SOURCE_PASS TARGET_DBSERVER TARGET_DATABASE TARGET_USER TARGET_PASS" >"$1"
# default it no to run setup
GETTING=0
echo ""
echo -ne "\n Would you like to add a set of sync databases? [y/N]: "
read -r answer
if [[ $answer == "y" ]]; then
# start Database tutorial
echo -ne "\n RUNNING DATABASE SETUP\n"
# set checker to get more
GETTING=1
fi
# start setup
while [ "$GETTING" -eq "1" ]; do
# get the Source Database IP/Domain
echo -ne "\n Set the Source Database IP/Domain\n"
read -e -p " # Example (127.0.0.1 | localhost): " -i "127.0.0.1" INPUT_SOURCE_DBSERVER
# check that we have a string
if [ ! ${#INPUT_SOURCE_DBSERVER} -ge 2 ]; then
echo -ne "\n YOU MUST ADD A SOURCE DATABASE IP/DOMAIN!\n\n"
# remove the file
rm "$1"
# start again
exit 1
fi
# get the Source Database Name
echo -ne "\n Set the Source Database Name\n"
echo -ne " # Example (database_name): "
read -r INPUT_SOURCE_DATABASE
# check that we have a string
if [ ! ${#INPUT_SOURCE_DATABASE} -ge 2 ]; then
echo -ne "\n YOU MUST ADD A SOURCE DATABASE NAME!\n\n"
# remove the file
rm "$1"
# start again
exit 1
fi
# get the Source Database User Name
echo -ne "\n Set the Source Database User Name\n"
echo -ne " # Example (database_user): "
read -r INPUT_SOURCE_USER
# check that we have a string
if [ ! ${#INPUT_SOURCE_USER} -ge 2 ]; then
echo -ne "\n YOU MUST ADD A SOURCE DATABASE USER NAME!\n\n"
# remove the file
rm "$1"
# start again
exit 1
fi
# get the Source Database User Password
echo -ne "\n Set the Source Database User Password\n"
echo -ne " # Example (realy..): "
read -s INPUT_SOURCE_PASSWORD
# check that we have a string
if [ ! ${#INPUT_SOURCE_PASSWORD} -ge 2 ]; then
echo -ne "\n YOU MUST ADD A SOURCE DATABASE USER PASSWORD!\n\n"
# remove the file
rm "$1"
# start again
exit 1
fi
# get the Target Database IP/Domain
echo -ne "\n Set the Target Database IP/Domain\n"
read -e -p " # Example (127.0.0.1 | localhost): " -i "127.0.0.1" INPUT_TARGET_DBSERVER
# check that we have a string
if [ ! ${#INPUT_TARGET_DBSERVER} -ge 2 ]; then
echo -ne "\n YOU MUST ADD A TARGET DATABASE IP/DOMAIN!\n\n"
# remove the file
rm "$1"
# start again
exit 1
fi
# get the Target Database Name
echo -ne "\n Set the Target Database Name\n"
echo -ne " # Example (database_name): "
read -r INPUT_TARGET_DATABASE
# check that we have a string
if [ ! ${#INPUT_TARGET_DATABASE} -ge 2 ]; then
echo -ne "\n YOU MUST ADD A TARGET DATABASE NAME!\n\n"
# remove the file
rm "$1"
# start again
exit 1
fi
# get the Target Database User Name
echo -ne "\n Set the Target Database User Name\n"
echo -ne " # Example (database_user): "
read -r INPUT_TARGET_USER
# check that we have a string
if [ ! ${#INPUT_TARGET_USER} -ge 2 ]; then
echo -ne "\n YOU MUST ADD A TARGET DATABASE USER NAME!\n\n"
# remove the file
rm "$1"
# start again
exit 1
fi
# get the Target Database User Password
echo -ne "\n Set the Target Database User Password\n"
echo -ne " # Example (realy..): "
read -s INPUT_TARGET_PASSWORD
# check that we have a string
if [ ! ${#INPUT_TARGET_PASSWORD} -ge 2 ]; then
echo -ne "\n YOU MUST ADD A TARGET DATABASE USER PASSWORD!\n\n"
# remove the file
rm "$1"
# start again
exit 1
fi
# add to the file
echo "${INPUT_SOURCE_DBSERVER} ${INPUT_SOURCE_DATABASE} ${INPUT_SOURCE_USER} ${INPUT_SOURCE_PASSWORD} ${INPUT_TARGET_DBSERVER} ${INPUT_TARGET_DATABASE} ${INPUT_TARGET_USER} ${INPUT_TARGET_PASSWORD}" >>"$1"
# check if another should be added
echo ""
echo -ne "\n Would you like to add another set of sync databases? [y/N]: "
read -r answer
if [[ $answer != "y" ]]; then
# end the loop
GETTING=0
fi
done
}
### setup sync folders file ###
function getSyncFolders() {
# start building the website folder details
echo "# SOURCE_PATH TARGET_PATH" >"$1"
# default it no to run setup
GETTING=0
echo ""
echo -ne "\n Would you like to add a set of sync folders? [y/N]: "
read -r answer
if [[ $answer == "y" ]]; then
# start Folder tutorial
echo -ne "\n RUNNING FOLDER SETUP\n"
# set checker to get more
GETTING=1
fi
# start setup
while [ "$GETTING" -eq "1" ]; do
# get source folder path path
echo -ne "\n Set the Source Folder Path\n"
echo -ne " # Example (/home/username_a/): "
read -r INPUT_SOURCE_PATH
# check that we have a source path
if [ ! -d "$INPUT_SOURCE_PATH" ]; then
echo -ne "\n YOU MUST ADD A SOURCE PATH!\n\n"
# remove the file
rm "$1"
# start again
exit 1
fi
# get target folder path path
echo -ne "\n Set the Target Folder Path\n"
echo -ne " # Example (/home/username_b): "
read -r INPUT_TARGET_PATH
# check that we have a target path
if [ ! -d "$INPUT_TARGET_PATH" ]; then
echo -ne "\n YOU MUST ADD A TARGET PATH!\n\n"
# remove the file
rm "$1"
# start again
exit 1
fi
# must set realpath
source_folder=$(realpath -s "${INPUT_SOURCE_PATH}")
target_folder=$(realpath -s "${INPUT_TARGET_PATH}")
# add to the file
echo "${source_folder} ${target_folder}" >>"$1"
# get hash
HASH=$(setMD5 "${source_folder}${target_folder}")
# check if exclusion is needed
getExcluded "${EXCLUDEPATH}${HASH}"
# check if another should be added
echo ""
echo -ne "\n Would you like to add another set of sync folders? [y/N]: "
read -r answer
if [[ $answer != "y" ]]; then
# end the loop
GETTING=0
fi
done
}
### setup sync folders file ###
function getExcluded() {
# default it no to run setup
GETTING=0
echo ""
echo -ne "\n Would you like to add excluded files/folders? [y/N]: "
read -r answer
if [[ $answer == "y" ]]; then
# start exclution
echo -ne "\n See for more details https://linuxize.com/post/how-to-exclude-files-and-directories-with-rsync/\n"
# set checker to get more
GETTING=1
fi
# start setup
while [ "$GETTING" -eq "1" ]; do
# get source folder path path
echo -ne "\n Add file or folder to exclude\n"
echo -ne " # Example (configuration.php or administrator/*): "
read -r INPUT_EXCLUDE
# add to file
echo "${INPUT_EXCLUDE}" >>"$1"
# check if another should be added
echo ""
echo -ne "\n Would you like to add another exclusion? [y/N]: "
read -r answer
if [[ $answer != "y" ]]; then
# end the loop
GETTING=0
fi
done
}
### sync databases ###
function syncDBs() {
while IFS=$'\t' read -r -a databases; do
[[ "$databases" =~ ^#.*$ ]] && continue
# SOURCE_DBSERVER SOURCE_DATABASE SOURCE_USER SOURCE_PASS TARGET_DBSERVER TARGET_DATABASE TARGET_USER TARGET_PASS
syncDB "${databases[0]}" "${databases[1]}" "${databases[2]}" "${databases[3]}" "${databases[4]}" "${databases[5]}" "${databases[6]}" "${databases[7]}"
done <$1
}
### sync database ###
function syncDB() {
# give the user log data
echoTweak "Syncing databases of [$2] with [$6]..."
# local source_server="$1"
# local source_db="$2"
# local source_user="$3"
# local source_pass="$4"
# local target_server="$5"
# local target_db="$6"
# local target_user="$7"
# local target_pass="$8"
# move tables from one database to the other
mysqldump --opt -q --host="$1" --user="$3" --password="$4" "$2" |
mysql --host="$5" --user="$7" --password="$8" -C "$6"
# we may want to look at passing the password more securly (TODO)
# done :)
echo "done"
}
### sync folders ###
function syncFolders() {
while IFS=$'\t' read -r -a folders; do
[[ "$folders" =~ ^#.*$ ]] && continue
# SOURCE_PATH TARGET_PATH
syncFolder "${folders[0]}" "${folders[1]}"
done <$1
}
### sync folder ###
function syncFolder() {
local source_folder="$1"
local target_folder="$2"
# get the owners
local source_owner=$(stat -c '%U' "${source_folder}")
local target_owner=$(stat -c '%U' "${target_folder}")
# give the user log data
echoTweak "Syncing folders of [${source_owner}] with [${target_owner}]..."
# get hash
HASH=$(setMD5 "${source_folder}${target_folder}")
# check if we have exclude file
local tmpName=$(getRandom)
local tmpPath="${HOMEPATH}.${tmpName}"
# local exclude=''
if [ -f "${EXCLUDEPATH}${HASH}" ]; then
##########################################################################
# I tried doing this exclude with rsync but it just does not work (TODO)
# while IFS= read -r line; do
# exclude+=" --exclude '${line}'"
# done < "${EXCLUDEPATH}${HASH}"
# IF YOU CAN HELP LET ME KNOW
#########################################################################
# make tmp dir
mkdir "$tmpPath"
# move file/folders out to tmp folder
for line in $(cat ${EXCLUDEPATH}${HASH}); do
line=$(echo $line | sed $'s/\r//')
mv -f $target_folder/$line $tmpPath
done
fi
# we use rsync to do all the sync work (very smart)
rsync -qrd --delete "${source_folder}/" "${target_folder}"
##########################################################################
# I tried doing this exclude with rsync but it just does not work (TODO)
# if [ ! ${#exclude} -ge 2 ]; then
# rsync -qrd --delete "${source_folder}/" "${target_folder}"
# else
# rsync -qrd $exclude --delete "${source_folder}/" "${target_folder}"
# fi
# IF YOU CAN HELP LET ME KNOW
#########################################################################
# move the files back
if [ -f "${EXCLUDEPATH}${HASH}" ]; then
# move file/folders out
for line in $(cat ${EXCLUDEPATH}${HASH}); do
line=$(echo $line | sed $'s/\r//')
mv -f $tmpPath/$line $target_folder
done
# remove tmp dir
rm -r "$tmpPath"
fi
# run chown again (will only work if script run as root)
chown -R "${target_owner}":"${target_owner}" "${target_folder}/"*
# done :)
echo "done"
}
##############################################################
############## ##########
############## MAIN ##########
############## ##########
##############################################################
main
View Git Blame Copy Permalink