accounting/frappe_docker
2
0
Fork 0
mirror of https://github.com/frappe/frappe_docker.git synced 2025-02-03 04:28:31 +00:00
Code Issues Releases Activity

Merge pull request #286 from Thunderbottom/restore-site_config

Browse Source 
fix: restore encryption key on site restore
This commit is contained in:
Revant Nandgaonkar 2020-06-26 11:00:13 +05:30 committed by GitHub
commit 64c0517665
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 162 additions and 38 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
.travis.yml
View File

@ -153,7 +153,10 @@ jobs:
- stage: "Build and test edge images"
if: type = pull_request
before_install:
- sudo apt-get update && sudo apt-get -y install docker-compose
- sudo curl -L "https://github.com/docker/compose/releases/download/1.26.0/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose
- sudo chmod +x /usr/local/bin/docker-compose
- sudo ln -s /usr/local/bin/docker-compose /usr/bin/docker-compose
- sudo apt-get update && sudo apt-get -y install w3m
script:
- docker build -t frappe/frappe-socketio:edge -f build/frappe-socketio/Dockerfile .
- docker build -t frappe/frappe-worker:develop -f build/frappe-worker/Dockerfile .
@ -164,7 +167,7 @@ jobs:
- stage: "Pull and test edge images"
if: branch = develop AND type != pull_request
before_install:
- sudo apt-get update && sudo apt-get -y install docker-compose
- sudo apt-get update && sudo apt-get -y install docker-compose w3m
script:
- ./tests/docker-test.sh
- stage: "Helm Chart Release"

21
build/common/commands/restore_backup.py
View File

@ -1,5 +1,6 @@
import os
import datetime
import json
import tarfile
import hashlib
import frappe
@ -8,7 +9,7 @@ import boto3
from new import get_password
from push_backup import DATE_FORMAT, check_environment_variables
from frappe.utils import get_sites, random_string
from frappe.installer import make_conf, get_conf_params, make_site_dirs
from frappe.installer import make_conf, get_conf_params, make_site_dirs, update_site_config
from check_connection import get_site_config, get_config
@ -38,7 +39,7 @@ def decompress_db(files_base, site):
os.system(command)
def restore_database(files_base, site):
def restore_database(files_base, site_config_path, site):
db_root_password = get_password('MYSQL_ROOT_PASSWORD')
if not db_root_password:
print('Variable MYSQL_ROOT_PASSWORD not set')
@ -96,6 +97,15 @@ def restore_database(files_base, site):
print('Restoring database for site: {}'.format(site))
os.system(command)
if os.path.exists(site_config_path):
with open(site_config_path, 'r') as sc:
site_config = json.load(sc)
encryption_key = site_config.get("encryption_key")
if encryption_key:
print('Restoring site config for site: {}'.format(site))
update_site_config('encryption_key', encryption_key,
site_config_path=os.path.join(os.getcwd(), site, "site_config.json"))
def restore_files(files_base):
public_files = files_base + '-files.tar'
@ -180,8 +190,11 @@ def main():
latest_backup = max(backups).strftime(DATE_FORMAT)
files_base = os.path.join(backup_dir, site, latest_backup, '')
files_base += latest_backup + '-' + site_slug
site_config_path = files_base + '-site_config_backup.json'
if not os.path.exists(site_config_path):
site_config_path = os.path.join(backup_dir, site, 'site_config.json')
if site in get_sites():
restore_database(files_base, site)
restore_database(files_base, site_config_path, site)
restore_private_files(files_base)
restore_files(files_base)
else:
@ -203,7 +216,7 @@ def main():
db_password=site_config.get('db_password'),
)
make_site_dirs()
restore_database(files_base, site)
restore_database(files_base, site_config_path, site)
restore_private_files(files_base)
restore_files(files_base)

172
tests/docker-test.sh
View File

@ -41,13 +41,21 @@ function loopHealthCheck() {
echo "Health check successful"
}
echo "Copy env-example file"
echo -e "\e[1m\e[4mCopy env-example file\e[0m"
cp env-example .env
echo -e "\n"
echo "Set version to v12"
echo -e "\e[1m\e[4mSet version to v12\e[0m"
sed -i -e "s/edge/v12/g" .env
echo -e "\n"
echo "Start Services"
echo -e "\e[1m\e[4mStart Services\e[0m"
docker-compose \
--project-name frappebench00 \
-f installation/docker-compose-common.yml \
-f installation/docker-compose-erpnext.yml \
-f installation/erpnext-publish.yml \
pull
docker-compose \
--project-name frappebench00 \
-f installation/docker-compose-common.yml \
@ -56,23 +64,30 @@ docker-compose \
up -d
loopHealthCheck
echo -e "\n"
echo "Create new site (v12)"
echo -e "\e[1m\e[4mCreate new site (v12)\e[0m"
docker run -it \
-e "SITE_NAME=test.localhost" \
-e "INSTALL_APPS=erpnext" \
-v frappebench00_sites-vol:/home/frappe/frappe-bench/sites \
--network frappebench00_default \
frappe/erpnext-worker:v12 new
echo -e "\n"
echo "Ping created site"
curl -S http://test.localhost/api/method/version
echo ""
echo -e "\e[1m\e[4mPing created site\e[0m"
curl -sS http://test.localhost/api/method/version
echo -e "\n"
echo "Set version to edge"
echo -e "\e[1m\e[4mCheck Created Site Index Page\e[0m"
curl -s http://test.localhost | w3m -T text/html -dump
echo -e "\n"
echo -e "\e[1m\e[4mSet version to edge\e[0m"
sed -i -e "s/v12/edge/g" .env
echo -e "\n"
echo "Restart containers with edge image"
echo -e "\e[1m\e[4mRestart containers with edge image\e[0m"
docker-compose \
--project-name frappebench00 \
-f installation/docker-compose-common.yml \
@ -87,31 +102,37 @@ docker-compose \
up -d
checkMigrationComplete
echo -e "\n"
echo "Ping migrated site"
echo -e "\e[1m\e[4mPing migrated site\e[0m"
sleep 3
curl -S http://test.localhost/api/method/version
echo ""
curl -sS http://test.localhost/api/method/version
echo -e "\n"
echo "Backup site"
echo -e "\e[1m\e[4mCheck Migrated Site Index Page\e[0m"
curl -s http://test.localhost | w3m -T text/html -dump
echo -e "\n"
echo -e "\e[1m\e[4mBackup site\e[0m"
docker run -it \
-e "WITH_FILES=1" \
-v frappebench00_sites-vol:/home/frappe/frappe-bench/sites \
--network frappebench00_default \
frappe/erpnext-worker:edge backup
echo -e "\n"
export MINIO_ACCESS_KEY="AKIAIOSFODNN7EXAMPLE"
export MINIO_SECRET_KEY="wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"
echo "Start MinIO container for s3 compatible storage"
echo -e "\e[1m\e[4mStart MinIO container for s3 compatible storage\e[0m"
docker run -d --name minio \
-e "MINIO_ACCESS_KEY=$MINIO_ACCESS_KEY" \
-e "MINIO_SECRET_KEY=$MINIO_SECRET_KEY" \
--network frappebench00_default \
minio/minio server /data
echo -e "\n"
echo "Create bucket named erpnext after 3 sec."
sleep 3
echo -e "\e[1m\e[4mCreate bucket named erpnext\e[0m"
docker run \
--network frappebench00_default \
vltgroup/s3cmd:latest s3cmd --access_key=$MINIO_ACCESS_KEY \
@ -121,8 +142,9 @@ docker run \
--host=minio:9000 \
--host-bucket=minio:9000 \
mb s3://erpnext
echo -e "\n"
echo "Push backup to MinIO s3"
echo -e "\e[1m\e[4mPush backup to MinIO s3\e[0m"
docker run \
-e BUCKET_NAME=erpnext \
-e REGION=us-east-1 \
@ -133,19 +155,22 @@ docker run \
-v frappebench00_sites-vol:/home/frappe/frappe-bench/sites \
--network frappebench00_default \
frappe/erpnext-worker:edge push-backup
echo -e "\n"
echo "Stop Services"
echo -e "\e[1m\e[4mStop Services\e[0m"
docker-compose \
--project-name frappebench00 \
-f installation/docker-compose-common.yml \
-f installation/docker-compose-erpnext.yml \
-f installation/erpnext-publish.yml \
stop
echo -e "\n"
echo "Prune Containers"
echo -e "\e[1m\e[4mPrune Containers\e[0m"
docker container prune -f && docker volume prune -f
echo -e "\n"
echo "Start Services"
echo -e "\e[1m\e[4mStart Services\e[0m"
docker-compose \
--project-name frappebench00 \
-f installation/docker-compose-common.yml \
@ -154,8 +179,9 @@ docker-compose \
up -d
loopHealthCheck
echo -e "\n"
echo "Restore backup from MinIO / S3"
echo -e "\e[1m\e[4mRestore backup from MinIO / S3\e[0m"
docker run \
-e MYSQL_ROOT_PASSWORD=admin \
-e BUCKET_NAME=erpnext \
@ -167,15 +193,16 @@ docker run \
-v frappebench00_sites-vol:/home/frappe/frappe-bench/sites \
--network frappebench00_default \
frappe/erpnext-worker:edge restore-backup
echo -e "\n"
echo "Check Restored Site"
echo -e "\e[1m\e[4mCheck Restored Site\e[0m"
sleep 3
RESTORE_STATUS=$(curl -S http://test.localhost/api/method/version || echo "")
RESTORE_STATUS=$(curl -sS http://test.localhost/api/method/version || echo "")
INCREMENT=0
while [[ -z "$RESTORE_STATUS" && $INCREMENT -lt 60 ]]; do
sleep 1
echo "Wait for restoration to complete ..."
RESTORE_STATUS=$(curl -S http://test.localhost/api/method/version || echo "")
RESTORE_STATUS=$(curl -sS http://test.localhost/api/method/version || echo "")
((INCREMENT=INCREMENT+1))
if [[ -z "$RESTORE_STATUS" && $INCREMENT -eq 60 ]]; then
CONTAINER_ID=$(docker-compose \
@ -189,10 +216,54 @@ while [[ -z "$RESTORE_STATUS" && $INCREMENT -lt 60 ]]; do
fi
done
echo "Ping restored site"
echo -e "\e[1m\e[4mPing restored site\e[0m"
echo $RESTORE_STATUS
echo -e "\n"
echo "Migrate command in edge container"
echo -e "\e[1m\e[4mCheck Restored Site Index Page\e[0m"
curl -s http://test.localhost | w3m -T text/html -dump
echo -e "\n"
echo -e "\e[1m\e[4mCreate new site (edge)\e[0m"
docker run -it \
-e "SITE_NAME=edge.localhost" \
-e "INSTALL_APPS=erpnext" \
-v frappebench00_sites-vol:/home/frappe/frappe-bench/sites \
--network frappebench00_default \
frappe/erpnext-worker:edge new
echo -e "\n"
echo -e "\e[1m\e[4mCheck New Edge Site\e[0m"
sleep 3
RESTORE_STATUS=$(curl -sS http://edge.localhost/api/method/version || echo "")
INCREMENT=0
while [[ -z "$RESTORE_STATUS" && $INCREMENT -lt 60 ]]; do
sleep 1
echo -e "\e[1m\e[4mWait for restoration to complete ..."
RESTORE_STATUS=$(curl -sS http://edge.localhost/api/method/version || echo "")
((INCREMENT=INCREMENT+1))
if [[ -z "$RESTORE_STATUS" && $INCREMENT -eq 60 ]]; then
CONTAINER_ID=$(docker-compose \
--project-name frappebench00 \
-f installation/docker-compose-common.yml \
-f installation/docker-compose-erpnext.yml \
-f installation/erpnext-publish.yml \
ps -q erpnext-python)
docker logs $CONTAINER_ID
exit 1
fi
done
echo -e "\n"
echo -e "\e[1m\e[4mPing new edge site\e[0m"
echo $RESTORE_STATUS
echo -e "\n"
echo -e "\e[1m\e[4mCheck New Edge Index Page\e[0m"
curl -s http://edge.localhost | w3m -T text/html -dump
echo -e "\n"
echo -e "\e[1m\e[4mMigrate command in edge container\e[0m"
docker run -it \
-e "MAINTENANCE_MODE=1" \
-v frappebench00_sites-vol:/home/frappe/frappe-bench/sites \
@ -200,16 +271,53 @@ docker run -it \
frappe/erpnext-worker:edge migrate
checkMigrationComplete
echo -e "\n"
echo "Create new site (edge)"
docker run -it \
-e "SITE_NAME=edge.localhost" \
-e "INSTALL_APPS=erpnext" \
echo -e "\e[1m\e[4mRestore backup from MinIO / S3 (Overwrite)\e[0m"
docker run \
-e MYSQL_ROOT_PASSWORD=admin \
-e BUCKET_NAME=erpnext \
-e BUCKET_DIR=local \
-e ACCESS_KEY_ID=$MINIO_ACCESS_KEY \
-e SECRET_ACCESS_KEY=$MINIO_SECRET_KEY \
-e ENDPOINT_URL=http://minio:9000 \
-e REGION=us-east-1 \
-v frappebench00_sites-vol:/home/frappe/frappe-bench/sites \
--network frappebench00_default \
frappe/erpnext-worker:edge new
frappe/erpnext-worker:edge restore-backup
echo -e "\n"
echo "Check console command for site test.localhost"
echo -e "\e[1m\e[4mCheck Overwritten Site\e[0m"
sleep 3
RESTORE_STATUS=$(curl -sS http://test.localhost/api/method/version || echo "")
INCREMENT=0
while [[ -z "$RESTORE_STATUS" && $INCREMENT -lt 60 ]]; do
sleep 1
echo -e "\e[1m\e[4mWait for restoration to complete ..."
RESTORE_STATUS=$(curl -sS http://test.localhost/api/method/version || echo "")
((INCREMENT=INCREMENT+1))
if [[ -z "$RESTORE_STATUS" && $INCREMENT -eq 60 ]]; then
CONTAINER_ID=$(docker-compose \
--project-name frappebench00 \
-f installation/docker-compose-common.yml \
-f installation/docker-compose-erpnext.yml \
-f installation/erpnext-publish.yml \
ps -q erpnext-python)
docker logs $CONTAINER_ID
exit 1
fi
done
echo -e "\n"
echo -e "\e[1m\e[4mPing overwritten site\e[0m"
echo $RESTORE_STATUS
echo -e "\n"
echo -e "\e[1m\e[4mCheck Overwritten Index Page\e[0m"
curl -s http://test.localhost | w3m -T text/html -dump
echo -e "\n"
echo -e "\e[1m\e[4mCheck console command for site test.localhost\e[0m"
docker run \
-v frappebench00_sites-vol:/home/frappe/frappe-bench/sites \
--network frappebench00_default \