From bdbcf40702d6ad6690a34d362d6adef2a5cb3c91 Mon Sep 17 00:00:00 2001 From: "Chinmay D. Pai" Date: Thu, 25 Jun 2020 23:23:31 +0530 Subject: [PATCH 1/5] fix: restore encryption key on site restore if encryption key exists for a site, it will now be restored to the site config while restoring site Signed-off-by: Chinmay D. Pai --- build/common/commands/restore_backup.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/build/common/commands/restore_backup.py b/build/common/commands/restore_backup.py index f50772a9..4d6f9c47 100644 --- a/build/common/commands/restore_backup.py +++ b/build/common/commands/restore_backup.py @@ -1,5 +1,6 @@ import os import datetime +import json import tarfile import hashlib import frappe @@ -8,7 +9,7 @@ import boto3 from new import get_password from push_backup import DATE_FORMAT, check_environment_variables from frappe.utils import get_sites, random_string -from frappe.installer import make_conf, get_conf_params, make_site_dirs +from frappe.installer import make_conf, get_conf_params, make_site_dirs, update_site_config from check_connection import get_site_config, get_config @@ -38,7 +39,7 @@ def decompress_db(files_base, site): os.system(command) -def restore_database(files_base, site): +def restore_database(files_base, site_config_path, site): db_root_password = get_password('MYSQL_ROOT_PASSWORD') if not db_root_password: print('Variable MYSQL_ROOT_PASSWORD not set') @@ -96,6 +97,15 @@ def restore_database(files_base, site): print('Restoring database for site: {}'.format(site)) os.system(command) + if os.path.exists(site_config_path): + with open(site_config_path, 'r') as sc: + site_config = json.load(sc) + encryption_key = site_config.get("encryption_key") + if encryption_key: + print('Restoring site config for site: {}'.format(site)) + update_site_config('encryption_key', encryption_key, + site_config_path=os.path.join(os.getcwd(), site, "site_config.json")) + def restore_files(files_base): public_files = files_base + '-files.tar' @@ -180,8 +190,9 @@ def main(): latest_backup = max(backups).strftime(DATE_FORMAT) files_base = os.path.join(backup_dir, site, latest_backup, '') files_base += latest_backup + '-' + site_slug + site_config_path = os.path.join(backup_dir, site, 'site_config.json') if site in get_sites(): - restore_database(files_base, site) + restore_database(files_base, site_config_path, site) restore_private_files(files_base) restore_files(files_base) else: From 95416ee5550e9a4e45606b79e647c49415242612 Mon Sep 17 00:00:00 2001 From: "Chinmay D. Pai" Date: Thu, 25 Jun 2020 23:42:55 +0530 Subject: [PATCH 2/5] fix: check for all possible site config backups Signed-off-by: Chinmay D. Pai --- build/common/commands/restore_backup.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/build/common/commands/restore_backup.py b/build/common/commands/restore_backup.py index 4d6f9c47..531950d9 100644 --- a/build/common/commands/restore_backup.py +++ b/build/common/commands/restore_backup.py @@ -190,7 +190,9 @@ def main(): latest_backup = max(backups).strftime(DATE_FORMAT) files_base = os.path.join(backup_dir, site, latest_backup, '') files_base += latest_backup + '-' + site_slug - site_config_path = os.path.join(backup_dir, site, 'site_config.json') + site_config_path = files_base + '-site_config_backup.json' + if not os.path.exists(site_config_path): + site_config_path = os.path.join(backup_dir, site, 'site_config.json') if site in get_sites(): restore_database(files_base, site_config_path, site) restore_private_files(files_base) From 1a1636df552bc1c90448954906b1091467836f67 Mon Sep 17 00:00:00 2001 From: Revant Nandgaonkar Date: Fri, 26 Jun 2020 05:47:17 +0530 Subject: [PATCH 3/5] fix: fresh restore site_config backup --- build/common/commands/restore_backup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/common/commands/restore_backup.py b/build/common/commands/restore_backup.py index 531950d9..131babb3 100644 --- a/build/common/commands/restore_backup.py +++ b/build/common/commands/restore_backup.py @@ -216,7 +216,7 @@ def main(): db_password=site_config.get('db_password'), ) make_site_dirs() - restore_database(files_base, site) + restore_database(files_base, site_config_path, site) restore_private_files(files_base) restore_files(files_base) From 8beb6a284f722c54c19225c99471aceae88afc4b Mon Sep 17 00:00:00 2001 From: Revant Nandgaonkar Date: Fri, 26 Jun 2020 06:03:54 +0530 Subject: [PATCH 4/5] test: add tests overwrite restore check index pages of created sites improve formatting --- tests/docker-test.sh | 178 +++++++++++++++++++++++++++++++++++-------- 1 file changed, 146 insertions(+), 32 deletions(-) diff --git a/tests/docker-test.sh b/tests/docker-test.sh index dded6bb7..fc080003 100755 --- a/tests/docker-test.sh +++ b/tests/docker-test.sh @@ -41,13 +41,21 @@ function loopHealthCheck() { echo "Health check successful" } -echo "Copy env-example file" +echo -e "\e[1m\e[4mCopy env-example file\e[0m" cp env-example .env +echo -e "\n" -echo "Set version to v12" +echo -e "\e[1m\e[4mSet version to v12\e[0m" sed -i -e "s/edge/v12/g" .env +echo -e "\n" -echo "Start Services" +echo -e "\e[1m\e[4mStart Services\e[0m" +docker-compose \ + --project-name frappebench00 \ + -f installation/docker-compose-common.yml \ + -f installation/docker-compose-erpnext.yml \ + -f installation/erpnext-publish.yml \ + pull docker-compose \ --project-name frappebench00 \ -f installation/docker-compose-common.yml \ @@ -56,29 +64,42 @@ docker-compose \ up -d loopHealthCheck +echo -e "\n" -echo "Create new site (v12)" +echo -e "\e[1m\e[4mCreate new site (v12)\e[0m" docker run -it \ -e "SITE_NAME=test.localhost" \ -e "INSTALL_APPS=erpnext" \ -v frappebench00_sites-vol:/home/frappe/frappe-bench/sites \ --network frappebench00_default \ frappe/erpnext-worker:v12 new +echo -e "\n" -echo "Ping created site" -curl -S http://test.localhost/api/method/version -echo "" +echo -e "\e[1m\e[4mPing created site\e[0m" +curl -sS http://test.localhost/api/method/version +echo -e "\n" -echo "Set version to edge" +echo -e "\e[1m\e[4mCheck Created Site Index Page\e[0m" +curl -s http://test.localhost # | w3m -T text/html -dump +echo -e "\n" + +echo -e "\e[1m\e[4mSet version to edge\e[0m" sed -i -e "s/v12/edge/g" .env +echo -e "\n" -echo "Restart containers with edge image" +echo -e "\e[1m\e[4mRestart containers with edge image\e[0m" docker-compose \ --project-name frappebench00 \ -f installation/docker-compose-common.yml \ -f installation/docker-compose-erpnext.yml \ -f installation/erpnext-publish.yml \ stop +docker-compose \ + --project-name frappebench00 \ + -f installation/docker-compose-common.yml \ + -f installation/docker-compose-erpnext.yml \ + -f installation/erpnext-publish.yml \ + pull docker-compose \ --project-name frappebench00 \ -f installation/docker-compose-common.yml \ @@ -87,31 +108,37 @@ docker-compose \ up -d checkMigrationComplete +echo -e "\n" -echo "Ping migrated site" +echo -e "\e[1m\e[4mPing migrated site\e[0m" sleep 3 -curl -S http://test.localhost/api/method/version -echo "" +curl -sS http://test.localhost/api/method/version +echo -e "\n" -echo "Backup site" +echo -e "\e[1m\e[4mCheck Migrated Site Index Page\e[0m" +curl -s http://test.localhost # | w3m -T text/html -dump +echo -e "\n" + +echo -e "\e[1m\e[4mBackup site\e[0m" docker run -it \ -e "WITH_FILES=1" \ -v frappebench00_sites-vol:/home/frappe/frappe-bench/sites \ --network frappebench00_default \ frappe/erpnext-worker:edge backup +echo -e "\n" export MINIO_ACCESS_KEY="AKIAIOSFODNN7EXAMPLE" export MINIO_SECRET_KEY="wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" -echo "Start MinIO container for s3 compatible storage" +echo -e "\e[1m\e[4mStart MinIO container for s3 compatible storage\e[0m" docker run -d --name minio \ -e "MINIO_ACCESS_KEY=$MINIO_ACCESS_KEY" \ -e "MINIO_SECRET_KEY=$MINIO_SECRET_KEY" \ --network frappebench00_default \ minio/minio server /data +echo -e "\n" -echo "Create bucket named erpnext after 3 sec." -sleep 3 +echo -e "\e[1m\e[4mCreate bucket named erpnext\e[0m" docker run \ --network frappebench00_default \ vltgroup/s3cmd:latest s3cmd --access_key=$MINIO_ACCESS_KEY \ @@ -121,8 +148,9 @@ docker run \ --host=minio:9000 \ --host-bucket=minio:9000 \ mb s3://erpnext +echo -e "\n" -echo "Push backup to MinIO s3" +echo -e "\e[1m\e[4mPush backup to MinIO s3\e[0m" docker run \ -e BUCKET_NAME=erpnext \ -e REGION=us-east-1 \ @@ -133,19 +161,22 @@ docker run \ -v frappebench00_sites-vol:/home/frappe/frappe-bench/sites \ --network frappebench00_default \ frappe/erpnext-worker:edge push-backup +echo -e "\n" -echo "Stop Services" +echo -e "\e[1m\e[4mStop Services\e[0m" docker-compose \ --project-name frappebench00 \ -f installation/docker-compose-common.yml \ -f installation/docker-compose-erpnext.yml \ -f installation/erpnext-publish.yml \ stop +echo -e "\n" -echo "Prune Containers" +echo -e "\e[1m\e[4mPrune Containers\e[0m" docker container prune -f && docker volume prune -f +echo -e "\n" -echo "Start Services" +echo -e "\e[1m\e[4mStart Services\e[0m" docker-compose \ --project-name frappebench00 \ -f installation/docker-compose-common.yml \ @@ -154,8 +185,9 @@ docker-compose \ up -d loopHealthCheck +echo -e "\n" -echo "Restore backup from MinIO / S3" +echo -e "\e[1m\e[4mRestore backup from MinIO / S3\e[0m" docker run \ -e MYSQL_ROOT_PASSWORD=admin \ -e BUCKET_NAME=erpnext \ @@ -167,15 +199,16 @@ docker run \ -v frappebench00_sites-vol:/home/frappe/frappe-bench/sites \ --network frappebench00_default \ frappe/erpnext-worker:edge restore-backup +echo -e "\n" -echo "Check Restored Site" +echo -e "\e[1m\e[4mCheck Restored Site\e[0m" sleep 3 -RESTORE_STATUS=$(curl -S http://test.localhost/api/method/version || echo "") +RESTORE_STATUS=$(curl -sS http://test.localhost/api/method/version || echo "") INCREMENT=0 while [[ -z "$RESTORE_STATUS" && $INCREMENT -lt 60 ]]; do sleep 1 echo "Wait for restoration to complete ..." - RESTORE_STATUS=$(curl -S http://test.localhost/api/method/version || echo "") + RESTORE_STATUS=$(curl -sS http://test.localhost/api/method/version || echo "") ((INCREMENT=INCREMENT+1)) if [[ -z "$RESTORE_STATUS" && $INCREMENT -eq 60 ]]; then CONTAINER_ID=$(docker-compose \ @@ -189,10 +222,54 @@ while [[ -z "$RESTORE_STATUS" && $INCREMENT -lt 60 ]]; do fi done -echo "Ping restored site" +echo -e "\e[1m\e[4mPing restored site\e[0m" echo $RESTORE_STATUS +echo -e "\n" -echo "Migrate command in edge container" +echo -e "\e[1m\e[4mCheck Restored Site Index Page\e[0m" +curl -s http://test.localhost # | w3m -T text/html -dump +echo -e "\n" + +echo -e "\e[1m\e[4mCreate new site (edge)\e[0m" +docker run -it \ + -e "SITE_NAME=edge.localhost" \ + -e "INSTALL_APPS=erpnext" \ + -v frappebench00_sites-vol:/home/frappe/frappe-bench/sites \ + --network frappebench00_default \ + frappe/erpnext-worker:edge new +echo -e "\n" + +echo -e "\e[1m\e[4mCheck New Edge Site\e[0m" +sleep 3 +RESTORE_STATUS=$(curl -sS http://edge.localhost/api/method/version || echo "") +INCREMENT=0 +while [[ -z "$RESTORE_STATUS" && $INCREMENT -lt 60 ]]; do + sleep 1 + echo -e "\e[1m\e[4mWait for restoration to complete ..." + RESTORE_STATUS=$(curl -sS http://edge.localhost/api/method/version || echo "") + ((INCREMENT=INCREMENT+1)) + if [[ -z "$RESTORE_STATUS" && $INCREMENT -eq 60 ]]; then + CONTAINER_ID=$(docker-compose \ + --project-name frappebench00 \ + -f installation/docker-compose-common.yml \ + -f installation/docker-compose-erpnext.yml \ + -f installation/erpnext-publish.yml \ + ps -q erpnext-python) + docker logs $CONTAINER_ID + exit 1 + fi +done +echo -e "\n" + +echo -e "\e[1m\e[4mPing new edge site\e[0m" +echo $RESTORE_STATUS +echo -e "\n" + +echo -e "\e[1m\e[4mCheck New Edge Index Page\e[0m" +curl -s http://edge.localhost # | w3m -T text/html -dump +echo -e "\n" + +echo -e "\e[1m\e[4mMigrate command in edge container\e[0m" docker run -it \ -e "MAINTENANCE_MODE=1" \ -v frappebench00_sites-vol:/home/frappe/frappe-bench/sites \ @@ -200,16 +277,53 @@ docker run -it \ frappe/erpnext-worker:edge migrate checkMigrationComplete +echo -e "\n" -echo "Create new site (edge)" -docker run -it \ - -e "SITE_NAME=edge.localhost" \ - -e "INSTALL_APPS=erpnext" \ +echo -e "\e[1m\e[4mRestore backup from MinIO / S3 (Overwrite)\e[0m" +docker run \ + -e MYSQL_ROOT_PASSWORD=admin \ + -e BUCKET_NAME=erpnext \ + -e BUCKET_DIR=local \ + -e ACCESS_KEY_ID=$MINIO_ACCESS_KEY \ + -e SECRET_ACCESS_KEY=$MINIO_SECRET_KEY \ + -e ENDPOINT_URL=http://minio:9000 \ + -e REGION=us-east-1 \ -v frappebench00_sites-vol:/home/frappe/frappe-bench/sites \ --network frappebench00_default \ - frappe/erpnext-worker:edge new + frappe/erpnext-worker:edge restore-backup +echo -e "\n" -echo "Check console command for site test.localhost" +echo -e "\e[1m\e[4mCheck Overwritten Site\e[0m" +sleep 3 +RESTORE_STATUS=$(curl -sS http://test.localhost/api/method/version || echo "") +INCREMENT=0 +while [[ -z "$RESTORE_STATUS" && $INCREMENT -lt 60 ]]; do + sleep 1 + echo -e "\e[1m\e[4mWait for restoration to complete ..." + RESTORE_STATUS=$(curl -sS http://test.localhost/api/method/version || echo "") + ((INCREMENT=INCREMENT+1)) + if [[ -z "$RESTORE_STATUS" && $INCREMENT -eq 60 ]]; then + CONTAINER_ID=$(docker-compose \ + --project-name frappebench00 \ + -f installation/docker-compose-common.yml \ + -f installation/docker-compose-erpnext.yml \ + -f installation/erpnext-publish.yml \ + ps -q erpnext-python) + docker logs $CONTAINER_ID + exit 1 + fi +done +echo -e "\n" + +echo -e "\e[1m\e[4mPing overwritten site\e[0m" +echo $RESTORE_STATUS +echo -e "\n" + +echo -e "\e[1m\e[4mCheck Overwritten Index Page\e[0m" +curl -s http://test.localhost # | w3m -T text/html -dump +echo -e "\n" + +echo -e "\e[1m\e[4mCheck console command for site test.localhost\e[0m" docker run \ -v frappebench00_sites-vol:/home/frappe/frappe-bench/sites \ --network frappebench00_default \ From f77ca7726c48bd5c723a79164c679b7e1973b610 Mon Sep 17 00:00:00 2001 From: Revant Nandgaonkar Date: Fri, 26 Jun 2020 10:19:27 +0530 Subject: [PATCH 5/5] test: fix cgo errors before building --- .travis.yml | 7 +++++-- tests/docker-test.sh | 16 +++++----------- 2 files changed, 10 insertions(+), 13 deletions(-) diff --git a/.travis.yml b/.travis.yml index 9e1c5951..fc5b100b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -153,7 +153,10 @@ jobs: - stage: "Build and test edge images" if: type = pull_request before_install: - - sudo apt-get update && sudo apt-get -y install docker-compose + - sudo curl -L "https://github.com/docker/compose/releases/download/1.26.0/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose + - sudo chmod +x /usr/local/bin/docker-compose + - sudo ln -s /usr/local/bin/docker-compose /usr/bin/docker-compose + - sudo apt-get update && sudo apt-get -y install w3m script: - docker build -t frappe/frappe-socketio:edge -f build/frappe-socketio/Dockerfile . - docker build -t frappe/frappe-worker:develop -f build/frappe-worker/Dockerfile . @@ -164,7 +167,7 @@ jobs: - stage: "Pull and test edge images" if: branch = develop AND type != pull_request before_install: - - sudo apt-get update && sudo apt-get -y install docker-compose + - sudo apt-get update && sudo apt-get -y install docker-compose w3m script: - ./tests/docker-test.sh - stage: "Helm Chart Release" diff --git a/tests/docker-test.sh b/tests/docker-test.sh index fc080003..597c3c2a 100755 --- a/tests/docker-test.sh +++ b/tests/docker-test.sh @@ -80,7 +80,7 @@ curl -sS http://test.localhost/api/method/version echo -e "\n" echo -e "\e[1m\e[4mCheck Created Site Index Page\e[0m" -curl -s http://test.localhost # | w3m -T text/html -dump +curl -s http://test.localhost | w3m -T text/html -dump echo -e "\n" echo -e "\e[1m\e[4mSet version to edge\e[0m" @@ -94,12 +94,6 @@ docker-compose \ -f installation/docker-compose-erpnext.yml \ -f installation/erpnext-publish.yml \ stop -docker-compose \ - --project-name frappebench00 \ - -f installation/docker-compose-common.yml \ - -f installation/docker-compose-erpnext.yml \ - -f installation/erpnext-publish.yml \ - pull docker-compose \ --project-name frappebench00 \ -f installation/docker-compose-common.yml \ @@ -116,7 +110,7 @@ curl -sS http://test.localhost/api/method/version echo -e "\n" echo -e "\e[1m\e[4mCheck Migrated Site Index Page\e[0m" -curl -s http://test.localhost # | w3m -T text/html -dump +curl -s http://test.localhost | w3m -T text/html -dump echo -e "\n" echo -e "\e[1m\e[4mBackup site\e[0m" @@ -227,7 +221,7 @@ echo $RESTORE_STATUS echo -e "\n" echo -e "\e[1m\e[4mCheck Restored Site Index Page\e[0m" -curl -s http://test.localhost # | w3m -T text/html -dump +curl -s http://test.localhost | w3m -T text/html -dump echo -e "\n" echo -e "\e[1m\e[4mCreate new site (edge)\e[0m" @@ -266,7 +260,7 @@ echo $RESTORE_STATUS echo -e "\n" echo -e "\e[1m\e[4mCheck New Edge Index Page\e[0m" -curl -s http://edge.localhost # | w3m -T text/html -dump +curl -s http://edge.localhost | w3m -T text/html -dump echo -e "\n" echo -e "\e[1m\e[4mMigrate command in edge container\e[0m" @@ -320,7 +314,7 @@ echo $RESTORE_STATUS echo -e "\n" echo -e "\e[1m\e[4mCheck Overwritten Index Page\e[0m" -curl -s http://test.localhost # | w3m -T text/html -dump +curl -s http://test.localhost | w3m -T text/html -dump echo -e "\n" echo -e "\e[1m\e[4mCheck console command for site test.localhost\e[0m"