Merge pull request #286 from Thunderbottom/restore-site_config

fix: restore encryption key on site restore

.travis.yml

@@ -153,7 +153,10 @@ jobs:
     - stage: "Build and test edge images"
       if: type = pull_request
       before_install:
-        - sudo apt-get update && sudo apt-get -y install docker-compose
+        - sudo curl -L "https://github.com/docker/compose/releases/download/1.26.0/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose
+        - sudo chmod +x /usr/local/bin/docker-compose
+        - sudo ln -s /usr/local/bin/docker-compose /usr/bin/docker-compose
+        - sudo apt-get update && sudo apt-get -y install w3m
       script:
         - docker build -t frappe/frappe-socketio:edge -f build/frappe-socketio/Dockerfile .
         - docker build -t frappe/frappe-worker:develop -f build/frappe-worker/Dockerfile .
@@ -164,7 +167,7 @@ jobs:
     - stage: "Pull and test edge images"
       if: branch = develop AND type != pull_request
       before_install:
-        - sudo apt-get update && sudo apt-get -y install docker-compose
+        - sudo apt-get update && sudo apt-get -y install docker-compose w3m
       script:
         - ./tests/docker-test.sh
     - stage: "Helm Chart Release"

build/common/commands/restore_backup.py

@@ -1,5 +1,6 @@
 import os
 import datetime
+import json
 import tarfile
 import hashlib
 import frappe
@@ -8,7 +9,7 @@ import boto3
 from new import get_password
 from push_backup import DATE_FORMAT, check_environment_variables
 from frappe.utils import get_sites, random_string
-from frappe.installer import make_conf, get_conf_params, make_site_dirs
+from frappe.installer import make_conf, get_conf_params, make_site_dirs, update_site_config
 from check_connection import get_site_config, get_config
 
 
@@ -38,7 +39,7 @@ def decompress_db(files_base, site):
     os.system(command)
 
 
-def restore_database(files_base, site):
+def restore_database(files_base, site_config_path, site):
     db_root_password = get_password('MYSQL_ROOT_PASSWORD')
     if not db_root_password:
         print('Variable MYSQL_ROOT_PASSWORD not set')
@@ -96,6 +97,15 @@ def restore_database(files_base, site):
     print('Restoring database for site: {}'.format(site))
     os.system(command)
 
+    if os.path.exists(site_config_path):
+        with open(site_config_path, 'r') as sc:
+            site_config = json.load(sc)
+        encryption_key = site_config.get("encryption_key")
+        if encryption_key:
+            print('Restoring site config for site: {}'.format(site))
+            update_site_config('encryption_key', encryption_key,
+                               site_config_path=os.path.join(os.getcwd(), site, "site_config.json"))
+
 
 def restore_files(files_base):
     public_files = files_base + '-files.tar'
@@ -180,8 +190,11 @@ def main():
         latest_backup = max(backups).strftime(DATE_FORMAT)
         files_base = os.path.join(backup_dir, site, latest_backup, '')
         files_base += latest_backup + '-' + site_slug
+        site_config_path = files_base + '-site_config_backup.json'
+        if not os.path.exists(site_config_path):
+            site_config_path = os.path.join(backup_dir, site, 'site_config.json')
         if site in get_sites():
-            restore_database(files_base, site)
+            restore_database(files_base, site_config_path, site)
             restore_private_files(files_base)
             restore_files(files_base)
         else:
@@ -203,7 +216,7 @@ def main():
                 db_password=site_config.get('db_password'),
             )
             make_site_dirs()
-            restore_database(files_base, site)
+            restore_database(files_base, site_config_path, site)
             restore_private_files(files_base)
             restore_files(files_base)
 

tests/docker-test.sh

@@ -41,13 +41,21 @@ function loopHealthCheck() {
     echo "Health check successful"
 }
 
-echo "Copy env-example file"
+echo -e "\e[1m\e[4mCopy env-example file\e[0m"
 cp env-example .env
+echo -e "\n"
 
-echo "Set version to v12"
+echo -e "\e[1m\e[4mSet version to v12\e[0m"
 sed -i -e "s/edge/v12/g" .env
+echo -e "\n"
 
-echo "Start Services"
+echo -e "\e[1m\e[4mStart Services\e[0m"
+docker-compose \
+    --project-name frappebench00 \
+    -f installation/docker-compose-common.yml \
+    -f installation/docker-compose-erpnext.yml \
+    -f installation/erpnext-publish.yml \
+    pull
 docker-compose \
     --project-name frappebench00 \
     -f installation/docker-compose-common.yml \
@@ -56,23 +64,30 @@ docker-compose \
     up -d
 
 loopHealthCheck
+echo -e "\n"
 
-echo "Create new site (v12)"
+echo -e "\e[1m\e[4mCreate new site (v12)\e[0m"
 docker run -it \
     -e "SITE_NAME=test.localhost" \
     -e "INSTALL_APPS=erpnext" \
     -v frappebench00_sites-vol:/home/frappe/frappe-bench/sites \
     --network frappebench00_default \
     frappe/erpnext-worker:v12 new
+echo -e "\n"
 
-echo "Ping created site"
+echo -e "\e[1m\e[4mPing created site\e[0m"
-curl -S http://test.localhost/api/method/version
+curl -sS http://test.localhost/api/method/version
-echo ""
+echo -e "\n"
 
-echo "Set version to edge"
+echo -e "\e[1m\e[4mCheck Created Site Index Page\e[0m"
+curl -s http://test.localhost | w3m -T text/html -dump
+echo -e "\n"
+
+echo -e "\e[1m\e[4mSet version to edge\e[0m"
 sed -i -e "s/v12/edge/g" .env
+echo -e "\n"
 
-echo "Restart containers with edge image"
+echo -e "\e[1m\e[4mRestart containers with edge image\e[0m"
 docker-compose \
     --project-name frappebench00 \
     -f installation/docker-compose-common.yml \
@@ -87,31 +102,37 @@ docker-compose \
     up -d
 
 checkMigrationComplete
+echo -e "\n"
 
-echo "Ping migrated site"
+echo -e "\e[1m\e[4mPing migrated site\e[0m"
 sleep 3
-curl -S http://test.localhost/api/method/version
+curl -sS http://test.localhost/api/method/version
-echo ""
+echo -e "\n"
 
-echo "Backup site"
+echo -e "\e[1m\e[4mCheck Migrated Site Index Page\e[0m"
+curl -s http://test.localhost | w3m -T text/html -dump
+echo -e "\n"
+
+echo -e "\e[1m\e[4mBackup site\e[0m"
 docker run -it \
     -e "WITH_FILES=1" \
     -v frappebench00_sites-vol:/home/frappe/frappe-bench/sites \
     --network frappebench00_default \
     frappe/erpnext-worker:edge backup
+echo -e "\n"
 
 export MINIO_ACCESS_KEY="AKIAIOSFODNN7EXAMPLE"
 export MINIO_SECRET_KEY="wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"
 
-echo "Start MinIO container for s3 compatible storage"
+echo -e "\e[1m\e[4mStart MinIO container for s3 compatible storage\e[0m"
 docker run -d --name minio \
   -e "MINIO_ACCESS_KEY=$MINIO_ACCESS_KEY" \
   -e "MINIO_SECRET_KEY=$MINIO_SECRET_KEY" \
   --network frappebench00_default \
   minio/minio server /data
+echo -e "\n"
 
-echo "Create bucket named erpnext after 3 sec."
+echo -e "\e[1m\e[4mCreate bucket named erpnext\e[0m"
-sleep 3
 docker run \
     --network frappebench00_default \
     vltgroup/s3cmd:latest s3cmd --access_key=$MINIO_ACCESS_KEY \
@@ -121,8 +142,9 @@ docker run \
     --host=minio:9000 \
     --host-bucket=minio:9000 \
     mb s3://erpnext
+echo -e "\n"
 
-echo "Push backup to MinIO s3"
+echo -e "\e[1m\e[4mPush backup to MinIO s3\e[0m"
 docker run \
     -e BUCKET_NAME=erpnext \
     -e REGION=us-east-1 \
@@ -133,19 +155,22 @@ docker run \
     -v frappebench00_sites-vol:/home/frappe/frappe-bench/sites \
     --network frappebench00_default \
     frappe/erpnext-worker:edge push-backup
+echo -e "\n"
 
-echo "Stop Services"
+echo -e "\e[1m\e[4mStop Services\e[0m"
 docker-compose \
     --project-name frappebench00 \
     -f installation/docker-compose-common.yml \
     -f installation/docker-compose-erpnext.yml \
     -f installation/erpnext-publish.yml \
     stop
+echo -e "\n"
 
-echo "Prune Containers"
+echo -e "\e[1m\e[4mPrune Containers\e[0m"
 docker container prune -f && docker volume prune -f
+echo -e "\n"
 
-echo "Start Services"
+echo -e "\e[1m\e[4mStart Services\e[0m"
 docker-compose \
     --project-name frappebench00 \
     -f installation/docker-compose-common.yml \
@@ -154,8 +179,9 @@ docker-compose \
     up -d
 
 loopHealthCheck
+echo -e "\n"
 
-echo "Restore backup from MinIO / S3"
+echo -e "\e[1m\e[4mRestore backup from MinIO / S3\e[0m"
 docker run \
     -e MYSQL_ROOT_PASSWORD=admin \
     -e BUCKET_NAME=erpnext \
@@ -167,15 +193,16 @@ docker run \
     -v frappebench00_sites-vol:/home/frappe/frappe-bench/sites \
     --network frappebench00_default \
     frappe/erpnext-worker:edge restore-backup
+echo -e "\n"
 
-echo "Check Restored Site"
+echo -e "\e[1m\e[4mCheck Restored Site\e[0m"
 sleep 3
-RESTORE_STATUS=$(curl -S http://test.localhost/api/method/version || echo "")
+RESTORE_STATUS=$(curl -sS http://test.localhost/api/method/version || echo "")
 INCREMENT=0
 while [[ -z "$RESTORE_STATUS" && $INCREMENT -lt 60 ]]; do
     sleep 1
     echo "Wait for restoration to complete ..."
-    RESTORE_STATUS=$(curl -S http://test.localhost/api/method/version || echo "")
+    RESTORE_STATUS=$(curl -sS http://test.localhost/api/method/version || echo "")
     ((INCREMENT=INCREMENT+1))
     if [[ -z "$RESTORE_STATUS" && $INCREMENT -eq 60 ]]; then
         CONTAINER_ID=$(docker-compose \
@@ -189,10 +216,54 @@ while [[ -z "$RESTORE_STATUS" && $INCREMENT -lt 60 ]]; do
     fi
 done
 
-echo "Ping restored site"
+echo -e "\e[1m\e[4mPing restored site\e[0m"
 echo $RESTORE_STATUS
+echo -e "\n"
 
-echo "Migrate command in edge container"
+echo -e "\e[1m\e[4mCheck Restored Site Index Page\e[0m"
+curl -s http://test.localhost | w3m -T text/html -dump
+echo -e "\n"
+
+echo -e "\e[1m\e[4mCreate new site (edge)\e[0m"
+docker run -it \
+    -e "SITE_NAME=edge.localhost" \
+    -e "INSTALL_APPS=erpnext" \
+    -v frappebench00_sites-vol:/home/frappe/frappe-bench/sites \
+    --network frappebench00_default \
+    frappe/erpnext-worker:edge new
+echo -e "\n"
+
+echo -e "\e[1m\e[4mCheck New Edge Site\e[0m"
+sleep 3
+RESTORE_STATUS=$(curl -sS http://edge.localhost/api/method/version || echo "")
+INCREMENT=0
+while [[ -z "$RESTORE_STATUS" && $INCREMENT -lt 60 ]]; do
+    sleep 1
+    echo -e "\e[1m\e[4mWait for restoration to complete ..."
+    RESTORE_STATUS=$(curl -sS http://edge.localhost/api/method/version || echo "")
+    ((INCREMENT=INCREMENT+1))
+    if [[ -z "$RESTORE_STATUS" && $INCREMENT -eq 60 ]]; then
+        CONTAINER_ID=$(docker-compose \
+            --project-name frappebench00 \
+            -f installation/docker-compose-common.yml \
+            -f installation/docker-compose-erpnext.yml \
+            -f installation/erpnext-publish.yml \
+            ps -q erpnext-python)
+        docker logs $CONTAINER_ID
+        exit 1
+    fi
+done
+echo -e "\n"
+
+echo -e "\e[1m\e[4mPing new edge site\e[0m"
+echo $RESTORE_STATUS
+echo -e "\n"
+
+echo -e "\e[1m\e[4mCheck New Edge Index Page\e[0m"
+curl -s http://edge.localhost | w3m -T text/html -dump
+echo -e "\n"
+
+echo -e "\e[1m\e[4mMigrate command in edge container\e[0m"
 docker run -it \
     -e "MAINTENANCE_MODE=1" \
     -v frappebench00_sites-vol:/home/frappe/frappe-bench/sites \
@@ -200,16 +271,53 @@ docker run -it \
     frappe/erpnext-worker:edge migrate
 
 checkMigrationComplete
+echo -e "\n"
 
-echo "Create new site (edge)"
+echo -e "\e[1m\e[4mRestore backup from MinIO / S3 (Overwrite)\e[0m"
-docker run -it \
+docker run \
-    -e "SITE_NAME=edge.localhost" \
+    -e MYSQL_ROOT_PASSWORD=admin \
-    -e "INSTALL_APPS=erpnext" \
+    -e BUCKET_NAME=erpnext \
+    -e BUCKET_DIR=local \
+    -e ACCESS_KEY_ID=$MINIO_ACCESS_KEY \
+    -e SECRET_ACCESS_KEY=$MINIO_SECRET_KEY \
+    -e ENDPOINT_URL=http://minio:9000 \
+    -e REGION=us-east-1 \
     -v frappebench00_sites-vol:/home/frappe/frappe-bench/sites \
     --network frappebench00_default \
-    frappe/erpnext-worker:edge new
+    frappe/erpnext-worker:edge restore-backup
+echo -e "\n"
 
-echo "Check console command for site test.localhost"
+echo -e "\e[1m\e[4mCheck Overwritten Site\e[0m"
+sleep 3
+RESTORE_STATUS=$(curl -sS http://test.localhost/api/method/version || echo "")
+INCREMENT=0
+while [[ -z "$RESTORE_STATUS" && $INCREMENT -lt 60 ]]; do
+    sleep 1
+    echo -e "\e[1m\e[4mWait for restoration to complete ..."
+    RESTORE_STATUS=$(curl -sS http://test.localhost/api/method/version || echo "")
+    ((INCREMENT=INCREMENT+1))
+    if [[ -z "$RESTORE_STATUS" && $INCREMENT -eq 60 ]]; then
+        CONTAINER_ID=$(docker-compose \
+            --project-name frappebench00 \
+            -f installation/docker-compose-common.yml \
+            -f installation/docker-compose-erpnext.yml \
+            -f installation/erpnext-publish.yml \
+            ps -q erpnext-python)
+        docker logs $CONTAINER_ID
+        exit 1
+    fi
+done
+echo -e "\n"
+
+echo -e "\e[1m\e[4mPing overwritten site\e[0m"
+echo $RESTORE_STATUS
+echo -e "\n"
+
+echo -e "\e[1m\e[4mCheck Overwritten Index Page\e[0m"
+curl -s http://test.localhost | w3m -T text/html -dump
+echo -e "\n"
+
+echo -e "\e[1m\e[4mCheck console command for site test.localhost\e[0m"
 docker run \
     -v frappebench00_sites-vol:/home/frappe/frappe-bench/sites \
     --network frappebench00_default \