From 4f12e61a7a04bd202d97bc7113c073e604c713f0 Mon Sep 17 00:00:00 2001 From: Revant Nandgaonkar Date: Fri, 1 May 2020 16:54:52 +0530 Subject: [PATCH 1/2] test: all worker commands --- .travis.yml | 11 +++ tests/docker-test.sh | 216 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 227 insertions(+) create mode 100755 tests/docker-test.sh diff --git a/.travis.yml b/.travis.yml index 73f9a74b..150acdc3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -110,3 +110,14 @@ jobs: - ./travis.py erpnext --nginx --git-version 11 - ./travis.py erpnext --nginx --tag v11 --tag-only - ./travis.py erpnext --nginx --tag version-11 --tag-only + - stage: "Build and test develop branch" + if: type = pull_request OR branch = develop + before_install: + - sudo apt-get update && sudo apt-get -y install docker-compose + script: + - docker build -t frappe/frappe-socketio:edge -f build/frappe-socketio/Dockerfile . + - docker build -t frappe/frappe-worker:develop -f build/frappe-worker/Dockerfile . + - docker build -t frappe/erpnext-worker:edge -f build/erpnext-worker/Dockerfile . + - docker build -t frappe/frappe-nginx:develop -f build/frappe-nginx/Dockerfile . + - docker build -t frappe/erpnext-nginx:edge -f build/erpnext-nginx/Dockerfile . + - ./tests/docker-test.sh diff --git a/tests/docker-test.sh b/tests/docker-test.sh new file mode 100755 index 00000000..dded6bb7 --- /dev/null +++ b/tests/docker-test.sh @@ -0,0 +1,216 @@ +#!/bin/bash + +function checkMigrationComplete() { + echo "Check Auto Migration" + CONTAINER_ID=$(docker-compose \ + --project-name frappebench00 \ + -f installation/docker-compose-common.yml \ + -f installation/docker-compose-erpnext.yml \ + -f installation/erpnext-publish.yml \ + ps -q erpnext-python) + + DOCKER_LOG=$(docker logs $CONTAINER_ID 2>&1 | grep "Starting gunicorn") + INCREMENT=0 + while [[ $DOCKER_LOG != *"Starting gunicorn"* && $INCREMENT -lt 60 ]]; do + sleep 3 + echo "Wait for migration to complete ..." + ((INCREMENT=INCREMENT+1)) + DOCKER_LOG=$(docker logs $CONTAINER_ID 2>&1 | grep "Starting gunicorn") + if [[ $DOCKER_LOG != *"Starting gunicorn"* && $INCREMENT -eq 60 ]]; then + docker logs $CONTAINER_ID + exit 1 + fi + done +} + +function loopHealthCheck() { + echo "Create Container to Check MariaDB" + docker run --name frappe_doctor \ + -v frappebench00_sites-vol:/home/frappe/frappe-bench/sites \ + --network frappebench00_default \ + frappe/erpnext-worker:edge doctor || true + + echo "Loop Health Check" + FRAPPE_LOG=$(docker logs frappe_doctor | grep "Health check successful" || echo "") + while [[ -z "$FRAPPE_LOG" ]]; do + sleep 1 + CONTAINER=$(docker start frappe_doctor) + echo "Restarting $CONTAINER ..." + FRAPPE_LOG=$(docker logs frappe_doctor | grep "Health check successful" || echo "") + done + echo "Health check successful" +} + +echo "Copy env-example file" +cp env-example .env + +echo "Set version to v12" +sed -i -e "s/edge/v12/g" .env + +echo "Start Services" +docker-compose \ + --project-name frappebench00 \ + -f installation/docker-compose-common.yml \ + -f installation/docker-compose-erpnext.yml \ + -f installation/erpnext-publish.yml \ + up -d + +loopHealthCheck + +echo "Create new site (v12)" +docker run -it \ + -e "SITE_NAME=test.localhost" \ + -e "INSTALL_APPS=erpnext" \ + -v frappebench00_sites-vol:/home/frappe/frappe-bench/sites \ + --network frappebench00_default \ + frappe/erpnext-worker:v12 new + +echo "Ping created site" +curl -S http://test.localhost/api/method/version +echo "" + +echo "Set version to edge" +sed -i -e "s/v12/edge/g" .env + +echo "Restart containers with edge image" +docker-compose \ + --project-name frappebench00 \ + -f installation/docker-compose-common.yml \ + -f installation/docker-compose-erpnext.yml \ + -f installation/erpnext-publish.yml \ + stop +docker-compose \ + --project-name frappebench00 \ + -f installation/docker-compose-common.yml \ + -f installation/docker-compose-erpnext.yml \ + -f installation/erpnext-publish.yml \ + up -d + +checkMigrationComplete + +echo "Ping migrated site" +sleep 3 +curl -S http://test.localhost/api/method/version +echo "" + +echo "Backup site" +docker run -it \ + -e "WITH_FILES=1" \ + -v frappebench00_sites-vol:/home/frappe/frappe-bench/sites \ + --network frappebench00_default \ + frappe/erpnext-worker:edge backup + +export MINIO_ACCESS_KEY="AKIAIOSFODNN7EXAMPLE" +export MINIO_SECRET_KEY="wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" + +echo "Start MinIO container for s3 compatible storage" +docker run -d --name minio \ + -e "MINIO_ACCESS_KEY=$MINIO_ACCESS_KEY" \ + -e "MINIO_SECRET_KEY=$MINIO_SECRET_KEY" \ + --network frappebench00_default \ + minio/minio server /data + +echo "Create bucket named erpnext after 3 sec." +sleep 3 +docker run \ + --network frappebench00_default \ + vltgroup/s3cmd:latest s3cmd --access_key=$MINIO_ACCESS_KEY \ + --secret_key=$MINIO_SECRET_KEY \ + --region=us-east-1 \ + --no-ssl \ + --host=minio:9000 \ + --host-bucket=minio:9000 \ + mb s3://erpnext + +echo "Push backup to MinIO s3" +docker run \ + -e BUCKET_NAME=erpnext \ + -e REGION=us-east-1 \ + -e BUCKET_DIR=local \ + -e ACCESS_KEY_ID=$MINIO_ACCESS_KEY \ + -e SECRET_ACCESS_KEY=$MINIO_SECRET_KEY \ + -e ENDPOINT_URL=http://minio:9000 \ + -v frappebench00_sites-vol:/home/frappe/frappe-bench/sites \ + --network frappebench00_default \ + frappe/erpnext-worker:edge push-backup + +echo "Stop Services" +docker-compose \ + --project-name frappebench00 \ + -f installation/docker-compose-common.yml \ + -f installation/docker-compose-erpnext.yml \ + -f installation/erpnext-publish.yml \ + stop + +echo "Prune Containers" +docker container prune -f && docker volume prune -f + +echo "Start Services" +docker-compose \ + --project-name frappebench00 \ + -f installation/docker-compose-common.yml \ + -f installation/docker-compose-erpnext.yml \ + -f installation/erpnext-publish.yml \ + up -d + +loopHealthCheck + +echo "Restore backup from MinIO / S3" +docker run \ + -e MYSQL_ROOT_PASSWORD=admin \ + -e BUCKET_NAME=erpnext \ + -e BUCKET_DIR=local \ + -e ACCESS_KEY_ID=$MINIO_ACCESS_KEY \ + -e SECRET_ACCESS_KEY=$MINIO_SECRET_KEY \ + -e ENDPOINT_URL=http://minio:9000 \ + -e REGION=us-east-1 \ + -v frappebench00_sites-vol:/home/frappe/frappe-bench/sites \ + --network frappebench00_default \ + frappe/erpnext-worker:edge restore-backup + +echo "Check Restored Site" +sleep 3 +RESTORE_STATUS=$(curl -S http://test.localhost/api/method/version || echo "") +INCREMENT=0 +while [[ -z "$RESTORE_STATUS" && $INCREMENT -lt 60 ]]; do + sleep 1 + echo "Wait for restoration to complete ..." + RESTORE_STATUS=$(curl -S http://test.localhost/api/method/version || echo "") + ((INCREMENT=INCREMENT+1)) + if [[ -z "$RESTORE_STATUS" && $INCREMENT -eq 60 ]]; then + CONTAINER_ID=$(docker-compose \ + --project-name frappebench00 \ + -f installation/docker-compose-common.yml \ + -f installation/docker-compose-erpnext.yml \ + -f installation/erpnext-publish.yml \ + ps -q erpnext-python) + docker logs $CONTAINER_ID + exit 1 + fi +done + +echo "Ping restored site" +echo $RESTORE_STATUS + +echo "Migrate command in edge container" +docker run -it \ + -e "MAINTENANCE_MODE=1" \ + -v frappebench00_sites-vol:/home/frappe/frappe-bench/sites \ + --network frappebench00_default \ + frappe/erpnext-worker:edge migrate + +checkMigrationComplete + +echo "Create new site (edge)" +docker run -it \ + -e "SITE_NAME=edge.localhost" \ + -e "INSTALL_APPS=erpnext" \ + -v frappebench00_sites-vol:/home/frappe/frappe-bench/sites \ + --network frappebench00_default \ + frappe/erpnext-worker:edge new + +echo "Check console command for site test.localhost" +docker run \ + -v frappebench00_sites-vol:/home/frappe/frappe-bench/sites \ + --network frappebench00_default \ + frappe/erpnext-worker:edge console test.localhost From da4f65438dba426e5c59133413c690c1028afbab Mon Sep 17 00:00:00 2001 From: Revant Nandgaonkar Date: Fri, 1 May 2020 19:42:19 +0530 Subject: [PATCH 2/2] test: separate builds for develop branch and PR Develop branch jobs pull edge images and test PR jobs build images and test --- .travis.yml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 150acdc3..355180a7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -110,8 +110,8 @@ jobs: - ./travis.py erpnext --nginx --git-version 11 - ./travis.py erpnext --nginx --tag v11 --tag-only - ./travis.py erpnext --nginx --tag version-11 --tag-only - - stage: "Build and test develop branch" - if: type = pull_request OR branch = develop + - stage: "Build and test edge images" + if: type = pull_request before_install: - sudo apt-get update && sudo apt-get -y install docker-compose script: @@ -121,3 +121,9 @@ jobs: - docker build -t frappe/frappe-nginx:develop -f build/frappe-nginx/Dockerfile . - docker build -t frappe/erpnext-nginx:edge -f build/erpnext-nginx/Dockerfile . - ./tests/docker-test.sh + - stage: "Pull and test edge images" + if: branch = develop AND type != pull_request + before_install: + - sudo apt-get update && sudo apt-get -y install docker-compose + script: + - ./tests/docker-test.sh