diff --git a/.github/workflows/build_stable.yml b/.github/workflows/build_stable.yml index e933b29b..07e5fef3 100644 --- a/.github/workflows/build_stable.yml +++ b/.github/workflows/build_stable.yml @@ -1,6 +1,18 @@ name: Build Stable on: + pull_request: + branches: + - main + paths: + - .github/** + - build/** + - installation/** + - tests/** + - .dockerignore + - docker-bake.hcl + - docker-compose.yml + - env* push: branches: - main @@ -19,6 +31,9 @@ on: workflow_dispatch: +env: + IS_AUTHORIZED_RUN: ${{ github.repository == 'frappe/frappe_docker' && github.event_name != 'pull_request' }} + jobs: build_frappe: name: Frappe @@ -36,7 +51,7 @@ jobs: - name: Login uses: docker/login-action@v1 - if: github.repository == 'frappe/frappe_docker' + if: env.IS_AUTHORIZED_RUN == 'true' with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} @@ -57,7 +72,7 @@ jobs: run: ./tests/test-frappe.sh - name: Push - if: github.repository == 'frappe/frappe_docker' + if: env.IS_AUTHORIZED_RUN == 'true' uses: docker/bake-action@v1.6.0 with: files: docker-bake.hcl @@ -83,7 +98,7 @@ jobs: - name: Login uses: docker/login-action@v1 - if: github.repository == 'frappe/frappe_docker' + if: env.IS_AUTHORIZED_RUN == 'true' with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} @@ -104,7 +119,7 @@ jobs: run: ./tests/test-erpnext.sh - name: Push - if: github.repository == 'frappe/frappe_docker' + if: env.IS_AUTHORIZED_RUN == 'true' uses: docker/bake-action@v1.6.0 with: files: docker-bake.hcl @@ -116,7 +131,7 @@ jobs: release_helm: name: Release Helm runs-on: ubuntu-latest - if: github.repository == 'frappe/frappe_docker' + if: github.repository == 'frappe/frappe_docker' && github.event_name != 'pull_request' needs: [build_frappe, build_erpnext] steps: diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index e3c7c1a7..376c9bda 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,9 +1,38 @@ -name: Test +name: Integration Test on: + push: + branches: + - main + paths: + - .github/** + - build/** + - installation/** + - tests/** + - .dockerignore + - docker-bake.hcl + - docker-compose.yml + - env* + + pull_request: + branches: + - main + paths: + - .github/** + - build/** + - installation/** + - tests/** + - .dockerignore + - docker-bake.hcl + - docker-compose.yml + - env* + + workflow_dispatch: + schedule: # Every day at 01:00 am # Develop images are built at 12:00 pm, we want to use them + # Also, we don't build new images on this event - cron: 0 1 * * * jobs: @@ -13,5 +42,21 @@ jobs: - name: Checkout uses: actions/checkout@v2 + - name: Get latest versions + if: github.event_name != 'schedule' + run: ./.github/scripts/get-latest-tags.sh + env: + VERSION: 13 + + - name: Build + if: github.event_name != 'schedule' + uses: docker/bake-action@v1.6.0 + with: + files: docker-bake.hcl + targets: frappe-develop,frappe-stable + load: true + env: + GIT_TAG: ${{ env.FRAPPE_VERSION }} + - name: Test run: ./tests/integration-test.sh diff --git a/build/frappe-nginx/docker-entrypoint.sh b/build/frappe-nginx/docker-entrypoint.sh index 94277fc3..0be9232e 100755 --- a/build/frappe-nginx/docker-entrypoint.sh +++ b/build/frappe-nginx/docker-entrypoint.sh @@ -8,7 +8,8 @@ rsync -a --delete /var/www/html/assets/* /assets /rsync -touch /var/www/html/sites/.build -r "$(find /assets -maxdepth 1 -name "*.*" | head -n 1)" +# shellcheck disable=SC2012 +touch /var/www/html/sites/.build -r "$(ls -td /assets/* | head -n 1)" [[ -z "${FRAPPE_PY}" ]] && FRAPPE_PY='0.0.0.0' diff --git a/build/frappe-worker/commands/backup.py b/build/frappe-worker/commands/backup.py new file mode 100644 index 00000000..b6aa60a2 --- /dev/null +++ b/build/frappe-worker/commands/backup.py @@ -0,0 +1,39 @@ +import os +import frappe +from frappe.utils.backups import scheduled_backup +from frappe.utils import cint, get_sites, now + + +def backup(sites, with_files=False): + for site in sites: + frappe.init(site) + frappe.connect() + odb = scheduled_backup( + ignore_files=not with_files, + backup_path_db=None, + backup_path_files=None, + backup_path_private_files=None, + force=True + ) + print("database backup taken -", odb.backup_path_db, "- on", now()) + if with_files: + print("files backup taken -", odb.backup_path_files, "- on", now()) + print("private files backup taken -", odb.backup_path_private_files, "- on", now()) + frappe.destroy() + + +def main(): + installed_sites = ":".join(get_sites()) + sites = os.environ.get("SITES", installed_sites).split(":") + with_files = cint(os.environ.get("WITH_FILES")) + + backup(sites, with_files) + + if frappe.redis_server: + frappe.redis_server.connection_pool.disconnect() + + exit(0) + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/build/frappe-worker/docker-entrypoint.sh b/build/frappe-worker/docker-entrypoint.sh index 5065d2a9..6695ea7b 100755 --- a/build/frappe-worker/docker-entrypoint.sh +++ b/build/frappe-worker/docker-entrypoint.sh @@ -159,13 +159,8 @@ doctor) ;; backup) - if [[ -n $WITH_FILES ]]; then - WITH_FILES=--with-files - fi - - for site in ${SITES//:/ }; do - bench --site "$site" backup $WITH_FILES - done + /home/frappe/frappe-bench/env/bin/python /home/frappe/frappe-bench/commands/backup.py + exit ;; console) diff --git a/tests/integration-test.sh b/tests/integration-test.sh index c9a609ba..af7a8831 100755 --- a/tests/integration-test.sh +++ b/tests/integration-test.sh @@ -1,5 +1,7 @@ #!/bin/bash +set -e + source tests/functions.sh project_name=frappe_bench_00 @@ -9,25 +11,25 @@ docker_compose_with_args() { docker-compose \ -p $project_name \ -f installation/docker-compose-common.yml \ - -f installation/docker-compose-erpnext.yml \ - -f installation/erpnext-publish.yml \ + -f installation/docker-compose-frappe.yml \ + -f installation/frappe-publish.yml \ $@ } check_migration_complete() { print_group Check migration - container_id=$(docker_compose_with_args ps -q erpnext-python) - thelogs=$(docker logs "${container_id}" 2>&1 | grep "Starting gunicorn") + container_id=$(docker_compose_with_args ps -q frappe-python) + cmd="docker logs ${container_id} 2>&1 | grep 'Starting gunicorn' || echo ''" + worker_log=$(eval "$cmd") INCREMENT=0 - while [[ ${thelogs} != *"Starting gunicorn"* && ${INCREMENT} -lt 120 ]]; do + while [[ ${worker_log} != *"Starting gunicorn"* && ${INCREMENT} -lt 120 ]]; do sleep 3 ((INCREMENT = INCREMENT + 1)) echo "Wait for migration to complete..." - thelogs=$(docker logs "${container_id}" 2>&1 | grep "Starting gunicorn") - - if [[ ${thelogs} != *"Starting gunicorn"* && ${INCREMENT} -eq 120 ]]; then + worker_log=$(eval "$cmd") + if [[ ${worker_log} != *"Starting gunicorn"* && ${INCREMENT} -eq 120 ]]; then echo Migration timeout docker logs "${container_id}" exit 1 @@ -44,7 +46,7 @@ check_health() { docker run --name frappe_doctor \ -v "${project_name}_sites-vol:/home/frappe/frappe-bench/sites" \ --network "${project_name}_default" \ - frappe/erpnext-worker:edge doctor || true + frappe/frappe-worker:edge doctor || true cmd='docker logs frappe_doctor | grep "Health check successful" || echo ""' doctor_log=$(eval "$cmd") @@ -68,6 +70,7 @@ check_health() { echo ::group::Setup .env cp env-example .env sed -i -e "s/edge/v13/g" .env +cat .env # shellcheck disable=SC2046 export $(cat .env) @@ -92,15 +95,15 @@ SITE_NAME=test.localhost docker run \ --rm \ -e SITE_NAME=$SITE_NAME \ - -e INSTALL_APPS=erpnext \ -v ${project_name}_sites-vol:/home/frappe/frappe-bench/sites \ --network ${project_name}_default \ - frappe/erpnext-worker:v13 new + frappe/frappe-worker:v13 new ping_site print_group "Update .env (v13 -> edge)" sed -i -e "s/v13/edge/g" .env +cat .env # shellcheck disable=SC2046 export $(cat .env) @@ -122,7 +125,7 @@ docker run \ -e POSTGRES_PASSWORD=admin \ -v ${project_name}_sites-vol:/home/frappe/frappe-bench/sites \ --network ${project_name}_default \ - frappe/erpnext-worker:edge new + frappe/frappe-worker:edge new check_migration_complete SITE_NAME=$PG_SITE_NAME ping_site @@ -133,7 +136,7 @@ docker run \ -e WITH_FILES=1 \ -v ${project_name}_sites-vol:/home/frappe/frappe-bench/sites \ --network ${project_name}_default \ - frappe/erpnext-worker:edge backup + frappe/frappe-worker:edge backup MINIO_ACCESS_KEY="AKIAIOSFODNN7EXAMPLE" MINIO_SECRET_KEY="wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" @@ -160,12 +163,12 @@ docker run \ --no-ssl \ --host=minio:9000 \ --host-bucket=minio:9000 \ - mb s3://erpnext + mb s3://frappe print_group Push backup docker run \ --rm \ - -e BUCKET_NAME=erpnext \ + -e BUCKET_NAME=frappe \ -e REGION=us-east-1 \ -e BUCKET_DIR=local \ -e ACCESS_KEY_ID=$MINIO_ACCESS_KEY \ @@ -173,7 +176,7 @@ docker run \ -e ENDPOINT_URL=http://minio:9000 \ -v ${project_name}_sites-vol:/home/frappe/frappe-bench/sites \ --network ${project_name}_default \ - frappe/erpnext-worker:edge push-backup + frappe/frappe-worker:edge push-backup print_group Prune and restart services docker_compose_with_args stop @@ -186,7 +189,7 @@ print_group Restore backup from S3 docker run \ --rm \ -e MYSQL_ROOT_PASSWORD=admin \ - -e BUCKET_NAME=erpnext \ + -e BUCKET_NAME=frappe \ -e BUCKET_DIR=local \ -e ACCESS_KEY_ID=$MINIO_ACCESS_KEY \ -e SECRET_ACCESS_KEY=$MINIO_SECRET_KEY \ @@ -194,7 +197,7 @@ docker run \ -e REGION=us-east-1 \ -v ${project_name}_sites-vol:/home/frappe/frappe-bench/sites \ --network ${project_name}_default \ - frappe/erpnext-worker:edge restore-backup + frappe/frappe-worker:edge restore-backup check_health ping_site @@ -205,10 +208,10 @@ print_group "Create new site (edge)" docker run \ --rm \ -e SITE_NAME=$EDGE_SITE_NAME \ - -e INSTALL_APPS=erpnext \ + -e INSTALL_APPS=frappe \ -v ${project_name}_sites-vol:/home/frappe/frappe-bench/sites \ --network ${project_name}_default \ - frappe/erpnext-worker:edge new + frappe/frappe-worker:edge new check_health SITE_NAME=$EDGE_SITE_NAME ping_site @@ -220,7 +223,7 @@ docker run \ -v ${project_name}_sites-vol:/home/frappe/frappe-bench/sites \ -v ${project_name}_assets-vol:/home/frappe/frappe-bench/sites/assets \ --network ${project_name}_default \ - frappe/erpnext-worker:edge migrate + frappe/frappe-worker:edge migrate check_migration_complete @@ -228,7 +231,7 @@ print_group "Restore backup S3 (overwrite)" docker run \ --rm \ -e MYSQL_ROOT_PASSWORD=admin \ - -e BUCKET_NAME=erpnext \ + -e BUCKET_NAME=frappe \ -e BUCKET_DIR=local \ -e ACCESS_KEY_ID=$MINIO_ACCESS_KEY \ -e SECRET_ACCESS_KEY=$MINIO_SECRET_KEY \ @@ -236,7 +239,7 @@ docker run \ -e REGION=us-east-1 \ -v ${project_name}_sites-vol:/home/frappe/frappe-bench/sites \ --network ${project_name}_default \ - frappe/erpnext-worker:edge restore-backup + frappe/frappe-worker:edge restore-backup check_migration_complete ping_site @@ -246,14 +249,14 @@ docker run \ --rm \ -v ${project_name}_sites-vol:/home/frappe/frappe-bench/sites \ --network ${project_name}_default \ - frappe/erpnext-worker:edge console $SITE_NAME + frappe/frappe-worker:edge console $SITE_NAME print_group "Check console for $PG_SITE_NAME" docker run \ --rm \ -v ${project_name}_sites-vol:/home/frappe/frappe-bench/sites \ --network ${project_name}_default \ - frappe/erpnext-worker:edge console $PG_SITE_NAME + frappe/frappe-worker:edge console $PG_SITE_NAME print_group "Check drop site for $SITE_NAME (MariaDB)" docker run \ @@ -261,7 +264,7 @@ docker run \ -e SITE_NAME=$SITE_NAME \ -v ${project_name}_sites-vol:/home/frappe/frappe-bench/sites \ --network ${project_name}_default \ - frappe/erpnext-worker:edge drop + frappe/frappe-worker:edge drop print_group "Check drop site for $PG_SITE_NAME (Postgres)" docker run \ @@ -269,7 +272,7 @@ docker run \ -e SITE_NAME=$PG_SITE_NAME \ -v ${project_name}_sites-vol:/home/frappe/frappe-bench/sites \ --network ${project_name}_default \ - frappe/erpnext-worker:edge drop + frappe/frappe-worker:edge drop print_group Check bench --help docker run \ @@ -277,4 +280,4 @@ docker run \ -v ${project_name}_sites-vol:/home/frappe/frappe-bench/sites \ --network ${project_name}_default \ --user frappe \ - frappe/erpnext-worker:edge bench --help + frappe/frappe-worker:edge bench --help