fix: Improve CI, fix builds (#554)

* ci(Test): Add `workflow_dispatch` trigger

* ci(Test): Build and run on push and pr

* test: Set failfast, remove ERPNext reference

* test: Reveal .env file for logs

* ci(Test): Rename step

* ci(Test): Fix git tag

* test: Add -x flag for debugging

* test: Fix failing test

* ci(Test): Check if built images are used

* ci(Test): Configure trigger paths, don't build on schedule

* test: Remove bug

* ci: Build and test stable images on PR

* fix(frappe-nginx): Get back to `ls` from `find`

* check if test passes with erpnext images

* fix(frappe-worker): Default sites in backup command

* get integration test changes back

* chore: Fix linting

.github/workflows/build_stable.yml

@@ -1,6 +1,18 @@
 name: Build Stable
 
 on:
+  pull_request:
+    branches:
+      - main
+    paths:
+      - .github/**
+      - build/**
+      - installation/**
+      - tests/**
+      - .dockerignore
+      - docker-bake.hcl
+      - docker-compose.yml
+      - env*
   push:
     branches:
       - main
@@ -19,6 +31,9 @@ on:
 
   workflow_dispatch:
 
+env:
+  IS_AUTHORIZED_RUN: ${{ github.repository == 'frappe/frappe_docker' && github.event_name != 'pull_request' }}
+
 jobs:
   build_frappe:
     name: Frappe
@@ -36,7 +51,7 @@ jobs:
 
       - name: Login
         uses: docker/login-action@v1
-        if: github.repository == 'frappe/frappe_docker'
+        if: env.IS_AUTHORIZED_RUN == 'true'
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
@@ -57,7 +72,7 @@ jobs:
         run: ./tests/test-frappe.sh
 
       - name: Push
-        if: github.repository == 'frappe/frappe_docker'
+        if: env.IS_AUTHORIZED_RUN == 'true'
         uses: docker/bake-action@v1.6.0
         with:
           files: docker-bake.hcl
@@ -83,7 +98,7 @@ jobs:
 
       - name: Login
         uses: docker/login-action@v1
-        if: github.repository == 'frappe/frappe_docker'
+        if: env.IS_AUTHORIZED_RUN == 'true'
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
@@ -104,7 +119,7 @@ jobs:
         run: ./tests/test-erpnext.sh
 
       - name: Push
-        if: github.repository == 'frappe/frappe_docker'
+        if: env.IS_AUTHORIZED_RUN == 'true'
         uses: docker/bake-action@v1.6.0
         with:
           files: docker-bake.hcl
@@ -116,7 +131,7 @@ jobs:
   release_helm:
     name: Release Helm
     runs-on: ubuntu-latest
-    if: github.repository == 'frappe/frappe_docker'
+    if: github.repository == 'frappe/frappe_docker' && github.event_name != 'pull_request'
     needs: [build_frappe, build_erpnext]
 
     steps:

.github/workflows/test.yml

@@ -1,9 +1,38 @@
-name: Test
+name: Integration Test
 
 on:
+  push:
+    branches:
+      - main
+    paths:
+      - .github/**
+      - build/**
+      - installation/**
+      - tests/**
+      - .dockerignore
+      - docker-bake.hcl
+      - docker-compose.yml
+      - env*
+
+  pull_request:
+    branches:
+      - main
+    paths:
+      - .github/**
+      - build/**
+      - installation/**
+      - tests/**
+      - .dockerignore
+      - docker-bake.hcl
+      - docker-compose.yml
+      - env*
+
+  workflow_dispatch:
+
   schedule:
     # Every day at 01:00 am
     # Develop images are built at 12:00 pm, we want to use them
+    # Also, we don't build new images on this event
     - cron: 0 1 * * *
 
 jobs:
@@ -13,5 +42,21 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v2
 
+      - name: Get latest versions
+        if: github.event_name != 'schedule'
+        run: ./.github/scripts/get-latest-tags.sh
+        env:
+          VERSION: 13
+
+      - name: Build
+        if: github.event_name != 'schedule'
+        uses: docker/bake-action@v1.6.0
+        with:
+          files: docker-bake.hcl
+          targets: frappe-develop,frappe-stable
+          load: true
+        env:
+          GIT_TAG: ${{ env.FRAPPE_VERSION }}
+
       - name: Test
         run: ./tests/integration-test.sh

build/frappe-nginx/docker-entrypoint.sh

@@ -8,7 +8,8 @@ rsync -a --delete /var/www/html/assets/* /assets
 
 /rsync
 
-touch /var/www/html/sites/.build -r "$(find /assets -maxdepth 1 -name "*.*" | head -n 1)"
+# shellcheck disable=SC2012
+touch /var/www/html/sites/.build -r "$(ls -td /assets/* | head -n 1)"
 
 [[ -z "${FRAPPE_PY}" ]] && FRAPPE_PY='0.0.0.0'
 

build/frappe-worker/commands/backup.py

@@ -0,0 +1,39 @@
+import os
+import frappe
+from frappe.utils.backups import scheduled_backup
+from frappe.utils import cint, get_sites, now
+
+
+def backup(sites, with_files=False):
+    for site in sites:
+        frappe.init(site)
+        frappe.connect()
+        odb = scheduled_backup(
+            ignore_files=not with_files,
+            backup_path_db=None,
+            backup_path_files=None,
+            backup_path_private_files=None,
+            force=True
+        )
+        print("database backup taken -", odb.backup_path_db, "- on", now())
+        if with_files:
+            print("files backup taken -", odb.backup_path_files, "- on", now())
+            print("private files backup taken -", odb.backup_path_private_files, "- on", now())
+        frappe.destroy()
+
+
+def main():
+    installed_sites = ":".join(get_sites())
+    sites = os.environ.get("SITES", installed_sites).split(":")
+    with_files = cint(os.environ.get("WITH_FILES"))
+
+    backup(sites, with_files)
+
+    if frappe.redis_server:
+        frappe.redis_server.connection_pool.disconnect()
+
+    exit(0)
+
+
+if __name__ == "__main__":
+    main()

build/frappe-worker/docker-entrypoint.sh

@@ -159,13 +159,8 @@ doctor)
   ;;
 
 backup)
-  if [[ -n $WITH_FILES ]]; then
+  /home/frappe/frappe-bench/env/bin/python /home/frappe/frappe-bench/commands/backup.py
-    WITH_FILES=--with-files
+  exit
-  fi
-
-  for site in ${SITES//:/ }; do
-    bench --site "$site" backup $WITH_FILES
-  done
   ;;
 
 console)

tests/integration-test.sh

@@ -1,5 +1,7 @@
 #!/bin/bash
 
+set -e
+
 source tests/functions.sh
 
 project_name=frappe_bench_00
@@ -9,25 +11,25 @@ docker_compose_with_args() {
     docker-compose \
         -p $project_name \
         -f installation/docker-compose-common.yml \
-        -f installation/docker-compose-erpnext.yml \
+        -f installation/docker-compose-frappe.yml \
-        -f installation/erpnext-publish.yml \
+        -f installation/frappe-publish.yml \
         $@
 }
 
 check_migration_complete() {
     print_group Check migration
 
-    container_id=$(docker_compose_with_args ps -q erpnext-python)
+    container_id=$(docker_compose_with_args ps -q frappe-python)
-    thelogs=$(docker logs "${container_id}" 2>&1 | grep "Starting gunicorn")
+    cmd="docker logs ${container_id} 2>&1 | grep 'Starting gunicorn' || echo ''"
+    worker_log=$(eval "$cmd")
     INCREMENT=0
 
-    while [[ ${thelogs} != *"Starting gunicorn"* && ${INCREMENT} -lt 120 ]]; do
+    while [[ ${worker_log} != *"Starting gunicorn"* && ${INCREMENT} -lt 120 ]]; do
         sleep 3
         ((INCREMENT = INCREMENT + 1))
         echo "Wait for migration to complete..."
-        thelogs=$(docker logs "${container_id}" 2>&1 | grep "Starting gunicorn")
+        worker_log=$(eval "$cmd")
-
+        if [[ ${worker_log} != *"Starting gunicorn"* && ${INCREMENT} -eq 120 ]]; then
-        if [[ ${thelogs} != *"Starting gunicorn"* && ${INCREMENT} -eq 120 ]]; then
             echo Migration timeout
             docker logs "${container_id}"
             exit 1
@@ -44,7 +46,7 @@ check_health() {
     docker run --name frappe_doctor \
         -v "${project_name}_sites-vol:/home/frappe/frappe-bench/sites" \
         --network "${project_name}_default" \
-        frappe/erpnext-worker:edge doctor || true
+        frappe/frappe-worker:edge doctor || true
 
     cmd='docker logs frappe_doctor | grep "Health check successful" || echo ""'
     doctor_log=$(eval "$cmd")
@@ -68,6 +70,7 @@ check_health() {
 echo ::group::Setup .env
 cp env-example .env
 sed -i -e "s/edge/v13/g" .env
+cat .env
 # shellcheck disable=SC2046
 export $(cat .env)
 
@@ -92,15 +95,15 @@ SITE_NAME=test.localhost
 docker run \
     --rm \
     -e SITE_NAME=$SITE_NAME \
-    -e INSTALL_APPS=erpnext \
     -v ${project_name}_sites-vol:/home/frappe/frappe-bench/sites \
     --network ${project_name}_default \
-    frappe/erpnext-worker:v13 new
+    frappe/frappe-worker:v13 new
 
 ping_site
 
 print_group "Update .env (v13 -> edge)"
 sed -i -e "s/v13/edge/g" .env
+cat .env
 # shellcheck disable=SC2046
 export $(cat .env)
 
@@ -122,7 +125,7 @@ docker run \
     -e POSTGRES_PASSWORD=admin \
     -v ${project_name}_sites-vol:/home/frappe/frappe-bench/sites \
     --network ${project_name}_default \
-    frappe/erpnext-worker:edge new
+    frappe/frappe-worker:edge new
 
 check_migration_complete
 SITE_NAME=$PG_SITE_NAME ping_site
@@ -133,7 +136,7 @@ docker run \
     -e WITH_FILES=1 \
     -v ${project_name}_sites-vol:/home/frappe/frappe-bench/sites \
     --network ${project_name}_default \
-    frappe/erpnext-worker:edge backup
+    frappe/frappe-worker:edge backup
 
 MINIO_ACCESS_KEY="AKIAIOSFODNN7EXAMPLE"
 MINIO_SECRET_KEY="wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"
@@ -160,12 +163,12 @@ docker run \
     --no-ssl \
     --host=minio:9000 \
     --host-bucket=minio:9000 \
-    mb s3://erpnext
+    mb s3://frappe
 
 print_group Push backup
 docker run \
     --rm \
-    -e BUCKET_NAME=erpnext \
+    -e BUCKET_NAME=frappe \
     -e REGION=us-east-1 \
     -e BUCKET_DIR=local \
     -e ACCESS_KEY_ID=$MINIO_ACCESS_KEY \
@@ -173,7 +176,7 @@ docker run \
     -e ENDPOINT_URL=http://minio:9000 \
     -v ${project_name}_sites-vol:/home/frappe/frappe-bench/sites \
     --network ${project_name}_default \
-    frappe/erpnext-worker:edge push-backup
+    frappe/frappe-worker:edge push-backup
 
 print_group Prune and restart services
 docker_compose_with_args stop
@@ -186,7 +189,7 @@ print_group Restore backup from S3
 docker run \
     --rm \
     -e MYSQL_ROOT_PASSWORD=admin \
-    -e BUCKET_NAME=erpnext \
+    -e BUCKET_NAME=frappe \
     -e BUCKET_DIR=local \
     -e ACCESS_KEY_ID=$MINIO_ACCESS_KEY \
     -e SECRET_ACCESS_KEY=$MINIO_SECRET_KEY \
@@ -194,7 +197,7 @@ docker run \
     -e REGION=us-east-1 \
     -v ${project_name}_sites-vol:/home/frappe/frappe-bench/sites \
     --network ${project_name}_default \
-    frappe/erpnext-worker:edge restore-backup
+    frappe/frappe-worker:edge restore-backup
 
 check_health
 ping_site
@@ -205,10 +208,10 @@ print_group "Create new site (edge)"
 docker run \
     --rm \
     -e SITE_NAME=$EDGE_SITE_NAME \
-    -e INSTALL_APPS=erpnext \
+    -e INSTALL_APPS=frappe \
     -v ${project_name}_sites-vol:/home/frappe/frappe-bench/sites \
     --network ${project_name}_default \
-    frappe/erpnext-worker:edge new
+    frappe/frappe-worker:edge new
 
 check_health
 SITE_NAME=$EDGE_SITE_NAME ping_site
@@ -220,7 +223,7 @@ docker run \
     -v ${project_name}_sites-vol:/home/frappe/frappe-bench/sites \
     -v ${project_name}_assets-vol:/home/frappe/frappe-bench/sites/assets \
     --network ${project_name}_default \
-    frappe/erpnext-worker:edge migrate
+    frappe/frappe-worker:edge migrate
 
 check_migration_complete
 
@@ -228,7 +231,7 @@ print_group "Restore backup S3 (overwrite)"
 docker run \
     --rm \
     -e MYSQL_ROOT_PASSWORD=admin \
-    -e BUCKET_NAME=erpnext \
+    -e BUCKET_NAME=frappe \
     -e BUCKET_DIR=local \
     -e ACCESS_KEY_ID=$MINIO_ACCESS_KEY \
     -e SECRET_ACCESS_KEY=$MINIO_SECRET_KEY \
@@ -236,7 +239,7 @@ docker run \
     -e REGION=us-east-1 \
     -v ${project_name}_sites-vol:/home/frappe/frappe-bench/sites \
     --network ${project_name}_default \
-    frappe/erpnext-worker:edge restore-backup
+    frappe/frappe-worker:edge restore-backup
 
 check_migration_complete
 ping_site
@@ -246,14 +249,14 @@ docker run \
     --rm \
     -v ${project_name}_sites-vol:/home/frappe/frappe-bench/sites \
     --network ${project_name}_default \
-    frappe/erpnext-worker:edge console $SITE_NAME
+    frappe/frappe-worker:edge console $SITE_NAME
 
 print_group "Check console for $PG_SITE_NAME"
 docker run \
     --rm \
     -v ${project_name}_sites-vol:/home/frappe/frappe-bench/sites \
     --network ${project_name}_default \
-    frappe/erpnext-worker:edge console $PG_SITE_NAME
+    frappe/frappe-worker:edge console $PG_SITE_NAME
 
 print_group "Check drop site for $SITE_NAME (MariaDB)"
 docker run \
@@ -261,7 +264,7 @@ docker run \
     -e SITE_NAME=$SITE_NAME \
     -v ${project_name}_sites-vol:/home/frappe/frappe-bench/sites \
     --network ${project_name}_default \
-    frappe/erpnext-worker:edge drop
+    frappe/frappe-worker:edge drop
 
 print_group "Check drop site for $PG_SITE_NAME (Postgres)"
 docker run \
@@ -269,7 +272,7 @@ docker run \
     -e SITE_NAME=$PG_SITE_NAME \
     -v ${project_name}_sites-vol:/home/frappe/frappe-bench/sites \
     --network ${project_name}_default \
-    frappe/erpnext-worker:edge drop
+    frappe/frappe-worker:edge drop
 
 print_group Check bench --help
 docker run \
@@ -277,4 +280,4 @@ docker run \
     -v ${project_name}_sites-vol:/home/frappe/frappe-bench/sites \
     --network ${project_name}_default \
     --user frappe \
-    frappe/erpnext-worker:edge bench --help
+    frappe/frappe-worker:edge bench --help