diff --git a/.github/scripts/get-latest-tag.sh b/.github/scripts/get-latest-tag.sh deleted file mode 100755 index 1af75311..00000000 --- a/.github/scripts/get-latest-tag.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash - -TAGS=$(git ls-remote --refs --tags --sort='v:refname' https://github.com/$REPO "v$VERSION.*") -TAG=$(echo $TAGS | tail -n1 | sed 's/.*\///') - -echo "GIT_TAG=$TAG" >> $GITHUB_ENV -echo "GIT_BRANCH=version-$VERSION" >> $GITHUB_ENV -echo "VERSION=$VERSION" >> $GITHUB_ENV \ No newline at end of file diff --git a/.github/scripts/get-latest-tags.sh b/.github/scripts/get-latest-tags.sh new file mode 100755 index 00000000..809b2120 --- /dev/null +++ b/.github/scripts/get-latest-tags.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +set -e +set -x + +get_tag() { + tags=$(git ls-remote --refs --tags --sort='v:refname' https://github.com/$1 "v$2.*") + tag=$(echo "$tags" | tail -n1 | sed 's/.*\///') + echo "$tag" +} + +FRAPPE_VERSION=$(get_tag frappe/frappe "$VERSION") +ERPNEXT_VERSION=$(get_tag frappe/erpnext "$VERSION") + +# shellcheck disable=SC2086 +echo "FRAPPE_VERSION=$FRAPPE_VERSION" >>$GITHUB_ENV +# shellcheck disable=SC2086 +echo "ERPNEXT_VERSION=$ERPNEXT_VERSION" >>$GITHUB_ENV diff --git a/.github/scripts/install-deps-and-test.sh b/.github/scripts/install-deps-and-test.sh deleted file mode 100755 index 39d5e0f1..00000000 --- a/.github/scripts/install-deps-and-test.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash - -set -e - -sudo apt-get install -y w3m - -./tests/check-format.sh -./tests/docker-test.sh - -# This is done to not to rebuild images in the next step -git clean -fdx \ No newline at end of file diff --git a/.github/workflows/build_develop.yml b/.github/workflows/build_develop.yml index b89c1a76..d3f7c55b 100644 --- a/.github/workflows/build_develop.yml +++ b/.github/workflows/build_develop.yml @@ -14,61 +14,115 @@ on: - docker-compose.yml - env* - # Nightly builds at 12:00 am schedule: + # Every day at 12:00 pm - cron: 0 0 * * * workflow_dispatch: +env: + IS_AUTHORIZED_RUN: ${{ github.repository == 'frappe/frappe_docker' && github.event_name != 'pull_request' }} + jobs: build_bench: - name: Bench image + name: Bench runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - - uses: docker/setup-buildx-action@v1 + - name: Checkout + uses: actions/checkout@v2 - - uses: docker/login-action@v1 - if: github.repository == 'frappe/frappe_docker' && github.event_name != 'pull_request' + - name: Setup Buildx + uses: docker/setup-buildx-action@v1 + + - name: Login + uses: docker/login-action@v1 + if: env.IS_AUTHORIZED_RUN == 'true' with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Build and push + - name: Build and test uses: docker/bake-action@v1.6.0 with: files: docker-bake.hcl - targets: frappe-bench - push: ${{ github.repository == 'frappe/frappe_docker' && github.event_name != 'pull_request' }} + targets: bench-test - build_main: - name: Frappe and ERPNext images + - name: Push + uses: docker/bake-action@v1.6.0 + if: env.IS_AUTHORIZED_RUN == 'true' + with: + files: docker-bake.hcl + targets: bench-build + push: true + + build_frappe: + name: Frappe runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - - uses: docker/setup-buildx-action@v1 + - name: Checkout + uses: actions/checkout@v2 - - uses: docker/login-action@v1 - if: github.repository == 'frappe/frappe_docker' && github.event_name != 'pull_request' + - name: Setup Buildx + uses: docker/setup-buildx-action@v1 + + - name: Login + uses: docker/login-action@v1 + if: env.IS_AUTHORIZED_RUN == 'true' with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Build Frappe images + - name: Build uses: docker/bake-action@v1.6.0 with: files: docker-bake.hcl - targets: frappe-develop - load: true - push: ${{ github.repository == 'frappe/frappe_docker' && github.event_name != 'pull_request' }} - - - name: Build ERPNext images - uses: docker/bake-action@v1.6.0 - with: - files: docker-bake.hcl - targets: erpnext-develop + targets: frappe-develop-test load: true push: ${{ github.repository == 'frappe/frappe_docker' && github.event_name != 'pull_request' }} - name: Test - run: ./.github/scripts/install-deps-and-test.sh + run: ./tests/test-frappe.sh + + - name: Push + if: env.IS_AUTHORIZED_RUN == 'true' + uses: docker/bake-action@v1.6.0 + with: + files: docker-bake.hcl + targets: frappe-develop + push: true + + build_erpnext: + name: ERPNext + runs-on: ubuntu-latest + needs: build_frappe + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Setup Buildx + uses: docker/setup-buildx-action@v1 + + - name: Login + uses: docker/login-action@v1 + if: env.IS_AUTHORIZED_RUN == 'true' + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Build + uses: docker/bake-action@v1.6.0 + with: + files: docker-bake.hcl + targets: erpnext-develop-test + load: true + + - name: Test + run: FRAPPE_VERSION=develop ./tests/test-erpnext.sh + + - name: Push + if: env.IS_AUTHORIZED_RUN == 'true' + uses: docker/bake-action@v1.6.0 + with: + files: docker-bake.hcl + targets: erpnext-develop + push: true diff --git a/.github/workflows/build_stable.yml b/.github/workflows/build_stable.yml index cda94413..399ab42a 100644 --- a/.github/workflows/build_stable.yml +++ b/.github/workflows/build_stable.yml @@ -20,62 +20,117 @@ on: workflow_dispatch: jobs: - build: - name: Frappe and ERPNext images + build_frappe: + name: Frappe runs-on: ubuntu-latest strategy: matrix: version: [12, 13] steps: - - uses: actions/checkout@v2 - - uses: docker/setup-buildx-action@v1 - - uses: docker/login-action@v1 + - name: Checkout + uses: actions/checkout@v2 + + - name: Setup Buildx + uses: docker/setup-buildx-action@v1 + + - name: Login + uses: docker/login-action@v1 if: github.repository == 'frappe/frappe_docker' with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Setup Frappe variables - run: ./.github/scripts/get-latest-tag.sh + - name: Get latest versions + run: | + ./.github/scripts/get-latest-tags.sh + echo "VERSION=$VERSION" >>$GITHUB_ENV + echo "GIT_TAG=$FRAPPE_VERSION" >>$GITHUB_ENV + echo "GIT_BRANCH=version-$VERSION" >>$GITHUB_ENV env: - REPO: frappe/frappe VERSION: ${{ matrix.version }} - - name: Build Frappe images + - name: Build + uses: docker/bake-action@v1.6.0 + with: + files: docker-bake.hcl + targets: frappe-stable-test + load: true + + - name: Test + run: ./tests/test-frappe.sh + + - name: Push + if: github.repository == 'frappe/frappe_docker' uses: docker/bake-action@v1.6.0 with: files: docker-bake.hcl targets: frappe-stable push: true - - name: Setup ERPNext variables - run: ./.github/scripts/get-latest-tag.sh + build_erpnext: + name: ERPNext + runs-on: ubuntu-latest + needs: build_frappe + strategy: + matrix: + version: [12, 13] + + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Setup Buildx + uses: docker/setup-buildx-action@v1 + + - name: Login + uses: docker/login-action@v1 + if: github.repository == 'frappe/frappe_docker' + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Get latest versions + run: | + ./.github/scripts/get-latest-tags.sh + echo "VERSION=$VERSION" >>$GITHUB_ENV + echo "GIT_TAG=$ERPNEXT" >>$GITHUB_ENV + echo "GIT_BRANCH=version-$VERSION" >>$GITHUB_ENV env: - REPO: frappe/erpnext VERSION: ${{ matrix.version }} - - name: Build ERPNext images + - name: Build + uses: docker/bake-action@v1.6.0 + with: + files: docker-bake.hcl + targets: erpnext-stable-test + load: true + + - name: Test + run: ./tests/test-erpnext.sh + + - name: Push + if: github.repository == 'frappe/frappe_docker' uses: docker/bake-action@v1.6.0 with: files: docker-bake.hcl targets: erpnext-stable push: true - - name: Test - if: ${{ matrix.version != 12 }} - run: ./.github/scripts/install-deps-and-test.sh + release_helm: + name: Release Helm + runs-on: ubuntu-latest + if: github.repository == 'frappe/frappe_docker' + needs: [build_frappe, build_erpnext] - - name: Setup Helm deploy key - if: github.repository == 'frappe/frappe_docker' + steps: + - name: Setup deploy key uses: webfactory/ssh-agent@v0.5.3 with: ssh-private-key: ${{ secrets.HELM_DEPLOY_KEY }} - - name: Release Helm Chart - if: github.repository == 'frappe/frappe_docker' + - name: Release run: | - pip install --upgrade pip git clone git@github.com:frappe/helm.git && cd helm pip install -r release_wizard/requirements.txt ./release_wizard/wizard 13 patch --remote origin --ci diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 00000000..d7320dff --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,16 @@ +name: Lint + +on: + push: + branches: + - main + pull_request: + branches: + - main + +jobs: + lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: ludeeus/action-shellcheck@master diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index d45b0707..14d7593a 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -2,18 +2,17 @@ name: Mark stale issues and pull requests on: schedule: - - cron: "0 0 * * *" + # Every day at 12:00 pm + - cron: 0 0 * * * jobs: stale: - runs-on: ubuntu-latest - steps: - - uses: actions/stale@v4 - with: - repo-token: ${{ secrets.GITHUB_TOKEN }} - stale-issue-message: 'This issue has been automatically marked as stale. You have a week to explain why you believe this is an error.' - stale-pr-message: 'This PR has been automatically marked as stale. You have a week to explain why you believe this is an error.' - stale-issue-label: 'no-issue-activity' - stale-pr-label: 'no-pr-activity' \ No newline at end of file + - uses: actions/stale@v4 + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} + stale-issue-message: This issue has been automatically marked as stale. You have a week to explain why you believe this is an error. + stale-pr-message: This PR has been automatically marked as stale. You have a week to explain why you believe this is an error. + stale-issue-label: no-issue-activity + stale-pr-label: no-pr-activity diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 00000000..e3c7c1a7 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,17 @@ +name: Test + +on: + schedule: + # Every day at 01:00 am + # Develop images are built at 12:00 pm, we want to use them + - cron: 0 1 * * * + +jobs: + test: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Test + run: ./tests/integration-test.sh diff --git a/build/bench/Dockerfile b/build/bench/Dockerfile index d4c9d6d0..58d143d9 100644 --- a/build/bench/Dockerfile +++ b/build/bench/Dockerfile @@ -1,5 +1,5 @@ # Frappe Bench Dockerfile -FROM debian:buster-slim +FROM debian:buster-slim as build ARG GIT_REPO=https://github.com/frappe/bench.git ARG GIT_BRANCH=develop @@ -58,7 +58,7 @@ RUN apt-get update -y && apt-get install \ rm -rf /var/lib/apt/lists/* RUN sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen \ - && dpkg-reconfigure --frontend=noninteractive locales + && dpkg-reconfigure --frontend=noninteractive locales # Detect arch, download and install wkhtmltox RUN if [ `uname -m` = 'aarch64' ]; then export ARCH=arm64; fi \ @@ -83,11 +83,6 @@ ENV PATH=/home/frappe/.local/bin:$PATH # Export python executables for interactive shell RUN echo "export PATH=/home/frappe/.local/bin:\$PATH" >> /home/frappe/.bashrc -# Print version and verify bashrc is properly sourced so that everything works in the Dockerfile -RUN bench --version -# Print version and verify bashrc is properly sourced so that everything works in the interactive shell -RUN bash -c "bench --version" - # !!! UPDATE NODEJS PERIODICALLY WITH LATEST VERSIONS !!! # https://nodejs.org/en/about/releases/ # https://nodejs.org/download/release/latest-v10.x/ @@ -110,6 +105,10 @@ RUN . "$NVM_DIR/nvm.sh" && nvm use v${NODE_VERSION} && npm install -g yarn RUN . "$NVM_DIR/nvm.sh" && nvm alias default v${NODE_VERSION} ENV PATH="/home/frappe/.nvm/versions/node/v${NODE_VERSION}/bin/:${PATH}" +EXPOSE 8000-8005 9000-9005 6787 + +FROM build as test + # Print version and verify bashrc is properly sourced so that everything works in the Dockerfile RUN node --version \ && npm --version \ @@ -119,4 +118,5 @@ RUN bash -c "node --version" \ && bash -c "npm --version" \ && bash -c "yarn --version" -EXPOSE 8000-8005 9000-9005 6787 +RUN bench --help +RUN bash -c "bench --help" \ No newline at end of file diff --git a/docker-bake.hcl b/docker-bake.hcl index 4de80173..8f05ac72 100644 --- a/docker-bake.hcl +++ b/docker-bake.hcl @@ -1,8 +1,14 @@ # Images -target "frappe-bench" { +target "bench-build" { tags = ["frappe/bench:latest"] dockerfile = "build/bench/Dockerfile" + target = "build" +} + +target "bench-test" { + inherits = ["bench-build"] + target = "test" } target "frappe-nginx" { @@ -46,14 +52,19 @@ target "stable-args" { } } +function "set_develop_tags" { + params = [repo] + result = ["${repo}:latest", "${repo}:edge", "${repo}:develop"] +} + function "set_stable_tags" { params = [repo] result = ["${repo}:${GIT_TAG}", "${repo}:v${VERSION}", "${repo}:${GIT_BRANCH}"] } -function "set_develop_tags" { +function "set_test_tags" { params = [repo] - result = ["${repo}:latest", "${repo}:edge", "${repo}:develop"] + result = ["${repo}:test"] } @@ -92,6 +103,41 @@ group "erpnext-develop" { targets = ["erpnext-nginx-develop", "erpnext-worker-develop"] } +# Develop test + +target "frappe-nginx-develop-test" { + inherits = ["frappe-nginx-develop"] + tags = set_test_tags("frappe/frappe-nginx") +} + +target "frappe-worker-develop-test" { + inherits = ["frappe-worker-develop"] + tags = set_test_tags("frappe/frappe-worker") +} + +target "frappe-socketio-develop-test" { + inherits = ["frappe-socketio-develop"] + tags = set_test_tags("frappe/frappe-socketio") +} + +target "erpnext-nginx-develop-test" { + inherits = ["erpnext-nginx-develop"] + tags = set_test_tags("frappe/erpnext-nginx") +} + +target "erpnext-worker-develop-test" { + inherits = ["erpnext-worker-develop"] + tags = set_test_tags("frappe/erpnext-worker") +} + +group "frappe-develop-test" { + targets = ["frappe-nginx-develop-test", "frappe-worker-develop-test", "frappe-socketio-develop-test"] +} + +group "erpnext-develop-test" { + targets = ["erpnext-nginx-develop-test", "erpnext-worker-develop-test"] +} + # Stable @@ -127,3 +173,37 @@ group "frappe-stable" { group "erpnext-stable" { targets = ["erpnext-nginx-stable", "erpnext-worker-stable"] } + +# Stable test +target "frappe-nginx-stable-test" { + inherits = ["frappe-nginx-stable"] + tags = set_test_tags("frappe/frappe-nginx") +} + +target "frappe-worker-stable-test" { + inherits = ["frappe-worker-stable"] + tags = set_test_tags("frappe/frappe-worker") +} + +target "frappe-socketio-stable-test" { + inherits = ["frappe-socketio-stable"] + tags = set_test_tags("frappe/frappe-socketio") +} + +target "erpnext-nginx-stable-test" { + inherits = ["erpnext-nginx-stable"] + tags = set_test_tags("frappe/erpnext-nginx") +} + +target "erpnext-worker-stable-test" { + inherits = ["erpnext-worker-stable"] + tags = set_test_tags("frappe/erpnext-worker") +} + +group "frappe-stable-test" { + targets = ["frappe-nginx-stable-test", "frappe-worker-stable-test", "frappe-socketio-stable-test"] +} + +group "erpnext-stable-test" { + targets = ["erpnext-nginx-stable-test", "erpnext-worker-stable-test"] +} diff --git a/tests/check-format.sh b/tests/check-format.sh deleted file mode 100755 index 144984b3..00000000 --- a/tests/check-format.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash - -echo "Checking bash scripts with shellcheck" >&2 - -while IFS= read -r shellfile; do - shellcheck --check-sourced --severity=style --color=always --exclude=SC2164,SC2086,SC2012,SC2016 ${shellfile} -done < <(find ./build -name "*.sh") diff --git a/tests/docker-test.sh b/tests/docker-test.sh deleted file mode 100755 index 7693111e..00000000 --- a/tests/docker-test.sh +++ /dev/null @@ -1,403 +0,0 @@ -#!/bin/bash -ULINE='\e[1m\e[4m' -ENDULINE='\e[0m' -NEWLINE='\n' - -function checkMigrationComplete() { - echo "Check Migration" - CONTAINER_ID=$(docker-compose \ - --project-name frappebench00 \ - -f installation/docker-compose-common.yml \ - -f installation/docker-compose-erpnext.yml \ - -f installation/erpnext-publish.yml \ - ps -q erpnext-python) - - DOCKER_LOG=$(docker logs ${CONTAINER_ID} 2>&1 | grep "Starting gunicorn") - INCREMENT=0 - while [[ ${DOCKER_LOG} != *"Starting gunicorn"* && ${INCREMENT} -lt 60 ]]; do - sleep 3 - echo "Wait for migration to complete ..." - ((INCREMENT = INCREMENT + 1)) - DOCKER_LOG=$(docker logs ${CONTAINER_ID} 2>&1 | grep "Starting gunicorn") - if [[ ${DOCKER_LOG} != *"Starting gunicorn"* && ${INCREMENT} -eq 60 ]]; then - docker logs ${CONTAINER_ID} - exit 1 - fi - done - - echo -e "${ULINE}Migration Log${ENDULINE}" - docker logs ${CONTAINER_ID} -} - -function loopHealthCheck() { - echo "Create Container to Check MariaDB" - docker run --name frappe_doctor \ - -v frappebench00_sites-vol:/home/frappe/frappe-bench/sites \ - --network frappebench00_default \ - frappe/erpnext-worker:edge doctor || true - - echo "Loop Health Check" - FRAPPE_LOG=$(docker logs frappe_doctor | grep "Health check successful" || echo "") - while [[ -z "${FRAPPE_LOG}" ]]; do - sleep 1 - CONTAINER=$(docker start frappe_doctor) - echo "Restarting ${CONTAINER} ..." - FRAPPE_LOG=$(docker logs frappe_doctor | grep "Health check successful" || echo "") - done - echo "Health check successful" -} - -echo -e "${ULINE}Copy env-example file${ENDULINE}" -cp env-example .env -export $(cat .env) - -echo -e "${NEWLINE}${ULINE}Set version to v13${ENDULINE}" -sed -i -e "s/edge/v13/g" .env -export $(cat .env) - -echo -e "${NEWLINE}${ULINE}Start Services${ENDULINE}" -docker-compose \ - --project-name frappebench00 \ - -f installation/docker-compose-common.yml \ - -f installation/docker-compose-erpnext.yml \ - -f installation/erpnext-publish.yml \ - pull -docker pull postgres:11.8 -docker-compose \ - --project-name frappebench00 \ - -f installation/docker-compose-common.yml \ - -f installation/docker-compose-erpnext.yml \ - -f installation/erpnext-publish.yml \ - up -d -# Start postgres -docker run --name postgresql -d \ - -e "POSTGRES_PASSWORD=admin" \ - -v frappebench00_sites-vol:/home/frappe/frappe-bench/sites \ - --network frappebench00_default \ - postgres:11.8 - -loopHealthCheck - -echo -e "${NEWLINE}${ULINE}Create new site (v13)${ENDULINE}" -docker run -it \ - -e "SITE_NAME=test.localhost" \ - -e "INSTALL_APPS=erpnext" \ - -v frappebench00_sites-vol:/home/frappe/frappe-bench/sites \ - --network frappebench00_default \ - frappe/erpnext-worker:v13 new - -echo -e "${NEWLINE}${ULINE}Ping created site${ENDULINE}" -curl -sS http://test.localhost/api/method/version - -echo -e "${NEWLINE}${ULINE}Check Created Site Index Page${ENDULINE}" -curl -s http://test.localhost | w3m -T text/html -dump - -echo -e "${NEWLINE}${ULINE}Set version to edge${ENDULINE}" -sed -i -e "s/v13/edge/g" .env -export $(cat .env) - -echo -e "${NEWLINE}${ULINE}Restart containers with edge image${ENDULINE}" -docker-compose \ - --project-name frappebench00 \ - -f installation/docker-compose-common.yml \ - -f installation/docker-compose-erpnext.yml \ - -f installation/erpnext-publish.yml \ - stop -docker-compose \ - --project-name frappebench00 \ - -f installation/docker-compose-common.yml \ - -f installation/docker-compose-erpnext.yml \ - -f installation/erpnext-publish.yml \ - up -d - -checkMigrationComplete - -echo -e "${NEWLINE}${ULINE}Ping migrated site${ENDULINE}" -sleep 3 -curl -sS http://test.localhost/api/method/version - -echo -e "${NEWLINE}${ULINE}Check Migrated Site Index Page${ENDULINE}" -curl -s http://test.localhost | w3m -T text/html -dump - -echo -e "${NEWLINE}${ULINE}Create new site (pgsql)${ENDULINE}" -docker run -it \ - -e "SITE_NAME=pgsql.localhost" \ - -e "POSTGRES_HOST=postgresql" \ - -e "DB_ROOT_USER=postgres" \ - -e "POSTGRES_PASSWORD=admin" \ - -v frappebench00_sites-vol:/home/frappe/frappe-bench/sites \ - --network frappebench00_default \ - frappe/erpnext-worker:edge new - -echo -e "${NEWLINE}${ULINE}Check New PGSQL Site${ENDULINE}" -sleep 3 -RESTORE_STATUS=$(curl -sS http://pgsql.localhost/api/method/version || echo "") -INCREMENT=0 -while [[ -z "${RESTORE_STATUS}" && ${INCREMENT} -lt 60 ]]; do - sleep 1 - echo -e "${ULINE}Wait for restoration to complete ..." - RESTORE_STATUS=$(curl -sS http://pgsql.localhost/api/method/version || echo "") - ((INCREMENT = INCREMENT + 1)) - if [[ -z "${RESTORE_STATUS}" && ${INCREMENT} -eq 60 ]]; then - CONTAINER_ID=$(docker-compose \ - --project-name frappebench00 \ - -f installation/docker-compose-common.yml \ - -f installation/docker-compose-erpnext.yml \ - -f installation/erpnext-publish.yml \ - ps -q erpnext-python) - docker logs ${CONTAINER_ID} - exit 1 - fi -done - -echo -e "${NEWLINE}${ULINE}Ping new pgsql site${ENDULINE}" -echo $RESTORE_STATUS - -echo -e "${NEWLINE}${ULINE}Check New PGSQL Index Page${ENDULINE}" -curl -s http://pgsql.localhost | w3m -T text/html -dump - -echo -e "${NEWLINE}${ULINE}Backup site${ENDULINE}" -docker run -it \ - -e "WITH_FILES=1" \ - -v frappebench00_sites-vol:/home/frappe/frappe-bench/sites \ - --network frappebench00_default \ - frappe/erpnext-worker:edge backup - -MINIO_ACCESS_KEY="AKIAIOSFODNN7EXAMPLE" -MINIO_SECRET_KEY="wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" - -echo -e "${ULINE}Start MinIO container for s3 compatible storage${ENDULINE}" -docker run -d --name minio \ - -e "MINIO_ACCESS_KEY=${MINIO_ACCESS_KEY}" \ - -e "MINIO_SECRET_KEY=${MINIO_SECRET_KEY}" \ - --network frappebench00_default \ - minio/minio server /data - -echo -e "${NEWLINE}${ULINE}Create bucket named erpnext${ENDULINE}" -docker run \ - --network frappebench00_default \ - vltgroup/s3cmd:latest s3cmd --access_key=${MINIO_ACCESS_KEY} \ - --secret_key=${MINIO_SECRET_KEY} \ - --region=us-east-1 \ - --no-ssl \ - --host=minio:9000 \ - --host-bucket=minio:9000 \ - mb s3://erpnext - -echo -e "${NEWLINE}${NEWLINE}${ULINE}Push backup to MinIO s3${ENDULINE}" -docker run \ - -e BUCKET_NAME=erpnext \ - -e REGION=us-east-1 \ - -e BUCKET_DIR=local \ - -e ACCESS_KEY_ID=${MINIO_ACCESS_KEY} \ - -e SECRET_ACCESS_KEY=${MINIO_SECRET_KEY} \ - -e ENDPOINT_URL=http://minio:9000 \ - -v frappebench00_sites-vol:/home/frappe/frappe-bench/sites \ - --network frappebench00_default \ - frappe/erpnext-worker:edge push-backup - -echo -e "${NEWLINE}${ULINE}Stop Services${ENDULINE}" -docker-compose \ - --project-name frappebench00 \ - -f installation/docker-compose-common.yml \ - -f installation/docker-compose-erpnext.yml \ - -f installation/erpnext-publish.yml \ - stop - -echo -e "${NEWLINE}${ULINE}Prune Containers${ENDULINE}" -docker container prune -f && docker volume prune -f - -echo -e "${NEWLINE}${ULINE}Start Services${ENDULINE}" -docker-compose \ - --project-name frappebench00 \ - -f installation/docker-compose-common.yml \ - -f installation/docker-compose-erpnext.yml \ - -f installation/erpnext-publish.yml \ - up -d - -loopHealthCheck - -echo -e "${NEWLINE}${ULINE}Restore backup from MinIO / S3${ENDULINE}" -docker run \ - -e MYSQL_ROOT_PASSWORD=admin \ - -e BUCKET_NAME=erpnext \ - -e BUCKET_DIR=local \ - -e ACCESS_KEY_ID=${MINIO_ACCESS_KEY} \ - -e SECRET_ACCESS_KEY=${MINIO_SECRET_KEY} \ - -e ENDPOINT_URL=http://minio:9000 \ - -e REGION=us-east-1 \ - -v frappebench00_sites-vol:/home/frappe/frappe-bench/sites \ - --network frappebench00_default \ - frappe/erpnext-worker:edge restore-backup - -echo -e "${NEWLINE}${ULINE}Check Restored Site (test)${ENDULINE}" -sleep 3 -RESTORE_STATUS=$(curl -sS http://test.localhost/api/method/version || echo "") -INCREMENT=0 -while [[ -z "${RESTORE_STATUS}" && ${INCREMENT} -lt 60 ]]; do - sleep 1 - echo "Wait for restoration to complete ..." - RESTORE_STATUS=$(curl -sS http://test.localhost/api/method/version || echo "") - ((INCREMENT = INCREMENT + 1)) - if [[ -z "${RESTORE_STATUS}" && ${INCREMENT} -eq 60 ]]; then - CONTAINER_ID=$(docker-compose \ - --project-name frappebench00 \ - -f installation/docker-compose-common.yml \ - -f installation/docker-compose-erpnext.yml \ - -f installation/erpnext-publish.yml \ - ps -q erpnext-python) - docker logs ${CONTAINER_ID} - exit 1 - fi -done - -echo -e "${ULINE}Ping restored site (test)${ENDULINE}" -echo ${RESTORE_STATUS} - -echo -e "${NEWLINE}${ULINE}Check Restored Site Index Page (test)${ENDULINE}" -curl -s http://test.localhost | w3m -T text/html -dump - -echo -e "${NEWLINE}${ULINE}Check Restored Site (pgsql)${ENDULINE}" -sleep 3 -RESTORE_STATUS=$(curl -sS http://pgsql.localhost/api/method/version || echo "") -INCREMENT=0 -while [[ -z "${RESTORE_STATUS}" && ${INCREMENT} -lt 60 ]]; do - sleep 1 - echo "Wait for restoration to complete ..." - RESTORE_STATUS=$(curl -sS http://pgsql.localhost/api/method/version || echo "") - ((INCREMENT = INCREMENT + 1)) - if [[ -z "${RESTORE_STATUS}" && ${INCREMENT} -eq 60 ]]; then - CONTAINER_ID=$(docker-compose \ - --project-name frappebench00 \ - -f installation/docker-compose-common.yml \ - -f installation/docker-compose-erpnext.yml \ - -f installation/erpnext-publish.yml \ - ps -q erpnext-python) - docker logs ${CONTAINER_ID} - exit 1 - fi -done - -echo -e "${ULINE}Ping restored site (pgsql)${ENDULINE}" -echo ${RESTORE_STATUS} - -echo -e "${NEWLINE}${ULINE}Check Restored Site Index Page (pgsql)${ENDULINE}" -curl -s http://pgsql.localhost | w3m -T text/html -dump - -echo -e "${NEWLINE}${ULINE}Create new site (edge)${ENDULINE}" -docker run -it \ - -e "SITE_NAME=edge.localhost" \ - -e "INSTALL_APPS=erpnext" \ - -v frappebench00_sites-vol:/home/frappe/frappe-bench/sites \ - --network frappebench00_default \ - frappe/erpnext-worker:edge new - -echo -e "${NEWLINE}${ULINE}Check New Edge Site${ENDULINE}" -sleep 3 -RESTORE_STATUS=$(curl -sS http://edge.localhost/api/method/version || echo "") -INCREMENT=0 -while [[ -z "${RESTORE_STATUS}" && ${INCREMENT} -lt 60 ]]; do - sleep 1 - echo -e "${ULINE}Wait for restoration to complete ...${ENDULINE}" - RESTORE_STATUS=$(curl -sS http://edge.localhost/api/method/version || echo "") - ((INCREMENT = INCREMENT + 1)) - if [[ -z "${RESTORE_STATUS}" && ${INCREMENT} -eq 60 ]]; then - CONTAINER_ID=$(docker-compose \ - --project-name frappebench00 \ - -f installation/docker-compose-common.yml \ - -f installation/docker-compose-erpnext.yml \ - -f installation/erpnext-publish.yml \ - ps -q erpnext-python) - docker logs ${CONTAINER_ID} - exit 1 - fi -done - -echo -e "${NEWLINE}${ULINE}Ping new edge site${ENDULINE}" -echo ${RESTORE_STATUS} - -echo -e "${NEWLINE}${ULINE}Check New Edge Index Page${ENDULINE}" -curl -s http://edge.localhost | w3m -T text/html -dump - -echo -e "${NEWLINE}${ULINE}Migrate command in edge container${ENDULINE}" -docker run -it \ - -e "MAINTENANCE_MODE=1" \ - -v frappebench00_sites-vol:/home/frappe/frappe-bench/sites \ - -v frappebench00_assets-vol:/home/frappe/frappe-bench/sites/assets \ - --network frappebench00_default \ - frappe/erpnext-worker:edge migrate - -checkMigrationComplete - -echo -e "${NEWLINE}${ULINE}Restore backup from MinIO / S3 (Overwrite)${ENDULINE}" -docker run \ - -e MYSQL_ROOT_PASSWORD=admin \ - -e BUCKET_NAME=erpnext \ - -e BUCKET_DIR=local \ - -e ACCESS_KEY_ID=${MINIO_ACCESS_KEY} \ - -e SECRET_ACCESS_KEY=${MINIO_SECRET_KEY} \ - -e ENDPOINT_URL=http://minio:9000 \ - -e REGION=us-east-1 \ - -v frappebench00_sites-vol:/home/frappe/frappe-bench/sites \ - --network frappebench00_default \ - frappe/erpnext-worker:edge restore-backup - -echo -e "${NEWLINE}${ULINE}Check Overwritten Site${ENDULINE}" -sleep 3 -RESTORE_STATUS=$(curl -sS http://test.localhost/api/method/version || echo "") -INCREMENT=0 -while [[ -z "${RESTORE_STATUS}" && ${INCREMENT} -lt 60 ]]; do - sleep 1 - echo -e "${ULINE}Wait for restoration to complete ..." - RESTORE_STATUS=$(curl -sS http://test.localhost/api/method/version || echo "") - ((INCREMENT = INCREMENT + 1)) - if [[ -z "${RESTORE_STATUS}" && ${INCREMENT} -eq 60 ]]; then - CONTAINER_ID=$(docker-compose \ - --project-name frappebench00 \ - -f installation/docker-compose-common.yml \ - -f installation/docker-compose-erpnext.yml \ - -f installation/erpnext-publish.yml \ - ps -q erpnext-python) - docker logs ${CONTAINER_ID} - exit 1 - fi -done - -echo -e "${NEWLINE}${ULINE}Ping overwritten site${ENDULINE}" -echo ${RESTORE_STATUS} - -echo -e "${NEWLINE}${ULINE}Check Overwritten Index Page${ENDULINE}" -curl -s http://test.localhost | w3m -T text/html -dump - -echo -e "${NEWLINE}${ULINE}Check console command for site test.localhost${ENDULINE}" -docker run \ - -v frappebench00_sites-vol:/home/frappe/frappe-bench/sites \ - --network frappebench00_default \ - frappe/erpnext-worker:edge console test.localhost - -echo -e "${NEWLINE}${ULINE}Check console command for site pgsql.localhost${ENDULINE}" -docker run \ - -v frappebench00_sites-vol:/home/frappe/frappe-bench/sites \ - --network frappebench00_default \ - frappe/erpnext-worker:edge console pgsql.localhost - -echo -e "${NEWLINE}${ULINE}Check drop site: test.localhost (mariadb)${ENDULINE}" -docker run \ - -e SITE_NAME=test.localhost \ - -v frappebench00_sites-vol:/home/frappe/frappe-bench/sites \ - --network frappebench00_default \ - frappe/erpnext-worker:edge drop - -echo -e "${NEWLINE}${ULINE}Check drop site: pgsql.localhost (pgsql)${ENDULINE}" -docker run \ - -e SITE_NAME=pgsql.localhost \ - -v frappebench00_sites-vol:/home/frappe/frappe-bench/sites \ - --network frappebench00_default \ - frappe/erpnext-worker:edge drop - -echo -e "${NEWLINE}${ULINE}Check bench --help${ENDULINE}" -docker run \ - -v frappebench00_sites-vol:/home/frappe/frappe-bench/sites \ - --network frappebench00_default \ - --user frappe \ - frappe/erpnext-worker:edge bench --help diff --git a/tests/functions.sh b/tests/functions.sh new file mode 100644 index 00000000..8453c499 --- /dev/null +++ b/tests/functions.sh @@ -0,0 +1,26 @@ +#!/bin/bash + +print_group() { + echo ::endgroup:: + echo "::group::$*" +} + +ping_site() { + print_group "Ping site $SITE_NAME" + + echo Ping version + ping_res=$(curl -sS "http://$SITE_NAME/api/method/version") + echo "$ping_res" + if [[ -z $(echo "$ping_res" | grep "message" || echo "") ]]; then + echo "Ping failed" + exit 1 + fi + + echo Check index + index_res=$(curl -sS "http://$SITE_NAME") + if [[ -n $(echo "$index_res" | grep "Internal Server Error" || echo "") ]]; then + echo "Index check failed" + echo "$index_res" + exit 1 + fi +} diff --git a/tests/integration-test.sh b/tests/integration-test.sh new file mode 100755 index 00000000..14483f26 --- /dev/null +++ b/tests/integration-test.sh @@ -0,0 +1,281 @@ +#!/bin/bash + +# shellcheck source=functions.sh +source tests/functions.sh + +project_name=frappe_bench_00 + +docker_compose_with_args() { + # shellcheck disable=SC2068 + docker-compose \ + -p $project_name \ + -f installation/docker-compose-common.yml \ + -f installation/docker-compose-erpnext.yml \ + -f installation/erpnext-publish.yml \ + $@ +} + +check_migration_complete() { + print_group Check migration + + container_id=$(docker_compose_with_args ps -q erpnext-python) + thelogs=$(docker logs "${container_id}" 2>&1 | grep "Starting gunicorn") + INCREMENT=0 + + while [[ ${thelogs} != *"Starting gunicorn"* && ${INCREMENT} -lt 120 ]]; do + sleep 3 + ((INCREMENT = INCREMENT + 1)) + echo "Wait for migration to complete..." + thelogs=$(docker logs "${container_id}" 2>&1 | grep "Starting gunicorn") + + if [[ ${thelogs} != *"Starting gunicorn"* && ${INCREMENT} -eq 120 ]]; then + echo Migration timeout + docker logs "${container_id}" + exit 1 + fi + done + + echo Migration Log + docker logs "${container_id}" +} + +check_health() { + print_group Loop health check + + docker run --name frappe_doctor \ + -v "${project_name}_sites-vol:/home/frappe/frappe-bench/sites" \ + --network "${project_name}_default" \ + frappe/erpnext-worker:edge doctor || true + + cmd='docker logs frappe_doctor | grep "Health check successful" || echo ""' + doctor_log=$(eval "$cmd") + INCREMENT=0 + + while [[ -z "${doctor_log}" && ${INCREMENT} -lt 60 ]]; do + sleep 1 + ((INCREMENT = INCREMENT + 1)) + container=$(docker start frappe_doctor) + echo "Restarting ${container}..." + doctor_log=$(eval "$cmd") + + if [[ ${INCREMENT} -eq 60 ]]; then + docker logs "${container}" + exit 1 + fi + done +} + +# Initial group +echo ::group::Setup .env +cp env-example .env +sed -i -e "s/edge/v13/g" .env +# shellcheck disable=SC2046 +export $(cat .env) + +print_group Start services +echo Start main services +docker_compose_with_args up -d --quiet-pull + +echo Start postgres +docker pull postgres:11.8 -q +docker run \ + --name postgresql \ + -d \ + -e POSTGRES_PASSWORD=admin \ + -v ${project_name}_sites-vol:/home/frappe/frappe-bench/sites \ + --network ${project_name}_default \ + postgres:11.8 + +check_health + +print_group "Create new site " +SITE_NAME=test.localhost +docker run \ + --rm \ + -e SITE_NAME=$SITE_NAME \ + -e INSTALL_APPS=erpnext \ + -v ${project_name}_sites-vol:/home/frappe/frappe-bench/sites \ + --network ${project_name}_default \ + frappe/erpnext-worker:v13 new + +ping_site + +print_group "Update .env (v13 -> edge)" +sed -i -e "s/v13/edge/g" .env +# shellcheck disable=SC2046 +export $(cat .env) + +print_group Restart containers +docker_compose_with_args stop +docker_compose_with_args up -d + +check_migration_complete +sleep 5 +ping_site + +PG_SITE_NAME=pgsql.localhost +print_group "Create new site (Postgres)" +docker run \ + --rm \ + -e SITE_NAME=$PG_SITE_NAME \ + -e POSTGRES_HOST=postgresql \ + -e DB_ROOT_USER=postgres \ + -e POSTGRES_PASSWORD=admin \ + -v ${project_name}_sites-vol:/home/frappe/frappe-bench/sites \ + --network ${project_name}_default \ + frappe/erpnext-worker:edge new + +check_migration_complete +SITE_NAME=$PG_SITE_NAME ping_site + +print_group Backup site +docker run \ + --rm \ + -e WITH_FILES=1 \ + -v ${project_name}_sites-vol:/home/frappe/frappe-bench/sites \ + --network ${project_name}_default \ + frappe/erpnext-worker:edge backup + +MINIO_ACCESS_KEY="AKIAIOSFODNN7EXAMPLE" +MINIO_SECRET_KEY="wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" + +print_group Prepare S3 server +echo Start S3 server +docker run \ + --name minio \ + -d \ + -e "MINIO_ACCESS_KEY=$MINIO_ACCESS_KEY" \ + -e "MINIO_SECRET_KEY=$MINIO_SECRET_KEY" \ + --network ${project_name}_default \ + minio/minio server /data + +echo Create bucket +docker run \ + --rm \ + --network ${project_name}_default \ + vltgroup/s3cmd:latest \ + s3cmd \ + --access_key=$MINIO_ACCESS_KEY \ + --secret_key=$MINIO_SECRET_KEY \ + --region=us-east-1 \ + --no-ssl \ + --host=minio:9000 \ + --host-bucket=minio:9000 \ + mb s3://erpnext + +print_group Push backup +docker run \ + --rm \ + -e BUCKET_NAME=erpnext \ + -e REGION=us-east-1 \ + -e BUCKET_DIR=local \ + -e ACCESS_KEY_ID=$MINIO_ACCESS_KEY \ + -e SECRET_ACCESS_KEY=$MINIO_SECRET_KEY \ + -e ENDPOINT_URL=http://minio:9000 \ + -v ${project_name}_sites-vol:/home/frappe/frappe-bench/sites \ + --network ${project_name}_default \ + frappe/erpnext-worker:edge push-backup + +print_group Prune and restart services +docker_compose_with_args stop +docker container prune -f && docker volume prune -f +docker_compose_with_args up -d + +check_health + +print_group Restore backup from S3 +docker run \ + --rm \ + -e MYSQL_ROOT_PASSWORD=admin \ + -e BUCKET_NAME=erpnext \ + -e BUCKET_DIR=local \ + -e ACCESS_KEY_ID=$MINIO_ACCESS_KEY \ + -e SECRET_ACCESS_KEY=$MINIO_SECRET_KEY \ + -e ENDPOINT_URL=http://minio:9000 \ + -e REGION=us-east-1 \ + -v ${project_name}_sites-vol:/home/frappe/frappe-bench/sites \ + --network ${project_name}_default \ + frappe/erpnext-worker:edge restore-backup + +check_health +ping_site +SITE_NAME=$PG_SITE_NAME ping_site + +EDGE_SITE_NAME=edge.localhost +print_group "Create new site (edge)" +docker run \ + --rm \ + -e SITE_NAME=$EDGE_SITE_NAME \ + -e INSTALL_APPS=erpnext \ + -v ${project_name}_sites-vol:/home/frappe/frappe-bench/sites \ + --network ${project_name}_default \ + frappe/erpnext-worker:edge new + +check_health +SITE_NAME=$EDGE_SITE_NAME ping_site + +print_group Migrate edge site +docker run \ + --rm \ + -e MAINTENANCE_MODE=1 \ + -v ${project_name}_sites-vol:/home/frappe/frappe-bench/sites \ + -v ${project_name}_assets-vol:/home/frappe/frappe-bench/sites/assets \ + --network ${project_name}_default \ + frappe/erpnext-worker:edge migrate + +check_migration_complete + +print_group "Restore backup S3 (overwrite)" +docker run \ + --rm \ + -e MYSQL_ROOT_PASSWORD=admin \ + -e BUCKET_NAME=erpnext \ + -e BUCKET_DIR=local \ + -e ACCESS_KEY_ID=$MINIO_ACCESS_KEY \ + -e SECRET_ACCESS_KEY=$MINIO_SECRET_KEY \ + -e ENDPOINT_URL=http://minio:9000 \ + -e REGION=us-east-1 \ + -v ${project_name}_sites-vol:/home/frappe/frappe-bench/sites \ + --network ${project_name}_default \ + frappe/erpnext-worker:edge restore-backup + +check_migration_complete +ping_site + +print_group "Check console for $SITE_NAME" +docker run \ + --rm \ + -v ${project_name}_sites-vol:/home/frappe/frappe-bench/sites \ + --network ${project_name}_default \ + frappe/erpnext-worker:edge console $SITE_NAME + +print_group "Check console for $PG_SITE_NAME" +docker run \ + --rm \ + -v ${project_name}_sites-vol:/home/frappe/frappe-bench/sites \ + --network ${project_name}_default \ + frappe/erpnext-worker:edge console $PG_SITE_NAME + +print_group "Check drop site for $SITE_NAME (MariaDB)" +docker run \ + --rm \ + -e SITE_NAME=$SITE_NAME \ + -v ${project_name}_sites-vol:/home/frappe/frappe-bench/sites \ + --network ${project_name}_default \ + frappe/erpnext-worker:edge drop + +print_group "Check drop site for $PG_SITE_NAME (Postgres)" +docker run \ + --rm \ + -e SITE_NAME=$PG_SITE_NAME \ + -v ${project_name}_sites-vol:/home/frappe/frappe-bench/sites \ + --network ${project_name}_default \ + frappe/erpnext-worker:edge drop + +print_group Check bench --help +docker run \ + --rm \ + -v ${project_name}_sites-vol:/home/frappe/frappe-bench/sites \ + --network ${project_name}_default \ + --user frappe \ + frappe/erpnext-worker:edge bench --help diff --git a/tests/test-erpnext.sh b/tests/test-erpnext.sh new file mode 100755 index 00000000..ac734164 --- /dev/null +++ b/tests/test-erpnext.sh @@ -0,0 +1,38 @@ +#!/bin/bash + +set -e + +# shellcheck source=functions.sh +source tests/functions.sh + +project_name="test_erpnext" +SITE_NAME="test_erpnext.localhost" + +echo ::group::Setup env +cp env-example .env +sed -i -e "s/FRAPPE_VERSION=edge/FRAPPE_VERSION=$FRAPPE_VERSION/g" .env +sed -i -e "s/ERPNEXT_VERSION=edge/ERPNEXT_VERSION=test/g" .env +# shellcheck disable=SC2046 +export $(cat .env) +cat .env + +print_group Start services +FRAPPE_VERSION=$FRAPPE_VERSION ERPNEXT_VERSION="test" \ + docker-compose \ + -p $project_name \ + -f installation/docker-compose-common.yml \ + -f installation/docker-compose-erpnext.yml \ + -f installation/erpnext-publish.yml \ + up -d + +print_group Create site +docker run \ + --rm \ + -e "SITE_NAME=$SITE_NAME" \ + -e "INSTALL_APPS=erpnext" \ + -v ${project_name}_sites-vol:/home/frappe/frappe-bench/sites \ + --network ${project_name}_default \ + frappe/erpnext-worker:test new + +ping_site +rm .env diff --git a/tests/test-frappe.sh b/tests/test-frappe.sh new file mode 100755 index 00000000..c4998300 --- /dev/null +++ b/tests/test-frappe.sh @@ -0,0 +1,36 @@ +#!/bin/bash + +set -e + +# shellcheck source=functions.sh +source tests/functions.sh + +project_name="test_frappe" +SITE_NAME="test_frappe.localhost" + +echo ::group::Setup env +cp env-example .env +sed -i -e "s/edge/test/g" .env +# shellcheck disable=SC2046 +export $(cat .env) +cat .env + +print_group Start services +FRAPPE_VERSION="test" \ + docker-compose \ + -p $project_name \ + -f installation/docker-compose-common.yml \ + -f installation/docker-compose-frappe.yml \ + -f installation/frappe-publish.yml \ + up -d + +print_group Create site +docker run \ + --rm \ + -e "SITE_NAME=$SITE_NAME" \ + -v ${project_name}_sites-vol:/home/frappe/frappe-bench/sites \ + --network ${project_name}_default \ + frappe/frappe-worker:test new + +ping_site +rm .env