accounting/frappe_docker
2
0
Fork 0
mirror of https://github.com/frappe/frappe_docker.git synced 2024-09-19 18:49:01 +00:00
Code Issues Releases Activity

fix: Improve CI, fix builds (#554)

Browse Source 
* ci(Test): Add `workflow_dispatch` trigger

* ci(Test): Build and run on push and pr

* test: Set failfast, remove ERPNext reference

* test: Reveal .env file for logs

* ci(Test): Rename step

* ci(Test): Fix git tag

* test: Add -x flag for debugging

* test: Fix failing test

* ci(Test): Check if built images are used

* ci(Test): Configure trigger paths, don't build on schedule

* test: Remove bug

* ci: Build and test stable images on PR

* fix(frappe-nginx): Get back to `ls` from `find`

* check if test passes with erpnext images

* fix(frappe-worker): Default sites in backup command

* get integration test changes back

* chore: Fix linting
This commit is contained in:
Lev 2021-11-09 15:15:33 +03:00 committed by GitHub
parent b57371edf3
commit 5254e2aad3
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 140 additions and 42 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
.github/workflows/build_stable.yml vendored
View File

@ -1,6 +1,18 @@
name: Build Stable
on:
pull_request:
branches:
- main
paths:
- .github/**
- build/**
- installation/**
- tests/**
- .dockerignore
- docker-bake.hcl
- docker-compose.yml
- env*
push:
branches:
- main
@ -19,6 +31,9 @@ on:
workflow_dispatch:
env:
IS_AUTHORIZED_RUN: ${{ github.repository == 'frappe/frappe_docker' && github.event_name != 'pull_request' }}
jobs:
build_frappe:
name: Frappe
@ -36,7 +51,7 @@ jobs:
- name: Login
uses: docker/login-action@v1
if: github.repository == 'frappe/frappe_docker'
if: env.IS_AUTHORIZED_RUN == 'true'
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
@ -57,7 +72,7 @@ jobs:
run: ./tests/test-frappe.sh
- name: Push
if: github.repository == 'frappe/frappe_docker'
if: env.IS_AUTHORIZED_RUN == 'true'
uses: docker/bake-action@v1.6.0
with:
files: docker-bake.hcl
@ -83,7 +98,7 @@ jobs:
- name: Login
uses: docker/login-action@v1
if: github.repository == 'frappe/frappe_docker'
if: env.IS_AUTHORIZED_RUN == 'true'
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
@ -104,7 +119,7 @@ jobs:
run: ./tests/test-erpnext.sh
- name: Push
if: github.repository == 'frappe/frappe_docker'
if: env.IS_AUTHORIZED_RUN == 'true'
uses: docker/bake-action@v1.6.0
with:
files: docker-bake.hcl
@ -116,7 +131,7 @@ jobs:
release_helm:
name: Release Helm
runs-on: ubuntu-latest
if: github.repository == 'frappe/frappe_docker'
if: github.repository == 'frappe/frappe_docker' && github.event_name != 'pull_request'
needs: [build_frappe, build_erpnext]
steps:

47
.github/workflows/test.yml vendored
View File

@ -1,9 +1,38 @@
name: Test
name: Integration Test
on:
push:
branches:
- main
paths:
- .github/**
- build/**
- installation/**
- tests/**
- .dockerignore
- docker-bake.hcl
- docker-compose.yml
- env*
pull_request:
branches:
- main
paths:
- .github/**
- build/**
- installation/**
- tests/**
- .dockerignore
- docker-bake.hcl
- docker-compose.yml
- env*
workflow_dispatch:
schedule:
# Every day at 01:00 am
# Develop images are built at 12:00 pm, we want to use them
# Also, we don't build new images on this event
- cron: 0 1 * * *
jobs:
@ -13,5 +42,21 @@ jobs:
- name: Checkout
uses: actions/checkout@v2
- name: Get latest versions
if: github.event_name != 'schedule'
run: ./.github/scripts/get-latest-tags.sh
env:
VERSION: 13
- name: Build
if: github.event_name != 'schedule'
uses: docker/bake-action@v1.6.0
with:
files: docker-bake.hcl
targets: frappe-develop,frappe-stable
load: true
env:
GIT_TAG: ${{ env.FRAPPE_VERSION }}
- name: Test
run: ./tests/integration-test.sh

3
build/frappe-nginx/docker-entrypoint.sh
View File

@ -8,7 +8,8 @@ rsync -a --delete /var/www/html/assets/* /assets
/rsync
touch /var/www/html/sites/.build -r "$(find /assets -maxdepth 1 -name "*.*" | head -n 1)"
# shellcheck disable=SC2012
touch /var/www/html/sites/.build -r "$(ls -td /assets/* | head -n 1)"
[[ -z "${FRAPPE_PY}" ]] && FRAPPE_PY='0.0.0.0'

39
build/frappe-worker/commands/backup.py Normal file
View File

@ -0,0 +1,39 @@
import os
import frappe
from frappe.utils.backups import scheduled_backup
from frappe.utils import cint, get_sites, now
def backup(sites, with_files=False):
for site in sites:
frappe.init(site)
frappe.connect()
odb = scheduled_backup(
ignore_files=not with_files,
backup_path_db=None,
backup_path_files=None,
backup_path_private_files=None,
force=True
)
print("database backup taken -", odb.backup_path_db, "- on", now())
if with_files:
print("files backup taken -", odb.backup_path_files, "- on", now())
print("private files backup taken -", odb.backup_path_private_files, "- on", now())
frappe.destroy()
def main():
installed_sites = ":".join(get_sites())
sites = os.environ.get("SITES", installed_sites).split(":")
with_files = cint(os.environ.get("WITH_FILES"))
backup(sites, with_files)
if frappe.redis_server:
frappe.redis_server.connection_pool.disconnect()
exit(0)
if __name__ == "__main__":
main()

9
build/frappe-worker/docker-entrypoint.sh
View File

@ -159,13 +159,8 @@ doctor)
;;
backup)
if [[ -n $WITH_FILES ]]; then
WITH_FILES=--with-files
fi
for site in ${SITES//:/ }; do
bench --site "$site" backup $WITH_FILES
done
/home/frappe/frappe-bench/env/bin/python /home/frappe/frappe-bench/commands/backup.py
exit
;;
console)

59
tests/integration-test.sh
View File

@ -1,5 +1,7 @@
#!/bin/bash
set -e
source tests/functions.sh
project_name=frappe_bench_00
@ -9,25 +11,25 @@ docker_compose_with_args() {
docker-compose \
-p $project_name \
-f installation/docker-compose-common.yml \
-f installation/docker-compose-erpnext.yml \
-f installation/erpnext-publish.yml \
-f installation/docker-compose-frappe.yml \
-f installation/frappe-publish.yml \
$@
}
check_migration_complete() {
print_group Check migration
container_id=$(docker_compose_with_args ps -q erpnext-python)
thelogs=$(docker logs "${container_id}" 2>&1 | grep "Starting gunicorn")
container_id=$(docker_compose_with_args ps -q frappe-python)
cmd="docker logs ${container_id} 2>&1 | grep 'Starting gunicorn' || echo ''"
worker_log=$(eval "$cmd")
INCREMENT=0
while [[ ${thelogs} != *"Starting gunicorn"* && ${INCREMENT} -lt 120 ]]; do
while [[ ${worker_log} != *"Starting gunicorn"* && ${INCREMENT} -lt 120 ]]; do
sleep 3
((INCREMENT = INCREMENT + 1))
echo "Wait for migration to complete..."
thelogs=$(docker logs "${container_id}" 2>&1 | grep "Starting gunicorn")
if [[ ${thelogs} != *"Starting gunicorn"* && ${INCREMENT} -eq 120 ]]; then
worker_log=$(eval "$cmd")
if [[ ${worker_log} != *"Starting gunicorn"* && ${INCREMENT} -eq 120 ]]; then
echo Migration timeout
docker logs "${container_id}"
exit 1
@ -44,7 +46,7 @@ check_health() {
docker run --name frappe_doctor \
-v "${project_name}_sites-vol:/home/frappe/frappe-bench/sites" \
--network "${project_name}_default" \
frappe/erpnext-worker:edge doctor || true
frappe/frappe-worker:edge doctor || true
cmd='docker logs frappe_doctor | grep "Health check successful" || echo ""'
doctor_log=$(eval "$cmd")
@ -68,6 +70,7 @@ check_health() {
echo ::group::Setup .env
cp env-example .env
sed -i -e "s/edge/v13/g" .env
cat .env
# shellcheck disable=SC2046
export $(cat .env)
@ -92,15 +95,15 @@ SITE_NAME=test.localhost
docker run \
--rm \
-e SITE_NAME=$SITE_NAME \
-e INSTALL_APPS=erpnext \
-v ${project_name}_sites-vol:/home/frappe/frappe-bench/sites \
--network ${project_name}_default \
frappe/erpnext-worker:v13 new
frappe/frappe-worker:v13 new
ping_site
print_group "Update .env (v13 -> edge)"
sed -i -e "s/v13/edge/g" .env
cat .env
# shellcheck disable=SC2046
export $(cat .env)
@ -122,7 +125,7 @@ docker run \
-e POSTGRES_PASSWORD=admin \
-v ${project_name}_sites-vol:/home/frappe/frappe-bench/sites \
--network ${project_name}_default \
frappe/erpnext-worker:edge new
frappe/frappe-worker:edge new
check_migration_complete
SITE_NAME=$PG_SITE_NAME ping_site
@ -133,7 +136,7 @@ docker run \
-e WITH_FILES=1 \
-v ${project_name}_sites-vol:/home/frappe/frappe-bench/sites \
--network ${project_name}_default \
frappe/erpnext-worker:edge backup
frappe/frappe-worker:edge backup
MINIO_ACCESS_KEY="AKIAIOSFODNN7EXAMPLE"
MINIO_SECRET_KEY="wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"
@ -160,12 +163,12 @@ docker run \
--no-ssl \
--host=minio:9000 \
--host-bucket=minio:9000 \
mb s3://erpnext
mb s3://frappe
print_group Push backup
docker run \
--rm \
-e BUCKET_NAME=erpnext \
-e BUCKET_NAME=frappe \
-e REGION=us-east-1 \
-e BUCKET_DIR=local \
-e ACCESS_KEY_ID=$MINIO_ACCESS_KEY \
@ -173,7 +176,7 @@ docker run \
-e ENDPOINT_URL=http://minio:9000 \
-v ${project_name}_sites-vol:/home/frappe/frappe-bench/sites \
--network ${project_name}_default \
frappe/erpnext-worker:edge push-backup
frappe/frappe-worker:edge push-backup
print_group Prune and restart services
docker_compose_with_args stop
@ -186,7 +189,7 @@ print_group Restore backup from S3
docker run \
--rm \
-e MYSQL_ROOT_PASSWORD=admin \
-e BUCKET_NAME=erpnext \
-e BUCKET_NAME=frappe \
-e BUCKET_DIR=local \
-e ACCESS_KEY_ID=$MINIO_ACCESS_KEY \
-e SECRET_ACCESS_KEY=$MINIO_SECRET_KEY \
@ -194,7 +197,7 @@ docker run \
-e REGION=us-east-1 \
-v ${project_name}_sites-vol:/home/frappe/frappe-bench/sites \
--network ${project_name}_default \
frappe/erpnext-worker:edge restore-backup
frappe/frappe-worker:edge restore-backup
check_health
ping_site
@ -205,10 +208,10 @@ print_group "Create new site (edge)"
docker run \
--rm \
-e SITE_NAME=$EDGE_SITE_NAME \
-e INSTALL_APPS=erpnext \
-e INSTALL_APPS=frappe \
-v ${project_name}_sites-vol:/home/frappe/frappe-bench/sites \
--network ${project_name}_default \
frappe/erpnext-worker:edge new
frappe/frappe-worker:edge new
check_health
SITE_NAME=$EDGE_SITE_NAME ping_site
@ -220,7 +223,7 @@ docker run \
-v ${project_name}_sites-vol:/home/frappe/frappe-bench/sites \
-v ${project_name}_assets-vol:/home/frappe/frappe-bench/sites/assets \
--network ${project_name}_default \
frappe/erpnext-worker:edge migrate
frappe/frappe-worker:edge migrate
check_migration_complete
@ -228,7 +231,7 @@ print_group "Restore backup S3 (overwrite)"
docker run \
--rm \
-e MYSQL_ROOT_PASSWORD=admin \
-e BUCKET_NAME=erpnext \
-e BUCKET_NAME=frappe \
-e BUCKET_DIR=local \
-e ACCESS_KEY_ID=$MINIO_ACCESS_KEY \
-e SECRET_ACCESS_KEY=$MINIO_SECRET_KEY \
@ -236,7 +239,7 @@ docker run \
-e REGION=us-east-1 \
-v ${project_name}_sites-vol:/home/frappe/frappe-bench/sites \
--network ${project_name}_default \
frappe/erpnext-worker:edge restore-backup
frappe/frappe-worker:edge restore-backup
check_migration_complete
ping_site
@ -246,14 +249,14 @@ docker run \
--rm \
-v ${project_name}_sites-vol:/home/frappe/frappe-bench/sites \
--network ${project_name}_default \
frappe/erpnext-worker:edge console $SITE_NAME
frappe/frappe-worker:edge console $SITE_NAME
print_group "Check console for $PG_SITE_NAME"
docker run \
--rm \
-v ${project_name}_sites-vol:/home/frappe/frappe-bench/sites \
--network ${project_name}_default \
frappe/erpnext-worker:edge console $PG_SITE_NAME
frappe/frappe-worker:edge console $PG_SITE_NAME
print_group "Check drop site for $SITE_NAME (MariaDB)"
docker run \
@ -261,7 +264,7 @@ docker run \
-e SITE_NAME=$SITE_NAME \
-v ${project_name}_sites-vol:/home/frappe/frappe-bench/sites \
--network ${project_name}_default \
frappe/erpnext-worker:edge drop
frappe/frappe-worker:edge drop
print_group "Check drop site for $PG_SITE_NAME (Postgres)"
docker run \
@ -269,7 +272,7 @@ docker run \
-e SITE_NAME=$PG_SITE_NAME \
-v ${project_name}_sites-vol:/home/frappe/frappe-bench/sites \
--network ${project_name}_default \
frappe/erpnext-worker:edge drop
frappe/frappe-worker:edge drop
print_group Check bench --help
docker run \
@ -277,4 +280,4 @@ docker run \
-v ${project_name}_sites-vol:/home/frappe/frappe-bench/sites \
--network ${project_name}_default \
--user frappe \
frappe/erpnext-worker:edge bench --help
frappe/frappe-worker:edge bench --help