From e6088af885390e89a9f5d3e596c47f683cf5a6fd Mon Sep 17 00:00:00 2001 From: Revant Nandgaonkar Date: Mon, 16 Jan 2023 04:20:09 +0530 Subject: [PATCH] refactor: build only one frappe/erpnext image (#1032) * ci: skip frappe builds * refactor: build only one frappe/erpnext image * fix: lint nginx entrypoint script * docs: update and organize docs * docs: fix lint errors * fix(custom): pass base64 encoded apps json * ci: update dependabot * docs: update contributing * docs: remove info about multi image setup * fix: initiate empty common_site_config.json default config has host keys set to localhost causes connection errors * docs: add details for pwd volumes * fix: symlink assets instead of copy * fix: nginx private files * ci: skip docker compose v2 install for ubuntu-latest * fix: organize layers * feat: allow remove git remote for custom image * docs: allow remove git remote for custom image * fix: remove duplicate --apps_path --- .github/dependabot.yml | 14 +- .github/scripts/update_pwd.py | 9 +- .github/workflows/docker-build-push.yml | 3 - CONTRIBUTING.md | 6 - README.md | 43 +++--- compose.yaml | 28 +++- custom_app/README.md | 46 ------ custom_app/backend.Dockerfile | 14 -- custom_app/compose.override.yaml | 21 --- custom_app/docker-bake.hcl | 27 ---- custom_app/frontend.Dockerfile | 37 ----- docker-bake.hcl | 52 ++----- docs/backup-and-push-cronjob.md | 2 +- docs/custom-apps.md | 101 ++++++++++++ development/README.md => docs/development.md | 4 +- docs/environment-variables.md | 60 +++++++ docs/images-and-compose-files.md | 101 ------------ docs/list-of-containers.md | 58 +++++++ docs/migrate-from-multi-image-setup.md | 115 ++++++++++++++ docs/patch-code-from-images.md | 34 ---- docs/port-based-multi-tenancy.md | 89 +++++++---- docs/setup-options.md | 3 - docs/single-compose-setup.md | 40 +++++ docs/single-server-example.md | 30 ++-- docs/site-operations.md | 4 +- docs/troubleshoot.md | 4 +- images/custom/Containerfile | 145 +++++++++++++++++ images/nginx/Dockerfile | 62 -------- images/nginx/entrypoint.sh | 9 -- images/production/Containerfile | 132 ++++++++++++++++ images/socketio/Dockerfile | 34 ---- images/worker/Dockerfile | 146 ------------------ images/worker/configure.py | 56 ------- images/worker/gevent_patch.py | 3 - images/worker/install-app.sh | 11 -- images/worker/patched_bench_helper.py | 48 ------ images/worker/pretend-bench.sh | 5 - .../compose.custom-domain-ssl.yaml | 0 .../compose.custom-domain.yaml | 0 overrides/compose.erpnext.yaml | 27 ---- .../compose.mariadb-shared.yaml | 0 .../compose.multi-bench-ssl.yaml | 0 .../compose.multi-bench.yaml | 0 overrides/compose.redis.yaml | 28 +++- .../compose.traefik-ssl.yaml | 0 .../compose.traefik.yaml | 0 pwd.yml | 74 +++++---- requirements-dev.txt | 3 - resources/nginx-entrypoint.sh | 52 +++++++ .../nginx => resources}/nginx-template.conf | 26 ++-- {images/worker => resources}/push_backup.py | 1 + tests/compose.ci-erpnext.yaml | 24 --- tests/compose.ci.yaml | 16 +- tests/conftest.py | 12 +- tests/test_frappe_docker.py | 8 +- 55 files changed, 931 insertions(+), 936 deletions(-) delete mode 100644 custom_app/README.md delete mode 100644 custom_app/backend.Dockerfile delete mode 100644 custom_app/compose.override.yaml delete mode 100644 custom_app/docker-bake.hcl delete mode 100644 custom_app/frontend.Dockerfile create mode 100644 docs/custom-apps.md rename development/README.md => docs/development.md (98%) create mode 100644 docs/environment-variables.md delete mode 100644 docs/images-and-compose-files.md create mode 100644 docs/list-of-containers.md create mode 100644 docs/migrate-from-multi-image-setup.md delete mode 100644 docs/patch-code-from-images.md create mode 100644 docs/single-compose-setup.md create mode 100644 images/custom/Containerfile delete mode 100644 images/nginx/Dockerfile delete mode 100755 images/nginx/entrypoint.sh create mode 100644 images/production/Containerfile delete mode 100644 images/socketio/Dockerfile delete mode 100644 images/worker/Dockerfile delete mode 100755 images/worker/configure.py delete mode 100644 images/worker/gevent_patch.py delete mode 100755 images/worker/install-app.sh delete mode 100644 images/worker/patched_bench_helper.py delete mode 100755 images/worker/pretend-bench.sh rename {docs/compose => overrides}/compose.custom-domain-ssl.yaml (100%) rename {docs/compose => overrides}/compose.custom-domain.yaml (100%) delete mode 100644 overrides/compose.erpnext.yaml rename {docs/compose => overrides}/compose.mariadb-shared.yaml (100%) rename {docs/compose => overrides}/compose.multi-bench-ssl.yaml (100%) rename {docs/compose => overrides}/compose.multi-bench.yaml (100%) rename {docs/compose => overrides}/compose.traefik-ssl.yaml (100%) rename {docs/compose => overrides}/compose.traefik.yaml (100%) delete mode 100644 requirements-dev.txt create mode 100755 resources/nginx-entrypoint.sh rename {images/nginx => resources}/nginx-template.conf (80%) rename {images/worker => resources}/push_backup.py (99%) delete mode 100644 tests/compose.ci-erpnext.yaml diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 8ff402c2..8fb9a8c8 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -11,22 +11,12 @@ updates: interval: daily - package-ecosystem: docker - directory: images/nginx + directory: images/production schedule: interval: daily - package-ecosystem: docker - directory: images/worker - schedule: - interval: daily - - - package-ecosystem: docker - directory: images/socketio - schedule: - interval: daily - - - package-ecosystem: npm - directory: images/socketio + directory: images/custom schedule: interval: daily diff --git a/.github/scripts/update_pwd.py b/.github/scripts/update_pwd.py index 914e7850..998612c5 100644 --- a/.github/scripts/update_pwd.py +++ b/.github/scripts/update_pwd.py @@ -13,12 +13,9 @@ def get_versions(): def update_pwd(frappe_version: str, erpnext_version: str): with open("pwd.yml", "r+") as f: content = f.read() - for image, version in ( - ("frappe/frappe-socketio", frappe_version), - ("frappe/erpnext-worker", erpnext_version), - ("frappe/erpnext-nginx", erpnext_version), - ): - content = re.sub(rf"{image}:.*", f"{image}:{version}", content) + content = re.sub( + rf"frappe/erpnext:.*", f"frappe/erpnext:{erpnext_version}", content + ) f.seek(0) f.truncate() f.write(content) diff --git a/.github/workflows/docker-build-push.yml b/.github/workflows/docker-build-push.yml index 99a7e632..8b43242a 100644 --- a/.github/workflows/docker-build-push.yml +++ b/.github/workflows/docker-build-push.yml @@ -54,9 +54,6 @@ jobs: with: python-version: "3.10" - - name: Install Docker Compose v2 - uses: ndeloof/install-compose-action@4a33bc31f327b8231c4f343f6fba704fedc0fa23 - - name: Install dependencies run: | python -m venv venv diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index a152d8a8..cb315ca4 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -61,16 +61,10 @@ Run pytest: pytest ``` -> We also have `requirements-dev.txt` file that contains development requirements for backend image (you can find it in `images/worker/` directory). - # Documentation Place relevant markdown files in the `docs` directory and index them in README.md located at the root of repo. -# Wiki - -Add alternatives that can be used optionally along with frappe_docker. Add articles to list on home page as well. - # Frappe and ERPNext updates Each Frappe/ERPNext release triggers new stable images builds as well as bump to helm chart. diff --git a/README.md b/README.md index 2e847bc0..933d70a8 100644 --- a/README.md +++ b/README.md @@ -21,40 +21,39 @@ cd frappe_docker Wait for 5 minutes for ERPNext site to be created or check `create-site` container logs before opening browser on port 8080. (username: `Administrator`, password: `admin`) -# Development +# Documentation -We have baseline for developing in VSCode devcontainer with [frappe/bench](https://github.com/frappe/bench). [Start development](development). +### [Production](#production) -# Production +- [List of containers](docs/list-of-containers.md) +- [Single Compose Setup](docs/single-compose-setup.md) +- [Environment Variables](docs/environment-variables.md) +- [Single Server Example](docs/single-server-example.md) +- [Setup Options](docs/setup-options.md) +- [Site Operations](docs/site-operations.md) +- [Backup and Push Cron Job](docs/backup-and-push-cronjob.md) +- [Port Based Multi Tenancy](docs/port-based-multi-tenancy.md) +- [Migrate from multi-image setup](docs/migrate-from-multi-image-setup.md) -We provide simple and intuitive production setup with prebuilt Frappe and ERPNext images and compose files. To learn more about those, [read the docs](docs/images-and-compose-files.md). +### [Custom Images](#custom-images) -Also, there's docs to help with deployment: +- [Custom Apps](docs/custom-apps.md) +- [Build Version 10 Images](docs/build-version-10-images.md) -- Examples: - - [Single Server](docs/single-server-example.md) - - [Setup options](docs/setup-options.md) - - [Kubernetes (frappe/helm)](https://helm.erpnext.com) -- [Site operations](docs/site-operations.md). -- Other - - [backup and push cron jobs](docs/backup-and-push-cronjob.md) - - [bench console and vscode debugger](docs/bench-console-and-vscode-debugger.md) - - [build version 10](docs/build-version-10-images.md) - - [connect to localhost services from containers for local app development](docs/connect-to-localhost-services-from-containers-for-local-app-development.md) - - [patch code from images](docs/patch-code-from-images.md) - - [port based multi tenancy](docs/port-based-multi-tenancy.md) -- [Troubleshoot](docs/troubleshoot.md) +### [Development](#development) -# Custom app +- [Development using containers](docs/development.md) +- [Bench Console and VSCode Debugger](docs/bench-console-and-vscode-debugger.md) +- [Connect to localhost services](docs/connect-to-localhost-services-from-containers-for-local-app-development.md) -Learn how to containerize your custom Frappe app(s) in [this guide](custom_app/README.md). +### [Troubleshoot](docs/troubleshoot.md) # Contributing If you want to contribute to this repo refer to [CONTRIBUTING.md](CONTRIBUTING.md) -This repository is only for Docker related stuff. You also might want to contribute to: +This repository is only for container related stuff. You also might want to contribute to: - [Frappe framework](https://github.com/frappe/frappe#contributing), - [ERPNext](https://github.com/frappe/erpnext#contributing), -- or [Frappe Bench](https://github.com/frappe/bench). +- [Frappe Bench](https://github.com/frappe/bench). diff --git a/compose.yaml b/compose.yaml index c27a9072..ad8bdce5 100644 --- a/compose.yaml +++ b/compose.yaml @@ -5,15 +5,24 @@ x-depends-on-configurator: &depends_on_configurator x-backend-defaults: &backend_defaults <<: *depends_on_configurator - image: frappe/frappe-worker:${FRAPPE_VERSION:?No Frappe version set} + image: frappe/erpnext:${ERPNEXT_VERSION:?No ERPNext version set} volumes: - sites:/home/frappe/frappe-bench/sites - - assets:/home/frappe/frappe-bench/sites/assets:ro services: configurator: <<: *backend_defaults - command: configure.py + entrypoint: + - bash + - -c + command: + - > + bench set-config -g db_host $$DB_HOST; + bench set-config -gp db_port $$DB_PORT; + bench set-config -g redis_cache "redis://$$REDIS_CACHE"; + bench set-config -g redis_queue "redis://$$REDIS_QUEUE"; + bench set-config -g redis_socketio "redis://$$REDIS_SOCKETIO"; + bench set-config -gp socketio_port $$SOCKETIO_PORT; environment: DB_HOST: ${DB_HOST} DB_PORT: ${DB_PORT} @@ -27,7 +36,9 @@ services: <<: *backend_defaults frontend: - image: frappe/frappe-nginx:${FRAPPE_VERSION} + image: frappe/erpnext:${ERPNEXT_VERSION} + command: + - nginx-entrypoint.sh environment: BACKEND: backend:8000 SOCKETIO: websocket:9000 @@ -38,15 +49,17 @@ services: PROXY_READ_TIMOUT: ${PROXY_READ_TIMOUT:-120} CLIENT_MAX_BODY_SIZE: ${CLIENT_MAX_BODY_SIZE:-50m} volumes: - - sites:/usr/share/nginx/html/sites - - assets:/usr/share/nginx/html/assets + - sites:/home/frappe/frappe-bench/sites depends_on: - backend - websocket websocket: <<: *depends_on_configurator - image: frappe/frappe-socketio:${FRAPPE_VERSION} + image: frappe/erpnext:${ERPNEXT_VERSION} + command: + - node + - /home/frappe/frappe-bench/apps/frappe/socketio.js volumes: - sites:/home/frappe/frappe-bench/sites @@ -69,4 +82,3 @@ services: # ERPNext requires local assets access (Frappe does not) volumes: sites: - assets: diff --git a/custom_app/README.md b/custom_app/README.md deleted file mode 100644 index 07ada233..00000000 --- a/custom_app/README.md +++ /dev/null @@ -1,46 +0,0 @@ -This is basic configuration for building images and testing custom apps that use Frappe. - -You can see that there's four files in this folder: - -- `backend.Dockerfile`, -- `frontend.Dockerfile`, -- `docker-bake.hcl`, -- `compose.override.yaml`. - -Python code will be built in `backend.Dockerfile`. JS and CSS (and other fancy frontend stuff) files will be built in `frontend.Dockerfile`. - -`docker-bake.hcl` is reference file for [Buildx Bake](https://github.com/docker/buildx/blob/master/docs/reference/buildx_bake.md). It helps to build images without having to remember all build arguments. - -`compose.override.yaml` is [Compose](https://docs.docker.com/compose/compose-file/) override that replaces images from [main compose file](https://github.com/frappe/frappe_docker/blob/main/compose.yaml) so it would use your own images. - -To get started, install Docker and [Buildx](https://github.com/docker/buildx#installing). Then copy all content of this folder (except this README) to your app's root directory. Also copy `compose.yaml` in the root of this repository. - -Before the next step—to build images—replace "custom_app" with your app's name in `docker-bake.hcl`. After that, let's try to build: - -```bash -FRAPPE_VERSION=... ERPNEXT_VERSION=... docker buildx bake -``` - -> 💡 We assume that majority of our users use ERPNext, that's why images in this tutorial are based on ERPNext images. If don't want ERPNext, change base image in Dockerfile and remove ERPNEXT_VERSION from bake file. To know more about steps used to build frontend image read comments in `frontend.Dockerfile`. - -If something goes wrong feel free to leave an issue. - -To test if site works, setup `.env` file (check [example](<(https://github.com/frappe/frappe_docker/blob/main/example.env)>)) and run: - -```bash -docker-compose -f compose.yaml -f overrides/compose.noproxy.yaml -f overrides/compose.mariadb.yaml -f overrides/compose.redis.yaml -f custom_app/compose.override.yaml up -d -docker-compose exec backend \ - bench new-site 127.0.0.1 \ - --mariadb-root-password 123 \ - --admin-password admin \ - --install-app -docker-compose restart backend -``` - -Cool! You just containerized your app! - -## Installing multiple apps - -Backend builds contain `install-app` script that places app where it should be. Each call to script installs given app. Usage: `install-app [APP_NAME]`. - -If you want to install an app from git, clone it locally, COPY in Dockerfile. diff --git a/custom_app/backend.Dockerfile b/custom_app/backend.Dockerfile deleted file mode 100644 index 3bd32834..00000000 --- a/custom_app/backend.Dockerfile +++ /dev/null @@ -1,14 +0,0 @@ -# syntax=docker/dockerfile:1.3 - -ARG ERPNEXT_VERSION -FROM frappe/erpnext-worker:${ERPNEXT_VERSION} - -USER root - -ARG APP_NAME -COPY . ../apps/${APP_NAME} - -RUN --mount=type=cache,target=/root/.cache/pip \ - install-app ${APP_NAME} - -USER frappe diff --git a/custom_app/compose.override.yaml b/custom_app/compose.override.yaml deleted file mode 100644 index 002e0795..00000000 --- a/custom_app/compose.override.yaml +++ /dev/null @@ -1,21 +0,0 @@ -services: - configurator: - image: custom_app/worker:${VERSION} - - backend: - image: custom_app/worker:${VERSION} - - frontend: - image: custom_app/nginx:${VERSION} - - queue-short: - image: custom_app/worker:${VERSION} - - queue-default: - image: custom_app/worker:${VERSION} - - queue-long: - image: custom_app/worker:${VERSION} - - scheduler: - image: custom_app/worker:${VERSION} diff --git a/custom_app/docker-bake.hcl b/custom_app/docker-bake.hcl deleted file mode 100644 index 64971aa7..00000000 --- a/custom_app/docker-bake.hcl +++ /dev/null @@ -1,27 +0,0 @@ -APP_NAME="custom_app" - -variable "FRAPPE_VERSION" {} -variable "ERPNEXT_VERSION" {} - -group "default" { - targets = ["backend", "frontend"] -} - -target "backend" { - dockerfile = "backend.Dockerfile" - tags = ["custom_app/worker:latest"] - args = { - "ERPNEXT_VERSION" = ERPNEXT_VERSION - "APP_NAME" = APP_NAME - } -} - -target "frontend" { - dockerfile = "frontend.Dockerfile" - tags = ["custom_app/nginx:latest"] - args = { - "FRAPPE_VERSION" = FRAPPE_VERSION - "ERPNEXT_VERSION" = ERPNEXT_VERSION - "APP_NAME" = APP_NAME - } -} diff --git a/custom_app/frontend.Dockerfile b/custom_app/frontend.Dockerfile deleted file mode 100644 index 7b3c8473..00000000 --- a/custom_app/frontend.Dockerfile +++ /dev/null @@ -1,37 +0,0 @@ -ARG FRAPPE_VERSION=version-14 -# Prepare builder image -FROM frappe/bench:latest as assets - -ARG FRAPPE_VERSION=version-14 -ARG ERPNEXT_VERSION=version-14 -ARG APP_NAME - -# Setup frappe-bench using FRAPPE_VERSION -RUN bench init --version=${FRAPPE_VERSION} --skip-redis-config-generation --verbose --skip-assets /home/frappe/frappe-bench -WORKDIR /home/frappe/frappe-bench - -# Comment following if ERPNext is not required -RUN bench get-app --branch=${ERPNEXT_VERSION} --skip-assets --resolve-deps erpnext - -# Copy custom app(s) -COPY --chown=frappe:frappe . apps/${APP_NAME} - -# Setup dependencies -RUN bench setup requirements - -# Build static assets, copy files instead of symlink -RUN bench build --production --verbose --hard-link - - -# Use frappe-nginx image with nginx template and env vars -FROM frappe/frappe-nginx:${FRAPPE_VERSION} - -# Remove existing assets -USER root -RUN rm -fr /usr/share/nginx/html/assets - -# Copy built assets -COPY --from=assets /home/frappe/frappe-bench/sites/assets /usr/share/nginx/html/assets - -# Use non-root user -USER 1000 diff --git a/docker-bake.hcl b/docker-bake.hcl index 72eb9d9a..9d542136 100644 --- a/docker-bake.hcl +++ b/docker-bake.hcl @@ -44,16 +44,8 @@ target "bench-test" { # Main images # Base for all other targets -group "frappe" { - targets = ["frappe-worker", "frappe-nginx", "frappe-socketio"] -} - -group "erpnext" { - targets = ["erpnext-worker", "erpnext-nginx"] -} - group "default" { - targets = ["frappe", "erpnext"] + targets = ["erpnext"] } function "tag" { @@ -68,46 +60,20 @@ function "tag" { target "default-args" { args = { - FRAPPE_REPO = "${FRAPPE_REPO}" - ERPNEXT_REPO = "${ERPNEXT_REPO}" + FRAPPE_PATH = "${FRAPPE_REPO}" + ERPNEXT_PATH = "${ERPNEXT_REPO}" BENCH_REPO = "${BENCH_REPO}" - FRAPPE_VERSION = "${FRAPPE_VERSION}" - ERPNEXT_VERSION = "${ERPNEXT_VERSION}" + FRAPPE_BRANCH = "${FRAPPE_VERSION}" + ERPNEXT_BRANCH = "${ERPNEXT_VERSION}" PYTHON_VERSION = can(regex("v13", "${ERPNEXT_VERSION}")) ? "3.9.9" : "3.10.5" NODE_VERSION = can(regex("v13", "${FRAPPE_VERSION}")) ? "14.19.3" : "16.18.0" } } -target "frappe-worker" { +target "erpnext" { inherits = ["default-args"] - context = "images/worker" - target = "frappe" - tags = tag("frappe-worker", "${FRAPPE_VERSION}") -} - -target "erpnext-worker" { - inherits = ["default-args"] - context = "images/worker" + context = "." + dockerfile = "images/production/Containerfile" target = "erpnext" - tags = tag("erpnext-worker", "${ERPNEXT_VERSION}") -} - -target "frappe-nginx" { - inherits = ["default-args"] - context = "images/nginx" - target = "frappe" - tags = tag("frappe-nginx", "${FRAPPE_VERSION}") -} - -target "erpnext-nginx" { - inherits = ["default-args"] - context = "images/nginx" - target = "erpnext" - tags = tag("erpnext-nginx", "${ERPNEXT_VERSION}") -} - -target "frappe-socketio" { - inherits = ["default-args"] - context = "images/socketio" - tags = tag("frappe-socketio", "${FRAPPE_VERSION}") + tags = tag("erpnext", "${ERPNEXT_VERSION}") } diff --git a/docs/backup-and-push-cronjob.md b/docs/backup-and-push-cronjob.md index c6051ff8..a159dd8f 100644 --- a/docs/backup-and-push-cronjob.md +++ b/docs/backup-and-push-cronjob.md @@ -5,7 +5,7 @@ Create backup service or stack. version: "3.7" services: backup: - image: frappe/erpnext-worker:v14 + image: frappe/erpnext:v14 entrypoint: ["bash", "-c"] command: | for SITE in $(/home/frappe/frappe-bench/env/bin/python -c "import frappe;print(' '.join(frappe.utils.get_sites()))") diff --git a/docs/custom-apps.md b/docs/custom-apps.md new file mode 100644 index 00000000..7d79414f --- /dev/null +++ b/docs/custom-apps.md @@ -0,0 +1,101 @@ +### Clone frappe_docker and switch directory + +```shell +git clone https://github.com/frappe/frappe_docker +cd frappe_docker +``` + +### Load custom apps through json + +`apps.json` needs to be passed in as build arg environment variable. + +```shell +export APPS_JSON='[ + { + "url": "https://github.com/frappe/payments", + "branch": "develop" + }, + { + "url": "https://github.com/frappe/erpnext", + "branch": "version-14" + }, + { + "url": "https://user:password@git.example.com/project/repository.git", + "branch": "main" + } +]' +export APPS_JSON_BASE64=$(echo ${APPS_JSON} | base64 --wrap=0) +``` + +You can also generate base64 string from json file: + +```shell +export APPS_JSON_BASE64=$(base64 --wrap=0 /path/to/apps.json) +``` + +Note: + +- `url` needs to be http(s) git url with token/auth in case of private repo. +- add dependencies manually in `apps.json` e.g. add `payments` if you are installing `erpnext` +- use fork repo or branch for ERPNext in case you need to use your fork or test a PR. + +### Build Image + +```shell +buildah build \ + --build-arg=FRAPPE_PATH=https://github.com/frappe/frappe \ + --build-arg=FRAPPE_BRANCH=version-14 \ + --build-arg=PYTHON_VERSION=3.10.5 \ + --build-arg=NODE_VERSION=16.18.0 \ + --build-arg=APPS_JSON_BASE64=$APPS_JSON_BASE64 \ + --tag=ghcr.io/user/repo/custom:1.0.0 \ + --file=images/custom/Containerfile . +``` + +Note: + +- Use `docker` instead of `buildah` as per your setup. +- `FRAPPE_PATH` and `FRAPPE_BRANCH` build args are optional and can be overridden in case of fork/branch or test a PR. +- Make sure `APPS_JSON_BASE64` variable has correct base64 encoded JSON string. It is consumed as build arg, base64 encoding ensures it to be friendly with environment variables. Use `jq empty apps.json` to validate `apps.json` file. +- Make sure the `--tag` is valid image name that will be pushed to registry. +- Change `--build-arg` as per version of Python, NodeJS, Frappe Framework repo and branch +- Set `--build-arg=REMOVE_GIT_REMOTE=true` to remove git upstream remotes from all apps. Use this in case they have secrets or private tokens and you don't wish to ship them in final image. + +### Push image to use in yaml files + +Login to `docker` or `buildah` + +```shell +buildah login +``` + +Push image + +```shell +buildah push ghcr.io/user/repo/custom:1.0.0 +``` + +### Use Kaniko + +Following executor args are required. Example runs locally in docker container. +You can run it part of CI/CD or part of your cluster. + +```shell +podman run --rm -it \ + -v "$HOME"/.docker/config.json:/kaniko/.docker/config.json \ + gcr.io/kaniko-project/executor:latest \ + --dockerfile=images/custom/Containerfile \ + --context=git://github.com/frappe/frappe_docker \ + --build-arg=FRAPPE_PATH=https://github.com/frappe/frappe \ + --build-arg=FRAPPE_BRANCH=version-14 \ + --build-arg=PYTHON_VERSION=3.10.5 \ + --build-arg=NODE_VERSION=16.18.0 \ + --build-arg=APPS_JSON=$APPS_JSON \ + --destination=ghcr.io/user/repo/custom:1.0.0 +``` + +More about [kaniko](https://github.com/GoogleContainerTools/kaniko) + +### Use Images + +Make sure image name is correct to be pushed to registry. After the images are pushed, you can pull them to servers to be deployed. If the registry is private, additional auth is needed. diff --git a/development/README.md b/docs/development.md similarity index 98% rename from development/README.md rename to docs/development.md index 464218a7..476314a4 100644 --- a/development/README.md +++ b/docs/development.md @@ -14,9 +14,9 @@ It is recommended you allocate at least 4GB of RAM to docker: - [Instructions for macOS](https://docs.docker.com/desktop/settings/mac/#advanced) Here is a screenshot showing the relevant setting in the Help Manual -![image](/docs/images/Docker%20Manual%20Screenshot%20-%20Resources%20section.png) +![image](images/Docker%20Manual%20Screenshot%20-%20Resources%20section.png) Here is a screenshot showing the settings in Docker Desktop on Mac -![images](/docs/images/Docker%20Desktop%20Screenshot%20-%20Resources%20section.png) +![images](images/Docker%20Desktop%20Screenshot%20-%20Resources%20section.png) ## Bootstrap Containers for development diff --git a/docs/environment-variables.md b/docs/environment-variables.md new file mode 100644 index 00000000..15e61818 --- /dev/null +++ b/docs/environment-variables.md @@ -0,0 +1,60 @@ +## Environment Variables + +All of the commands are directly passed to container as per type of service. Only environment variables used in image are for `nginx-entrypoint.sh` command. They are as follows: + +- `BACKEND`: Set to `{host}:{port}`, defaults to `0.0.0.0:8000` +- `SOCKETIO`: Set to `{host}:{port}`, defaults to `0.0.0.0:9000` +- `UPSTREAM_REAL_IP_ADDRESS`: Set Nginx config for [ngx_http_realip_module#set_real_ip_from](http://nginx.org/en/docs/http/ngx_http_realip_module.html#set_real_ip_from), defaults to `127.0.0.1` +- `UPSTREAM_REAL_IP_HEADER`: Set Nginx config for [ngx_http_realip_module#real_ip_header](http://nginx.org/en/docs/http/ngx_http_realip_module.html#real_ip_header), defaults to `X-Forwarded-For` +- `UPSTREAM_REAL_IP_RECURSIVE`: Set Nginx config for [ngx_http_realip_module#real_ip_recursive](http://nginx.org/en/docs/http/ngx_http_realip_module.html#real_ip_recursive) Set defaults to `off` +- `FRAPPE_SITE_NAME_HEADER`: Set proxy header `X-Frappe-Site-Name` and serve site named in the header, defaults to `$host`, i.e. find site name from host header. More details [below](#frappe_site_name_header) +- `PROXY_READ_TIMEOUT`: Upstream gunicorn service timeout, defaults to `120` +- `CLIENT_MAX_BODY_SIZE`: Max body size for uploads, defaults to `50m` + +To bypass `nginx-entrypoint.sh`, mount desired `/etc/nginx/conf.d/default.conf` and run `nginx -g 'daemon off;'` as container command. + +## Configuration + +We use environment variables to configure our setup. docker-compose uses variables from `.env` file. To get started, copy `example.env` to `.env`. + +### `FRAPPE_VERSION` + +Frappe framework release. You can find all releases [here](https://github.com/frappe/frappe/releases). + +### `DB_PASSWORD` + +Password for MariaDB (or Postgres) database. + +### `DB_HOST` + +Hostname for MariaDB (or Postgres) database. Set only if external service for database is used. + +### `DB_PORT` + +Port for MariaDB (3306) or Postgres (5432) database. Set only if external service for database is used. + +### `REDIS_CACHE` + +Hostname for redis server to store cache. Set only if external service for redis is used. + +### `REDIS_QUEUE` + +Hostname for redis server to store queue data. Set only if external service for redis is used. + +### `REDIS_SOCKETIO` + +Hostname for redis server to store socketio data. Set only if external service for redis is used. + +### `ERPNEXT_VERSION` + +ERPNext [release](https://github.com/frappe/frappe/releases). This variable is required if you use ERPNext override. + +### `LETSENCRYPT_EMAIL` + +Email that used to register https certificate. This one is required only if you use HTTPS override. + +### `FRAPPE_SITE_NAME_HEADER` + +This environment variable is not required. Default value is `$$host` which resolves site by host. For example, if your host is `example.com`, site's name should be `example.com`, or if host is `127.0.0.1` (local debugging), it should be `127.0.0.1` This variable allows to override described behavior. Let's say you create site named `mysite` and do want to access it by `127.0.0.1` host. Than you would set this variable to `mysite`. + +There is other variables not mentioned here. They're somewhat internal and you don't have to worry about them except you want to change main compose file. diff --git a/docs/images-and-compose-files.md b/docs/images-and-compose-files.md deleted file mode 100644 index dafc46a6..00000000 --- a/docs/images-and-compose-files.md +++ /dev/null @@ -1,101 +0,0 @@ -# Images - -There's 4 images that you can find in `/images` directory: - -- `bench`. It is used for development. [Learn more how to start development](../development/README.md). -- `nginx`. This image contains JS and CSS assets. Container using this image also routes incoming requests using [nginx](https://www.nginx.com). -- `socketio`. Container using this image processes realtime websocket requests using [Socket.IO](https://socket.io). -- `worker`. Multi-purpose Python backend. Runs [Werkzeug server](https://werkzeug.palletsprojects.com/en/2.0.x/) with [gunicorn](https://gunicorn.org), queues (via `bench worker`), or schedule (via `bench schedule`). - -> `nginx`, `socketio` and `worker` images — everything we need to be able to run all processes that Frappe framework requires (take a look at [Bench Procfile reference](https://frappeframework.com/docs/v13/user/en/bench/resources/bench-procfile)). We follow [Docker best practices](https://docs.docker.com/develop/develop-images/dockerfile_best-practices/#decouple-applications) and split these processes to different containers. - -> ERPNext images don't have their own Dockerfiles. We use [multi-stage builds](https://docs.docker.com/develop/develop-images/multistage-build/) and [Docker Buildx](https://docs.docker.com/engine/reference/commandline/buildx/) to reuse as much things as possible and make our builds more efficient. - -# Compose files - -After building the images we have to run the containers. The best and simplest way to do this is to use [compose files](https://docs.docker.com/compose/compose-file/). - -We have one main compose file, `compose.yaml`. Services described, networking, volumes are also handled there. - -## Services - -All services are described in `compose.yaml` - -- `configurator`. Updates `common_site_config.json` so Frappe knows how to access db and redis. It is executed on every `docker-compose up` (and exited immediately). Other services start after this container exits successfully. -- `backend`. [Werkzeug server](https://werkzeug.palletsprojects.com/en/2.0.x/). -- `db`. Optional service that runs [MariaDB](https://mariadb.com) if you also use `overrides/compose.mariadb.yaml` or [Postgres](https://www.postgresql.org) if you also use `overrides/compose.postgres.yaml`. -- `redis`. Optional service that runs [Redis](https://redis.io) server with cache, [Socket.IO](https://socket.io) and queues data. -- `frontend`. [nginx](https://www.nginx.com) server that serves JS/CSS assets and routes incoming requests. -- `proxy`. [Traefik](https://traefik.io/traefik/) proxy. It is here for complicated setups or HTTPS override (with `overrides/compose.https.yaml`). -- `websocket`. Node server that runs [Socket.IO](https://socket.io). -- `queue-short`, `queue-default`, `queue-long`. Python servers that run job queues using [rq](https://python-rq.org). -- `scheduler`. Python server that runs tasks on schedule using [schedule](https://schedule.readthedocs.io/en/stable/). - -## Overrides - -We have several [overrides](https://docs.docker.com/compose/extends/): - -- `overrides/compose.proxy.yaml`. Adds traefik proxy to setup. -- `overrides/compose.noproxy.yaml`. Publishes `frontend` ports directly without any proxy. -- `overrides/compose.erpnext.yaml`. Replaces all Frappe images with ERPNext ones. ERPNext images are built on top of Frappe ones, so it is safe to replace them. -- `overrides/compose.https.yaml`. Automatically sets up Let's Encrypt certificate and redirects all requests to directed to http, to https. -- `overrides/compose.mariadb.yaml`. Adds `db` service and sets its image to MariaDB. -- `overrides/compose.postgres.yaml`. Adds `db` service and sets its image to Postgres. Note that ERPNext currently doesn't support Postgres. -- `overrides/compose.redis.yaml`. Adds `redis` service and sets its image to `redis`. - -It is quite simple to run overrides. All we need to do is to specify compose files that should be used by docker-compose. For example, we want ERPNext: - -```bash -# Point to main compose file (compose.yaml) and add one more. -docker-compose -f compose.yaml -f overrides/compose.erpnext.yaml config -``` - -⚠ Make sure to use docker-compose v2 (run `docker-compose -v` to check). If you want to use v1 make sure the correct `$`-signs as they get duplicated by the `config` command! - -That's it! Of course, we also have to setup `.env` before all of that, but that's not the point. - -## Configuration - -We use environment variables to configure our setup. docker-compose uses variables from `.env` file. To get started, copy `example.env` to `.env`. - -### `FRAPPE_VERSION` - -Frappe framework release. You can find all releases [here](https://github.com/frappe/frappe/releases). - -### `DB_PASSWORD` - -Password for MariaDB (or Postgres) database. - -### `DB_HOST` - -Hostname for MariaDB (or Postgres) database. Set only if external service for database is used. - -### `DB_PORT` - -Port for MariaDB (3306) or Postgres (5432) database. Set only if external service for database is used. - -### `REDIS_CACHE` - -Hostname for redis server to store cache. Set only if external service for redis is used. - -### `REDIS_QUEUE` - -Hostname for redis server to store queue data. Set only if external service for redis is used. - -### `REDIS_SOCKETIO` - -Hostname for redis server to store socketio data. Set only if external service for redis is used. - -### `ERPNEXT_VERSION` - -ERPNext [release](https://github.com/frappe/frappe/releases). This variable is required if you use ERPNext override. - -### `LETSENCRYPT_EMAIL` - -Email that used to register https certificate. This one is required only if you use HTTPS override. - -### `FRAPPE_SITE_NAME_HEADER` - -This environment variable is not required. Default value is `$$host` which resolves site by host. For example, if your host is `example.com`, site's name should be `example.com`, or if host is `127.0.0.1` (local debugging), it should be `127.0.0.1` This variable allows to override described behavior. Let's say you create site named `mysite` and do want to access it by `127.0.0.1` host. Than you would set this variable to `mysite`. - -There is other variables not mentioned here. They're somewhat internal and you don't have to worry about them except you want to change main compose file. diff --git a/docs/list-of-containers.md b/docs/list-of-containers.md new file mode 100644 index 00000000..c4c3e701 --- /dev/null +++ b/docs/list-of-containers.md @@ -0,0 +1,58 @@ +# Images + +There's 4 images that you can find in `/images` directory: + +- `bench`. It is used for development. [Learn more how to start development](../development/README.md). +- `production`. + - Multi-purpose Python backend. Runs [Werkzeug server](https://werkzeug.palletsprojects.com/en/2.0.x/) with [gunicorn](https://gunicorn.org), queues (via `bench worker`), or schedule (via `bench schedule`). + - Contains JS and CSS assets and routes incoming requests using [nginx](https://www.nginx.com). + - Processes realtime websocket requests using [Socket.IO](https://socket.io). +- `custom`. It is used to build bench using `apps.json` file set with `--apps_path` during bench initialization. `apps.json` is a json array. e.g. `[{"url":"{{repo_url}}","branch":"{{repo_branch}}"}]` + +Image has everything we need to be able to run all processes that Frappe framework requires (take a look at [Bench Procfile reference](https://frappeframework.com/docs/v14/user/en/bench/resources/bench-procfile)). We follow [Docker best practices](https://docs.docker.com/develop/develop-images/dockerfile_best-practices/#decouple-applications) and split these processes to different containers. + +> We use [multi-stage builds](https://docs.docker.com/develop/develop-images/multistage-build/) and [Docker Buildx](https://docs.docker.com/engine/reference/commandline/buildx/) to reuse as much things as possible and make our builds more efficient. + +# Compose files + +After building the images we have to run the containers. The best and simplest way to do this is to use [compose files](https://docs.docker.com/compose/compose-file/). + +We have one main compose file, `compose.yaml`. Services described, networking, volumes are also handled there. + +## Services + +All services are described in `compose.yaml` + +- `configurator`. Updates `common_site_config.json` so Frappe knows how to access db and redis. It is executed on every `docker-compose up` (and exited immediately). Other services start after this container exits successfully. +- `backend`. [Werkzeug server](https://werkzeug.palletsprojects.com/en/2.0.x/). +- `db`. Optional service that runs [MariaDB](https://mariadb.com) if you also use `overrides/compose.mariadb.yaml` or [Postgres](https://www.postgresql.org) if you also use `overrides/compose.postgres.yaml`. +- `redis`. Optional service that runs [Redis](https://redis.io) server with cache, [Socket.IO](https://socket.io) and queues data. +- `frontend`. [nginx](https://www.nginx.com) server that serves JS/CSS assets and routes incoming requests. +- `proxy`. [Traefik](https://traefik.io/traefik/) proxy. It is here for complicated setups or HTTPS override (with `overrides/compose.https.yaml`). +- `websocket`. Node server that runs [Socket.IO](https://socket.io). +- `queue-short`, `queue-default`, `queue-long`. Python servers that run job queues using [rq](https://python-rq.org). +- `scheduler`. Python server that runs tasks on schedule using [schedule](https://schedule.readthedocs.io/en/stable/). + +## Overrides + +We have several [overrides](https://docs.docker.com/compose/extends/): + +- `overrides/compose.proxy.yaml`. Adds traefik proxy to setup. +- `overrides/compose.noproxy.yaml`. Publishes `frontend` ports directly without any proxy. +- `overrides/compose.https.yaml`. Automatically sets up Let's Encrypt certificate and redirects all requests to directed to http, to https. +- `overrides/compose.mariadb.yaml`. Adds `db` service and sets its image to MariaDB. +- `overrides/compose.postgres.yaml`. Adds `db` service and sets its image to Postgres. Note that ERPNext currently doesn't support Postgres. +- `overrides/compose.redis.yaml`. Adds `redis` service and sets its image to `redis`. + +It is quite simple to run overrides. All we need to do is to specify compose files that should be used by docker-compose. For example, we want ERPNext: + +```bash +# Point to main compose file (compose.yaml) and add one more. +docker-compose -f compose.yaml -f overrides/compose.redis.yaml config +``` + +⚠ Make sure to use docker-compose v2 (run `docker-compose -v` to check). If you want to use v1 make sure the correct `$`-signs as they get duplicated by the `config` command! + +That's it! Of course, we also have to setup `.env` before all of that, but that's not the point. + +Check [environment variables](environment-variables.md) for more. diff --git a/docs/migrate-from-multi-image-setup.md b/docs/migrate-from-multi-image-setup.md new file mode 100644 index 00000000..5887d0c1 --- /dev/null +++ b/docs/migrate-from-multi-image-setup.md @@ -0,0 +1,115 @@ +## Migrate from multi-image setup + +All the containers now use same image. Use `frappe/erpnext` instead of `frappe/frappe-worker`, `frappe/frappe-nginx` , `frappe/frappe-socketio` , `frappe/erpnext-worker` and `frappe/erpnext-nginx`. + +Now you need to specify command and environment variables for following containers: + +### Frontend + +For `frontend` service to act as static assets frontend and reverse proxy, you need to pass `nginx-entrypoint.sh` as container `command` and `BACKEND` and `SOCKETIO` environment variables pointing `{host}:{port}` for gunicorn and websocket services. Check [environment variables](environment-variables.md) + +Now you only need to mount the `sites` volume at location `/home/frappe/frappe-bench/sites`. No need for `assets` volume and asset population script or steps. + +Example change: + +```yaml +# ... removed for brevity +frontend: + image: frappe/erpnext:${ERPNEXT_VERSION:?ERPNext version not set} + command: + - nginx-entrypoint.sh + environment: + BACKEND: backend:8000 + SOCKETIO: websocket:9000 + volumes: + - sites:/home/frappe/frappe-bench/sites +# ... removed for brevity +``` + +### Websocket + +For `websocket` service to act as socketio backend, you need to pass `["node", "/home/frappe/frappe-bench/apps/frappe/socketio.js"]` as container `command` + +Example change: + +```yaml +# ... removed for brevity +websocket: + image: frappe/erpnext:${ERPNEXT_VERSION:?ERPNext version not set} + command: + - node + - /home/frappe/frappe-bench/apps/frappe/socketio.js +# ... removed for brevity +``` + +### Configurator + +For `configurator` service to act as run once configuration job, you need to pass `["bash", "-c"]` as container `entrypoint` and bash script inline to yaml. There is no `configure.py` in the container now. + +Example change: + +```yaml +# ... removed for brevity +configurator: + image: frappe/erpnext:${ERPNEXT_VERSION:?ERPNext version not set} + restart: "no" + entrypoint: + - bash + - -c + command: + - > + bench set-config -g db_host $$DB_HOST; + bench set-config -gp db_port $$DB_PORT; + bench set-config -g redis_cache "redis://$$REDIS_CACHE"; + bench set-config -g redis_queue "redis://$$REDIS_QUEUE"; + bench set-config -g redis_socketio "redis://$$REDIS_SOCKETIO"; + bench set-config -gp socketio_port $$SOCKETIO_PORT; + environment: + DB_HOST: db + DB_PORT: "3306" + REDIS_CACHE: redis-cache:6379 + REDIS_QUEUE: redis-queue:6379 + REDIS_SOCKETIO: redis-socketio:6379 + SOCKETIO_PORT: "9000" +# ... removed for brevity +``` + +### Site Creation + +For `create-site` service to act as run once site creation job, you need to pass `["bash", "-c"]` as container `entrypoint` and bash script inline to yaml. Make sure to use `--no-mariadb-socket` as upstream bench is installed in container. + +The `WORKDIR` has changed to `/home/frappe/frappe-bench` like `bench` setup we are used to. So the path to find `common_site_config.json` has changed to `sites/common_site_config.json`. + +Example change: + +```yaml +# ... removed for brevity +create-site: + image: frappe/erpnext:${ERPNEXT_VERSION:?ERPNext version not set} + restart: "no" + entrypoint: + - bash + - -c + command: + - > + wait-for-it -t 120 db:3306; + wait-for-it -t 120 redis-cache:6379; + wait-for-it -t 120 redis-queue:6379; + wait-for-it -t 120 redis-socketio:6379; + export start=`date +%s`; + until [[ -n `grep -hs ^ sites/common_site_config.json | jq -r ".db_host // empty"` ]] && \ + [[ -n `grep -hs ^ sites/common_site_config.json | jq -r ".redis_cache // empty"` ]] && \ + [[ -n `grep -hs ^ sites/common_site_config.json | jq -r ".redis_queue // empty"` ]]; + do + echo "Waiting for sites/common_site_config.json to be created"; + sleep 5; + if (( `date +%s`-start > 120 )); then + echo "could not find sites/common_site_config.json with required keys"; + exit 1 + fi + done; + echo "sites/common_site_config.json found"; + bench new-site frontend --no-mariadb-socket --admin-password=admin --db-root-password=admin --install-app payments --install-app erpnext --set-default; + +# ... removed for brevity +``` diff --git a/docs/patch-code-from-images.md b/docs/patch-code-from-images.md deleted file mode 100644 index 67245098..00000000 --- a/docs/patch-code-from-images.md +++ /dev/null @@ -1,34 +0,0 @@ -Example: https://discuss.erpnext.com/t/sms-two-factor-authentication-otp-msg-change/47835 - -Above example needs following Dockerfile based patch - -```Dockerfile -FROM frappe/erpnext-worker:v12.17.0 - -... -USER root -RUN sed -i -e "s/Your verification code is/আপনার লগইন কোড/g" /home/frappe/frappe-bench/apps/frappe/frappe/twofactor.py -USER frappe -... - -``` - -Example for `nginx` image, - -```Dockerfile -FROM frappe/erpnext-nginx:v13.27.0 - -# Hack to use Frappe/ERPNext offline. -RUN sed -i 's/navigator.onLine/navigator.onLine||true/' \ - /usr/share/nginx/html/assets/js/desk.min.js \ - /usr/share/nginx/html/assets/js/dialog.min.js \ - /usr/share/nginx/html/assets/js/frappe-web.min.js -``` - -Alternatively copy the modified source code file directly over `/home/frappe/frappe-bench/apps/frappe/frappe/twofactor.py` - -```Dockerfile -... -COPY twofactor.py /home/frappe/frappe-bench/apps/frappe/frappe/twofactor.py -... -``` diff --git a/docs/port-based-multi-tenancy.md b/docs/port-based-multi-tenancy.md index 660c30bf..2f469bfd 100644 --- a/docs/port-based-multi-tenancy.md +++ b/docs/port-based-multi-tenancy.md @@ -6,43 +6,64 @@ Remove the traefik service from docker-compose.yml ### Step 2 -Create nginx config file `/opt/nginx/conf/serve-8001.conf`: +Add service for each port that needs to be exposed. -``` -server { - listen 8001; - server_name $http_host; +e.g. `port-site-1`, `port-site-2`, `port-site-3`. - location / { - - rewrite ^(.+)/$ $1 permanent; - rewrite ^(.+)/index\.html$ $1 permanent; - rewrite ^(.+)\.html$ $1 permanent; - - proxy_set_header X-Frappe-Site-Name mysite.localhost; - proxy_set_header Host mysite.localhost; - proxy_pass http://frontend; - } -} +```yaml +# ... removed for brevity +services: + # ... removed for brevity + port-site-1: + image: frappe/erpnext:v14.11.1 + deploy: + restart_policy: + condition: on-failure + command: + - nginx-entrypoint.sh + environment: + BACKEND: backend:8000 + FRAPPE_SITE_NAME_HEADER: site1.local + SOCKETIO: websocket:9000 + volumes: + - sites:/home/frappe/frappe-bench/sites + ports: + - "8080:8080" + port-site-2: + image: frappe/erpnext:v14.11.1 + deploy: + restart_policy: + condition: on-failure + command: + - nginx-entrypoint.sh + environment: + BACKEND: backend:8000 + FRAPPE_SITE_NAME_HEADER: site2.local + SOCKETIO: websocket:9000 + volumes: + - sites:/home/frappe/frappe-bench/sites + ports: + - "8081:8080" + port-site-3: + image: frappe/erpnext:v14.11.1 + deploy: + restart_policy: + condition: on-failure + command: + - nginx-entrypoint.sh + environment: + BACKEND: backend:8000 + FRAPPE_SITE_NAME_HEADER: site3.local + SOCKETIO: websocket:9000 + volumes: + - sites:/home/frappe/frappe-bench/sites + ports: + - "8082:8080" ``` Notes: -- Replace the port with any port of choice e.g. `listen 4200;` -- Change `mysite.localhost` to site name -- Repeat the server blocks for multiple ports and site names to get the effect of port based multi tenancy -- For old images use `proxy_pass http://erpnext-nginx` instead of `proxy_pass http://frontend` - -### Step 3 - -Run the docker container - -```shell -docker run --network=_default \ - -p 8001:8001 \ - --volume=/opt/nginx/conf/serve-8001.conf:/etc/nginx/conf.d/default.conf -d nginx -``` - -Note: Change the volumes, network and ports as needed - -With the above example configured site will be accessible on `http://localhost:8001` +- Above setup will expose `site1.local`, `site2.local`, `site3.local` on port `8080`, `8081`, `8082` respectively. +- Change `site1.local` to site name to serve from bench. +- Change the `BACKEND` and `SOCKETIO` environment variables as per your service names. +- Make sure `sites:` volume is available as part of yaml. diff --git a/docs/setup-options.md b/docs/setup-options.md index bc74d4d1..0fd9fbbe 100644 --- a/docs/setup-options.md +++ b/docs/setup-options.md @@ -60,7 +60,6 @@ environment variables or the `configurator` will fail. # Generate YAML docker compose -f compose.yaml \ -f overrides/compose.proxy.yaml \ - -f overrides/compose.erpnext.yaml \ config > ~/gitops/docker-compose.yml # Start containers @@ -86,7 +85,6 @@ docker compose --project-name -f ~/gitops/docker-compose.yml up - ```sh # Generate YAML docker compose -f compose.yaml \ - -f overrides/compose.erpnext.yaml \ -f overrides/compose.mariadb.yaml \ -f overrides/compose.redis.yaml \ -f overrides/compose.https.yaml \ @@ -110,7 +108,6 @@ nano .env # Pull new images docker compose -f compose.yaml \ - -f overrides/compose.erpnext.yaml \ # ... your other overrides config > ~/gitops/docker-compose.yml diff --git a/docs/single-compose-setup.md b/docs/single-compose-setup.md new file mode 100644 index 00000000..59bf4312 --- /dev/null +++ b/docs/single-compose-setup.md @@ -0,0 +1,40 @@ +# Single Compose Setup + +This setup is a very simple single compose file that does everything to start required services and a frappe-bench. It is used to start play with docker instance with a site. The file is located in the root of repo and named `pwd.yml`. + +## Services + +### frappe-bench components + +- backend, serves gunicorn backend +- frontend, serves static assets through nginx frontend reverse proxies websocket and gunicorn. +- queue-default, default rq worker. +- queue-long, long rq worker. +- queue-short, short rq worker. +- schedule, event scheduler. +- websocket, socketio websocket for realtime communication. + +### Run once configuration + +- configurator, configures `common_site_config.json` to set db and redis hosts. +- create-site, creates one site to serve as default site for the frappe-bench. + +### Service dependencies + +- db, mariadb, container with frappe specific configuration. +- redis-cache, redis for cache data. +- redis-queue, redis for rq data. +- redis-socketio, redis for socketio pubsub. + +## Volumes + +- sites: Volume for bench data. Common config, all sites, all site configs and site files will be stored here. +- logs: Volume for bench logs. all process logs are dumped here. No need to mount it. Each container will create a temporary volume for logs if not specified. + +## Adaptation + +If you understand containers use the `pwd.yml` as a reference to build more complex setup like, single server example, Docker Swarm stack, Kubernetes Helm chart, etc. + +This serves only site called `frontend` through the nginx. `FRAPPE_SITE_NAME_HEADER` is set to `frontend` and a default site called `frontend` is created. + +Change the `$$host` will allow container to accept any host header and serve that site. To escape `$` in compose yaml use it like `$$`. To unset default site remove `currentsite.txt` file from `sites` directory. diff --git a/docs/single-server-example.md b/docs/single-server-example.md index e125d2d3..11f5b12b 100644 --- a/docs/single-server-example.md +++ b/docs/single-server-example.md @@ -87,13 +87,13 @@ Deploy the traefik container with letsencrypt SSL ```shell docker compose --project-name traefik \ --env-file ~/gitops/traefik.env \ - -f docs/compose/compose.traefik.yaml \ - -f docs/compose/compose.traefik-ssl.yaml up -d + -f overrides/compose.traefik.yaml \ + -f overrides/compose.traefik-ssl.yaml up -d ``` This will make the traefik dashboard available on `traefik.example.com` and all certificates will reside in `/data/traefik/certificates` on host filesystem. -For LAN setup deploy the traefik container without overriding `docs/compose/compose.traefik-ssl.yaml`. +For LAN setup deploy the traefik container without overriding `overrides/compose.traefik-ssl.yaml`. ### Install MariaDB @@ -120,7 +120,7 @@ Note: Change the password from `changeit` to more secure one. Deploy the mariadb container ```shell -docker compose --project-name mariadb --env-file ~/gitops/mariadb.env -f docs/compose/compose.mariadb-shared.yaml up -d +docker compose --project-name mariadb --env-file ~/gitops/mariadb.env -f overrides/compose.mariadb-shared.yaml up -d ``` This will make `mariadb-database` service available under `mariadb-network`. Data will reside in `/data/mariadb`. @@ -155,10 +155,9 @@ Create a yaml file called `erpnext-one.yaml` in `~/gitops` directory: docker compose --project-name erpnext-one \ --env-file ~/gitops/erpnext-one.env \ -f compose.yaml \ - -f overrides/compose.erpnext.yaml \ -f overrides/compose.redis.yaml \ - -f docs/compose/compose.multi-bench.yaml \ - -f docs/compose/compose.multi-bench-ssl.yaml config > ~/gitops/erpnext-one.yaml + -f overrides/compose.multi-bench.yaml \ + -f overrides/compose.multi-bench-ssl.yaml config > ~/gitops/erpnext-one.yaml ``` For LAN setup do not override `compose.multi-bench-ssl.yaml`. @@ -176,7 +175,7 @@ Create sites `one.example.com` and `two.example.com`: ```shell # one.example.com docker compose --project-name erpnext-one exec backend \ - bench new-site one.example.com --mariadb-root-password changeit --install-app erpnext --admin-password changeit + bench new-site one.example.com --no-mariadb-socket --mariadb-root-password changeit --install-app erpnext --admin-password changeit ``` You can stop here and have a single bench single site setup complete. Continue to add one more site to the current bench. @@ -184,7 +183,7 @@ You can stop here and have a single bench single site setup complete. Continue t ```shell # two.example.com docker compose --project-name erpnext-one exec backend \ - bench new-site two.example.com --mariadb-root-password changeit --install-app erpnext --admin-password changeit + bench new-site two.example.com --no-mariadb-socket --mariadb-root-password changeit --install-app erpnext --admin-password changeit ``` #### Create second bench @@ -217,10 +216,9 @@ Create a yaml file called `erpnext-two.yaml` in `~/gitops` directory: docker compose --project-name erpnext-two \ --env-file ~/gitops/erpnext-two.env \ -f compose.yaml \ - -f overrides/compose.erpnext.yaml \ -f overrides/compose.redis.yaml \ - -f docs/compose/compose.multi-bench.yaml \ - -f docs/compose/compose.multi-bench-ssl.yaml config > ~/gitops/erpnext-two.yaml + -f overrides/compose.multi-bench.yaml \ + -f overrides/compose.multi-bench-ssl.yaml config > ~/gitops/erpnext-two.yaml ``` Use the above command after any changes are made to `erpnext-two.env` file to regenerate `~/gitops/erpnext-two.yaml`. e.g. after changing version to migrate the bench. @@ -236,10 +234,10 @@ Create sites `three.example.com` and `four.example.com`: ```shell # three.example.com docker compose --project-name erpnext-two exec backend \ - bench new-site three.example.com --mariadb-root-password changeit --install-app erpnext --admin-password changeit + bench new-site three.example.com --no-mariadb-socket --mariadb-root-password changeit --install-app erpnext --admin-password changeit # four.example.com docker compose --project-name erpnext-two exec backend \ - bench new-site four.example.com --mariadb-root-password changeit --install-app erpnext --admin-password changeit + bench new-site four.example.com --no-mariadb-socket --mariadb-root-password changeit --install-app erpnext --admin-password changeit ``` #### Create custom domain to existing site @@ -271,8 +269,8 @@ Generate yaml to reverse proxy: ```shell docker compose --project-name custom-one-example \ --env-file ~/gitops/custom-one-example.env \ - -f docs/compose/compose.custom-domain.yaml \ - -f docs/compose/compose.custom-domain-ssl.yaml config > ~/gitops/custom-one-example.yaml + -f overrides/compose.custom-domain.yaml \ + -f overrides/compose.custom-domain-ssl.yaml config > ~/gitops/custom-one-example.yaml ``` For LAN setup do not override `compose.custom-domain-ssl.yaml`. diff --git a/docs/site-operations.md b/docs/site-operations.md index f012350c..fc81add4 100644 --- a/docs/site-operations.md +++ b/docs/site-operations.md @@ -9,7 +9,7 @@ Note: - Wait for the `db` service to start and `configurator` to exit before trying to create a new site. Usually this takes up to 10 seconds. ```sh -docker-compose exec backend bench new-site --mariadb-root-password --admin-password +docker-compose exec backend bench new-site --no-mariadb-socket --mariadb-root-password --admin-password ``` If you need to install some app, specify `--install-app`. To see all options, just run `bench new-site --help`. @@ -24,7 +24,7 @@ docker-compose exec backend bench set-config -g root_password Also command is slightly different: ```sh -docker-compose exec backend bench new-site --db-type postgres --admin-password +docker-compose exec backend bench new-site --no-mariadb-socket --db-type postgres --admin-password ``` ## Push backup to S3 storage diff --git a/docs/troubleshoot.md b/docs/troubleshoot.md index ca7d9bc7..eb5a9167 100644 --- a/docs/troubleshoot.md +++ b/docs/troubleshoot.md @@ -1,5 +1,4 @@ 1. [Fixing MariaDB issues after rebuilding the container](#fixing-mariadb-issues-after-rebuilding-the-container) -1. [Letsencrypt companion not working](#letsencrypt-companion-not-working) 1. [docker-compose does not recognize variables from `.env` file](#docker-compose-does-not-recognize-variables-from-env-file) 1. [Windows Based Installation](#windows-based-installation) @@ -46,12 +45,11 @@ Note: For MariaDB 10.4 and above use `mysql.global_priv` instead of `mysql.user` ### docker-compose does not recognize variables from `.env` file -If you are using old version of `docker-compose` the .env file needs to be located in directory from where the docker-compose command is executed. There may also be difference in official `docker-compose` and the one packaged by distro. +If you are using old version of `docker-compose` the .env file needs to be located in directory from where the docker-compose command is executed. There may also be difference in official `docker-compose` and the one packaged by distro. Use `--env-file=.env` if available to explicitly specify the path to file. ### Windows Based Installation - Set environment variable `COMPOSE_CONVERT_WINDOWS_PATHS` e.g. `set COMPOSE_CONVERT_WINDOWS_PATHS=1` -- Make the `frappe-mariadb.cnf` read-only for mariadb container to pick it up. - While using docker machine, port-forward the port 80 of VM to port 80 of host machine - Name all the sites ending with `.localhost`. and access it via browser locally. e.g. `http://site1.localhost` - related issue comment https://github.com/frappe/frappe_docker/issues/448#issuecomment-851723912 diff --git a/images/custom/Containerfile b/images/custom/Containerfile new file mode 100644 index 00000000..66260330 --- /dev/null +++ b/images/custom/Containerfile @@ -0,0 +1,145 @@ +ARG PYTHON_VERSION=3.10.5 +FROM python:${PYTHON_VERSION}-slim-bullseye AS base + +ARG WKHTMLTOPDF_VERSION=0.12.6-1 +ARG NODE_VERSION=16.18.0 +ENV NVM_DIR=/home/frappe/.nvm +ENV PATH ${NVM_DIR}/versions/node/v${NODE_VERSION}/bin/:${PATH} + +RUN useradd -ms /bin/bash frappe \ + && apt-get update \ + && apt-get install --no-install-recommends -y \ + curl \ + git \ + vim \ + nginx \ + gettext-base \ + # MariaDB + mariadb-client \ + # Postgres + libpq-dev \ + postgresql-client \ + # For healthcheck + wait-for-it \ + jq \ + # NodeJS + && mkdir -p ${NVM_DIR} \ + && curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.2/install.sh | bash \ + && . ${NVM_DIR}/nvm.sh \ + && nvm install ${NODE_VERSION} \ + && nvm use v${NODE_VERSION} \ + && npm install -g yarn \ + && nvm alias default v${NODE_VERSION} \ + && rm -rf ${NVM_DIR}/.cache \ + && echo 'export NVM_DIR="/home/frappe/.nvm"' >>/home/frappe/.bashrc \ + && echo '[ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" # This loads nvm' >>/home/frappe/.bashrc \ + && echo '[ -s "$NVM_DIR/bash_completion" ] && \. "$NVM_DIR/bash_completion" # This loads nvm bash_completion' >>/home/frappe/.bashrc \ + # Install wkhtmltopdf with patched qt + && if [ "$(uname -m)" = "aarch64" ]; then export ARCH=arm64; fi \ + && if [ "$(uname -m)" = "x86_64" ]; then export ARCH=amd64; fi \ + && downloaded_file=wkhtmltox_$WKHTMLTOPDF_VERSION.buster_${ARCH}.deb \ + && curl -sLO https://github.com/wkhtmltopdf/packaging/releases/download/$WKHTMLTOPDF_VERSION/$downloaded_file \ + && apt-get install -y ./$downloaded_file \ + && rm $downloaded_file \ + # Clean up + && rm -rf /var/lib/apt/lists/* \ + && rm -fr /etc/nginx/sites-enabled/default \ + && pip3 install frappe-bench \ + # Fixes for non-root nginx and logs to stdout + && sed -i '/user www-data/d' /etc/nginx/nginx.conf \ + && ln -sf /dev/stdout /var/log/nginx/access.log && ln -sf /dev/stderr /var/log/nginx/error.log \ + && touch /run/nginx.pid \ + && chown -R frappe:frappe /etc/nginx/conf.d \ + && chown -R frappe:frappe /etc/nginx/nginx.conf \ + && chown -R frappe:frappe /var/log/nginx \ + && chown -R frappe:frappe /var/lib/nginx \ + && chown -R frappe:frappe /run/nginx.pid + +COPY resources/nginx-template.conf /templates/nginx/frappe.conf.template +COPY resources/nginx-entrypoint.sh /usr/local/bin/nginx-entrypoint.sh +COPY resources/push_backup.py /usr/local/bin/push-backup + +FROM base AS builder + +RUN apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y \ + # For frappe framework + wget \ + # For psycopg2 + libpq-dev \ + # Other + libffi-dev \ + liblcms2-dev \ + libldap2-dev \ + libmariadb-dev \ + libsasl2-dev \ + libtiff5-dev \ + libwebp-dev \ + redis-tools \ + rlwrap \ + tk8.6-dev \ + cron \ + # For pandas + gcc \ + build-essential \ + libbz2-dev \ + && rm -rf /var/lib/apt/lists/* + +# apps.json includes +ARG APPS_JSON_BASE64 +RUN if [ -n "${APPS_JSON_BASE64}" ]; then \ + mkdir /opt/frappe && echo "${APPS_JSON_BASE64}" | base64 -d > /opt/frappe/apps.json; \ + fi + +USER frappe + +ARG REMOVE_GIT_REMOTE +ARG FRAPPE_BRANCH=version-14 +ARG FRAPPE_PATH=https://github.com/frappe/frappe +RUN export APP_INSTALL_ARGS="" && \ + if [ -n "${APPS_JSON_BASE64}" ]; then \ + export APP_INSTALL_ARGS="--apps_path=/opt/frappe/apps.json"; \ + fi && \ + bench init ${APP_INSTALL_ARGS}\ + --frappe-branch=${FRAPPE_BRANCH} \ + --frappe-path=${FRAPPE_PATH} \ + --no-procfile \ + --no-backups \ + --skip-redis-config-generation \ + --verbose \ + /home/frappe/frappe-bench && \ + cd /home/frappe/frappe-bench && \ + echo "$(jq 'del(.db_host, .redis_cache, .redis_queue, .redis_socketio)' sites/common_site_config.json)" \ + > sites/common_site_config.json && \ + if [ -n "${REMOVE_GIT_REMOTE}" ]; then \ + find apps -name .git -type d -prune | xargs -i git --git-dir {} remote rm upstream; \ + fi + +WORKDIR /home/frappe/frappe-bench + +FROM base as backend + +USER frappe + +COPY --from=builder --chown=frappe:frappe /home/frappe/frappe-bench /home/frappe/frappe-bench + +WORKDIR /home/frappe/frappe-bench + +VOLUME [ \ + "/home/frappe/frappe-bench/sites", \ + "/home/frappe/frappe-bench/sites/assets", \ + "/home/frappe/frappe-bench/logs" \ +] + +CMD [ \ + "/home/frappe/frappe-bench/env/bin/gunicorn", \ + "--chdir=/home/frappe/frappe-bench/sites", \ + "--bind=0.0.0.0:8000", \ + "--threads=4", \ + "--workers=2", \ + "--worker-class=gthread", \ + "--worker-tmp-dir=/dev/shm", \ + "--timeout=120", \ + "--preload", \ + "frappe.app:application" \ +] diff --git a/images/nginx/Dockerfile b/images/nginx/Dockerfile deleted file mode 100644 index 29d92b79..00000000 --- a/images/nginx/Dockerfile +++ /dev/null @@ -1,62 +0,0 @@ -FROM frappe/bench:latest as assets_builder - -ARG FRAPPE_VERSION -ARG FRAPPE_REPO=https://github.com/frappe/frappe -ARG PYTHON_VERSION -ARG NODE_VERSION -ENV NVM_DIR=/home/frappe/.nvm -ENV PATH ${NVM_DIR}/versions/node/v${NODE_VERSION}/bin/:${PATH} -RUN PYENV_VERSION=${PYTHON_VERSION} bench init --version=${FRAPPE_VERSION} --frappe-path=${FRAPPE_REPO} --skip-redis-config-generation --verbose --skip-assets /home/frappe/frappe-bench - -WORKDIR /home/frappe/frappe-bench - - -FROM assets_builder as frappe_assets - -RUN bench setup requirements \ - && if [ -z "${FRAPPE_VERSION##*v14*}" ] || [ "$FRAPPE_VERSION" = "develop" ]; then \ - export BUILD_OPTS="--production";\ - fi \ - && FRAPPE_ENV=production bench build --verbose --hard-link ${BUILD_OPTS} - - -FROM assets_builder as erpnext_assets - -ARG ERPNEXT_VERSION -ARG ERPNEXT_REPO=https://github.com/frappe/erpnext -RUN bench get-app --branch=${ERPNEXT_VERSION} --skip-assets --resolve-deps erpnext ${ERPNEXT_REPO}\ - && if [ -z "${ERPNEXT_VERSION##*v14*}" ] || [ "$ERPNEXT_VERSION" = "develop" ]; then \ - export BUILD_OPTS="--production"; \ - fi \ - && FRAPPE_ENV=production bench build --verbose --hard-link ${BUILD_OPTS} - - -FROM alpine/git as bench - -# Error pages -ARG BENCH_REPO=https://github.com/frappe/bench -RUN git clone --depth 1 ${BENCH_REPO} /tmp/bench \ - && mkdir /out \ - && mv /tmp/bench/bench/config/templates/502.html /out \ - && touch /out/.build - -FROM nginxinc/nginx-unprivileged:1.23.3-alpine as frappe - -# Set default ENV variables for backwards compatibility -ENV PROXY_READ_TIMOUT=120 -ENV CLIENT_MAX_BODY_SIZE=50m - -# https://github.com/nginxinc/docker-nginx-unprivileged/blob/main/stable/alpine/20-envsubst-on-templates.sh -COPY nginx-template.conf /etc/nginx/templates/default.conf.template -# https://github.com/nginxinc/docker-nginx-unprivileged/blob/main/stable/alpine/docker-entrypoint.sh -COPY entrypoint.sh /docker-entrypoint.d/frappe-entrypoint.sh - -COPY --from=bench /out /usr/share/nginx/html/ -COPY --from=frappe_assets /home/frappe/frappe-bench/sites/assets /usr/share/nginx/html/assets - -USER 1000 - - -FROM frappe as erpnext - -COPY --from=erpnext_assets /home/frappe/frappe-bench/sites/assets /usr/share/nginx/html/assets diff --git a/images/nginx/entrypoint.sh b/images/nginx/entrypoint.sh deleted file mode 100755 index 78c15c9b..00000000 --- a/images/nginx/entrypoint.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/sh - -set -e - -# Update timestamp for ".build" file to enable caching assets: -# https://github.com/frappe/frappe/blob/52d8e6d952130eea64a9990b9fd5b1f6877be1b7/frappe/utils/__init__.py#L799-L805 -if [ -d /usr/share/nginx/html/sites ]; then - touch /usr/share/nginx/html/sites/.build -r /usr/share/nginx/html/.build -fi diff --git a/images/production/Containerfile b/images/production/Containerfile new file mode 100644 index 00000000..853f12ff --- /dev/null +++ b/images/production/Containerfile @@ -0,0 +1,132 @@ +ARG PYTHON_VERSION=3.10.5 +FROM python:${PYTHON_VERSION}-slim-bullseye AS base + +ARG WKHTMLTOPDF_VERSION=0.12.6-1 +ARG NODE_VERSION=16.18.0 +ENV NVM_DIR=/home/frappe/.nvm +ENV PATH ${NVM_DIR}/versions/node/v${NODE_VERSION}/bin/:${PATH} + +RUN useradd -ms /bin/bash frappe \ + && apt-get update \ + && apt-get install --no-install-recommends -y \ + curl \ + git \ + vim \ + nginx \ + gettext-base \ + # MariaDB + mariadb-client \ + # Postgres + libpq-dev \ + postgresql-client \ + # For healthcheck + wait-for-it \ + jq \ + # NodeJS + && mkdir -p ${NVM_DIR} \ + && curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.2/install.sh | bash \ + && . ${NVM_DIR}/nvm.sh \ + && nvm install ${NODE_VERSION} \ + && nvm use v${NODE_VERSION} \ + && npm install -g yarn \ + && nvm alias default v${NODE_VERSION} \ + && rm -rf ${NVM_DIR}/.cache \ + && echo 'export NVM_DIR="/home/frappe/.nvm"' >>/home/frappe/.bashrc \ + && echo '[ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" # This loads nvm' >>/home/frappe/.bashrc \ + && echo '[ -s "$NVM_DIR/bash_completion" ] && \. "$NVM_DIR/bash_completion" # This loads nvm bash_completion' >>/home/frappe/.bashrc \ + # Install wkhtmltopdf with patched qt + && if [ "$(uname -m)" = "aarch64" ]; then export ARCH=arm64; fi \ + && if [ "$(uname -m)" = "x86_64" ]; then export ARCH=amd64; fi \ + && downloaded_file=wkhtmltox_$WKHTMLTOPDF_VERSION.buster_${ARCH}.deb \ + && curl -sLO https://github.com/wkhtmltopdf/packaging/releases/download/$WKHTMLTOPDF_VERSION/$downloaded_file \ + && apt-get install -y ./$downloaded_file \ + && rm $downloaded_file \ + # Clean up + && rm -rf /var/lib/apt/lists/* \ + && rm -fr /etc/nginx/sites-enabled/default \ + && pip3 install frappe-bench \ + # Fixes for non-root nginx and logs to stdout + && sed -i '/user www-data/d' /etc/nginx/nginx.conf \ + && ln -sf /dev/stdout /var/log/nginx/access.log && ln -sf /dev/stderr /var/log/nginx/error.log \ + && touch /run/nginx.pid \ + && chown -R frappe:frappe /etc/nginx/conf.d \ + && chown -R frappe:frappe /etc/nginx/nginx.conf \ + && chown -R frappe:frappe /var/log/nginx \ + && chown -R frappe:frappe /var/lib/nginx \ + && chown -R frappe:frappe /run/nginx.pid + +COPY resources/nginx-template.conf /templates/nginx/frappe.conf.template +COPY resources/nginx-entrypoint.sh /usr/local/bin/nginx-entrypoint.sh +COPY resources/push_backup.py /usr/local/bin/push-backup + +FROM base AS builder + +RUN apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y \ + # For frappe framework + wget \ + # For psycopg2 + libpq-dev \ + # Other + libffi-dev \ + liblcms2-dev \ + libldap2-dev \ + libmariadb-dev \ + libsasl2-dev \ + libtiff5-dev \ + libwebp-dev \ + redis-tools \ + rlwrap \ + tk8.6-dev \ + cron \ + # For pandas + gcc \ + build-essential \ + libbz2-dev \ + && rm -rf /var/lib/apt/lists/* + +USER frappe + +ARG FRAPPE_BRANCH=version-14 +ARG FRAPPE_PATH=https://github.com/frappe/frappe +ARG ERPNEXT_REPO=https://github.com/frappe/erpnext +ARG ERPNEXT_BRANCH=version-14 +RUN bench init \ + --frappe-branch=${FRAPPE_BRANCH} \ + --frappe-path=${FRAPPE_PATH} \ + --no-procfile \ + --no-backups \ + --skip-redis-config-generation \ + --verbose \ + /home/frappe/frappe-bench && \ + cd /home/frappe/frappe-bench && \ + bench get-app --branch=${ERPNEXT_BRANCH} --resolve-deps erpnext ${ERPNEXT_REPO} && \ + echo "$(jq 'del(.db_host, .redis_cache, .redis_queue, .redis_socketio)' sites/common_site_config.json)" \ + > sites/common_site_config.json + +FROM base as erpnext + +USER frappe + +COPY --from=builder --chown=frappe:frappe /home/frappe/frappe-bench /home/frappe/frappe-bench + +WORKDIR /home/frappe/frappe-bench + +VOLUME [ \ + "/home/frappe/frappe-bench/sites", \ + "/home/frappe/frappe-bench/sites/assets", \ + "/home/frappe/frappe-bench/logs" \ +] + +CMD [ \ + "/home/frappe/frappe-bench/env/bin/gunicorn", \ + "--chdir=/home/frappe/frappe-bench/sites", \ + "--bind=0.0.0.0:8000", \ + "--threads=4", \ + "--workers=2", \ + "--worker-class=gthread", \ + "--worker-tmp-dir=/dev/shm", \ + "--timeout=120", \ + "--preload", \ + "frappe.app:application" \ +] diff --git a/images/socketio/Dockerfile b/images/socketio/Dockerfile deleted file mode 100644 index fa28b92a..00000000 --- a/images/socketio/Dockerfile +++ /dev/null @@ -1,34 +0,0 @@ -FROM alpine/git as builder - -ARG FRAPPE_VERSION -ARG FRAPPE_REPO=https://github.com/frappe/frappe -RUN apk add -U jq -RUN git clone --depth 1 -b ${FRAPPE_VERSION} ${FRAPPE_REPO} /opt/frappe -RUN jq --argjson dependencies "$(jq '.dependencies | INDEX( "express", "redis", "socket.io", "superagent" ) as $keep | \ - del( \ - . | objects | \ - .[ \ - keys_unsorted[] | \ - select( $keep[ . ] | not ) \ - ] \ - )' /opt/frappe/package.json)" '.dependencies = $dependencies | del(.scripts.prepare)' /opt/frappe/package.json > /opt/frappe/dependencies.json && \ - mv /opt/frappe/dependencies.json /opt/frappe/package.json - -# NodeJS LTS -FROM node:18-alpine - -RUN addgroup -S frappe \ - && adduser -S frappe -G frappe -USER frappe - -WORKDIR /home/frappe/frappe-bench -RUN mkdir -p sites apps/frappe - -COPY --chown=frappe:frappe --from=builder /opt/frappe/package.json /opt/frappe/socketio.js /opt/frappe/node_utils.js apps/frappe/ - -RUN cd apps/frappe \ - && npm install --omit=dev - -WORKDIR /home/frappe/frappe-bench/sites - -CMD [ "node", "/home/frappe/frappe-bench/apps/frappe/socketio.js" ] diff --git a/images/worker/Dockerfile b/images/worker/Dockerfile deleted file mode 100644 index c5479765..00000000 --- a/images/worker/Dockerfile +++ /dev/null @@ -1,146 +0,0 @@ -# syntax=docker/dockerfile:1.3 - -ARG PYTHON_VERSION -FROM python:${PYTHON_VERSION}-slim-bullseye as base - -RUN apt-get update \ - && apt-get install --no-install-recommends -y \ - # Postgres - libpq-dev \ - && rm -rf /var/lib/apt/lists/* - -RUN useradd -ms /bin/bash frappe -USER frappe -RUN mkdir -p /home/frappe/frappe-bench/apps /home/frappe/frappe-bench/logs /home/frappe/frappe-bench/sites /home/frappe/frappe-bench/config -WORKDIR /home/frappe/frappe-bench - -USER root -RUN pip install --no-cache-dir -U pip wheel \ - && python -m venv env \ - && env/bin/pip install --no-cache-dir -U pip wheel - -COPY install-app.sh /usr/local/bin/install-app - - -FROM base as build_deps - -RUN apt-get update \ - && apt-get install --no-install-recommends -y \ - # Install git here because it is not required in production - git \ - # gcc and g++ are required for building different packages across different versions - # of Frappe and ERPNext and also on different platforms (for example, linux/arm64). - # It is safe to install build deps even if they are not required - # because they won't be included in final images. - gcc \ - g++ \ - # Make is required to build wheels of ERPNext deps in develop branch for linux/arm64 - make \ - && rm -rf /var/lib/apt/lists/* - - -FROM build_deps as frappe_builder - -ARG FRAPPE_VERSION -ARG FRAPPE_REPO=https://github.com/frappe/frappe -RUN --mount=type=cache,target=/root/.cache/pip \ - git clone --depth 1 -b ${FRAPPE_VERSION} ${FRAPPE_REPO} apps/frappe \ - && install-app frappe \ - && env/bin/pip install -U gevent \ - # Link Frappe's node_modules/ to make Website Theme work - && mkdir -p /home/frappe/frappe-bench/sites/assets/frappe/node_modules \ - && ln -s /home/frappe/frappe-bench/sites/assets/frappe/node_modules /home/frappe/frappe-bench/apps/frappe/node_modules - - -FROM frappe_builder as erpnext_builder - -ARG PAYMENTS_VERSION=develop -ARG PAYMENTS_REPO=https://github.com/frappe/payments -ARG ERPNEXT_VERSION -ARG ERPNEXT_REPO=https://github.com/frappe/erpnext -RUN --mount=type=cache,target=/root/.cache/pip \ - if [ -z "${ERPNEXT_VERSION##*v14*}" ] || [ "$ERPNEXT_VERSION" = "develop" ]; then \ - git clone --depth 1 -b ${PAYMENTS_VERSION} ${PAYMENTS_REPO} apps/payments && install-app payments; \ - fi \ - && git clone --depth 1 -b ${ERPNEXT_VERSION} ${ERPNEXT_REPO} apps/erpnext \ - && install-app erpnext - -FROM base as configured_base - -ARG WKHTMLTOPDF_VERSION=0.12.6-1 -ARG NODE_VERSION -ENV NVM_DIR=/home/frappe/.nvm -ENV PATH ${NVM_DIR}/versions/node/v${NODE_VERSION}/bin/:${PATH} -RUN apt-get update \ - # Setup Node lists - && apt-get install --no-install-recommends -y curl git \ - # NodeJS with NVM - && mkdir -p ${NVM_DIR} \ - && curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.0/install.sh | bash \ - && . ${NVM_DIR}/nvm.sh \ - && nvm install ${NODE_VERSION} \ - && nvm use v${NODE_VERSION} \ - && npm install -g yarn \ - && nvm alias default v${NODE_VERSION} \ - && rm -rf ${NVM_DIR}/.cache \ - && echo 'export NVM_DIR="/home/frappe/.nvm"' >>~/.bashrc \ - && echo '[ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" # This loads nvm' >> ~/.bashrc \ - && echo '[ -s "$NVM_DIR/bash_completion" ] && \. "$NVM_DIR/bash_completion" # This loads nvm bash_completion' >> ~/.bashrc \ - # Install wkhtmltopdf with patched qt - && if [ "$(uname -m)" = "aarch64" ]; then export ARCH=arm64; fi \ - && if [ "$(uname -m)" = "x86_64" ]; then export ARCH=amd64; fi \ - && downloaded_file=wkhtmltox_$WKHTMLTOPDF_VERSION.buster_${ARCH}.deb \ - && curl -sLO https://github.com/wkhtmltopdf/packaging/releases/download/$WKHTMLTOPDF_VERSION/$downloaded_file \ - && apt-get install -y ./$downloaded_file \ - && rm $downloaded_file \ - # Cleanup - && apt-get purge -y --auto-remove curl \ - && apt-get update \ - && apt-get install --no-install-recommends -y \ - # MariaDB - mariadb-client \ - # Postgres - postgresql-client \ - # For healthcheck - wait-for-it \ - jq \ - # Clean up - && rm -rf /var/lib/apt/lists/* - -COPY pretend-bench.sh /usr/local/bin/bench -COPY push_backup.py /usr/local/bin/push-backup -COPY configure.py patched_bench_helper.py /usr/local/bin/ -COPY gevent_patch.py /opt/patches/ - -WORKDIR /home/frappe/frappe-bench/sites - -CMD [ "/home/frappe/frappe-bench/env/bin/gunicorn", \ - "--bind=0.0.0.0:8000", \ - "--threads=4", \ - "--workers=2", \ - "--worker-class=gthread", \ - "--worker-tmp-dir=/dev/shm", \ - "--timeout=120", \ - "--preload", \ - "frappe.app:application" \ -] - - -FROM configured_base as frappe - -COPY --from=frappe_builder /home/frappe/frappe-bench/apps/frappe /home/frappe/frappe-bench/apps/frappe -COPY --from=frappe_builder /home/frappe/frappe-bench/env /home/frappe/frappe-bench/env -COPY --from=frappe_builder /home/frappe/frappe-bench/sites/apps.txt /home/frappe/frappe-bench/sites/ - -USER frappe - - -# Split frappe and erpnext to reduce image size (because of frappe-bench/env/ directory) -FROM configured_base as erpnext - -COPY --from=erpnext_builder --chown=frappe:frappe /home/frappe/frappe-bench/apps /home/frappe/frappe-bench/apps -COPY --from=erpnext_builder --chown=frappe:frappe /home/frappe/frappe-bench/env /home/frappe/frappe-bench/env -COPY --from=erpnext_builder --chown=frappe:frappe /home/frappe/frappe-bench/sites/apps.txt /home/frappe/frappe-bench/sites/ - - -USER frappe diff --git a/images/worker/configure.py b/images/worker/configure.py deleted file mode 100755 index 5b599e7e..00000000 --- a/images/worker/configure.py +++ /dev/null @@ -1,56 +0,0 @@ -#!/usr/local/bin/python -from __future__ import annotations - -import json -import os -from typing import Any, TypeVar - - -def update_config(**values: Any): - fname = "common_site_config.json" - if not os.path.exists(fname): - with open(fname, "a") as f: - json.dump({}, f) - - with open(fname, "r+") as f: - config: dict[str, Any] = json.load(f) - config.update(values) - f.seek(0) - f.truncate() - json.dump(config, f) - - -_T = TypeVar("_T") - - -def env(name: str, type_: type[_T] = str) -> _T: - value = os.getenv(name) - if not value: - raise RuntimeError(f'Required environment variable "{name}" not set') - try: - value = type_(value) - except Exception: - raise RuntimeError( - f'Cannot convert environment variable "{name}" to type "{type_}"' - ) - return value - - -def generate_redis_url(url: str): - return f"redis://{url}" - - -def main() -> int: - update_config( - db_host=env("DB_HOST"), - db_port=env("DB_PORT", int), - redis_cache=generate_redis_url(env("REDIS_CACHE")), - redis_queue=generate_redis_url(env("REDIS_QUEUE")), - redis_socketio=generate_redis_url(env("REDIS_SOCKETIO")), - socketio_port=env("SOCKETIO_PORT", int), - ) - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/images/worker/gevent_patch.py b/images/worker/gevent_patch.py deleted file mode 100644 index 09f65ce7..00000000 --- a/images/worker/gevent_patch.py +++ /dev/null @@ -1,3 +0,0 @@ -import gevent.monkey - -gevent.monkey.patch_all() diff --git a/images/worker/install-app.sh b/images/worker/install-app.sh deleted file mode 100755 index 320f14fd..00000000 --- a/images/worker/install-app.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash -set -e -set -x - -APP=$1 - -cd /home/frappe/frappe-bench - -env/bin/pip install -e "apps/$APP" - -echo "$APP" >>sites/apps.txt diff --git a/images/worker/patched_bench_helper.py b/images/worker/patched_bench_helper.py deleted file mode 100644 index f3f75195..00000000 --- a/images/worker/patched_bench_helper.py +++ /dev/null @@ -1,48 +0,0 @@ -from __future__ import annotations - -import click -import click.exceptions -import frappe.app -import frappe.database.db_manager -import frappe.utils.bench_helper - - -def patch_database_creator(): - """ - We need to interrupt Frappe site database creation to monkeypatch - functions that resolve host for user that owns site database. - In frappe_docker this was implemented in "new" command: - https://github.com/frappe/frappe_docker/blob/c808ad1767feaf793a2d14541ac0f4d9cbab45b3/build/frappe-worker/commands/new.py#L87 - """ - - frappe.database.db_manager.DbManager.get_current_host = lambda self: "%" - - -def patch_click_usage_error(): - bits: tuple[str, ...] = ( - click.style( - "Only Frappe framework bench commands are available in container setup.", - fg="yellow", - bold=True, - ), - "https://frappeframework.com/docs/v13/user/en/bench/frappe-commands", - ) - notice = "\n".join(bits) - - def format_message(self: click.exceptions.UsageError): - if "No such command" in self.message: - return f"{notice}\n\n{self.message}" - return self.message - - click.exceptions.UsageError.format_message = format_message - - -def main() -> int: - patch_database_creator() - patch_click_usage_error() - frappe.utils.bench_helper.main() - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/images/worker/pretend-bench.sh b/images/worker/pretend-bench.sh deleted file mode 100755 index fe39f0a3..00000000 --- a/images/worker/pretend-bench.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash -set -e - -# shellcheck disable=SC2068 -~/frappe-bench/env/bin/python /usr/local/bin/patched_bench_helper.py frappe $@ diff --git a/docs/compose/compose.custom-domain-ssl.yaml b/overrides/compose.custom-domain-ssl.yaml similarity index 100% rename from docs/compose/compose.custom-domain-ssl.yaml rename to overrides/compose.custom-domain-ssl.yaml diff --git a/docs/compose/compose.custom-domain.yaml b/overrides/compose.custom-domain.yaml similarity index 100% rename from docs/compose/compose.custom-domain.yaml rename to overrides/compose.custom-domain.yaml diff --git a/overrides/compose.erpnext.yaml b/overrides/compose.erpnext.yaml deleted file mode 100644 index 72c7b906..00000000 --- a/overrides/compose.erpnext.yaml +++ /dev/null @@ -1,27 +0,0 @@ -x-erpnext-backend-image: &erpnext_backend_image - image: frappe/erpnext-worker:${ERPNEXT_VERSION:?No ERPNext version set} - volumes: - - sites:/home/frappe/frappe-bench/sites - - assets:/home/frappe/frappe-bench/sites/assets:ro - -services: - configurator: - <<: *erpnext_backend_image - - backend: - <<: *erpnext_backend_image - - frontend: - image: frappe/erpnext-nginx:${ERPNEXT_VERSION} - - queue-short: - <<: *erpnext_backend_image - - queue-default: - <<: *erpnext_backend_image - - queue-long: - <<: *erpnext_backend_image - - scheduler: - <<: *erpnext_backend_image diff --git a/docs/compose/compose.mariadb-shared.yaml b/overrides/compose.mariadb-shared.yaml similarity index 100% rename from docs/compose/compose.mariadb-shared.yaml rename to overrides/compose.mariadb-shared.yaml diff --git a/docs/compose/compose.multi-bench-ssl.yaml b/overrides/compose.multi-bench-ssl.yaml similarity index 100% rename from docs/compose/compose.multi-bench-ssl.yaml rename to overrides/compose.multi-bench-ssl.yaml diff --git a/docs/compose/compose.multi-bench.yaml b/overrides/compose.multi-bench.yaml similarity index 100% rename from docs/compose/compose.multi-bench.yaml rename to overrides/compose.multi-bench.yaml diff --git a/overrides/compose.redis.yaml b/overrides/compose.redis.yaml index 5bd3a0aa..2be6386d 100644 --- a/overrides/compose.redis.yaml +++ b/overrides/compose.redis.yaml @@ -1,16 +1,30 @@ services: configurator: environment: - REDIS_CACHE: redis:6379/0 - REDIS_QUEUE: redis:6379/1 - REDIS_SOCKETIO: redis:6379/2 + REDIS_CACHE: redis-cache:6379 + REDIS_QUEUE: redis-queue:6379 + REDIS_SOCKETIO: redis-socketio:6379 depends_on: - - redis + - redis-cache + - redis-queue + - redis-socketio - redis: + redis-cache: image: redis:6.2-alpine volumes: - - redis-data:/data + - redis-cache-data:/data + + redis-queue: + image: redis:6.2-alpine + volumes: + - redis-queue-data:/data + + redis-socketio: + image: redis:6.2-alpine + volumes: + - redis-socketio-data:/data volumes: - redis-data: + redis-cache-data: + redis-queue-data: + redis-socketio-data: diff --git a/docs/compose/compose.traefik-ssl.yaml b/overrides/compose.traefik-ssl.yaml similarity index 100% rename from docs/compose/compose.traefik-ssl.yaml rename to overrides/compose.traefik-ssl.yaml diff --git a/docs/compose/compose.traefik.yaml b/overrides/compose.traefik.yaml similarity index 100% rename from docs/compose/compose.traefik.yaml rename to overrides/compose.traefik.yaml diff --git a/pwd.yml b/pwd.yml index 362d3dcf..c24f1914 100644 --- a/pwd.yml +++ b/pwd.yml @@ -2,18 +2,30 @@ version: "3" services: backend: - image: frappe/erpnext-worker:v14.12.1 + image: frappe/erpnext:v14.12.1 deploy: restart_policy: condition: on-failure volumes: - sites:/home/frappe/frappe-bench/sites - - assets:/home/frappe/frappe-bench/sites/assets + - logs:/home/frappe/frappe-bench/logs configurator: - image: frappe/erpnext-worker:v14.12.1 + image: frappe/erpnext:v14.12.1 + deploy: + restart_policy: + condition: none + entrypoint: + - bash + - -c command: - - configure.py + - > + bench set-config -g db_host $$DB_HOST; + bench set-config -gp db_port $$DB_PORT; + bench set-config -g redis_cache "redis://$$REDIS_CACHE"; + bench set-config -g redis_queue "redis://$$REDIS_QUEUE"; + bench set-config -g redis_socketio "redis://$$REDIS_SOCKETIO"; + bench set-config -gp socketio_port $$SOCKETIO_PORT; environment: DB_HOST: db DB_PORT: "3306" @@ -23,15 +35,16 @@ services: SOCKETIO_PORT: "9000" volumes: - sites:/home/frappe/frappe-bench/sites + - logs:/home/frappe/frappe-bench/logs create-site: - image: frappe/erpnext-worker:v14.12.1 + image: frappe/erpnext:v14.12.1 deploy: restart_policy: - condition: on-failure + condition: none volumes: - sites:/home/frappe/frappe-bench/sites - - assets:/home/frappe/frappe-bench/sites/assets + - logs:/home/frappe/frappe-bench/logs entrypoint: - bash - -c @@ -42,19 +55,19 @@ services: wait-for-it -t 120 redis-queue:6379; wait-for-it -t 120 redis-socketio:6379; export start=`date +%s`; - until [[ -n `grep -hs ^ common_site_config.json | jq -r ".db_host // empty"` ]] && \ - [[ -n `grep -hs ^ common_site_config.json | jq -r ".redis_cache // empty"` ]] && \ - [[ -n `grep -hs ^ common_site_config.json | jq -r ".redis_queue // empty"` ]]; + until [[ -n `grep -hs ^ sites/common_site_config.json | jq -r ".db_host // empty"` ]] && \ + [[ -n `grep -hs ^ sites/common_site_config.json | jq -r ".redis_cache // empty"` ]] && \ + [[ -n `grep -hs ^ sites/common_site_config.json | jq -r ".redis_queue // empty"` ]]; do - echo "Waiting for common_site_config.json to be created"; + echo "Waiting for sites/common_site_config.json to be created"; sleep 5; if (( `date +%s`-start > 120 )); then - echo "could not find common_site_config.json with required keys"; + echo "could not find sites/common_site_config.json with required keys"; exit 1 fi done; - echo "common_site_config.json found"; - bench new-site frontend --admin-password=admin --db-root-password=admin --install-app payments --install-app erpnext --set-default; + echo "sites/common_site_config.json found"; + bench new-site frontend --no-mariadb-socket --admin-password=admin --db-root-password=admin --install-app payments --install-app erpnext --set-default; db: image: mariadb:10.6 @@ -76,10 +89,12 @@ services: - db-data:/var/lib/mysql frontend: - image: frappe/erpnext-nginx:v14.12.1 + image: frappe/erpnext:v14.12.1 deploy: restart_policy: condition: on-failure + command: + - nginx-entrypoint.sh environment: BACKEND: backend:8000 FRAPPE_SITE_NAME_HEADER: frontend @@ -90,13 +105,13 @@ services: PROXY_READ_TIMOUT: 120 CLIENT_MAX_BODY_SIZE: 50m volumes: - - sites:/usr/share/nginx/html/sites - - assets:/usr/share/nginx/html/assets + - sites:/home/frappe/frappe-bench/sites + - logs:/home/frappe/frappe-bench/logs ports: - "8080:8080" queue-default: - image: frappe/erpnext-worker:v14.12.1 + image: frappe/erpnext:v14.12.1 deploy: restart_policy: condition: on-failure @@ -107,10 +122,10 @@ services: - default volumes: - sites:/home/frappe/frappe-bench/sites - - assets:/home/frappe/frappe-bench/sites/assets + - logs:/home/frappe/frappe-bench/logs queue-long: - image: frappe/erpnext-worker:v14.12.1 + image: frappe/erpnext:v14.12.1 deploy: restart_policy: condition: on-failure @@ -121,10 +136,10 @@ services: - long volumes: - sites:/home/frappe/frappe-bench/sites - - assets:/home/frappe/frappe-bench/sites/assets + - logs:/home/frappe/frappe-bench/logs queue-short: - image: frappe/erpnext-worker:v14.12.1 + image: frappe/erpnext:v14.12.1 deploy: restart_policy: condition: on-failure @@ -135,7 +150,7 @@ services: - short volumes: - sites:/home/frappe/frappe-bench/sites - - assets:/home/frappe/frappe-bench/sites/assets + - logs:/home/frappe/frappe-bench/logs redis-queue: image: redis:6.2-alpine @@ -162,7 +177,7 @@ services: - redis-socketio-data:/data scheduler: - image: frappe/erpnext-worker:v14.12.1 + image: frappe/erpnext:v14.12.1 deploy: restart_policy: condition: on-failure @@ -171,21 +186,24 @@ services: - schedule volumes: - sites:/home/frappe/frappe-bench/sites - - assets:/home/frappe/frappe-bench/sites/assets + - logs:/home/frappe/frappe-bench/logs websocket: - image: frappe/frappe-socketio:v14.22.0 + image: frappe/erpnext:v14.12.1 deploy: restart_policy: condition: on-failure + command: + - node + - /home/frappe/frappe-bench/apps/frappe/socketio.js volumes: - sites:/home/frappe/frappe-bench/sites - - assets:/home/frappe/frappe-bench/sites/assets + - logs:/home/frappe/frappe-bench/logs volumes: - assets: db-data: redis-queue-data: redis-cache-data: redis-socketio-data: sites: + logs: diff --git a/requirements-dev.txt b/requirements-dev.txt deleted file mode 100644 index 16b9a068..00000000 --- a/requirements-dev.txt +++ /dev/null @@ -1,3 +0,0 @@ -frappe @ git+https://github.com/frappe/frappe.git -boto3-stubs[s3] -black==22.12.0 diff --git a/resources/nginx-entrypoint.sh b/resources/nginx-entrypoint.sh new file mode 100755 index 00000000..50408e56 --- /dev/null +++ b/resources/nginx-entrypoint.sh @@ -0,0 +1,52 @@ +#!/bin/bash + +# Set variables that do not exist +if [[ -z "$BACKEND" ]]; then + echo "BACKEND defaulting to 0.0.0.0:8000" + export BACKEND=0.0.0.0:8000 +fi +if [[ -z "$SOCKETIO" ]]; then + echo "SOCKETIO defaulting to 0.0.0.0:9000" + export SOCKETIO=0.0.0.0:9000 +fi +if [[ -z "$UPSTREAM_REAL_IP_ADDRESS" ]]; then + echo "UPSTREAM_REAL_IP_ADDRESS defaulting to 127.0.0.1" + export UPSTREAM_REAL_IP_ADDRESS=127.0.0.1 +fi +if [[ -z "$UPSTREAM_REAL_IP_HEADER" ]]; then + echo "UPSTREAM_REAL_IP_HEADER defaulting to X-Forwarded-For" + export UPSTREAM_REAL_IP_HEADER=X-Forwarded-For +fi +if [[ -z "$UPSTREAM_REAL_IP_RECURSIVE" ]]; then + echo "UPSTREAM_REAL_IP_RECURSIVE defaulting to off" + export UPSTREAM_REAL_IP_RECURSIVE=off +fi +if [[ -z "$FRAPPE_SITE_NAME_HEADER" ]]; then + # shellcheck disable=SC2016 + echo 'FRAPPE_SITE_NAME_HEADER defaulting to $host' + # shellcheck disable=SC2016 + export FRAPPE_SITE_NAME_HEADER='$host' +fi + +if [[ -z "$PROXY_READ_TIMEOUT" ]]; then + echo "PROXY_READ_TIMEOUT defaulting to 120" + export PROXY_READ_TIMEOUT=120 +fi + +if [[ -z "$CLIENT_MAX_BODY_SIZE" ]]; then + echo "CLIENT_MAX_BODY_SIZE defaulting to 50m" + export CLIENT_MAX_BODY_SIZE=50m +fi + +# shellcheck disable=SC2016 +envsubst '${BACKEND} + ${SOCKETIO} + ${UPSTREAM_REAL_IP_ADDRESS} + ${UPSTREAM_REAL_IP_HEADER} + ${UPSTREAM_REAL_IP_RECURSIVE} + ${FRAPPE_SITE_NAME_HEADER} + ${PROXY_READ_TIMEOUT} + ${CLIENT_MAX_BODY_SIZE}' \ + /etc/nginx/conf.d/frappe.conf + +nginx -g 'daemon off;' diff --git a/images/nginx/nginx-template.conf b/resources/nginx-template.conf similarity index 80% rename from images/nginx/nginx-template.conf rename to resources/nginx-template.conf index 90eabeea..944bce3c 100644 --- a/images/nginx/nginx-template.conf +++ b/resources/nginx-template.conf @@ -14,8 +14,8 @@ map $http_x_forwarded_proto $proxy_x_forwarded_proto { server { listen 8080; - server_name $http_host; - root /usr/share/nginx/html; + server_name ${FRAPPE_SITE_NAME_HEADER}; + root /home/frappe/frappe-bench/sites; proxy_buffer_size 128k; proxy_buffers 4 256k; @@ -37,7 +37,7 @@ server { location ~ ^/protected/(.*) { internal; - try_files /sites/$http_host/$1 =404; + try_files /${FRAPPE_SITE_NAME_HEADER}/$1 =404; } location /socket.io { @@ -47,23 +47,23 @@ server { proxy_set_header Upgrade $http_upgrade; proxy_set_header Connection "upgrade"; proxy_set_header X-Frappe-Site-Name ${FRAPPE_SITE_NAME_HEADER}; - proxy_set_header Origin $scheme://$http_host; + proxy_set_header Origin $scheme://${FRAPPE_SITE_NAME_HEADER}; proxy_set_header Host $host; proxy_pass http://socketio-server; } location / { - rewrite ^(.+)/$ $proxy_x_forwarded_proto://$http_host$1 permanent; - rewrite ^(.+)/index\.html$ $proxy_x_forwarded_proto://$http_host$1 permanent; - rewrite ^(.+)\.html$ $proxy_x_forwarded_proto://$http_host$1 permanent; + rewrite ^(.+)/$ $proxy_x_forwarded_proto://${FRAPPE_SITE_NAME_HEADER}$1 permanent; + rewrite ^(.+)/index\.html$ $proxy_x_forwarded_proto://${FRAPPE_SITE_NAME_HEADER}$1 permanent; + rewrite ^(.+)\.html$ $proxy_x_forwarded_proto://${FRAPPE_SITE_NAME_HEADER}$1 permanent; location ~ ^/files/.*.(htm|html|svg|xml) { add_header Content-disposition "attachment"; - try_files /sites/${FRAPPE_SITE_NAME_HEADER}/public/$uri @webserver; + try_files /${FRAPPE_SITE_NAME_HEADER}/public/$uri @webserver; } - try_files /sites/${FRAPPE_SITE_NAME_HEADER}/public/$uri @webserver; + try_files /${FRAPPE_SITE_NAME_HEADER}/public/$uri @webserver; } location @webserver { @@ -72,18 +72,12 @@ server { proxy_set_header X-Frappe-Site-Name ${FRAPPE_SITE_NAME_HEADER}; proxy_set_header Host $host; proxy_set_header X-Use-X-Accel-Redirect True; - proxy_read_timeout ${PROXY_READ_TIMOUT}; + proxy_read_timeout ${PROXY_READ_TIMEOUT}; proxy_redirect off; proxy_pass http://backend-server; } - # error pages - error_page 502 /502.html; - location /502.html { - internal; - } - # optimizations sendfile on; keepalive_timeout 15; diff --git a/images/worker/push_backup.py b/resources/push_backup.py similarity index 99% rename from images/worker/push_backup.py rename to resources/push_backup.py index fb5f1f33..6d867f32 100755 --- a/images/worker/push_backup.py +++ b/resources/push_backup.py @@ -109,6 +109,7 @@ def parse_args(args: list[str]) -> Arguments: def main(args: list[str]) -> int: + os.chdir("sites") push_backup(parse_args(args)) return 0 diff --git a/tests/compose.ci-erpnext.yaml b/tests/compose.ci-erpnext.yaml deleted file mode 100644 index c767b9da..00000000 --- a/tests/compose.ci-erpnext.yaml +++ /dev/null @@ -1,24 +0,0 @@ -services: - configurator: - image: localhost:5000/frappe/erpnext-worker:${ERPNEXT_VERSION} - - backend: - image: localhost:5000/frappe/erpnext-worker:${ERPNEXT_VERSION} - - frontend: - image: localhost:5000/frappe/erpnext-nginx:${ERPNEXT_VERSION} - - websocket: - image: localhost:5000/frappe/frappe-socketio:${FRAPPE_VERSION} - - queue-short: - image: localhost:5000/frappe/erpnext-worker:${ERPNEXT_VERSION} - - queue-default: - image: localhost:5000/frappe/erpnext-worker:${ERPNEXT_VERSION} - - queue-long: - image: localhost:5000/frappe/erpnext-worker:${ERPNEXT_VERSION} - - scheduler: - image: localhost:5000/frappe/erpnext-worker:${ERPNEXT_VERSION} diff --git a/tests/compose.ci.yaml b/tests/compose.ci.yaml index c7e168f9..156c9018 100644 --- a/tests/compose.ci.yaml +++ b/tests/compose.ci.yaml @@ -1,24 +1,24 @@ services: configurator: - image: localhost:5000/frappe/frappe-worker:${FRAPPE_VERSION} + image: localhost:5000/frappe/erpnext:${ERPNEXT_VERSION} backend: - image: localhost:5000/frappe/frappe-worker:${FRAPPE_VERSION} + image: localhost:5000/frappe/erpnext:${ERPNEXT_VERSION} frontend: - image: localhost:5000/frappe/frappe-nginx:${FRAPPE_VERSION} + image: localhost:5000/frappe/erpnext:${ERPNEXT_VERSION} websocket: - image: localhost:5000/frappe/frappe-socketio:${FRAPPE_VERSION} + image: localhost:5000/frappe/erpnext:${ERPNEXT_VERSION} queue-short: - image: localhost:5000/frappe/frappe-worker:${FRAPPE_VERSION} + image: localhost:5000/frappe/erpnext:${ERPNEXT_VERSION} queue-default: - image: localhost:5000/frappe/frappe-worker:${FRAPPE_VERSION} + image: localhost:5000/frappe/erpnext:${ERPNEXT_VERSION} queue-long: - image: localhost:5000/frappe/frappe-worker:${FRAPPE_VERSION} + image: localhost:5000/frappe/erpnext:${ERPNEXT_VERSION} scheduler: - image: localhost:5000/frappe/frappe-worker:${FRAPPE_VERSION} + image: localhost:5000/frappe/erpnext:${ERPNEXT_VERSION} diff --git a/tests/conftest.py b/tests/conftest.py index f9885dd1..5c4e3d75 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -55,11 +55,12 @@ def frappe_site(compose: Compose): site_name = "tests" compose.bench( "new-site", - site_name, + "--no-mariadb-socket", "--mariadb-root-password", "123", "--admin-password", "admin", + site_name, ) compose("restart", "backend") yield site_name @@ -68,11 +69,7 @@ def frappe_site(compose: Compose): @pytest.fixture(scope="class") def erpnext_setup(compose: Compose): compose.stop() - - args = ["-f", "overrides/compose.erpnext.yaml"] - if CI: - args += ("-f", "tests/compose.ci-erpnext.yaml") - compose(*args, "up", "-d", "--quiet-pull") + compose("up", "-d", "--quiet-pull") yield compose.stop() @@ -83,13 +80,14 @@ def erpnext_site(compose: Compose): site_name = "test_erpnext_site" args = [ "new-site", - site_name, + "--no-mariadb-socket", "--mariadb-root-password", "123", "--admin-password", "admin", "--install-app", "erpnext", + site_name, ] erpnext_version = os.environ.get("ERPNEXT_VERSION") if erpnext_version in [ diff --git a/tests/test_frappe_docker.py b/tests/test_frappe_docker.py index 176fd194..f9992eaf 100644 --- a/tests/test_frappe_docker.py +++ b/tests/test_frappe_docker.py @@ -89,7 +89,13 @@ def test_frappe_connections_in_backends( ): filename = "_ping_frappe_connections.py" compose("cp", f"tests/{filename}", f"{service}:/tmp/") - compose.exec(service, python_path, f"/tmp/{filename}") + compose.exec( + "-w", + "/home/frappe/frappe-bench/sites", + service, + python_path, + f"/tmp/{filename}", + ) def test_push_backup(