diff --git a/.dockerignore b/.dockerignore index 0ed82517..3e68f8ad 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,7 +1,4 @@ -# frappe_docker .dockerignore file - -.travis.yml README.md -LICENSE.md +LICENSE .gitignore -docker-*.yml +compose*.yaml diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 70b20e02..8ff402c2 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -6,36 +6,31 @@ updates: interval: daily - package-ecosystem: docker - directory: build/bench + directory: images/bench schedule: interval: daily - package-ecosystem: docker - directory: build/erpnext-nginx + directory: images/nginx schedule: interval: daily - package-ecosystem: docker - directory: build/erpnext-worker + directory: images/worker schedule: interval: daily - package-ecosystem: docker - directory: build/frappe-nginx - schedule: - interval: daily - - - package-ecosystem: docker - directory: build/frappe-socketio - schedule: - interval: daily - - - package-ecosystem: docker - directory: build/frappe-worker + directory: images/socketio schedule: interval: daily - package-ecosystem: npm - directory: build/frappe-socketio + directory: images/socketio + schedule: + interval: daily + + - package-ecosystem: pip + directory: / schedule: interval: daily diff --git a/.github/scripts/get-latest-tags.sh b/.github/scripts/get-latest-tags.sh deleted file mode 100755 index 46784ba6..00000000 --- a/.github/scripts/get-latest-tags.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/bash - -set -e -set -x - -get_tag() { - tags=$(git ls-remote --refs --tags --sort='v:refname' "https://github.com/$1" "v$2.*") - tag=$(echo "$tags" | tail -n1 | sed 's/.*\///') - echo "$tag" -} - -FRAPPE_VERSION=$(get_tag frappe/frappe "$VERSION") -ERPNEXT_VERSION=$(get_tag frappe/erpnext "$VERSION") - -cat <>"$GITHUB_ENV" -FRAPPE_VERSION=$FRAPPE_VERSION -ERPNEXT_VERSION=$ERPNEXT_VERSION -GIT_BRANCH=version-$VERSION -VERSION=$VERSION -EOL diff --git a/.github/scripts/get_latest_tags.py b/.github/scripts/get_latest_tags.py new file mode 100644 index 00000000..e4a7b1dc --- /dev/null +++ b/.github/scripts/get_latest_tags.py @@ -0,0 +1,74 @@ +from __future__ import annotations + +import argparse +import json +import os +import re +import subprocess +import sys +from typing import Literal + +Repo = Literal["frappe", "erpnext"] +MajorVersion = Literal["12", "13", "develop"] + + +def get_latest_tag(repo: Repo, version: MajorVersion) -> str: + if version == "develop": + return "develop" + regex = rf"v{version}.*" + refs = subprocess.check_output( + ( + "git", + "ls-remote", + "--refs", + "--tags", + "--sort=v:refname", + f"https://github.com/frappe/{repo}", + str(regex), + ), + encoding="UTF-8", + ).split()[1::2] + + if not refs: + raise RuntimeError(f'No tags found for version "{regex}"') + ref = refs[-1] + matches: list[str] = re.findall(regex, ref) + if not matches: + raise RuntimeError(f'Can\'t parse tag from ref "{ref}"') + return matches[0] + + +def update_env(file_name: str, frappe_tag: str, erpnext_tag: str | None = None): + text = f"\nFRAPPE_VERSION={frappe_tag}" + if erpnext_tag: + text += f"\nERPNEXT_VERSION={erpnext_tag}" + + with open(file_name, "a") as f: + f.write(text) + + +def _print_resp(frappe_tag: str, erpnext_tag: str | None = None): + print(json.dumps({"frappe": frappe_tag, "erpnext": erpnext_tag})) + + +def main(_args: list[str]) -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--repo", choices=["frappe", "erpnext"], required=True) + parser.add_argument("--version", choices=["12", "13", "develop"], required=True) + args = parser.parse_args(_args) + + frappe_tag = get_latest_tag("frappe", args.version) + if args.repo == "erpnext": + erpnext_tag = get_latest_tag("erpnext", args.version) + else: + erpnext_tag = None + + file_name = os.getenv("GITHUB_ENV") + if file_name: + update_env(file_name, frappe_tag, erpnext_tag) + _print_resp(frappe_tag, erpnext_tag) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main(sys.argv[1:])) diff --git a/.github/scripts/update_example_env.py b/.github/scripts/update_example_env.py new file mode 100644 index 00000000..def9e0f8 --- /dev/null +++ b/.github/scripts/update_example_env.py @@ -0,0 +1,32 @@ +import os +import re + + +def get_versions(): + frappe_version = os.getenv("FRAPPE_VERSION") + erpnext_version = os.getenv("ERPNEXT_VERSION") + assert frappe_version, "No Frappe version set" + assert erpnext_version, "No ERPNext version set" + return frappe_version, erpnext_version + + +def update_env(frappe_version: str, erpnext_version: str): + with open("example.env", "r+") as f: + content = f.read() + for env, var in ( + ("FRAPPE_VERSION", frappe_version), + ("ERPNEXT_VERSION", erpnext_version), + ): + content = re.sub(rf"{env}=.*", f"{env}={var}", content) + f.seek(0) + f.truncate() + f.write(content) + + +def main() -> int: + update_env(*get_versions()) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.github/workflows/build_bench.yml b/.github/workflows/build_bench.yml index 5327b63f..1f664774 100644 --- a/.github/workflows/build_bench.yml +++ b/.github/workflows/build_bench.yml @@ -5,7 +5,7 @@ on: branches: - main paths: - - build/bench/** + - images/bench/** - docker-bake.hcl schedule: @@ -27,7 +27,6 @@ jobs: - name: Build and test uses: docker/bake-action@v1.7.0 with: - files: docker-bake.hcl targets: bench-test - name: Login @@ -42,5 +41,4 @@ jobs: uses: docker/bake-action@v1.7.0 with: targets: bench - files: docker-bake.hcl push: true diff --git a/.github/workflows/build_develop.yml b/.github/workflows/build_develop.yml index f0959192..9b7d0a90 100644 --- a/.github/workflows/build_develop.yml +++ b/.github/workflows/build_develop.yml @@ -1,17 +1,18 @@ -name: Build Develop +name: Develop build on: pull_request: branches: - main paths: - - .github/workflows/build_develop.yml - - build/** - - installation/** + - images/nginx/** + - images/socketio/** + - images/worker/** + - overrides/** - tests/** - - .dockerignore + - compose.yaml - docker-bake.hcl - - env-example + - example.env schedule: # Every day at 12:00 pm @@ -19,65 +20,13 @@ on: workflow_dispatch: -env: - IS_AUTHORIZED_RUN: ${{ github.repository == 'frappe/frappe_docker' && github.event_name != 'pull_request' }} - jobs: build: - name: Frappe & ERPNext - runs-on: ubuntu-latest - services: - registry: - image: registry:2 - ports: - - 5000:5000 - steps: - - name: Checkout - uses: actions/checkout@v3 - - - name: Setup Buildx - uses: docker/setup-buildx-action@v1 - with: - driver-opts: network=host - - - name: Login - uses: docker/login-action@v1 - if: env.IS_AUTHORIZED_RUN == 'true' - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - - name: Build Frappe - uses: docker/bake-action@v1.7.0 - with: - files: docker-bake.hcl - targets: frappe-develop-test - load: true - - - name: Push Frappe to local registry - uses: docker/bake-action@v1.7.0 - with: - files: docker-bake.hcl - targets: frappe-develop-test-local - push: true - - - name: Test Frappe - run: ./tests/test-frappe.sh - - - name: Build ERPNext - uses: docker/bake-action@v1.7.0 - with: - files: docker-bake.hcl - targets: erpnext-develop-test - load: true - - - name: Test ERPNext - run: ./tests/test-erpnext.sh - - - name: Push - if: env.IS_AUTHORIZED_RUN == 'true' - uses: docker/bake-action@v1.7.0 - with: - files: docker-bake.hcl - targets: frappe-develop,erpnext-develop - push: true + uses: frappe/frappe_docker/.github/workflows/docker-build-push.yml@main + with: + repo: erpnext + version: develop + push: ${{ github.repository == 'frappe/frappe_docker' && github.event_name != 'pull_request' }} + secrets: + DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} + DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} diff --git a/.github/workflows/build_stable.yml b/.github/workflows/build_stable.yml index bc265e01..855600f3 100644 --- a/.github/workflows/build_stable.yml +++ b/.github/workflows/build_stable.yml @@ -1,127 +1,98 @@ -name: Build Stable +name: Stable build on: pull_request: branches: - main paths: - - .github/workflows/build_stable.yml - - .github/scripts/get-latest-tags.sh - - build/** - - installation/** + - images/nginx/** + - images/socketio/** + - images/worker/** + - overrides/** - tests/** - - .dockerignore + - compose.yaml - docker-bake.hcl - - env-example + - example.env push: branches: - main paths: - - .github/workflows/build_stable.yml - - .github/scripts/get-latest-tags.sh - - build/** - - installation/** + - images/nginx/** + - images/socketio/** + - images/worker/** + - overrides/** - tests/** - - .dockerignore + - compose.yaml - docker-bake.hcl - - env-example + - example.env # Triggered from frappe/frappe and frappe/erpnext on releases repository_dispatch: workflow_dispatch: -env: - IS_AUTHORIZED_RUN: ${{ github.repository == 'frappe/frappe_docker' && github.event_name != 'pull_request' }} - jobs: - build: - name: Frappe & ERPNext + v12: + uses: frappe/frappe_docker/.github/workflows/docker-build-push.yml@main + with: + repo: erpnext + version: "13" + push: ${{ github.repository == 'frappe/frappe_docker' && github.event_name != 'pull_request' }} + secrets: + DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} + DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} + + v13: + uses: frappe/frappe_docker/.github/workflows/docker-build-push.yml@main + with: + repo: erpnext + version: "13" + push: ${{ github.repository == 'frappe/frappe_docker' && github.event_name != 'pull_request' }} + secrets: + DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} + DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} + + update_example_env: + name: Update example.env runs-on: ubuntu-latest - strategy: - matrix: - version: [12, 13] - services: - registry: - image: registry:2 - ports: - - 5000:5000 + if: ${{ github.repository == 'frappe/frappe_docker' && github.event_name != 'pull_request' }} + needs: v13 + steps: - name: Checkout uses: actions/checkout@v3 - - name: Setup Buildx - uses: docker/setup-buildx-action@v1 + - name: Setup Python + uses: actions/setup-python@v2 with: - driver-opts: network=host - - - name: Login - uses: docker/login-action@v1 - if: env.IS_AUTHORIZED_RUN == 'true' - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} + python-version: 3.9 - name: Get latest versions - run: ./.github/scripts/get-latest-tags.sh - env: - VERSION: ${{ matrix.version }} + run: python3 ./.github/scripts/get_latest_tags.py --repo erpnext --version 13 - - name: Build Frappe - uses: docker/bake-action@v1.7.0 - with: - files: docker-bake.hcl - targets: frappe-stable-test - load: true + - name: Update + run: python3 ./.github/scripts/update_example_env.py - - name: Push Frappe to local registry - uses: docker/bake-action@v1.7.0 - with: - files: docker-bake.hcl - targets: frappe-stable-test-local - push: true - - - name: Test Frappe - if: github.event_name == 'pull_request' - run: ./tests/test-frappe.sh - - - name: Build ERPNext - uses: docker/bake-action@v1.7.0 - with: - files: docker-bake.hcl - targets: erpnext-stable-test - load: true - - - name: Test ERPNext - if: github.event_name == 'pull_request' - run: ./tests/test-erpnext.sh - - - name: Push Frappe - if: env.IS_AUTHORIZED_RUN == 'true' - uses: docker/bake-action@v1.7.0 - with: - files: docker-bake.hcl - targets: frappe-stable - push: true - env: - GIT_TAG: ${{ env.FRAPPE_VERSION }} - - - name: Push ERPNext - if: env.IS_AUTHORIZED_RUN == 'true' - uses: docker/bake-action@v1.7.0 - with: - files: docker-bake.hcl - targets: erpnext-stable - push: true - env: - GIT_TAG: ${{ env.ERPNEXT_VERSION }} + - name: Push + run: | + git config --global user.name github-actions + git config --global user.email github-actions@github.com + git add example.env + if [ -z "$(git status --porcelain)" ]; then + echo "example.env did not change, exiting." + exit 0 + else + echo "example.env changed, pushing changes..." + git commit -m "chore: Update example.env" + git push origin main + fi release_helm: name: Release Helm runs-on: ubuntu-latest - if: github.repository == 'frappe/frappe_docker' && github.event_name != 'pull_request' - needs: build + if: ${{ github.repository == 'frappe/frappe_docker' && github.event_name != 'pull_request' }} + needs: v13 steps: - name: Setup deploy key diff --git a/.github/workflows/docker-build-push.yml b/.github/workflows/docker-build-push.yml new file mode 100644 index 00000000..9612342f --- /dev/null +++ b/.github/workflows/docker-build-push.yml @@ -0,0 +1,74 @@ +name: Build + +on: + workflow_call: + inputs: + repo: + required: true + type: string + description: "'erpnext' or 'frappe'" + version: + required: true + type: string + description: "Major version, git tags should match 'v{version}.*'; or 'develop'" + push: + required: true + type: boolean + secrets: + DOCKERHUB_USERNAME: + required: true + DOCKERHUB_TOKEN: + required: true + +jobs: + build: + name: Build + runs-on: ubuntu-latest + services: + registry: + image: registry:2 + ports: + - 5000:5000 + + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Setup Python + uses: actions/setup-python@v2 + with: + python-version: 3.9 + + - name: Setup Buildx + uses: docker/setup-buildx-action@v1 + with: + driver-opts: network=host + + - name: Install Docker Compose v2 + uses: ndeloof/install-compose-action@4a33bc31f327b8231c4f343f6fba704fedc0fa23 + + - name: Get latest versions + run: python3 ./.github/scripts/get_latest_tags.py --repo ${{ inputs.repo }} --version ${{ inputs.version }} + + - name: Build + uses: docker/bake-action@v1.6.0 + with: + push: true + env: + REGISTRY_USER: localhost:5000/frappe + + - name: Test + run: python3 tests/main.py + + - name: Login + if: ${{ inputs.push }} + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Push + if: ${{ inputs.push }} + uses: docker/bake-action@v1.6.0 + with: + push: true diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml deleted file mode 100644 index def9e204..00000000 --- a/.github/workflows/test.yml +++ /dev/null @@ -1,62 +0,0 @@ -name: Integration Test - -on: - push: - branches: - - main - paths: - - .github/workflows/test.yml - - .github/scripts/get-latest-tags.sh - - build/** - - installation/** - - tests/** - - .dockerignore - - docker-bake.hcl - - env-example - - pull_request: - branches: - - main - paths: - - .github/workflows/test.yml - - .github/scripts/get-latest-tags.sh - - build/** - - installation/** - - tests/** - - .dockerignore - - docker-bake.hcl - - env-example - - workflow_dispatch: - - schedule: - # Every day at 01:00 am - # Develop images are built at 12:00 pm, we want to use them - # Also, we don't build new images on this event - - cron: 0 1 * * * - -jobs: - test: - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v3 - - - name: Get latest versions - if: github.event_name != 'schedule' - run: ./.github/scripts/get-latest-tags.sh - env: - VERSION: 13 - - - name: Build - if: github.event_name != 'schedule' - uses: docker/bake-action@v1.7.0 - with: - files: docker-bake.hcl - targets: frappe-develop,frappe-stable - load: true - env: - GIT_TAG: ${{ env.FRAPPE_VERSION }} - - - name: Test - run: ./tests/integration-test.sh diff --git a/.gitignore b/.gitignore index e6ae4965..c2005695 100644 --- a/.gitignore +++ b/.gitignore @@ -1,18 +1,25 @@ -*.code-workspace - # Environment Variables .env # mounted volume sites -development +development/* !development/README.md -deploy_key -deploy_key.pub +!development/vscode-example/ # Pycharm .idea +# VS Code +.vscode/** +!.vscode/extensions.json + # VS Code devcontainer .devcontainer +*.code-workspace + +# Python +*.pyc +__pycache__ +venv diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 612c73e0..7d2f30c0 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -2,6 +2,8 @@ repos: - repo: https://github.com/pre-commit/pre-commit-hooks rev: v4.1.0 hooks: + - id: check-executables-have-shebangs + - id: check-shebang-scripts-are-executable - id: trailing-whitespace - id: end-of-file-fixer diff --git a/.prettierignore b/.prettierignore deleted file mode 100644 index abe1f008..00000000 --- a/.prettierignore +++ /dev/null @@ -1 +0,0 @@ -installation/docker-compose-custom.yml diff --git a/README.md b/README.md index 9580cda2..af05681c 100644 --- a/README.md +++ b/README.md @@ -1,69 +1,53 @@ [![Build Stable](https://github.com/frappe/frappe_docker/actions/workflows/build_stable.yml/badge.svg)](https://github.com/frappe/frappe_docker/actions/workflows/build_stable.yml) [![Build Develop](https://github.com/frappe/frappe_docker/actions/workflows/build_develop.yml/badge.svg)](https://github.com/frappe/frappe_docker/actions/workflows/build_develop.yml) -## Getting Started +Everything about [Frappe](https://github.com/frappe/frappe) and [ERPNext](https://github.com/frappe/erpnext) in containers. -### Try in Play With Docker +# Getting Started - - Try in PWD - - -Wait for 5 minutes for ERPNext site to be created or check `site-creator` container logs before opening browser on port 80. (username: `Administrator`, password: `admin`) - -### Setting up Pre-requisites - -This repository requires Docker, docker-compose and Git to be setup on the instance to be used. - -For Docker basics and best practices. Refer Docker [documentation](http://docs.docker.com). - -### Cloning the repository and preliminary steps - -Clone this repository somewhere in your system: +To get started, you need Docker, docker-compose and git setup on your machine. For Docker basics and best practices. Refer Docker [documentation](http://docs.docker.com). +After that, clone this repo: ```sh -git clone https://github.com/frappe/frappe_docker.git +git clone https://github.com/frappe/frappe_docker cd frappe_docker ``` -## Production Setup +# Development -It takes care of the following: +We have baseline for developing in VSCode devcontainer with [frappe/bench](https://github.com/frappe/bench). [Start development](development). -- Setting up the desired version of Frappe/ERPNext. -- Setting up all the system requirements: eg. MariaDB, Node, Redis. -- Configure networking for remote access and setting up LetsEncrypt. +# Production -It doesn't take care of the following: +We provide simple and intuitive production setup with prebuilt Frappe and ERPNext images and compose files. To learn more about those, [read the docs](docs/images-and-compose-files.md). -- Cron Job to backup sites is not created by default. -- Use `CronJob` on k8s or refer wiki for alternatives. +Also, there's docs to help with deployment: -1. Single Server Installs - 1. [Single bench](docs/single-bench.md). Easiest Install! - 2. [Multi bench](docs/multi-bench.md) -2. Multi Server Installs - 1. [Docker Swarm](docs/docker-swarm.md) - 2. [Kubernetes](https://helm.erpnext.com) -3. [Site Operations](docs/site-operations.md) -4. [Environment Variables](docs/environment-variables.md) -5. [Custom apps for production](docs/custom-apps-for-production.md) -6. [Tips for moving deployments](docs/tips-for-moving-deployments.md) -7. [Wiki for optional recipes](https://github.com/frappe/frappe_docker/wiki) +- [setup options](docs/setup-options.md), +- in cluster: + - [Docker Swarm](docs/docker-swarm.md), + - [Kubernetes (frappe/helm)](https://helm.erpnext.com), +- [site operations](docs/site-operations.md). +- Other + - [add custom domain using traefik](docs/add-custom-domain-using-traefik.md) + - [backup and push cron jobs](docs/backup-and-push-cronjob.md) + - [bench console and vscode debugger](docs/bench-console-and-vscode-debugger.md) + - [build version 10](docs/build-version-10-images.md) + - [connect to localhost services from containers for local app development](docs/connect-to-localhost-services-from-containers-for-local-app-development.md) + - [patch code from images](docs/patch-code-from-images.md) + - [port based multi tenancy](docs/port-based-multi-tenancy.md) +- [Troubleshoot](docs/troubleshoot.md) -## Development Setup +# Custom app -It takes care of complete setup to develop with Frappe/ERPNext and Bench, Including the following features: +Learn how to containerize your custom Frappe app in [this guide](custom_app/README.md). -- VSCode containers integration -- VSCode Python debugger -- Pre-configured Docker containers for an easy start +# Contributing -[Start development](development). +If you want to contribute to this repo refer to [CONTRIBUTING.md](CONTRIBUTING.md) -## Contributing +This repository is only for Docker related stuff. You also might want to contribute to: -- [Frappe Docker Images](CONTRIBUTING.md) -- [Frappe Framework](https://github.com/frappe/frappe#contributing) -- [ERPNext](https://github.com/frappe/erpnext#contributing) -- [frappe/bench](https://github.com/frappe/bench) +- [Frappe framework](https://github.com/frappe/frappe#contributing), +- [ERPNext](https://github.com/frappe/erpnext#contributing), +- or [Frappe Bench](https://github.com/frappe/bench). diff --git a/build/erpnext-nginx/Dockerfile b/build/erpnext-nginx/Dockerfile deleted file mode 100644 index 69e2614f..00000000 --- a/build/erpnext-nginx/Dockerfile +++ /dev/null @@ -1,35 +0,0 @@ -ARG NODE_IMAGE_TAG=14-bullseye-slim -ARG DOCKER_REGISTRY_PREFIX=frappe -ARG IMAGE_TAG=develop - -FROM node:${NODE_IMAGE_TAG} as builder - -ARG GIT_REPO=https://github.com/frappe/erpnext -ARG GIT_BRANCH=develop - -ARG FRAPPE_BRANCH=${GIT_BRANCH} - -RUN apt-get update \ - && apt-get install --no-install-recommends -y \ - python2 \ - git \ - build-essential \ - ca-certificates \ - && rm -rf /var/lib/apt/lists/* - -COPY build/erpnext-nginx/install_app.sh /install_app -RUN chmod +x /install_app \ - && /install_app erpnext ${GIT_REPO} ${GIT_BRANCH} ${FRAPPE_BRANCH} - - -FROM ${DOCKER_REGISTRY_PREFIX}/frappe-nginx:${IMAGE_TAG} - -COPY --from=builder --chown=1000:1000 /home/frappe/frappe-bench/sites/ /var/www/html/ -COPY --from=builder /rsync /rsync - -RUN echo "erpnext" >> /var/www/html/apps.txt - -VOLUME [ "/assets" ] - -ENTRYPOINT ["/docker-entrypoint.sh"] -CMD ["nginx", "-g", "daemon off;"] diff --git a/build/erpnext-nginx/install_app.sh b/build/erpnext-nginx/install_app.sh deleted file mode 100755 index 085dbaeb..00000000 --- a/build/erpnext-nginx/install_app.sh +++ /dev/null @@ -1,40 +0,0 @@ -#!/bin/bash - -set -e - -APP_NAME=${1} -APP_REPO=${2} -APP_BRANCH=${3} -FRAPPE_BRANCH=${4} - -[ "${APP_BRANCH}" ] && BRANCH="-b ${APP_BRANCH}" - -mkdir -p /home/frappe/frappe-bench -cd /home/frappe/frappe-bench -mkdir -p apps "sites/assets/${APP_NAME}" -echo -ne "frappe\n${APP_NAME}" >sites/apps.txt - -git clone --depth 1 -b "${FRAPPE_BRANCH}" https://github.com/frappe/frappe apps/frappe -# shellcheck disable=SC2086 -git clone --depth 1 ${BRANCH} ${APP_REPO} apps/${APP_NAME} - -echo "Install frappe NodeJS dependencies . . ." -cd apps/frappe -yarn --pure-lockfile - -echo "Install ${APP_NAME} NodeJS dependencies . . ." -yarn --pure-lockfile --cwd "../${APP_NAME}" - -echo "Build ${APP_NAME} assets . . ." -yarn production --app "${APP_NAME}" - -cd /home/frappe/frappe-bench -# shellcheck disable=SC2086 -cp -R apps/${APP_NAME}/${APP_NAME}/public/* sites/assets/${APP_NAME} - -# Add frappe and all the apps available under in frappe-bench here -echo "rsync -a --delete /var/www/html/assets/frappe /assets" >/rsync -echo "rsync -a --delete /var/www/html/assets/${APP_NAME} /assets" >>/rsync -chmod +x /rsync - -rm sites/apps.txt diff --git a/build/erpnext-worker/Dockerfile b/build/erpnext-worker/Dockerfile deleted file mode 100644 index a26b4f9e..00000000 --- a/build/erpnext-worker/Dockerfile +++ /dev/null @@ -1,15 +0,0 @@ -ARG IMAGE_TAG=develop -ARG DOCKER_REGISTRY_PREFIX=frappe -FROM ${DOCKER_REGISTRY_PREFIX}/frappe-worker:${IMAGE_TAG} - -ARG GIT_REPO=https://github.com/frappe/erpnext -ARG GIT_BRANCH=develop - -USER root -RUN apt-get update \ - && apt-get install --no-install-recommends -y \ - gcc \ - && rm -rf /var/lib/apt/lists/* - -USER frappe -RUN install_app erpnext ${GIT_REPO} ${GIT_BRANCH} diff --git a/build/frappe-nginx/Dockerfile b/build/frappe-nginx/Dockerfile deleted file mode 100644 index d1c9f635..00000000 --- a/build/frappe-nginx/Dockerfile +++ /dev/null @@ -1,75 +0,0 @@ -# This image uses nvm and same base image as the worker image. -# This is done to ensures that node-sass binary remains common. -# node-sass is required to enable website theme feature used -# by Website Manager role in Frappe Framework -ARG PYTHON_VERSION=3.9 -FROM python:${PYTHON_VERSION}-slim-bullseye as builder - -ARG GIT_REPO=https://github.com/frappe/frappe -ARG GIT_BRANCH=develop - -ENV NODE_VERSION=14.18.1 -ENV NVM_DIR=/root/.nvm -ENV PATH ${NVM_DIR}/versions/node/v${NODE_VERSION}/bin/:${PATH} - -RUN apt-get update \ - && apt-get install --no-install-recommends -y \ - git \ - build-essential \ - wget \ - # python2 for version-12 builds - python2 \ - && rm -rf /var/lib/apt/lists/* - -# Install nvm with node and yarn -RUN wget -qO- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.0/install.sh | bash \ - && . ${NVM_DIR}/nvm.sh \ - && nvm install ${NODE_VERSION} \ - && npm install -g yarn \ - && rm -rf ${NVM_DIR}/.cache - -WORKDIR /home/frappe/frappe-bench - -RUN mkdir -p apps sites/assets/css sites/assets/frappe /var/www/error_pages -RUN echo "frappe" > sites/apps.txt - -RUN git clone --depth 1 -b ${GIT_BRANCH} ${GIT_REPO} apps/frappe -RUN cd apps/frappe \ - && yarn \ - && yarn run production \ - && yarn install --production=true - -RUN git clone --depth 1 https://github.com/frappe/bench /tmp/bench \ - && cp -r /tmp/bench/bench/config/templates /var/www/error_pages - -RUN cp -R apps/frappe/frappe/public/* sites/assets/frappe \ - && cp -R apps/frappe/node_modules sites/assets/frappe/ - -FROM nginxinc/nginx-unprivileged:latest - -USER root - -RUN usermod -u 1000 nginx && groupmod -g 1000 nginx - -COPY --from=builder --chown=1000:1000 /home/frappe/frappe-bench/sites /var/www/html/ -COPY --from=builder --chown=1000:1000 /var/www/error_pages /var/www/ -COPY build/frappe-nginx/nginx-default.conf.template /etc/nginx/conf.d/default.conf.template -COPY build/frappe-nginx/docker-entrypoint.sh / - -RUN apt-get update \ - && apt-get install --no-install-recommends -y \ - rsync \ - && rm -rf /var/lib/apt/lists/* - -RUN echo "#!/bin/bash" > /rsync \ - && chmod +x /rsync - -RUN mkdir /assets -VOLUME [ "/assets" ] - -RUN chown -R nginx:nginx /assets /etc/nginx/conf.d/ /var/www/html/ - -USER nginx - -ENTRYPOINT ["/docker-entrypoint.sh"] -CMD ["nginx", "-g", "daemon off;"] diff --git a/build/frappe-nginx/docker-entrypoint.sh b/build/frappe-nginx/docker-entrypoint.sh deleted file mode 100755 index 976c4704..00000000 --- a/build/frappe-nginx/docker-entrypoint.sh +++ /dev/null @@ -1,66 +0,0 @@ -#!/bin/bash -ae - -## Thanks -# https://serverfault.com/a/919212 -## - -rsync -a --delete /var/www/html/assets/* /assets - -/rsync - -# shellcheck disable=SC2012 -touch /var/www/html/sites/.build -r "$(ls -td /assets/* | head -n 1)" - -[[ -z "${FRAPPE_PY}" ]] && FRAPPE_PY='0.0.0.0' - -[[ -z "${FRAPPE_PY_PORT}" ]] && FRAPPE_PY_PORT='8000' - -[[ -z "${FRAPPE_SOCKETIO}" ]] && FRAPPE_SOCKETIO='0.0.0.0' - -[[ -z "${SOCKETIO_PORT}" ]] && SOCKETIO_PORT='9000' - -[[ -z "${HTTP_TIMEOUT}" ]] && HTTP_TIMEOUT='120' - -[[ -z "${UPSTREAM_REAL_IP_ADDRESS}" ]] && UPSTREAM_REAL_IP_ADDRESS='127.0.0.1' - -[[ -z "${UPSTREAM_REAL_IP_RECURSIVE}" ]] && UPSTREAM_REAL_IP_RECURSIVE='off' - -[[ -z "${UPSTREAM_REAL_IP_HEADER}" ]] && UPSTREAM_REAL_IP_HEADER='X-Forwarded-For' - -[[ -z "${FRAPPE_SITE_NAME_HEADER}" ]] && FRAPPE_SITE_NAME_HEADER="\$host" - -[[ -z "${HTTP_HOST}" ]] && HTTP_HOST="\$http_host" - -[[ -z "${WEB_PORT}" ]] && WEB_PORT="8080" - -[[ -z "${SKIP_NGINX_TEMPLATE_GENERATION}" ]] && SKIP_NGINX_TEMPLATE_GENERATION='0' - -if [[ ${SKIP_NGINX_TEMPLATE_GENERATION} == 1 ]]; then - echo "Skipping default NGINX template generation. Please mount your own NGINX config file inside /etc/nginx/conf.d" -else - echo "Generating default template" - # shellcheck disable=SC2016 - envsubst '${FRAPPE_PY} - ${FRAPPE_PY_PORT} - ${FRAPPE_SOCKETIO} - ${SOCKETIO_PORT} - ${HTTP_TIMEOUT} - ${UPSTREAM_REAL_IP_ADDRESS} - ${UPSTREAM_REAL_IP_RECURSIVE} - ${FRAPPE_SITE_NAME_HEADER} - ${HTTP_HOST} - ${UPSTREAM_REAL_IP_HEADER} - ${WEB_PORT}' \ - /etc/nginx/conf.d/default.conf -fi - -echo "Waiting for frappe-python to be available on ${FRAPPE_PY} port ${FRAPPE_PY_PORT}" -# shellcheck disable=SC2016 -timeout 10 bash -c 'until printf "" 2>>/dev/null >>/dev/tcp/$0/$1; do sleep 1; done' ${FRAPPE_PY} ${FRAPPE_PY_PORT} -echo "Frappe-python available on ${FRAPPE_PY} port ${FRAPPE_PY_PORT}" -echo "Waiting for frappe-socketio to be available on ${FRAPPE_SOCKETIO} port ${SOCKETIO_PORT}" -# shellcheck disable=SC2016 -timeout 10 bash -c 'until printf "" 2>>/dev/null >>/dev/tcp/$0/$1; do sleep 1; done' ${FRAPPE_SOCKETIO} ${SOCKETIO_PORT} -echo "Frappe-socketio available on ${FRAPPE_SOCKETIO} port ${SOCKETIO_PORT}" - -exec "$@" diff --git a/build/frappe-nginx/nginx-default.conf.template b/build/frappe-nginx/nginx-default.conf.template deleted file mode 100644 index fd7c9719..00000000 --- a/build/frappe-nginx/nginx-default.conf.template +++ /dev/null @@ -1,118 +0,0 @@ -upstream frappe-server { - server ${FRAPPE_PY}:${FRAPPE_PY_PORT} fail_timeout=0; -} - -upstream socketio-server { - server ${FRAPPE_SOCKETIO}:${SOCKETIO_PORT} fail_timeout=0; -} - -# Parse the X-Forwarded-Proto header - if set - defaulting to $scheme. -map $http_x_forwarded_proto $proxy_x_forwarded_proto { - default $scheme; - https https; -} - -server { - listen ${WEB_PORT}; - server_name $http_host; - root /var/www/html; - - add_header X-Frame-Options "SAMEORIGIN"; - add_header Strict-Transport-Security "max-age=63072000; includeSubDomains; preload"; - add_header X-Content-Type-Options nosniff; - add_header X-XSS-Protection "1; mode=block"; - - # Define ${UPSTREAM_REAL_IP_ADDRESS} as our trusted upstream address, so we will be using - # its ${UPSTREAM_REAL_IP_HEADER} address as our remote address - set_real_ip_from ${UPSTREAM_REAL_IP_ADDRESS}; - real_ip_header ${UPSTREAM_REAL_IP_HEADER}; - real_ip_recursive ${UPSTREAM_REAL_IP_RECURSIVE}; - - location /assets { - try_files $uri =404; - } - - location ~ ^/protected/(.*) { - internal; - try_files /sites/$http_host/$1 =404; - } - - location /socket.io { - proxy_http_version 1.1; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $proxy_x_forwarded_proto; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection "upgrade"; - proxy_set_header Origin $proxy_x_forwarded_proto://$http_host; - proxy_set_header X-Frappe-Site-Name ${FRAPPE_SITE_NAME_HEADER}; - proxy_set_header Host ${HTTP_HOST}; - - proxy_pass http://socketio-server; - } - - location / { - rewrite ^(.+)/$ $1 permanent; - rewrite ^(.+)/index\.html$ $1 permanent; - rewrite ^(.+)\.html$ $1 permanent; - - location ~ ^/files/.*.(htm|html|svg|xml) { - add_header Content-disposition "attachment"; - try_files /sites/$http_host/public/$uri @webserver; - } - - try_files /sites/$http_host/public/$uri @webserver; - } - - location @webserver { - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $proxy_x_forwarded_proto; - proxy_set_header X-Frappe-Site-Name ${FRAPPE_SITE_NAME_HEADER}; - proxy_set_header Host ${HTTP_HOST}; - proxy_set_header X-Use-X-Accel-Redirect True; - proxy_read_timeout ${HTTP_TIMEOUT}; - proxy_redirect off; - - proxy_pass http://frappe-server; - } - - # error pages - error_page 502 /502.html; - location /502.html { - root /var/www/templates; - internal; - } - - # optimizations - sendfile on; - keepalive_timeout 15; - client_max_body_size 50m; - client_body_buffer_size 16K; - client_header_buffer_size 1k; - - # enable gzip compression - # based on https://mattstauffer.co/blog/enabling-gzip-on-nginx-servers-including-laravel-forge - gzip on; - gzip_http_version 1.1; - gzip_comp_level 5; - gzip_min_length 256; - gzip_proxied any; - gzip_vary on; - gzip_types - application/atom+xml - application/javascript - application/json - application/rss+xml - application/vnd.ms-fontobject - application/x-font-ttf - application/font-woff - application/x-web-app-manifest+json - application/xhtml+xml - application/xml - font/opentype - image/svg+xml - image/x-icon - text/css - text/plain - text/x-component; - # text/html is always compressed by HttpGzipModule -} diff --git a/build/frappe-socketio/Dockerfile b/build/frappe-socketio/Dockerfile deleted file mode 100644 index f9d5f817..00000000 --- a/build/frappe-socketio/Dockerfile +++ /dev/null @@ -1,35 +0,0 @@ -FROM alpine/git as builder - -ARG GIT_REPO=https://github.com/frappe/frappe.git -ARG GIT_BRANCH=develop - -RUN git clone --depth 1 -b ${GIT_BRANCH} ${GIT_REPO} /opt/frappe - -FROM node:bullseye-slim - -# Add frappe user -RUN useradd -ms /bin/bash frappe -WORKDIR /home/frappe/frappe-bench - -# Create bench directories and set ownership -RUN mkdir -p sites apps/frappe \ - && chown -R frappe:frappe /home/frappe - -# Download socketio -COPY build/frappe-socketio/package.json apps/frappe -COPY --from=builder /opt/frappe/socketio.js apps/frappe/socketio.js -COPY --from=builder /opt/frappe/node_utils.js apps/frappe/node_utils.js - -RUN cd apps/frappe \ - && npm install --only=prod - -# Setup docker-entrypoint -COPY build/frappe-socketio/docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh -# backwards compat -RUN ln -s /usr/local/bin/docker-entrypoint.sh / - -USER frappe -WORKDIR /home/frappe/frappe-bench/sites - -ENTRYPOINT ["docker-entrypoint.sh"] -CMD ["start"] diff --git a/build/frappe-socketio/docker-entrypoint.sh b/build/frappe-socketio/docker-entrypoint.sh deleted file mode 100755 index ec347813..00000000 --- a/build/frappe-socketio/docker-entrypoint.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/bash - -set -e - -function checkConfigExists() { - COUNTER=0 - while [[ ! -e /home/frappe/frappe-bench/sites/common_site_config.json && ${COUNTER} -le 30 ]]; do - sleep 1 - ((COUNTER = COUNTER + 1)) - echo "config file not created, retry ${COUNTER}" >&2 - done - - if [[ ! -e /home/frappe/frappe-bench/sites/common_site_config.json ]]; then - echo "timeout: config file not created" >&2 - exit 1 - fi -} - -if [[ "$1" == 'start' ]]; then - checkConfigExists - node /home/frappe/frappe-bench/apps/frappe/socketio.js -else - exec -c "$@" -fi diff --git a/build/frappe-worker/Dockerfile b/build/frappe-worker/Dockerfile deleted file mode 100644 index af1a71f6..00000000 --- a/build/frappe-worker/Dockerfile +++ /dev/null @@ -1,108 +0,0 @@ -ARG PYTHON_VERSION=3.9 -FROM python:${PYTHON_VERSION}-slim-bullseye - -# Add non root user without password -RUN useradd -ms /bin/bash frappe - -ARG GIT_REPO=https://github.com/frappe/frappe -ARG GIT_BRANCH=develop - -ARG ARCH=amd64 -ENV PYTHONUNBUFFERED 1 - -ENV NODE_VERSION=14.18.1 -ENV NVM_DIR /home/frappe/.nvm -ENV PATH ${NVM_DIR}/versions/node/v${NODE_VERSION}/bin/:${PATH} -ENV WKHTMLTOPDF_VERSION 0.12.6-1 - -# Install apt dependencies -RUN apt-get update \ - && apt-get install --no-install-recommends -y \ - # For frappe framework - git \ - mariadb-client \ - postgresql-client \ - gettext-base \ - wget \ - wait-for-it \ - # For PDF - libjpeg62-turbo \ - libx11-6 \ - libxcb1 \ - libxext6 \ - libxrender1 \ - libssl-dev \ - fonts-cantarell \ - xfonts-75dpi \ - xfonts-base \ - libxml2 \ - libffi-dev \ - libjpeg-dev \ - zlib1g-dev \ - # For psycopg2 - libpq-dev \ - # For arm64 python wheel builds - && if [ "$(uname -m)" = "aarch64" ]; then \ - apt-get install --no-install-recommends -y \ - gcc \ - g++; \ - fi \ - # Install additional requirements for develop branch - && if [ "${GIT_BRANCH}" = 'develop' ]; then \ - apt-get install --no-install-recommends -y \ - libcairo2 \ - python3-cffi \ - python3-brotli \ - libpango-1.0-0 \ - libpangoft2-1.0-0 \ - libpangocairo-1.0-0; \ - fi \ - && rm -rf /var/lib/apt/lists/* - -# Detect arch, download and install wkhtmltopdf -RUN if [ "$(uname -m)" = "aarch64" ]; then export ARCH=arm64; fi \ - && if [ "$(uname -m)" = "x86_64" ]; then export ARCH=amd64; fi \ - && downloaded_file=wkhtmltox_$WKHTMLTOPDF_VERSION.buster_${ARCH}.deb \ - && wget -q https://github.com/wkhtmltopdf/packaging/releases/download/$WKHTMLTOPDF_VERSION/$downloaded_file \ - && dpkg -i $downloaded_file \ - && rm $downloaded_file - -# Setup docker-entrypoint -COPY build/frappe-worker/docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh -RUN ln -s /usr/local/bin/docker-entrypoint.sh / # backwards compat - -WORKDIR /home/frappe/frappe-bench -RUN chown -R frappe:frappe /home/frappe -USER frappe - -# Create frappe-bench directories -RUN mkdir -p apps logs commands sites /home/frappe/backups - -# Setup python environment -RUN python -m venv env -RUN env/bin/pip install --no-cache-dir wheel gevent - -# Install nvm with node -RUN wget -qO- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.0/install.sh | bash \ - && . ${NVM_DIR}/nvm.sh \ - && nvm install ${NODE_VERSION} \ - && rm -rf ${NVM_DIR}/.cache - -# Install Frappe -RUN git clone --depth 1 -o upstream -b ${GIT_BRANCH} ${GIT_REPO} apps/frappe \ - && env/bin/pip install --no-cache-dir -e apps/frappe - -# Copy scripts and templates -COPY build/frappe-worker/commands/* /home/frappe/frappe-bench/commands/ -COPY build/frappe-worker/common_site_config.json.template /opt/frappe/common_site_config.json.template -COPY build/frappe-worker/install_app.sh /usr/local/bin/install_app -COPY build/frappe-worker/bench /usr/local/bin/bench -COPY build/frappe-worker/healthcheck.sh /usr/local/bin/healthcheck.sh - -# Use sites volume as working directory -WORKDIR /home/frappe/frappe-bench/sites - -VOLUME [ "/home/frappe/frappe-bench/sites", "/home/frappe/backups", "/home/frappe/frappe-bench/logs" ] - -ENTRYPOINT ["docker-entrypoint.sh"] -CMD ["start"] diff --git a/build/frappe-worker/bench b/build/frappe-worker/bench deleted file mode 100755 index f702d0d7..00000000 --- a/build/frappe-worker/bench +++ /dev/null @@ -1,22 +0,0 @@ -#!/home/frappe/frappe-bench/env/bin/python - -import os -import subprocess -import sys - -if __name__ == "__main__": - bench_dir = os.path.join(os.sep, "home", "frappe", "frappe-bench") - sites_dir = os.path.join(bench_dir, "sites") - bench_helper = os.path.join( - bench_dir, - "apps", - "frappe", - "frappe", - "utils", - "bench_helper.py", - ) - cwd = os.getcwd() - os.chdir(sites_dir) - subprocess.check_call( - [sys.executable, bench_helper, "frappe"] + sys.argv[1:], - ) diff --git a/build/frappe-worker/commands/auto_migrate.py b/build/frappe-worker/commands/auto_migrate.py deleted file mode 100644 index a9f84743..00000000 --- a/build/frappe-worker/commands/auto_migrate.py +++ /dev/null @@ -1,64 +0,0 @@ -import os - -import git -import semantic_version -from migrate import migrate_sites -from utils import ( - get_apps, - get_config, - get_container_versions, - get_version_file, - save_version_file, -) - - -def main(): - is_ready = False - apps = get_apps() - - container_versions = get_container_versions(apps) - - version_file = get_version_file() - - if not version_file: - version_file = container_versions - save_version_file(version_file) - - for app in apps: - container_version = None - file_version = None - version_file_hash = None - container_hash = None - - repo = git.Repo(os.path.join("..", "apps", app)) - branch = repo.active_branch.name - - if branch == "develop": - version_file_hash = version_file.get(app + "_git_hash") - container_hash = container_versions.get(app + "_git_hash") - if container_hash and version_file_hash: - if container_hash != version_file_hash: - is_ready = True - break - - if version_file.get(app): - file_version = semantic_version.Version(version_file.get(app)) - - if container_versions.get(app): - container_version = semantic_version.Version(container_versions.get(app)) - - if file_version and container_version: - if container_version > file_version: - is_ready = True - break - - config = get_config() - - if is_ready and config.get("maintenance_mode") != 1: - migrate_sites(maintenance_mode=True) - version_file = container_versions - save_version_file(version_file) - - -if __name__ == "__main__": - main() diff --git a/build/frappe-worker/commands/backup.py b/build/frappe-worker/commands/backup.py deleted file mode 100644 index 9338d207..00000000 --- a/build/frappe-worker/commands/backup.py +++ /dev/null @@ -1,45 +0,0 @@ -import os - -import frappe -from frappe.utils import cint, get_sites, now -from frappe.utils.backups import scheduled_backup - - -def backup(sites, with_files=False): - for site in sites: - frappe.init(site) - frappe.connect() - odb = scheduled_backup( - ignore_files=not with_files, - backup_path_db=None, - backup_path_files=None, - backup_path_private_files=None, - force=True, - ) - print("database backup taken -", odb.backup_path_db, "- on", now()) - if with_files: - print("files backup taken -", odb.backup_path_files, "- on", now()) - print( - "private files backup taken -", - odb.backup_path_private_files, - "- on", - now(), - ) - frappe.destroy() - - -def main(): - installed_sites = ":".join(get_sites()) - sites = os.environ.get("SITES", installed_sites).split(":") - with_files = cint(os.environ.get("WITH_FILES")) - - backup(sites, with_files) - - if frappe.redis_server: - frappe.redis_server.connection_pool.disconnect() - - exit(0) - - -if __name__ == "__main__": - main() diff --git a/build/frappe-worker/commands/check_connection.py b/build/frappe-worker/commands/check_connection.py deleted file mode 100644 index d2bae21f..00000000 --- a/build/frappe-worker/commands/check_connection.py +++ /dev/null @@ -1,124 +0,0 @@ -import socket -import time - -from constants import ( - DB_HOST_KEY, - DB_PORT, - DB_PORT_KEY, - REDIS_CACHE_KEY, - REDIS_QUEUE_KEY, - REDIS_SOCKETIO_KEY, -) -from six.moves.urllib.parse import urlparse -from utils import get_config - - -def is_open(ip, port, timeout=30): - s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - s.settimeout(timeout) - try: - s.connect((ip, int(port))) - s.shutdown(socket.SHUT_RDWR) - return True - except Exception: - return False - finally: - s.close() - - -def check_host(ip, port, retry=10, delay=3, print_attempt=True): - ipup = False - for i in range(retry): - if print_attempt: - print(f"Attempt {i+1} to connect to {ip}:{port}") - if is_open(ip, port): - ipup = True - break - else: - time.sleep(delay) - return ipup - - -# Check service -def check_service( - retry=10, delay=3, print_attempt=True, service_name=None, service_port=None -): - - config = get_config() - if not service_name: - service_name = config.get(DB_HOST_KEY, "mariadb") - if not service_port: - service_port = config.get(DB_PORT_KEY, DB_PORT) - - is_db_connected = False - is_db_connected = check_host( - service_name, service_port, retry, delay, print_attempt - ) - if not is_db_connected: - print( - "Connection to {service_name}:{service_port} timed out".format( - service_name=service_name, - service_port=service_port, - ) - ) - exit(1) - - -# Check redis queue -def check_redis_queue(retry=10, delay=3, print_attempt=True): - check_redis_queue = False - config = get_config() - redis_queue_url = urlparse( - config.get(REDIS_QUEUE_KEY, "redis://redis-queue:6379") - ).netloc - redis_queue, redis_queue_port = redis_queue_url.split(":") - check_redis_queue = check_host( - redis_queue, redis_queue_port, retry, delay, print_attempt - ) - if not check_redis_queue: - print("Connection to redis queue timed out") - exit(1) - - -# Check redis cache -def check_redis_cache(retry=10, delay=3, print_attempt=True): - check_redis_cache = False - config = get_config() - redis_cache_url = urlparse( - config.get(REDIS_CACHE_KEY, "redis://redis-cache:6379") - ).netloc - redis_cache, redis_cache_port = redis_cache_url.split(":") - check_redis_cache = check_host( - redis_cache, redis_cache_port, retry, delay, print_attempt - ) - if not check_redis_cache: - print("Connection to redis cache timed out") - exit(1) - - -# Check redis socketio -def check_redis_socketio(retry=10, delay=3, print_attempt=True): - check_redis_socketio = False - config = get_config() - redis_socketio_url = urlparse( - config.get(REDIS_SOCKETIO_KEY, "redis://redis-socketio:6379") - ).netloc - redis_socketio, redis_socketio_port = redis_socketio_url.split(":") - check_redis_socketio = check_host( - redis_socketio, redis_socketio_port, retry, delay, print_attempt - ) - if not check_redis_socketio: - print("Connection to redis socketio timed out") - exit(1) - - -def main(): - check_service() - check_redis_queue() - check_redis_cache() - check_redis_socketio() - print("Connections OK") - - -if __name__ == "__main__": - main() diff --git a/build/frappe-worker/commands/constants.py b/build/frappe-worker/commands/constants.py deleted file mode 100644 index fd70bbdc..00000000 --- a/build/frappe-worker/commands/constants.py +++ /dev/null @@ -1,13 +0,0 @@ -REDIS_QUEUE_KEY = "redis_queue" -REDIS_CACHE_KEY = "redis_cache" -REDIS_SOCKETIO_KEY = "redis_socketio" -DB_HOST_KEY = "db_host" -DB_PORT_KEY = "db_port" -DB_PORT = 3306 -APP_VERSIONS_JSON_FILE = "app_versions.json" -APPS_TXT_FILE = "apps.txt" -COMMON_SITE_CONFIG_FILE = "common_site_config.json" -DATE_FORMAT = "%Y%m%d_%H%M%S" -RDS_DB = "rds_db" -RDS_PRIVILEGES = "SELECT, INSERT, UPDATE, DELETE, CREATE, DROP, INDEX, ALTER, CREATE TEMPORARY TABLES, CREATE VIEW, EVENT, TRIGGER, SHOW VIEW, CREATE ROUTINE, ALTER ROUTINE, EXECUTE, LOCK TABLES" -ARCHIVE_SITES_PATH = "/home/frappe/frappe-bench/sites/archive_sites" diff --git a/build/frappe-worker/commands/doctor.py b/build/frappe-worker/commands/doctor.py deleted file mode 100644 index c7eac75f..00000000 --- a/build/frappe-worker/commands/doctor.py +++ /dev/null @@ -1,61 +0,0 @@ -import argparse - -from check_connection import ( - check_redis_cache, - check_redis_queue, - check_redis_socketio, - check_service, -) - - -def parse_args(): - parser = argparse.ArgumentParser() - parser.add_argument( - "-p", - "--ping-service", - dest="ping_services", - action="append", - type=str, - help='list of services to ping, e.g. doctor -p "postgres:5432" --ping-service "mariadb:3306"', - ) - args = parser.parse_args() - return args - - -def main(): - args = parse_args() - check_service(retry=1, delay=0, print_attempt=False) - print("Bench database Connected") - check_redis_cache(retry=1, delay=0, print_attempt=False) - print("Redis Cache Connected") - check_redis_queue(retry=1, delay=0, print_attempt=False) - print("Redis Queue Connected") - check_redis_socketio(retry=1, delay=0, print_attempt=False) - print("Redis SocketIO Connected") - - if args.ping_services: - for service in args.ping_services: - service_name = None - service_port = None - - try: - service_name, service_port = service.split(":") - except ValueError: - print("Service should be in format host:port, e.g postgres:5432") - exit(1) - - check_service( - retry=1, - delay=0, - print_attempt=False, - service_name=service_name, - service_port=service_port, - ) - print(f"{service_name}:{service_port} Connected") - - print("Health check successful") - exit(0) - - -if __name__ == "__main__": - main() diff --git a/build/frappe-worker/commands/migrate.py b/build/frappe-worker/commands/migrate.py deleted file mode 100644 index 2e99ed54..00000000 --- a/build/frappe-worker/commands/migrate.py +++ /dev/null @@ -1,52 +0,0 @@ -import os - -import frappe -from frappe.utils import cint, get_sites -from utils import get_config, save_config - - -def set_maintenance_mode(enable=True): - conf = get_config() - - if enable: - conf.update({"maintenance_mode": 1, "pause_scheduler": 1}) - save_config(conf) - - if not enable: - conf.update({"maintenance_mode": 0, "pause_scheduler": 0}) - save_config(conf) - - -def migrate_sites(maintenance_mode=False): - installed_sites = ":".join(get_sites()) - sites = os.environ.get("SITES", installed_sites).split(":") - if not maintenance_mode: - maintenance_mode = cint(os.environ.get("MAINTENANCE_MODE")) - - if maintenance_mode: - set_maintenance_mode(True) - - for site in sites: - print("Migrating", site) - frappe.init(site=site) - frappe.connect() - try: - from frappe.migrate import migrate - - migrate() - finally: - frappe.destroy() - - # Disable maintenance mode after migration - set_maintenance_mode(False) - - -def main(): - migrate_sites() - if frappe.redis_server: - frappe.redis_server.connection_pool.disconnect() - exit(0) - - -if __name__ == "__main__": - main() diff --git a/build/frappe-worker/commands/new.py b/build/frappe-worker/commands/new.py deleted file mode 100644 index 64e7213b..00000000 --- a/build/frappe-worker/commands/new.py +++ /dev/null @@ -1,132 +0,0 @@ -import os - -import frappe -import semantic_version -from constants import COMMON_SITE_CONFIG_FILE, RDS_DB, RDS_PRIVILEGES -from frappe.installer import update_site_config -from utils import get_config, get_password, get_site_config, run_command - -# try to import _new_site from frappe, which could possibly -# exist in either commands.py or installer.py, and so we need -# to maintain compatibility across all frappe versions. -try: - # <= version-{11,12} - from frappe.commands.site import _new_site -except ImportError: - # >= version-13 and develop - from frappe.installer import _new_site - - -def main(): - config = get_config() - db_type = "mariadb" - db_port = config.get("db_port", 3306) - db_host = config.get("db_host") - site_name = os.environ.get("SITE_NAME", "site1.localhost") - db_root_username = os.environ.get("DB_ROOT_USER", "root") - mariadb_root_password = get_password("MYSQL_ROOT_PASSWORD", "admin") - postgres_root_password = get_password("POSTGRES_PASSWORD") - db_root_password = mariadb_root_password - - if postgres_root_password: - db_type = "postgres" - db_host = os.environ.get("POSTGRES_HOST") - db_port = 5432 - db_root_password = postgres_root_password - if not db_host: - db_host = config.get("db_host") - print("Environment variable POSTGRES_HOST not found.") - print("Using db_host from common_site_config.json") - - sites_path = os.getcwd() - common_site_config_path = os.path.join(sites_path, COMMON_SITE_CONFIG_FILE) - update_site_config( - "root_login", - db_root_username, - validate=False, - site_config_path=common_site_config_path, - ) - update_site_config( - "root_password", - db_root_password, - validate=False, - site_config_path=common_site_config_path, - ) - - force = True if os.environ.get("FORCE", None) else False - install_apps = os.environ.get("INSTALL_APPS", None) - install_apps = install_apps.split(",") if install_apps else [] - frappe.init(site_name, new_site=True) - - if semantic_version.Version(frappe.__version__).major > 11: - _new_site( - None, - site_name, - db_root_username, - db_root_password, - admin_password=get_password("ADMIN_PASSWORD", "admin"), - verbose=True, - install_apps=install_apps, - source_sql=None, - force=force, - db_type=db_type, - reinstall=False, - db_host=db_host, - db_port=db_port, - ) - else: - _new_site( - None, - site_name, - mariadb_root_username=db_root_username, - mariadb_root_password=db_root_password, - admin_password=get_password("ADMIN_PASSWORD", "admin"), - verbose=True, - install_apps=install_apps, - source_sql=None, - force=force, - reinstall=False, - ) - - if db_type == "mariadb": - site_config = get_site_config(site_name) - db_name = site_config.get("db_name") - db_password = site_config.get("db_password") - - mysql_command = [ - "mysql", - f"-h{db_host}", - f"-u{db_root_username}", - f"-p{mariadb_root_password}", - "-e", - ] - - # Drop User if exists - command = mysql_command + [ - f"DROP USER IF EXISTS '{db_name}'; FLUSH PRIVILEGES;" - ] - run_command(command) - - # Grant permission to database and set password - grant_privileges = "ALL PRIVILEGES" - - # for Amazon RDS - if config.get(RDS_DB) or site_config.get(RDS_DB): - grant_privileges = RDS_PRIVILEGES - - command = mysql_command + [ - f"\ - CREATE USER IF NOT EXISTS '{db_name}'@'%' IDENTIFIED BY '{db_password}'; \ - GRANT {grant_privileges} ON `{db_name}`.* TO '{db_name}'@'%'; \ - FLUSH PRIVILEGES;" - ] - run_command(command) - - if frappe.redis_server: - frappe.redis_server.connection_pool.disconnect() - - exit(0) - - -if __name__ == "__main__": - main() diff --git a/build/frappe-worker/commands/push_backup.py b/build/frappe-worker/commands/push_backup.py deleted file mode 100644 index 78337ade..00000000 --- a/build/frappe-worker/commands/push_backup.py +++ /dev/null @@ -1,183 +0,0 @@ -import datetime -import os -import time -from glob import glob - -import boto3 -from constants import DATE_FORMAT -from frappe.utils import get_sites -from utils import check_s3_environment_variables, get_s3_config, upload_file_to_s3 - - -def get_file_ext(): - return { - "database": "-database.sql.gz", - "private_files": "-private-files.tar", - "public_files": "-files.tar", - "site_config": "-site_config_backup.json", - } - - -def get_backup_details(sitename): - backup_details = dict() - file_ext = get_file_ext() - - # add trailing slash https://stackoverflow.com/a/15010678 - site_backup_path = os.path.join(os.getcwd(), sitename, "private", "backups", "") - - if os.path.exists(site_backup_path): - for filetype, ext in file_ext.items(): - site_slug = sitename.replace(".", "_") - pattern = site_backup_path + "*-" + site_slug + ext - backup_files = list(filter(os.path.isfile, glob(pattern))) - - if len(backup_files) > 0: - backup_files.sort( - key=lambda file: os.stat( - os.path.join(site_backup_path, file) - ).st_ctime - ) - backup_date = datetime.datetime.strptime( - time.ctime(os.path.getmtime(backup_files[0])), - "%a %b %d %H:%M:%S %Y", - ) - backup_details[filetype] = { - "sitename": sitename, - "file_size_in_bytes": os.stat(backup_files[-1]).st_size, - "file_path": os.path.abspath(backup_files[-1]), - "filename": os.path.basename(backup_files[-1]), - "backup_date": backup_date.date().strftime("%Y-%m-%d %H:%M:%S"), - } - - return backup_details - - -def delete_old_backups(limit, bucket, site_name): - all_backups = list() - all_backup_dates = list() - backup_limit = int(limit) - check_s3_environment_variables() - bucket_dir = os.environ.get("BUCKET_DIR") - oldest_backup_date = None - - s3 = boto3.resource( - "s3", - region_name=os.environ.get("REGION"), - aws_access_key_id=os.environ.get("ACCESS_KEY_ID"), - aws_secret_access_key=os.environ.get("SECRET_ACCESS_KEY"), - endpoint_url=os.environ.get("ENDPOINT_URL"), - ) - - bucket = s3.Bucket(bucket) - objects = bucket.meta.client.list_objects_v2(Bucket=bucket.name, Delimiter="/") - - if objects: - for obj in objects.get("CommonPrefixes"): - if obj.get("Prefix") == bucket_dir + "/": - for backup_obj in bucket.objects.filter(Prefix=obj.get("Prefix")): - if backup_obj.get()["ContentType"] == "application/x-directory": - continue - try: - # backup_obj.key is bucket_dir/site/date_time/backupfile.extension - ( - bucket_dir, - site_slug, - date_time, - backupfile, - ) = backup_obj.key.split("/") - date_time_object = datetime.datetime.strptime( - date_time, DATE_FORMAT - ) - - if site_name in backup_obj.key: - all_backup_dates.append(date_time_object) - all_backups.append(backup_obj.key) - except IndexError as error: - print(error) - exit(1) - - if len(all_backup_dates) > 0: - oldest_backup_date = min(all_backup_dates) - - if len(all_backups) / 3 > backup_limit: - oldest_backup = None - for backup in all_backups: - try: - # backup is bucket_dir/site/date_time/backupfile.extension - backup_dir, site_slug, backup_dt_string, filename = backup.split("/") - backup_datetime = datetime.datetime.strptime( - backup_dt_string, DATE_FORMAT - ) - if backup_datetime == oldest_backup_date: - oldest_backup = backup - - except IndexError as error: - print(error) - exit(1) - - if oldest_backup: - for obj in bucket.objects.filter(Prefix=oldest_backup): - # delete all keys that are inside the oldest_backup - if bucket_dir in obj.key: - print("Deleting " + obj.key) - s3.Object(bucket.name, obj.key).delete() - - -def main(): - details = dict() - sites = get_sites() - conn, bucket = get_s3_config() - - for site in sites: - details = get_backup_details(site) - db_file = details.get("database", {}).get("file_path") - folder = os.environ.get("BUCKET_DIR") + "/" + site + "/" - if db_file: - folder = ( - os.environ.get("BUCKET_DIR") - + "/" - + site - + "/" - + os.path.basename(db_file)[:15] - + "/" - ) - upload_file_to_s3(db_file, folder, conn, bucket) - - # Archive site_config.json - site_config_file = details.get("site_config", {}).get("file_path") - if not site_config_file: - site_config_file = os.path.join(os.getcwd(), site, "site_config.json") - upload_file_to_s3(site_config_file, folder, conn, bucket) - - public_files = details.get("public_files", {}).get("file_path") - if public_files: - folder = ( - os.environ.get("BUCKET_DIR") - + "/" - + site - + "/" - + os.path.basename(public_files)[:15] - + "/" - ) - upload_file_to_s3(public_files, folder, conn, bucket) - - private_files = details.get("private_files", {}).get("file_path") - if private_files: - folder = ( - os.environ.get("BUCKET_DIR") - + "/" - + site - + "/" - + os.path.basename(private_files)[:15] - + "/" - ) - upload_file_to_s3(private_files, folder, conn, bucket) - - delete_old_backups(os.environ.get("BACKUP_LIMIT", "3"), bucket, site) - - print("push-backup complete") - exit(0) - - -if __name__ == "__main__": - main() diff --git a/build/frappe-worker/commands/restore_backup.py b/build/frappe-worker/commands/restore_backup.py deleted file mode 100644 index 1462cd27..00000000 --- a/build/frappe-worker/commands/restore_backup.py +++ /dev/null @@ -1,313 +0,0 @@ -import datetime -import hashlib -import os -import tarfile - -import boto3 -import frappe -from constants import COMMON_SITE_CONFIG_FILE, DATE_FORMAT, RDS_DB, RDS_PRIVILEGES -from frappe.installer import ( - get_conf_params, - make_conf, - make_site_dirs, - update_site_config, -) -from frappe.utils import get_sites, random_string -from utils import ( - check_s3_environment_variables, - get_config, - get_password, - get_site_config, - list_directories, - run_command, - set_key_in_site_config, -) - - -def get_backup_dir(): - return os.path.join(os.path.expanduser("~"), "backups") - - -def decompress_db(database_file, site): - command = ["gunzip", "-c", database_file] - with open(database_file.replace(".gz", ""), "w") as db_file: - print(f"Extract Database GZip for site {site}") - run_command(command, stdout=db_file) - - -def restore_database(files_base, site_config_path, site): - # restore database - database_file = files_base + "-database.sql.gz" - decompress_db(database_file, site) - config = get_config() - - # Set db_type if it exists in backup site_config.json - set_key_in_site_config("db_type", site, site_config_path) - # Set db_host if it exists in backup site_config.json - set_key_in_site_config("db_host", site, site_config_path) - # Set db_port if it exists in backup site_config.json - set_key_in_site_config("db_port", site, site_config_path) - - # get updated site_config - site_config = get_site_config(site) - - # if no db_type exists, default to mariadb - db_type = site_config.get("db_type", "mariadb") - is_database_restored = False - - if db_type == "mariadb": - restore_mariadb( - config=config, site_config=site_config, database_file=database_file - ) - is_database_restored = True - elif db_type == "postgres": - restore_postgres( - config=config, site_config=site_config, database_file=database_file - ) - is_database_restored = True - - if is_database_restored: - # Set encryption_key if it exists in backup site_config.json - set_key_in_site_config("encryption_key", site, site_config_path) - - -def restore_files(files_base): - public_files = files_base + "-files.tar" - # extract tar - public_tar = tarfile.open(public_files) - print(f"Extracting {public_files}") - public_tar.extractall() - - -def restore_private_files(files_base): - private_files = files_base + "-private-files.tar" - private_tar = tarfile.open(private_files) - print(f"Extracting {private_files}") - private_tar.extractall() - - -def pull_backup_from_s3(): - check_s3_environment_variables() - - # https://stackoverflow.com/a/54672690 - s3 = boto3.resource( - "s3", - region_name=os.environ.get("REGION"), - aws_access_key_id=os.environ.get("ACCESS_KEY_ID"), - aws_secret_access_key=os.environ.get("SECRET_ACCESS_KEY"), - endpoint_url=os.environ.get("ENDPOINT_URL"), - ) - - bucket_dir = os.environ.get("BUCKET_DIR") - bucket_name = os.environ.get("BUCKET_NAME") - bucket = s3.Bucket(bucket_name) - - # Change directory to /home/frappe/backups - os.chdir(get_backup_dir()) - - backup_files = [] - sites = set() - site_timestamps = set() - download_backups = [] - - for obj in bucket.objects.filter(Prefix=bucket_dir): - if obj.get()["ContentType"] == "application/x-directory": - continue - backup_file = obj.key.replace(os.path.join(bucket_dir, ""), "") - backup_files.append(backup_file) - site_name, timestamp, backup_type = backup_file.split("/") - site_timestamp = site_name + "/" + timestamp - sites.add(site_name) - site_timestamps.add(site_timestamp) - - # sort sites for latest backups - for site in sites: - backup_timestamps = [] - for site_timestamp in site_timestamps: - site_name, timestamp = site_timestamp.split("/") - if site == site_name: - timestamp_datetime = datetime.datetime.strptime(timestamp, DATE_FORMAT) - backup_timestamps.append(timestamp) - download_backups.append(site + "/" + max(backup_timestamps)) - - # Only download latest backups - for backup_file in backup_files: - for backup in download_backups: - if backup in backup_file: - if not os.path.exists(os.path.dirname(backup_file)): - os.makedirs(os.path.dirname(backup_file)) - print(f"Downloading {backup_file}") - bucket.download_file(bucket_dir + "/" + backup_file, backup_file) - - os.chdir(os.path.join(os.path.expanduser("~"), "frappe-bench", "sites")) - - -def restore_postgres(config, site_config, database_file): - # common config - common_site_config_path = os.path.join(os.getcwd(), COMMON_SITE_CONFIG_FILE) - - db_root_user = config.get("root_login") - if not db_root_user: - postgres_user = os.environ.get("DB_ROOT_USER") - if not postgres_user: - print("Variable DB_ROOT_USER not set") - exit(1) - - db_root_user = postgres_user - update_site_config( - "root_login", - db_root_user, - validate=False, - site_config_path=common_site_config_path, - ) - - db_root_password = config.get("root_password") - if not db_root_password: - root_password = get_password("POSTGRES_PASSWORD") - if not root_password: - print("Variable POSTGRES_PASSWORD not set") - exit(1) - - db_root_password = root_password - update_site_config( - "root_password", - db_root_password, - validate=False, - site_config_path=common_site_config_path, - ) - - # site config - db_host = site_config.get("db_host") - db_port = site_config.get("db_port", 5432) - db_name = site_config.get("db_name") - db_password = site_config.get("db_password") - - psql_command = ["psql"] - psql_uri = f"postgres://{db_root_user}:{db_root_password}@{db_host}:{db_port}" - - print("Restoring PostgreSQL") - run_command(psql_command + [psql_uri, "-c", f'DROP DATABASE IF EXISTS "{db_name}"']) - run_command(psql_command + [psql_uri, "-c", f"DROP USER IF EXISTS {db_name}"]) - run_command(psql_command + [psql_uri, "-c", f'CREATE DATABASE "{db_name}"']) - run_command( - psql_command - + [psql_uri, "-c", f"CREATE user {db_name} password '{db_password}'"] - ) - run_command( - psql_command - + [psql_uri, "-c", f'GRANT ALL PRIVILEGES ON DATABASE "{db_name}" TO {db_name}'] - ) - with open(database_file.replace(".gz", "")) as db_file: - run_command(psql_command + [f"{psql_uri}/{db_name}", "<"], stdin=db_file) - - -def restore_mariadb(config, site_config, database_file): - db_root_password = get_password("MYSQL_ROOT_PASSWORD") - if not db_root_password: - print("Variable MYSQL_ROOT_PASSWORD not set") - exit(1) - - db_root_user = os.environ.get("DB_ROOT_USER", "root") - - db_host = site_config.get("db_host", config.get("db_host")) - db_port = site_config.get("db_port", config.get("db_port", 3306)) - db_name = site_config.get("db_name") - db_password = site_config.get("db_password") - - # mysql command prefix - mysql_command = [ - "mysql", - f"-u{db_root_user}", - f"-h{db_host}", - f"-p{db_root_password}", - f"-P{db_port}", - ] - - # drop db if exists for clean restore - drop_database = mysql_command + ["-e", f"DROP DATABASE IF EXISTS `{db_name}`;"] - run_command(drop_database) - - # create db - create_database = mysql_command + [ - "-e", - f"CREATE DATABASE IF NOT EXISTS `{db_name}`;", - ] - run_command(create_database) - - # create user - create_user = mysql_command + [ - "-e", - f"CREATE USER IF NOT EXISTS '{db_name}'@'%' IDENTIFIED BY '{db_password}'; FLUSH PRIVILEGES;", - ] - run_command(create_user) - - # grant db privileges to user - - grant_privileges = "ALL PRIVILEGES" - - # for Amazon RDS - if config.get(RDS_DB) or site_config.get(RDS_DB): - grant_privileges = RDS_PRIVILEGES - - grant_privileges_command = mysql_command + [ - "-e", - f"GRANT {grant_privileges} ON `{db_name}`.* TO '{db_name}'@'%' IDENTIFIED BY '{db_password}'; FLUSH PRIVILEGES;", - ] - run_command(grant_privileges_command) - - print("Restoring MariaDB") - with open(database_file.replace(".gz", "")) as db_file: - run_command(mysql_command + [f"{db_name}"], stdin=db_file) - - -def main(): - backup_dir = get_backup_dir() - - if len(list_directories(backup_dir)) == 0: - pull_backup_from_s3() - - for site in list_directories(backup_dir): - site_slug = site.replace(".", "_") - backups = [ - datetime.datetime.strptime(backup, DATE_FORMAT) - for backup in list_directories(os.path.join(backup_dir, site)) - ] - latest_backup = max(backups).strftime(DATE_FORMAT) - files_base = os.path.join(backup_dir, site, latest_backup, "") - files_base += latest_backup + "-" + site_slug - site_config_path = files_base + "-site_config_backup.json" - if not os.path.exists(site_config_path): - site_config_path = os.path.join(backup_dir, site, "site_config.json") - if site in get_sites(): - print(f"Overwrite site {site}") - restore_database(files_base, site_config_path, site) - restore_private_files(files_base) - restore_files(files_base) - else: - site_config = get_conf_params( - db_name="_" + hashlib.sha1(site.encode()).hexdigest()[:16], - db_password=random_string(16), - ) - - frappe.local.site = site - frappe.local.sites_path = os.getcwd() - frappe.local.site_path = os.getcwd() + "/" + site - make_conf( - db_name=site_config.get("db_name"), - db_password=site_config.get("db_password"), - ) - make_site_dirs() - - print(f"Create site {site}") - restore_database(files_base, site_config_path, site) - restore_private_files(files_base) - restore_files(files_base) - - if frappe.redis_server: - frappe.redis_server.connection_pool.disconnect() - - exit(0) - - -if __name__ == "__main__": - main() diff --git a/build/frappe-worker/commands/utils.py b/build/frappe-worker/commands/utils.py deleted file mode 100644 index 7ea48225..00000000 --- a/build/frappe-worker/commands/utils.py +++ /dev/null @@ -1,208 +0,0 @@ -import json -import os -import subprocess - -import boto3 -import git -from constants import APP_VERSIONS_JSON_FILE, APPS_TXT_FILE, COMMON_SITE_CONFIG_FILE -from frappe.installer import update_site_config - - -def run_command(command, stdout=None, stdin=None, stderr=None): - stdout = stdout or subprocess.PIPE - stderr = stderr or subprocess.PIPE - stdin = stdin or subprocess.PIPE - process = subprocess.Popen(command, stdout=stdout, stdin=stdin, stderr=stderr) - out, error = process.communicate() - if process.returncode: - print("Something went wrong:") - print(f"return code: {process.returncode}") - print(f"stdout:\n{out}") - print(f"\nstderr:\n{error}") - exit(process.returncode) - - -def save_version_file(versions): - with open(APP_VERSIONS_JSON_FILE, "w") as f: - return json.dump(versions, f, indent=1, sort_keys=True) - - -def get_apps(): - apps = [] - try: - with open(APPS_TXT_FILE) as apps_file: - for app in apps_file.readlines(): - if app.strip(): - apps.append(app.strip()) - - except FileNotFoundError as exception: - print(exception) - exit(1) - except Exception: - print(APPS_TXT_FILE + " is not valid") - exit(1) - - return apps - - -def get_container_versions(apps): - versions = {} - for app in apps: - try: - version = __import__(app).__version__ - versions.update({app: version}) - except Exception: - pass - - try: - path = os.path.join("..", "apps", app) - repo = git.Repo(path) - commit_hash = repo.head.object.hexsha - versions.update({app + "_git_hash": commit_hash}) - except Exception: - pass - - return versions - - -def get_version_file(): - versions = None - try: - with open(APP_VERSIONS_JSON_FILE) as versions_file: - versions = json.load(versions_file) - except Exception: - pass - return versions - - -def get_config(): - config = None - try: - with open(COMMON_SITE_CONFIG_FILE) as config_file: - config = json.load(config_file) - except FileNotFoundError as exception: - print(exception) - exit(1) - except Exception: - print(COMMON_SITE_CONFIG_FILE + " is not valid") - exit(1) - return config - - -def get_site_config(site_name): - site_config = None - with open(f"{site_name}/site_config.json") as site_config_file: - site_config = json.load(site_config_file) - return site_config - - -def save_config(config): - with open(COMMON_SITE_CONFIG_FILE, "w") as f: - return json.dump(config, f, indent=1, sort_keys=True) - - -def get_password(env_var, default=None): - return ( - os.environ.get(env_var) - or get_password_from_secret(f"{env_var}_FILE") - or default - ) - - -def get_password_from_secret(env_var): - """Fetches the secret value from the docker secret file - usually located inside /run/secrets/ - Arguments: - env_var {str} -- Name of the environment variable - containing the path to the secret file. - Returns: - [str] -- Secret value - """ - passwd = None - secret_file_path = os.environ.get(env_var) - if secret_file_path: - with open(secret_file_path) as secret_file: - passwd = secret_file.read().strip() - - return passwd - - -def get_s3_config(): - check_s3_environment_variables() - bucket = os.environ.get("BUCKET_NAME") - - conn = boto3.client( - "s3", - region_name=os.environ.get("REGION"), - aws_access_key_id=os.environ.get("ACCESS_KEY_ID"), - aws_secret_access_key=os.environ.get("SECRET_ACCESS_KEY"), - endpoint_url=os.environ.get("ENDPOINT_URL"), - ) - - return conn, bucket - - -def upload_file_to_s3(filename, folder, conn, bucket): - - destpath = os.path.join(folder, os.path.basename(filename)) - try: - print("Uploading file:", filename) - conn.upload_file(filename, bucket, destpath) - - except Exception as e: - print("Error uploading: %s" % (e)) - exit(1) - - -def list_directories(path): - directories = [] - for name in os.listdir(path): - if os.path.isdir(os.path.join(path, name)): - directories.append(name) - return directories - - -def get_site_config_from_path(site_config_path): - site_config = dict() - if os.path.exists(site_config_path): - with open(site_config_path) as sc: - site_config = json.load(sc) - return site_config - - -def set_key_in_site_config(key, site, site_config_path): - site_config = get_site_config_from_path(site_config_path) - value = site_config.get(key) - if value: - print(f"Set {key} in site config for site: {site}") - update_site_config( - key, - value, - site_config_path=os.path.join(os.getcwd(), site, "site_config.json"), - ) - - -def check_s3_environment_variables(): - if "BUCKET_NAME" not in os.environ: - print("Variable BUCKET_NAME not set") - exit(1) - - if "ACCESS_KEY_ID" not in os.environ: - print("Variable ACCESS_KEY_ID not set") - exit(1) - - if "SECRET_ACCESS_KEY" not in os.environ: - print("Variable SECRET_ACCESS_KEY not set") - exit(1) - - if "ENDPOINT_URL" not in os.environ: - print("Variable ENDPOINT_URL not set") - exit(1) - - if "BUCKET_DIR" not in os.environ: - print("Variable BUCKET_DIR not set") - exit(1) - - if "REGION" not in os.environ: - print("Variable REGION not set") - exit(1) diff --git a/build/frappe-worker/common_site_config.json.template b/build/frappe-worker/common_site_config.json.template deleted file mode 100755 index f7272fd6..00000000 --- a/build/frappe-worker/common_site_config.json.template +++ /dev/null @@ -1,8 +0,0 @@ -{ - "db_host": "${DB_HOST}", - "db_port": ${DB_PORT}, - "redis_cache": "redis://${REDIS_CACHE}", - "redis_queue": "redis://${REDIS_QUEUE}", - "redis_socketio": "redis://${REDIS_SOCKETIO}", - "socketio_port": ${SOCKETIO_PORT} -} diff --git a/build/frappe-worker/docker-entrypoint.sh b/build/frappe-worker/docker-entrypoint.sh deleted file mode 100755 index 74247208..00000000 --- a/build/frappe-worker/docker-entrypoint.sh +++ /dev/null @@ -1,191 +0,0 @@ -#!/bin/bash - -function configureEnv() { - if [[ ! -f /home/frappe/frappe-bench/sites/common_site_config.json ]]; then - - if [[ -z "${MARIADB_HOST}" && -z "${POSTGRES_HOST}" ]]; then - echo "MARIADB_HOST or POSTGRES_HOST is not set" >&2 - exit 1 - fi - - if [[ -z "${REDIS_CACHE}" ]]; then - echo "REDIS_CACHE is not set" >&2 - exit 1 - fi - - if [[ -z "${REDIS_QUEUE}" ]]; then - echo "REDIS_QUEUE is not set" >&2 - exit 1 - fi - - if [[ -z "${REDIS_SOCKETIO}" ]]; then - echo "REDIS_SOCKETIO is not set" >&2 - exit 1 - fi - - if [[ -z "${SOCKETIO_PORT}" ]]; then - echo "SOCKETIO_PORT is not set" >&2 - exit 1 - fi - - if [[ -z "${DB_PORT}" ]]; then - export DB_PORT=3306 - fi - - export DB_HOST="${MARIADB_HOST:-$POSTGRES_HOST}" - - # shellcheck disable=SC2016 - envsubst '${DB_HOST} - ${DB_PORT} - ${REDIS_CACHE} - ${REDIS_QUEUE} - ${REDIS_SOCKETIO} - ${SOCKETIO_PORT}' /home/frappe/frappe-bench/sites/common_site_config.json - fi -} - -function checkConnection() { - /home/frappe/frappe-bench/env/bin/python /home/frappe/frappe-bench/commands/check_connection.py -} - -function checkConfigExists() { - COUNTER=0 - while [[ ! -e /home/frappe/frappe-bench/sites/common_site_config.json && ${COUNTER} -le 30 ]]; do - sleep 1 - ((COUNTER = COUNTER + 1)) - echo "config file not created, retry ${COUNTER}" >&2 - done - - if [[ ! -e /home/frappe/frappe-bench/sites/common_site_config.json ]]; then - echo "timeout: config file not created" >&2 - exit 1 - fi -} - -if [[ ! -e /home/frappe/frappe-bench/sites/apps.txt ]]; then - find /home/frappe/frappe-bench/apps -mindepth 1 -maxdepth 1 -type d -printf '%f\n' | - sort -r >/home/frappe/frappe-bench/sites/apps.txt -fi - -# symlink node_modules -ln -sfn /home/frappe/frappe-bench/sites/assets/frappe/node_modules \ - /home/frappe/frappe-bench/apps/frappe/node_modules - -case "$1" in - -start) - configureEnv - checkConnection - - [[ -z "${WORKERS}" ]] && WORKERS='2' - - [[ -z "${FRAPPE_PORT}" ]] && FRAPPE_PORT='8000' - - [[ -z "${WORKER_CLASS}" ]] && WORKER_CLASS='gthread' - - LOAD_CONFIG_FILE="" - [[ "${WORKER_CLASS}" == "gevent" ]] && - LOAD_CONFIG_FILE="-c /home/frappe/frappe-bench/commands/gevent_patch.py" - - if [[ -n "${AUTO_MIGRATE}" ]]; then - /home/frappe/frappe-bench/env/bin/python /home/frappe/frappe-bench/commands/auto_migrate.py - fi - - # shellcheck disable=SC2086 - /home/frappe/frappe-bench/env/bin/gunicorn ${LOAD_CONFIG_FILE} -b 0.0.0.0:${FRAPPE_PORT} \ - --worker-tmp-dir /dev/shm \ - --threads=4 \ - --workers ${WORKERS} \ - --worker-class=${WORKER_CLASS} \ - --log-file=- \ - -t 120 frappe.app:application --preload - ;; - -worker) - checkConfigExists - checkConnection - - : "${WORKER_TYPE:=default}" - bench worker --queue $WORKER_TYPE - ;; - -schedule) - checkConfigExists - checkConnection - - bench schedule - ;; - -new) - checkConfigExists - checkConnection - - /home/frappe/frappe-bench/env/bin/python /home/frappe/frappe-bench/commands/new.py - exit - ;; - -drop) - checkConfigExists - checkConnection - - : "${SITE_NAME:=site1.localhost}" - : "${DB_ROOT_USER:=root}" - : "${DB_ROOT_PASSWORD:=$POSTGRES_PASSWORD}" - : "${DB_ROOT_PASSWORD:=$MYSQL_ROOT_PASSWORD}" - : "${DB_ROOT_PASSWORD:=admin}" - - FLAGS= - if [[ ${NO_BACKUP} == 1 ]]; then - FLAGS="${FLAGS} --no-backup" - fi - if [[ ${FORCE} == 1 ]]; then - FLAGS="${FLAGS} --force" - fi - - # shellcheck disable=SC2086 - bench drop-site \ - ${SITE_NAME} \ - --root-login ${DB_ROOT_USER} \ - --root-password ${DB_ROOT_PASSWORD} \ - --archived-sites-path /home/frappe/frappe-bench/sites/archive_sites \ - ${FLAGS} - ;; - -migrate) - /home/frappe/frappe-bench/env/bin/python /home/frappe/frappe-bench/commands/migrate.py - exit - ;; - -doctor) - /home/frappe/frappe-bench/env/bin/python /home/frappe/frappe-bench/commands/doctor.py "${@:2}" - exit - ;; - -backup) - /home/frappe/frappe-bench/env/bin/python /home/frappe/frappe-bench/commands/backup.py - exit - ;; - -console) - if [[ -z "$2" ]]; then - echo "Need to specify a sitename with the command:" >&2 - echo "console " >&2 - exit 1 - fi - - bench --site "$2" console - ;; - -push-backup) - /home/frappe/frappe-bench/env/bin/python /home/frappe/frappe-bench/commands/push_backup.py - exit - ;; - -restore-backup) - /home/frappe/frappe-bench/env/bin/python /home/frappe/frappe-bench/commands/restore_backup.py - exit - ;; -*) - exec "$@" - ;; -esac diff --git a/build/frappe-worker/healthcheck.sh b/build/frappe-worker/healthcheck.sh deleted file mode 100755 index 3c4266ee..00000000 --- a/build/frappe-worker/healthcheck.sh +++ /dev/null @@ -1,45 +0,0 @@ -#!/bin/bash -set -ea - -function getUrl() { - grep "$2" "$1" | awk -v word="$2" '$word { gsub(/[",]/,"",$2); print $2}' | tr -d '\n' -} - -COMMON_SITE_CONFIG_JSON='/home/frappe/frappe-bench/sites/common_site_config.json' - -# Set DB Host and port -DB_HOST=$(getUrl "${COMMON_SITE_CONFIG_JSON}" "db_host") -DB_PORT=$(getUrl "${COMMON_SITE_CONFIG_JSON}" "db_port") -if [[ -z "${DB_PORT}" ]]; then - DB_PORT=3306 -fi - -# Set REDIS host:port -REDIS_CACHE=$(getUrl "${COMMON_SITE_CONFIG_JSON}" "redis_cache" | sed 's|redis://||g') -if [[ "${REDIS_CACHE}" == *"/"* ]]; then - REDIS_CACHE=$(echo "${REDIS_CACHE}" | cut -f1 -d"/") -fi - -REDIS_QUEUE=$(getUrl "${COMMON_SITE_CONFIG_JSON}" "redis_queue" | sed 's|redis://||g') -if [[ "${REDIS_QUEUE}" == *"/"* ]]; then - REDIS_QUEUE=$(echo "${REDIS_QUEUE}" | cut -f1 -d"/") -fi - -REDIS_SOCKETIO=$(getUrl "${COMMON_SITE_CONFIG_JSON}" "redis_socketio" | sed 's|redis://||g') -if [[ "${REDIS_SOCKETIO}" == *"/"* ]]; then - REDIS_SOCKETIO=$(echo "${REDIS_SOCKETIO}" | cut -f1 -d"/") -fi - -echo "Check ${DB_HOST}:${DB_PORT}" -wait-for-it "${DB_HOST}:${DB_PORT}" -t 1 -echo "Check ${REDIS_CACHE}" -wait-for-it "${REDIS_CACHE}" -t 1 -echo "Check ${REDIS_QUEUE}" -wait-for-it "${REDIS_QUEUE}" -t 1 -echo "Check ${REDIS_SOCKETIO}" -wait-for-it "${REDIS_SOCKETIO}" -t 1 - -if [[ "$1" = "-p" || "$1" = "--ping-service" ]]; then - echo "Check $2" - wait-for-it "$2" -t 1 -fi diff --git a/build/frappe-worker/install_app.sh b/build/frappe-worker/install_app.sh deleted file mode 100755 index 6e55ec74..00000000 --- a/build/frappe-worker/install_app.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash -ex - -APP_NAME=${1} -APP_REPO=${2} -APP_BRANCH=${3} - -[[ -n "${APP_BRANCH}" ]] && BRANCH="-b ${APP_BRANCH}" - -# shellcheck disable=SC2086 -git clone --depth 1 -o upstream "${APP_REPO}" ${BRANCH} "/home/frappe/frappe-bench/apps/${APP_NAME}" -/home/frappe/frappe-bench/env/bin/pip3 install --no-cache-dir -e "/home/frappe/frappe-bench/apps/${APP_NAME}" diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 00000000..da3decfd --- /dev/null +++ b/compose.yaml @@ -0,0 +1,77 @@ +x-depends-on-configurator: &depends_on_configurator + depends_on: + configurator: + condition: service_completed_successfully + +x-backend-defaults: &backend_defaults + <<: *depends_on_configurator + image: frappe/frappe-worker:${FRAPPE_VERSION:?No Frappe version set} + volumes: + - sites:/home/frappe/frappe-bench/sites + +services: + configurator: + <<: *backend_defaults + command: configure.py + environment: + DB_HOST: ${DB_HOST} + DB_PORT: ${DB_PORT} + REDIS_CACHE: ${REDIS_CACHE} + REDIS_QUEUE: ${REDIS_QUEUE} + REDIS_SOCKETIO: ${REDIS_SOCKETIO} + SOCKETIO_PORT: 9000 + depends_on: {} + + backend: + <<: *backend_defaults + volumes: + - sites:/home/frappe/frappe-bench/sites + - assets:/home/frappe/frappe-bench/sites/assets:ro + + frontend: + image: frappe/frappe-nginx:${FRAPPE_VERSION} + environment: + BACKEND: backend:8000 + SOCKETIO: websocket:9000 + FRAPPE_SITE_NAME_HEADER: ${FRAPPE_SITE_NAME_HEADER:-$$host} + UPSTREAM_REAL_IP_ADDRESS: ${UPSTREAM_REAL_IP_ADDRESS:-127.0.0.1} + UPSTREAM_REAL_IP_HEADER: ${UPSTREAM_REAL_IP_HEADER:-X-Forwarded-For} + UPSTREAM_REAL_IP_RECURSIVE: ${UPSTREAM_REAL_IP_RECURSIVE:-off} + volumes: + - sites:/usr/share/nginx/html/sites + - assets:/usr/share/nginx/html/assets + depends_on: + - backend + - websocket + labels: + - traefik.enable=true + - traefik.http.services.frontend.loadbalancer.server.port=8080 + - traefik.http.routers.frontend-http.entrypoints=web + - traefik.http.routers.frontend-http.rule=HostRegexp(`{any:.+}`) + + websocket: + <<: *depends_on_configurator + image: frappe/frappe-socketio:${FRAPPE_VERSION} + volumes: + - sites:/home/frappe/frappe-bench/sites + + queue-short: + <<: *backend_defaults + command: bench worker --queue short + + queue-default: + <<: *backend_defaults + command: bench worker --queue default + + queue-long: + <<: *backend_defaults + command: bench worker --queue long + + scheduler: + <<: *backend_defaults + command: bench schedule + +# ERPNext requires local assets access (Frappe does not) +volumes: + sites: + assets: diff --git a/custom_app/README.md b/custom_app/README.md new file mode 100644 index 00000000..7ebd919c --- /dev/null +++ b/custom_app/README.md @@ -0,0 +1,38 @@ +This is basic configuration for building images and testing custom apps that use Frappe. + +You can see that there's four files in this folder: + +- `backend.Dockerfile`, +- `frontend.Dockerfile`, +- `docker-bake.hcl`, +- `compose.override.yaml`. + +Python code will `backend.Dockerfile`. JS and CSS (and other fancy frontend stuff) files will be built in `frontend.Dockerfile` if required and served from there. + +`docker-bake.hcl` is reference file for cool new [Buildx Bake](https://github.com/docker/buildx/blob/master/docs/reference/buildx_bake.md). It helps to build images without having to remember all build arguments. + +`compose.override.yaml` is [Compose](https://docs.docker.com/compose/compose-file/) override that replaces images from [main compose file](https://github.com/frappe/frappe_docker/blob/main/compose.yaml) so it would use your own images. + +To get started, install Docker and [Buildx](https://github.com/docker/buildx#installing). Then copy all content of this folder (except this README) to your app's root directory. Also copy `compose.yaml` in the root of this repository. + +Before the next step—to build images—replace "custom_app" with your app's name in `docker-bake.hcl`. After that, let's try to build: + +```bash +FRAPPE_VERSION= docker buildx bake +``` + +If something goes wrong feel free to leave an issue. + +To test if site works, setup `.env` file (check [example](<(https://github.com/frappe/frappe_docker/blob/main/example.env)>)) and run: + +```bash +docker-compose up -d +docker-compose exec backend \ + bench new-site 127.0.0.1 \ + --mariadb-root-password 123 \ + --admin-password admin \ + --install-app +docker-compose restart backend +``` + +Cool! You just containerized your app! diff --git a/custom_app/backend.Dockerfile b/custom_app/backend.Dockerfile new file mode 100644 index 00000000..62a487d0 --- /dev/null +++ b/custom_app/backend.Dockerfile @@ -0,0 +1,8 @@ +ARG FRAPPE_VERSION +FROM frappe/frappe-worker:${FRAPPE_VERSION} + +ARG APP_NAME +COPY --chown=frappe . ../apps/${APP_NAME} + +RUN echo "frappe\n${APP_NAME}" >/home/frappe/frappe-bench/sites/apps.txt \ + && ../env/bin/pip install --no-cache-dir -e ../apps/${APP_NAME} diff --git a/custom_app/compose.override.yaml b/custom_app/compose.override.yaml new file mode 100644 index 00000000..002e0795 --- /dev/null +++ b/custom_app/compose.override.yaml @@ -0,0 +1,21 @@ +services: + configurator: + image: custom_app/worker:${VERSION} + + backend: + image: custom_app/worker:${VERSION} + + frontend: + image: custom_app/nginx:${VERSION} + + queue-short: + image: custom_app/worker:${VERSION} + + queue-default: + image: custom_app/worker:${VERSION} + + queue-long: + image: custom_app/worker:${VERSION} + + scheduler: + image: custom_app/worker:${VERSION} diff --git a/custom_app/docker-bake.hcl b/custom_app/docker-bake.hcl new file mode 100644 index 00000000..12aa7d0e --- /dev/null +++ b/custom_app/docker-bake.hcl @@ -0,0 +1,25 @@ +APP_NAME="custom_app" + +variable "FRAPPE_VERSION" {} + +group "default" { + targets = ["backend", "frontend"] +} + +target "backend" { + dockerfile = "backend.Dockerfile" + tags = ["custom_app/worker:latest"] + args = { + "FRAPPE_VERSION" = FRAPPE_VERSION + "APP_NAME" = APP_NAME + } +} + +target "frontend" { + dockerfile = "frontend.Dockerfile" + tags = ["custom_app/nginx:latest"] + args = { + "FRAPPE_VERSION" = FRAPPE_VERSION + "APP_NAME" = APP_NAME + } +} diff --git a/custom_app/frontend.Dockerfile b/custom_app/frontend.Dockerfile new file mode 100644 index 00000000..382608f5 --- /dev/null +++ b/custom_app/frontend.Dockerfile @@ -0,0 +1,52 @@ +ARG FRAPPE_VERSION +FROM node:14-bullseye-slim as prod_node_modules + +RUN apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y \ + git \ + build-essential \ + python \ + ca-certificates \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /root/frappe-bench +RUN mkdir -p sites/assets + +ARG FRAPPE_VERSION +RUN git clone --depth 1 -b ${FRAPPE_VERSION} https://github.com/frappe/frappe apps/frappe + +RUN yarn --cwd apps/frappe + + +ARG APP_NAME +COPY . apps/${APP_NAME} + +# Install production node modules +RUN yarn --cwd apps/${APP_NAME} --prod + + + +FROM prod_node_modules as assets + +ARG APP_NAME + +# Install development node modules +RUN yarn --cwd apps/${APP_NAME} + +# Build assets +RUN echo "frappe\n${APP_NAME}" >sites/apps.txt \ + && yarn --cwd apps/frappe production --app ${APP_NAME} \ + && rm sites/apps.txt + + + +FROM frappe/frappe-nginx:${FRAPPE_VERSION} + +ARG APP_NAME + +# Copy all not built assets +COPY --from=prod_node_modules /root/frappe-bench/apps/${APP_NAME}/${APP_NAME}/public /usr/share/nginx/html/assets/${APP_NAME} +# Copy production node modules +COPY --from=prod_node_modules /root/frappe-bench/apps/${APP_NAME}/node_modules /usr/share/nginx/html/assets/${APP_NAME}/node_modules +# Copy built assets +COPY --from=assets /root/frappe-bench/sites /usr/share/nginx/html diff --git a/devcontainer-example/devcontainer.json b/devcontainer-example/devcontainer.json index 0640f28b..e54fc138 100644 --- a/devcontainer-example/devcontainer.json +++ b/devcontainer-example/devcontainer.json @@ -3,7 +3,8 @@ "appPort": [8000, 9000, 6787], "remoteUser": "frappe", "settings": { - "terminal.integrated.shell.linux": "/bin/bash" + "terminal.integrated.shell.linux": "/bin/bash", + "debug.node.autoAttach": "disabled" }, "dockerComposeFile": "./docker-compose.yml", "service": "frappe", diff --git a/devcontainer-example/docker-compose.yml b/devcontainer-example/docker-compose.yml index 66c171b2..42efdc3f 100644 --- a/devcontainer-example/docker-compose.yml +++ b/devcontainer-example/docker-compose.yml @@ -8,22 +8,17 @@ services: - --skip-character-set-client-handshake - --skip-innodb-read-only-compressed # Temporary fix for MariaDB 10.6 environment: - - MYSQL_ROOT_PASSWORD=123 - - MYSQL_USER=root - # Sometimes db initialization takes longer than 10 seconds and site-creator goes away. - # Frappe doesn't use CONVERT_TZ() function that requires time zone info, so we can just skip it. - - MYSQL_INITDB_SKIP_TZINFO=1 + MYSQL_ROOT_PASSWORD: 123 volumes: - - mariadb-vol:/var/lib/mysql + - mariadb-data:/var/lib/mysql # Enable PostgreSQL only if you use it, see development/README.md for more information. # postgresql: # image: postgres:11.8 - # restart: on-failure # environment: - # - POSTGRES_PASSWORD=123 + # POSTGRES_PASSWORD: 123 # volumes: - # - postgresql-vol:/var/lib/postgresql/data + # - postgresql-data:/var/lib/postgresql/data redis-cache: image: redis:alpine @@ -43,9 +38,9 @@ services: - ..:/workspace:cached working_dir: /workspace/development ports: - - "8000-8005:8000-8005" - - "9000-9005:9000-9005" + - 8000-8005:8000-8005 + - 9000-9005:9000-9005 volumes: - mariadb-vol: - postgresql-vol: + mariadb-data: + postgresql-data: diff --git a/development/README.md b/development/README.md index 9ef0cb52..46fe1afa 100644 --- a/development/README.md +++ b/development/README.md @@ -52,12 +52,12 @@ After the extensions are installed, you can: - Open frappe_docker folder in VS Code. - `code .` -- Launch the command, from Command Palette (Ctrl + Shift + P) `Execute Remote Containers : Reopen in Container`. You can also click in the bottom left corner to access the remote container menu. +- Launch the command, from Command Palette (Ctrl + Shift + P) `Remote-Containers: Reopen in Container`. You can also click in the bottom left corner to access the remote container menu. Notes: - The `development` directory is ignored by git. It is mounted and available inside the container. Create all your benches (installations of bench, the tool that manages frappe) inside this directory. -- nvm with node v12 and v10 is installed. Check with `nvm ls`. Node v12 is used by default. +- Node v14 and v10 are installed. Check with `nvm ls`. Node v14 is used by default. ### Setup first bench @@ -68,7 +68,7 @@ bench init --skip-redis-config-generation --frappe-branch version-13 frappe-benc cd frappe-bench ``` -Note: For version 12 use python 3.7 by passing option to `bench init` command, e.g. `bench init --skip-redis-config-generation --frappe-branch version-12 --python python3.7 frappe-bench` +Note: For version 12 use Python 3.7 by passing option to `bench init` command, e.g. `bench init --skip-redis-config-generation --frappe-branch version-12 --python python3.7 frappe-bench` ### Setup hosts @@ -162,14 +162,14 @@ To install custom app ```shell # --branch is optional, use it to point to branch on custom app repository -bench get-app --branch version-12 myapp https://github.com/myusername/myapp.git +bench get --branch version-12 https://github.com/myusername/myapp bench --site mysite.localhost install-app myapp ``` To install ERPNext (from the version-12 branch): ```shell -bench get-app --branch version-12 erpnext https://github.com/frappe/erpnext.git +bench get --branch version-12 erpnext bench --site mysite.localhost install-app erpnext ``` @@ -242,6 +242,7 @@ Replace `mysite.localhost` with your site and run the following code in a Jupyte ```python import frappe + frappe.init(site='mysite.localhost', sites_path='/workspace/development/frappe-bench/sites') frappe.connect() frappe.local.lang = frappe.db.get_default('lang') @@ -250,38 +251,6 @@ frappe.db.connect() The first command can take a few seconds to be executed, this is to be expected. -### Fixing MariaDB issues after rebuilding the container - -For any reason after rebuilding the container if you are not be able to access MariaDB correctly with the previous configuration. Follow these instructions. - -The parameter `'db_name'@'%'` needs to be set in MariaDB and permission to the site database suitably assigned to the user. - -This step has to be repeated for all sites available under the current bench. -Example shows the queries to be executed for site `localhost` - -Open sites/localhost/site_config.json: - -```shell -code sites/localhost/site_config.json -``` - -and take note of the parameters `db_name` and `db_password`. - -Enter MariaDB Interactive shell: - -```shell -mysql -uroot -p123 -hmariadb -``` - -Execute following queries replacing `db_name` and `db_password` with the values found in site_config.json. - -```sql -UPDATE mysql.global_priv SET Host = '%' where User = 'db_name'; FLUSH PRIVILEGES; -SET PASSWORD FOR 'db_name'@'%' = PASSWORD('db_password'); FLUSH PRIVILEGES; -GRANT ALL PRIVILEGES ON `db_name`.* TO 'db_name'@'%'; FLUSH PRIVILEGES; -EXIT; -``` - ## Manually start containers In case you don't use VSCode, you may start the containers manually with the following command: diff --git a/development/vscode-example/settings.json b/development/vscode-example/settings.json deleted file mode 100644 index d0877e6f..00000000 --- a/development/vscode-example/settings.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "debug.node.autoAttach": "disabled", - "python.pythonPath": "/workspace/frappe-bench/env/bin/python", - "python.analysis.extraPaths": ["./frappe-bench/apps/frappe"] -} diff --git a/docker-bake.hcl b/docker-bake.hcl index 51f69ba3..28355ada 100644 --- a/docker-bake.hcl +++ b/docker-bake.hcl @@ -1,11 +1,22 @@ # Docker Buildx Bake build definition file # Reference: https://github.com/docker/buildx/blob/master/docs/reference/buildx_bake.md +variable "REGISTRY_USER" { + default = "frappe" +} + +variable "FRAPPE_VERSION" { + default = "develop" +} + +variable "ERPNEXT_VERSION" { + default = "develop" +} # Bench image target "bench" { - context = "build/bench" + context = "images/bench" target = "bench" tags = ["frappe/bench:latest"] } @@ -18,255 +29,67 @@ target "bench-test" { # Main images # Base for all other targets -target "frappe-nginx" { - dockerfile = "build/frappe-nginx/Dockerfile" +group "frappe" { + targets = ["frappe-worker", "frappe-nginx", "frappe-socketio"] +} + +group "erpnext" { + targets = ["erpnext-worker", "erpnext-nginx"] +} + +group "default" { + targets = ["frappe", "erpnext"] +} + +function "tag" { + params = [repo, version] + result = [ + # If `version` param is develop (development build) then use tag `latest` + "${version}" == "develop" ? "${REGISTRY_USER}/${repo}:latest" : "${REGISTRY_USER}/${repo}:${version}", + # Make short tag for major version if possible. For example, from v13.16.0 make v13. + can(regex("(v[0-9]+)[.]", "${version}")) ? "${REGISTRY_USER}/${repo}:${regex("(v[0-9]+)[.]", "${version}")[0]}" : "", + ] +} + +target "default-args" { + args = { + FRAPPE_VERSION = "${FRAPPE_VERSION}" + ERPNEXT_VERSION = "${ERPNEXT_VERSION}" + # If `ERPNEXT_VERSION` variable contains "v12" use Python 3.7. Else — 3.9. + PYTHON_VERSION = can(regex("v12", "${ERPNEXT_VERSION}")) ? "3.7" : "3.9" + } } target "frappe-worker" { - dockerfile = "build/frappe-worker/Dockerfile" -} - -target "frappe-socketio" { - dockerfile = "build/frappe-socketio/Dockerfile" -} - -target "erpnext-nginx" { - dockerfile = "build/erpnext-nginx/Dockerfile" + inherits = ["default-args"] + context = "images/worker" + target = "frappe" + tags = tag("frappe-worker", "${FRAPPE_VERSION}") } target "erpnext-worker" { - dockerfile = "build/erpnext-worker/Dockerfile" + inherits = ["default-args"] + context = "images/worker" + target = "erpnext" + tags = tag("erpnext-worker", "${ERPNEXT_VERSION}") } - -# Helpers - -target "develop-args" { - args = { - GIT_BRANCH = "develop" - IMAGE_TAG = "develop" - } +target "frappe-nginx" { + inherits = ["default-args"] + context = "images/nginx" + target = "frappe" + tags = tag("frappe-nginx", "${FRAPPE_VERSION}") } -function "set_develop_tags" { - params = [repo] - result = ["${repo}:latest", "${repo}:edge", "${repo}:develop"] +target "erpnext-nginx" { + inherits = ["default-args"] + context = "images/nginx" + target = "erpnext" + tags = tag("erpnext-nginx", "${ERPNEXT_VERSION}") } -# NOTE: Variable are used only for stable builds -variable "GIT_TAG" {} # git tag, e.g. v13.15.0 -variable "GIT_BRANCH" {} # git branch, e.g. version-13 -variable "VERSION" {} # Frappe and ERPNext version, e.g. 13 - -target "stable-args" { - args = { - GIT_BRANCH = "${GIT_BRANCH}" - IMAGE_TAG = "${GIT_BRANCH}" - # ERPNext build fails on v12 - # TODO: Remove PYTHON_VERSION argument when v12 will stop being supported - PYTHON_VERSION = "${VERSION}" == "12" ? "3.7" : "3.9" - } -} - -function "set_stable_tags" { - # e.g. base_image:v13.15.0, base_image:v13, base_image:version-13 - params = [repo] - result = ["${repo}:${GIT_TAG}", "${repo}:v${VERSION}", "${repo}:${GIT_BRANCH}"] -} - -target "test-erpnext-args" { - args = { - IMAGE_TAG = "test" - DOCKER_REGISTRY_PREFIX = "localhost:5000/frappe" - } -} - -function "set_local_test_tags" { - params = [repo] - result = ["localhost:5000/${repo}:test"] -} - -function "set_test_tags" { - params = [repo] - result = ["${repo}:test"] -} - - -# Develop images - -target "frappe-nginx-develop" { - inherits = ["frappe-nginx", "develop-args"] - tags = set_develop_tags("frappe/frappe-nginx") -} - -target "frappe-worker-develop" { - inherits = ["frappe-worker", "develop-args"] - tags = set_develop_tags("frappe/frappe-worker") -} - -target "frappe-socketio-develop" { - inherits = ["frappe-socketio", "develop-args"] - tags = set_develop_tags("frappe/frappe-socketio") -} - -target "erpnext-nginx-develop" { - inherits = ["erpnext-nginx", "develop-args"] - tags = set_develop_tags("frappe/erpnext-nginx") -} - -target "erpnext-worker-develop" { - inherits = ["erpnext-worker", "develop-args"] - tags = set_develop_tags("frappe/erpnext-worker") -} - -group "frappe-develop" { - targets = ["frappe-nginx-develop", "frappe-worker-develop", "frappe-socketio-develop"] -} - -group "erpnext-develop" { - targets = ["erpnext-nginx-develop", "erpnext-worker-develop"] -} - -# Test develop images - -target "frappe-nginx-develop-test-local" { - inherits = ["frappe-nginx-develop"] - tags = set_local_test_tags("frappe/frappe-nginx") -} - -target "frappe-worker-develop-test-local" { - inherits = ["frappe-worker-develop"] - tags = set_local_test_tags("frappe/frappe-worker") -} - -target "frappe-socketio-develop-test-local" { - inherits = ["frappe-socketio-develop"] - tags = set_local_test_tags("frappe/frappe-socketio") -} - -target "frappe-nginx-develop-test" { - inherits = ["frappe-nginx-develop"] - tags = set_test_tags("frappe/frappe-nginx") -} - -target "frappe-worker-develop-test" { - inherits = ["frappe-worker-develop"] - tags = set_test_tags("frappe/frappe-worker") -} - -target "frappe-socketio-develop-test" { - inherits = ["frappe-socketio-develop"] - tags = set_test_tags("frappe/frappe-socketio") -} - -target "erpnext-nginx-develop-test" { - inherits = ["erpnext-nginx-develop", "test-erpnext-args"] - tags = set_test_tags("frappe/erpnext-nginx") -} - -target "erpnext-worker-develop-test" { - inherits = ["erpnext-worker-develop", "test-erpnext-args"] - tags = set_test_tags("frappe/erpnext-worker") -} - -group "frappe-develop-test-local" { - targets = ["frappe-nginx-develop-test-local", "frappe-worker-develop-test-local", "frappe-socketio-develop-test-local"] -} - -group "frappe-develop-test" { - targets = ["frappe-nginx-develop-test", "frappe-worker-develop-test", "frappe-socketio-develop-test"] -} - -group "erpnext-develop-test" { - targets = ["erpnext-nginx-develop-test", "erpnext-worker-develop-test"] -} - - -# Stable images - -target "frappe-nginx-stable" { - inherits = ["frappe-nginx", "stable-args"] - tags = set_stable_tags("frappe/frappe-nginx") -} - -target "frappe-worker-stable" { - inherits = ["frappe-worker", "stable-args"] - tags = set_stable_tags("frappe/frappe-worker") -} - -target "frappe-socketio-stable" { - inherits = ["frappe-socketio", "stable-args"] - tags = set_stable_tags("frappe/frappe-socketio") -} - -target "erpnext-nginx-stable" { - inherits = ["erpnext-nginx", "stable-args"] - tags = set_stable_tags("frappe/erpnext-nginx") -} - -target "erpnext-worker-stable" { - inherits = ["erpnext-worker", "stable-args"] - tags = set_stable_tags("frappe/erpnext-worker") -} - -group "frappe-stable" { - targets = ["frappe-nginx-stable", "frappe-worker-stable", "frappe-socketio-stable"] -} - -group "erpnext-stable" { - targets = ["erpnext-nginx-stable", "erpnext-worker-stable"] -} - -# Test stable images - -target "frappe-nginx-stable-test-local" { - inherits = ["frappe-nginx-stable"] - tags = set_local_test_tags("frappe/frappe-nginx") -} - -target "frappe-worker-stable-test-local" { - inherits = ["frappe-worker-stable"] - tags = set_local_test_tags("frappe/frappe-worker") -} - -target "frappe-socketio-stable-test-local" { - inherits = ["frappe-socketio-stable"] - tags = set_local_test_tags("frappe/frappe-socketio") -} - -target "frappe-nginx-stable-test" { - inherits = ["frappe-nginx-stable"] - tags = set_test_tags("frappe/frappe-nginx") -} - -target "frappe-worker-stable-test" { - inherits = ["frappe-worker-stable"] - tags = set_test_tags("frappe/frappe-worker") -} - -target "frappe-socketio-stable-test" { - inherits = ["frappe-socketio-stable"] - tags = set_test_tags("frappe/frappe-socketio") -} - -target "erpnext-nginx-stable-test" { - inherits = ["erpnext-nginx-stable", "test-erpnext-args"] - tags = set_test_tags("frappe/erpnext-nginx") -} - -target "erpnext-worker-stable-test" { - inherits = ["erpnext-worker-stable", "test-erpnext-args"] - tags = set_test_tags("frappe/erpnext-worker") -} - -group "frappe-stable-test-local" { - targets = ["frappe-nginx-stable-test-local", "frappe-worker-stable-test-local", "frappe-socketio-stable-test-local"] -} - -group "frappe-stable-test" { - targets = ["frappe-nginx-stable-test", "frappe-worker-stable-test", "frappe-socketio-stable-test"] -} - -group "erpnext-stable-test" { - targets = ["erpnext-nginx-stable-test", "erpnext-worker-stable-test"] +target "frappe-socketio" { + inherits = ["default-args"] + context = "images/socketio" + tags = tag("frappe-socketio", "${FRAPPE_VERSION}") } diff --git a/docker-compose.yml b/docker-compose.yml deleted file mode 100644 index d6fd92d9..00000000 --- a/docker-compose.yml +++ /dev/null @@ -1,180 +0,0 @@ -version: "3" - -services: - traefik: - image: "traefik:v2.2" - command: - - "--providers.docker=true" - - "--providers.docker.exposedbydefault=false" - - "--entrypoints.web.address=:80" - - "--entrypoints.websecure.address=:443" - - "--certificatesresolvers.myresolver.acme.httpchallenge=true" - - "--certificatesresolvers.myresolver.acme.httpchallenge.entrypoint=web" - - "--certificatesresolvers.myresolver.acme.email=${LETSENCRYPT_EMAIL}" - - "--certificatesresolvers.myresolver.acme.storage=/letsencrypt/acme.json" - labels: - # enable traefik - - "traefik.enable=true" - # global redirect to https for production only - - "${HTTPS_REDIRECT_RULE_LABEL}" - - "${HTTPS_REDIRECT_ENTRYPOINT_LABEL}" - - "${HTTPS_REDIRECT_MIDDLEWARE_LABEL}" - # middleware redirect for production only - - "${HTTPS_USE_REDIRECT_MIDDLEWARE_LABEL}" - ports: - - "80:80" - - "443:443" - volumes: - - cert-vol:/letsencrypt - - /var/run/docker.sock:/var/run/docker.sock:ro - userns_mode: "host" - - erpnext-nginx: - image: frappe/erpnext-nginx:${ERPNEXT_VERSION} - restart: on-failure - environment: - - FRAPPE_PY=erpnext-python - - FRAPPE_PY_PORT=8000 - - FRAPPE_SOCKETIO=frappe-socketio - - SOCKETIO_PORT=9000 - - SKIP_NGINX_TEMPLATE_GENERATION=${SKIP_NGINX_TEMPLATE_GENERATION} - labels: - - "traefik.enable=true" - - "traefik.http.routers.erpnext-nginx.rule=Host(${SITES})" - - "${ENTRYPOINT_LABEL}" - - "${CERT_RESOLVER_LABEL}" - - "traefik.http.services.erpnext-nginx.loadbalancer.server.port=8080" - volumes: - - sites-vol:/var/www/html/sites:rw - - assets-vol:/assets:rw - - erpnext-python: - image: frappe/erpnext-worker:${ERPNEXT_VERSION} - restart: on-failure - environment: - - MARIADB_HOST=${MARIADB_HOST} - - REDIS_CACHE=redis-cache:6379 - - REDIS_QUEUE=redis-queue:6379 - - REDIS_SOCKETIO=redis-socketio:6379 - - SOCKETIO_PORT=9000 - - AUTO_MIGRATE=1 - - WORKER_CLASS=${WORKER_CLASS} - volumes: - - sites-vol:/home/frappe/frappe-bench/sites:rw - - assets-vol:/home/frappe/frappe-bench/sites/assets:rw - - frappe-socketio: - image: frappe/frappe-socketio:${FRAPPE_VERSION} - restart: on-failure - depends_on: - - redis-socketio - volumes: - - sites-vol:/home/frappe/frappe-bench/sites:rw - - logs-vol:/home/frappe/frappe-bench/logs:rw - - erpnext-worker-default: - image: frappe/erpnext-worker:${ERPNEXT_VERSION} - restart: on-failure - command: worker - depends_on: - - redis-queue - - redis-cache - volumes: - - sites-vol:/home/frappe/frappe-bench/sites:rw - - logs-vol:/home/frappe/frappe-bench/logs:rw - - erpnext-worker-short: - image: frappe/erpnext-worker:${ERPNEXT_VERSION} - restart: on-failure - command: worker - environment: - - WORKER_TYPE=short - depends_on: - - redis-queue - - redis-cache - volumes: - - sites-vol:/home/frappe/frappe-bench/sites:rw - - logs-vol:/home/frappe/frappe-bench/logs:rw - - erpnext-worker-long: - image: frappe/erpnext-worker:${ERPNEXT_VERSION} - restart: on-failure - command: worker - environment: - - WORKER_TYPE=long - depends_on: - - redis-queue - - redis-cache - volumes: - - sites-vol:/home/frappe/frappe-bench/sites:rw - - erpnext-schedule: - image: frappe/erpnext-worker:${ERPNEXT_VERSION} - restart: on-failure - command: schedule - depends_on: - - redis-queue - - redis-cache - volumes: - - sites-vol:/home/frappe/frappe-bench/sites:rw - - logs-vol:/home/frappe/frappe-bench/logs:rw - - redis-cache: - image: redis:latest - restart: on-failure - volumes: - - redis-cache-vol:/data - - redis-queue: - image: redis:latest - restart: on-failure - volumes: - - redis-queue-vol:/data - - redis-socketio: - image: redis:latest - restart: on-failure - volumes: - - redis-socketio-vol:/data - - mariadb: - image: mariadb:10.6 - restart: on-failure - command: - - --character-set-server=utf8mb4 - - --collation-server=utf8mb4_unicode_ci - - --skip-character-set-client-handshake - - --skip-innodb-read-only-compressed # Temporary fix for MariaDB 10.6 - environment: - - MYSQL_ROOT_PASSWORD=${MYSQL_ROOT_PASSWORD} - # Sometimes db initialization takes longer than 10 seconds and site-creator goes away. - # Frappe doesn't use CONVERT_TZ() function that requires time zone info, so we can just skip it. - - MYSQL_INITDB_SKIP_TZINFO=1 - volumes: - - mariadb-vol:/var/lib/mysql - - site-creator: - image: frappe/erpnext-worker:${ERPNEXT_VERSION} - restart: "no" - command: new - depends_on: - - erpnext-python - environment: - - SITE_NAME=${SITE_NAME} - - DB_ROOT_USER=${DB_ROOT_USER} - - MYSQL_ROOT_PASSWORD=${MYSQL_ROOT_PASSWORD} - - ADMIN_PASSWORD=${ADMIN_PASSWORD} - - INSTALL_APPS=${INSTALL_APPS} - volumes: - - sites-vol:/home/frappe/frappe-bench/sites:rw - - logs-vol:/home/frappe/frappe-bench/logs:rw - -volumes: - mariadb-vol: - redis-cache-vol: - redis-queue-vol: - redis-socketio-vol: - assets-vol: - sites-vol: - cert-vol: - logs-vol: diff --git a/docs/add-custom-domain-using-traefik.md b/docs/add-custom-domain-using-traefik.md new file mode 100644 index 00000000..59299bec --- /dev/null +++ b/docs/add-custom-domain-using-traefik.md @@ -0,0 +1,28 @@ +Add following labels to `frontend` service + +```yaml +traefik.http.routers.custom-domain.rule: Host(`custom.localhost`) +# Comment the entrypoints label if traefik already has default entrypoint set +traefik.http.routers.custom-domain.entrypoints: web +traefik.http.middlewares.custom-domain.headers.customrequestheaders.Host: mysite.localhost +traefik.http.routers.custom-domain.middlewares: custom-domain +# Add following header only if TLS is needed in case of live server +traefik.http.routers.custom-domain.tls.certresolver: main-resolver +``` + +Example: + +```yaml +frontend: + ... + labels: + ... + traefik.http.routers.custom-domain.rule: Host(`custom.localhost`) + traefik.http.routers.custom-domain.entrypoints: web + traefik.http.middlewares.custom-domain.headers.customrequestheaders.Host: mysite.localhost + traefik.http.routers.custom-domain.middlewares: custom-domain + traefik.http.routers.custom-domain.tls.certresolver: main-resolver + ... +``` + +This will add `custom.localhost` as custom domain for `mysite.localhost` diff --git a/docs/backup-and-push-cronjob.md b/docs/backup-and-push-cronjob.md new file mode 100644 index 00000000..ec2f3827 --- /dev/null +++ b/docs/backup-and-push-cronjob.md @@ -0,0 +1,63 @@ +Create backup service or stack. + +```yaml +# backup-job.yml +version: "3.7" +services: + backup: + image: frappe/erpnext-worker:v13 + entrypoint: ["bash", "-c"] + command: | + for $SITE in $(/home/frappe/frappe-bench/env/bin/python -c "import frappe;print(' '.join(frappe.utils.get_sites()))") + do + bench --site $SITE backup --with-files + push-backup \ + --site $SITE \ + --bucket $BUCKET_NAME \ + --region-name $REGION \ + --endpoint-url $ENDPOINT_URL \ + --aws-access-key-id $ACCESS_KEY_ID \ + --aws-secret-access-key $SECRET_ACCESS_KEY + done + environment: + - BUCKET_NAME=erpnext + - REGION=us-east-1 + - ACCESS_KEY_ID=RANDOMACCESSKEY + - SECRET_ACCESS_KEY=RANDOMSECRETKEY + - ENDPOINT_URL=https://endpoint.url + volumes: + - "sites-vol:/home/frappe/frappe-bench/sites" + # Uncomment in case of Docker Swarm with crazy-max/swarm-cronjob + # deploy: + # labels: + # - "swarm.cronjob.enable=true" + # - "swarm.cronjob.schedule=0 */3 * * *" + # - "swarm.cronjob.skip-running=true" + # replicas: 0 + # restart_policy: + # condition: none + networks: + - erpnext-network + +networks: + erpnext-network: + external: true + name: _default + +volumes: + sites-vol: + external: true + name: _sites-vol +``` + +In case of single docker host setup, add crontab entry for backup every 6 hours. + +``` +0 */6 * * * /usr/local/bin/docker-compose -f /path/to/backup-job.yml up -d > /dev/null +``` + +Notes: + +- Install [crazy-max/swarm-cronjob](https://github.com/crazy-max/swarm-cronjob) for docker swarm. +- Uncomment `deploy` section in case of usage on docker swarm. +- Change the cron string as per need. diff --git a/docs/bench-console-and-vscode-debugger.md b/docs/bench-console-and-vscode-debugger.md new file mode 100644 index 00000000..b1668392 --- /dev/null +++ b/docs/bench-console-and-vscode-debugger.md @@ -0,0 +1,16 @@ +Add the following configuration to `launch.json` `configurations` array to start bench console and use debugger. Replace `mysite.localhost` with appropriate site. Also replace `frappe-bench` with name of the bench directory. + +```json +{ + "name": "Bench Console", + "type": "python", + "request": "launch", + "program": "${workspaceFolder}/frappe-bench/apps/frappe/frappe/utils/bench_helper.py", + "args": ["frappe", "--site", "mysite.localhost", "console"], + "pythonPath": "${workspaceFolder}/frappe-bench/env/bin/python", + "cwd": "${workspaceFolder}/frappe-bench/sites", + "env": { + "DEV_SERVER": "1" + } +} +``` diff --git a/docs/build-version-10-images.md b/docs/build-version-10-images.md new file mode 100644 index 00000000..613f06ee --- /dev/null +++ b/docs/build-version-10-images.md @@ -0,0 +1,16 @@ +Clone the version-10 branch of this repo + +```shell +git clone https://github.com/frappe/frappe_docker.git -b version-10 && cd frappe_docker +``` + +Build the images + +```shell +export DOCKER_REGISTRY_PREFIX=frappe +docker build -t ${DOCKER_REGISTRY_PREFIX}/frappe-socketio:v10 -f build/frappe-socketio/Dockerfile . +docker build -t ${DOCKER_REGISTRY_PREFIX}/frappe-nginx:v10 -f build/frappe-nginx/Dockerfile . +docker build -t ${DOCKER_REGISTRY_PREFIX}/erpnext-nginx:v10 -f build/erpnext-nginx/Dockerfile . +docker build -t ${DOCKER_REGISTRY_PREFIX}/frappe-worker:v10 -f build/frappe-worker/Dockerfile . +docker build -t ${DOCKER_REGISTRY_PREFIX}/erpnext-worker:v10 -f build/erpnext-worker/Dockerfile . +``` diff --git a/docs/connect-to-localhost-services-from-containers-for-local-app-development.md b/docs/connect-to-localhost-services-from-containers-for-local-app-development.md new file mode 100644 index 00000000..4f151937 --- /dev/null +++ b/docs/connect-to-localhost-services-from-containers-for-local-app-development.md @@ -0,0 +1,10 @@ +Not using separate container +Add following to frappe container from the `.devcontainer/docker-compose.yml`: + +```yaml +extra_hosts: + app1.localhost: 172.17.0.1 + app2.localhost: 172.17.0.1 +``` + +This is makes the domain names `app1.localhost` and `app2.localhost` connect to docker host and connect to services running on `localhost`. diff --git a/docs/custom-apps-for-production.md b/docs/custom-apps-for-production.md deleted file mode 100644 index d00bd071..00000000 --- a/docs/custom-apps-for-production.md +++ /dev/null @@ -1,58 +0,0 @@ -# Custom apps - -To add your own Frappe/ERPNext apps to the image, we'll need to create a custom image with the help of a unique wrapper script - -> For the sake of simplicity, in this example, we'll be using a place holder called `[custom]`, and we'll be building off the edge image. - -Create two directories called `[custom]-worker` and `[custom]-nginx` in the `build` directory. - -```shell -cd frappe_docker -mkdir ./build/[custom]-worker ./build/[custom]-nginx -``` - -Create a `Dockerfile` in `./build/[custom]-worker` with the following content: - -```Dockerfile -FROM frappe/erpnext-worker:edge - -RUN install_app [custom] https://github.com/[username]/[custom] [branch] -# Only add the branch if you are using a specific tag or branch. -``` - -**Note:** Replace `https://github.com/[username]/[custom]` above with your custom app's Git repository URL (may include credentials if needed). Your custom app Git repository **must** be named exactly as the custom app's name, and use the same branch name as Frappe/ERPNext branch name that you use. - -Create a `Dockerfile` in `./build/[custom]-nginx` with the following content: - -```Dockerfile -FROM bitnami/node:12-prod - -COPY build/[custom]-nginx/install_app.sh /install_app - -RUN /install_app [custom] https://github.com/[username]/[custom] [branch] - -FROM frappe/erpnext-nginx:edge - -COPY --from=0 /home/frappe/frappe-bench/sites/ /var/www/html/ -COPY --from=0 /rsync /rsync -RUN echo -n "\n[custom]" >> /var/www/html/apps.txt - -VOLUME [ "/assets" ] - -ENTRYPOINT ["/docker-entrypoint.sh"] -CMD ["nginx", "-g", "daemon off;"] -``` - -Copy over the `install_app.sh` file from `./build/erpnext-nginx` - -```shell -cp ./build/erpnext-nginx/install_app.sh ./build/[custom]-nginx -``` - -Open up `./installation/docker-compose-custom.yml` and replace all instances of `[app]` with the name of your app. - -```shell -sed -i "s#\[app\]#[custom]#" ./installation/docker-compose-custom.yml -``` - -Install like usual, except that when you set the `INSTALL_APPS` variable to `erpnext,[custom]`. diff --git a/docs/docker-swarm.md b/docs/docker-swarm.md index 7bf2a412..1efb5177 100644 --- a/docs/docker-swarm.md +++ b/docs/docker-swarm.md @@ -1,295 +1,55 @@ -### Prerequisites +## Prerequisites -IMPORTANT: All commands are executed on live server with public IP and DNS Configured. +- [yq](https://mikefarah.gitbook.io/yq) +- [docker-compose](https://docs.docker.com/compose/) +- [docker swarm](https://docs.docker.com/engine/swarm/) -#### Setup docker swarm +#### Generate setup for docker swarm -Follow [dockerswarm.rocks](https://dockerswarm.rocks) guide to setup Docker swarm, Traefik and Portainer. +Generate the swarm compatible YAML, -Use Portainer for rest of the guide - -### Create Config - -Configs > Add Config > `frappe-mariadb-config` - -``` -[mysqld] -character-set-client-handshake = FALSE -character-set-server = utf8mb4 -collation-server = utf8mb4_unicode_ci - -[mysql] -default-character-set = utf8mb4 +```bash +docker-compose -f compose.yaml \ + -f overrides/compose.erpnext.yaml \ + -f overrides/compose.swarm.yaml \ + -f overrides/compose.https.yaml \ + config \ + | yq eval 'del(.services.*.depends_on) | del(.services.frontend.labels)' - \ + | yq eval '.services.proxy.command += "--providers.docker.swarmmode"' - > \ + ~/gitops/compose.yaml ``` -### Create Secret +In case you need to generate config for multiple benches. Install the proxy separately only once and generate stacks for each bench as follows: -Secret > Add Secret > `frappe-mariadb-root-password` - -``` -longsecretpassword +```bash +# Setup Bench $BENCH_SUFFIX +export BENCH_SUFFIX=one +docker-compose -f compose.yaml \ + -f overrides/compose.erpnext.yaml \ + -f overrides/compose.swarm.yaml \ + config \ + | yq eval 'del(.services.*.depends_on) | del(.services.frontend.labels)' - \ + | sed "s|frontend|frontend-${BENCH_SUFFIX}|g" \ + | yq eval ".services.frontend-${BENCH_SUFFIX}.\"networks\"=[\"traefik-public\",\"default\"]" - \ + | yq eval ".\"networks\"={\"traefik-public\":{\"external\":true}}" - > \ + ~/gitops/compose-${BENCH_SUFFIX}.yaml ``` -Note down this password. -It is only available in mariadb containers at location `/run/secrets/frappe-mariadb-root-password` later +Commands explained: -### Deploy MariaDB Replication +- `docker-compose -f ... -f ... config`, this command generates the YAML based on the overrides +- `yq eval 'del(.services.*.depends_on) | del(.services.frontend.labels)'`, this command removes the `depends_on` from all services and `labels` from frontend generated from previous command. +- `yq eval '.services.proxy.command += "--providers.docker.swarmmode"'`, this command enables swarmmode for traefik proxy. +- `sed "s|frontend|frontend-${BENCH_SUFFIX}|g"`, this command replaces the service name `frontend` with `frontend-` and `BENCH_SUFFIX` provided. +- `yq eval ".services.frontend-${BENCH_SUFFIX}.\"networks\"=[\"traefik-public\",\"default\"]"`, this command attaches `traefik-public` and `default` network to frontend service. +- `yq eval ".\"networks\"={\"traefik-public\":{\"external\":true}}"`, this commands adds external network `traefik-public` to the stack -Stacks > Add Stacks > `frappe-mariadb` +Notes: -```yaml -version: "3.7" +- Set `BENCH_SUFFIX` to the stack name. the stack will be located at `~/gitops/compose-${BENCH_SUFFIX}.yaml`. +- `traefik-public` is assumed to be the network for traefik loadbalancer for swarm. +- Once the stack YAML is generated, you can edit it further for advance setup and commit it to your gitops -services: - mariadb-master: - image: "bitnami/mariadb:10.3" - deploy: - restart_policy: - condition: on-failure - configs: - - source: frappe-mariadb-config - target: /opt/bitnami/mariadb/conf/bitnami/my_custom.cnf - networks: - - frappe-network - secrets: - - frappe-mariadb-root-password - volumes: - - "mariadb_master_data:/bitnami/mariadb" - environment: - - MARIADB_REPLICATION_MODE=master - - MARIADB_REPLICATION_USER=repl_user - - MARIADB_REPLICATION_PASSWORD_FILE=/run/secrets/frappe-mariadb-root-password - - MARIADB_ROOT_PASSWORD_FILE=/run/secrets/frappe-mariadb-root-password +#### Site Operations - mariadb-slave: - image: "bitnami/mariadb:10.3" - deploy: - restart_policy: - condition: on-failure - configs: - - source: frappe-mariadb-config - target: /opt/bitnami/mariadb/conf/bitnami/my_custom.cnf - networks: - - frappe-network - secrets: - - frappe-mariadb-root-password - volumes: - - "mariadb_slave_data:/bitnami/mariadb" - environment: - - MARIADB_REPLICATION_MODE=slave - - MARIADB_REPLICATION_USER=repl_user - - MARIADB_REPLICATION_PASSWORD_FILE=/run/secrets/frappe-mariadb-root-password - - MARIADB_MASTER_HOST=mariadb-master - - MARIADB_MASTER_PORT_NUMBER=3306 - - MARIADB_MASTER_ROOT_PASSWORD_FILE=/run/secrets/frappe-mariadb-root-password - -volumes: - mariadb_master_data: - mariadb_slave_data: - -configs: - frappe-mariadb-config: - external: true - -secrets: - frappe-mariadb-root-password: - external: true - -networks: - frappe-network: - name: frappe-network - attachable: true -``` - -### Deploy Frappe/ERPNext - -Stacks > Add Stacks > `frappe-bench-v13` - -```yaml -version: "3.7" - -services: - redis-cache: - image: redis:latest - volumes: - - redis-cache-vol:/data - deploy: - restart_policy: - condition: on-failure - networks: - - frappe-network - - redis-queue: - image: redis:latest - volumes: - - redis-queue-vol:/data - deploy: - restart_policy: - condition: on-failure - networks: - - frappe-network - - redis-socketio: - image: redis:latest - volumes: - - redis-socketio-vol:/data - deploy: - restart_policy: - condition: on-failure - networks: - - frappe-network - - erpnext-nginx: - image: frappe/erpnext-nginx:${ERPNEXT_VERSION?Variable ERPNEXT_VERSION not set} - environment: - - UPSTREAM_REAL_IP_ADDRESS=10.0.0.0/8 - - FRAPPE_PY=erpnext-python - - FRAPPE_PY_PORT=8000 - - FRAPPE_SOCKETIO=frappe-socketio - - SOCKETIO_PORT=9000 - volumes: - - sites-vol:/var/www/html/sites:rw - - assets-vol:/assets:rw - networks: - - frappe-network - - traefik-public - deploy: - restart_policy: - condition: on-failure - labels: - - "traefik.docker.network=traefik-public" - - "traefik.enable=true" - - "traefik.constraint-label=traefik-public" - - "traefik.http.routers.erpnext-nginx.rule=Host(${SITES?Variable SITES not set})" - - "traefik.http.routers.erpnext-nginx.entrypoints=http" - - "traefik.http.routers.erpnext-nginx.middlewares=https-redirect" - - "traefik.http.routers.erpnext-nginx-https.rule=Host(${SITES?Variable SITES not set})" - - "traefik.http.routers.erpnext-nginx-https.entrypoints=https" - - "traefik.http.routers.erpnext-nginx-https.tls=true" - - "traefik.http.routers.erpnext-nginx-https.tls.certresolver=le" - - "traefik.http.services.erpnext-nginx.loadbalancer.server.port=8080" - - erpnext-python: - image: frappe/erpnext-worker:${ERPNEXT_VERSION?Variable ERPNEXT_VERSION not set} - deploy: - restart_policy: - condition: on-failure - environment: - - MARIADB_HOST=${MARIADB_HOST?Variable MARIADB_HOST not set} - - REDIS_CACHE=redis-cache:6379 - - REDIS_QUEUE=redis-queue:6379 - - REDIS_SOCKETIO=redis-socketio:6379 - - SOCKETIO_PORT=9000 - - AUTO_MIGRATE=1 - volumes: - - sites-vol:/home/frappe/frappe-bench/sites:rw - - assets-vol:/home/frappe/frappe-bench/sites/assets:rw - networks: - - frappe-network - - frappe-socketio: - image: frappe/frappe-socketio:${FRAPPE_VERSION?Variable FRAPPE_VERSION not set} - deploy: - restart_policy: - condition: on-failure - volumes: - - sites-vol:/home/frappe/frappe-bench/sites:rw - networks: - - frappe-network - - erpnext-worker-default: - image: frappe/erpnext-worker:${ERPNEXT_VERSION?Variable ERPNEXT_VERSION not set} - deploy: - restart_policy: - condition: on-failure - command: worker - volumes: - - sites-vol:/home/frappe/frappe-bench/sites:rw - networks: - - frappe-network - - erpnext-worker-short: - image: frappe/erpnext-worker:${ERPNEXT_VERSION?Variable ERPNEXT_VERSION not set} - deploy: - restart_policy: - condition: on-failure - command: worker - environment: - - WORKER_TYPE=short - volumes: - - sites-vol:/home/frappe/frappe-bench/sites:rw - networks: - - frappe-network - - erpnext-worker-long: - image: frappe/erpnext-worker:${ERPNEXT_VERSION?Variable ERPNEXT_VERSION not set} - deploy: - restart_policy: - condition: on-failure - command: worker - environment: - - WORKER_TYPE=long - volumes: - - sites-vol:/home/frappe/frappe-bench/sites:rw - networks: - - frappe-network - - frappe-schedule: - image: frappe/erpnext-worker:${ERPNEXT_VERSION?Variable ERPNEXT_VERSION not set} - deploy: - restart_policy: - condition: on-failure - command: schedule - volumes: - - sites-vol:/home/frappe/frappe-bench/sites:rw - networks: - - frappe-network - -volumes: - redis-cache-vol: - redis-queue-vol: - redis-socketio-vol: - assets-vol: - sites-vol: - -networks: - traefik-public: - external: true - frappe-network: - external: true -``` - -Use environment variables: - -- `ERPNEXT_VERSION` variable to be set to desired version of ERPNext. e.g. 12.10.0 -- `FRAPPE_VERSION` variable to be set to desired version of Frappe Framework. e.g. 12.7.0 -- `MARIADB_HOST=frappe-mariadb_mariadb-master` -- `SITES` variable is list of sites in back tick and separated by comma - -``` -SITES=`site1.example.com`,`site2.example.com` -``` - -### Create new site job - -1. Containers > Add Container > `add-site1-example-com` -2. Select Image frappe/erpnext-worker:v13 -3. Set command as `new` -4. Select network `frappe-network` -5. Select Volume `frappe-bench-v13_sites-vol` and mount in container `/home/frappe/frappe-bench/sites` -6. Env variables: - - MYSQL_ROOT_PASSWORD=longsecretpassword - - SITE_NAME=site1.example.com - - INSTALL_APPS=erpnext -7. Start container - -### Migrate Sites job - -1. Containers > Add Container > `migrate-sites` -2. Select Image frappe/erpnext-worker:v13 -3. Set command as `migrate` -4. Select network `frappe-network` -5. Select Volume `frappe-bench-v13_sites-vol` and mount in container `/home/frappe/frappe-bench/sites` -6. Env variables: - - MAINTENANCE_MODE=1 -7. Start container +Refer [site operations documentation](./site-operations) to create new site, migrate site, drop site and perform other site operations. diff --git a/docs/environment-variables.md b/docs/environment-variables.md deleted file mode 100644 index b5ebb89d..00000000 --- a/docs/environment-variables.md +++ /dev/null @@ -1,33 +0,0 @@ -List of environment variables for containers - -### frappe-worker and erpnext-worker - -Following environment variables are set into sites volume as `common_site_config.json`. It means once the file is created in volume, the variables won't have any effect, you need to edit the common_site_config.json to update the configuration - -- `DB_HOST`: MariaDB host, domain name or ip address. -- `DB_PORT`: MariaDB port. -- `REDIS_CACHE`: Redis cache host, domain name or ip address. -- `REDIS_QUEUE`: Redis queue host, domain name or ip address. -- `REDIS_SOCKETIO`: Redis queue host, domain name or ip address. -- `SOCKETIO_PORT: `: Port on which the SocketIO should start. -- `WORKER_CLASS`: Optionally set gunicorn worker-class. Supports gevent only, defaults to gthread. - -### frappe-nginx and erpnext-nginx - -These variables are set on every container start. Change in these variables will reflect on every container start. - -- `FRAPPE_PY`: Gunicorn host to reverse proxy. Default: 0.0.0.0 -- `FRAPPE_PY_PORT`: Gunicorn port to reverse proxy. Default: 8000 -- `FRAPPE_SOCKETIO`: SocketIO host to reverse proxy. Default: 0.0.0.0 -- `SOCKETIO_PORT`: SocketIO port to reverse proxy. Default: 9000 -- `HTTP_TIMEOUT`: Nginx http timeout. Default: 120 -- `UPSTREAM_REAL_IP_ADDRESS `: The trusted address (or ip range) of upstream proxy servers. If set, this will tell nginx to trust the X-Forwarded-For header from these proxy servers in determining the real IP address of connecting clients. Default: 127.0.0.1 -- `UPSTREAM_REAL_IP_RECURSIVE`: When set to `on`, this will tell nginx to not just look to the last upstream proxy server in determining the real IP address. Default: off -- `UPSTREAM_REAL_IP_HEADER`: Set this to the header name sent by your upstream proxy server to indicate the real IP of connecting clients. Default: X-Forwarded-For -- `FRAPPE_SITE_NAME_HEADER`: NGINX `X-Frappe-Site-Name` header in the HTTP request which matches a site name. Default: `$host` -- `HTTP_HOST`: NGINX `Host` header in the HTTP request which matches a site name. Default: `$http_host` -- `SKIP_NGINX_TEMPLATE_GENERATION`: When set to `1`, this will not generate a default NGINX configuration. The config file must be mounted inside the container (`/etc/nginx/conf.d`) by the user in this case. Default: `0` - -### frappe-socketio - -This container takes configuration from `common_site_config.json` which is already created by erpnext gunicorn container. It doesn't use any environment variables. diff --git a/docs/images-and-compose-files.md b/docs/images-and-compose-files.md new file mode 100644 index 00000000..d763e31d --- /dev/null +++ b/docs/images-and-compose-files.md @@ -0,0 +1,100 @@ +# Images + +There's 4 images that you can find in `/images` directory: + +- `bench`. It is used for development. [Learn more how to start development](../development/README.md). +- `nginx`. This image contains JS and CSS assets. Container using this image also routes incoming requests using [nginx](https://www.nginx.com). +- `socketio`. Container using this image processes realtime websocket requests using [Socket.IO](https://socket.io). +- `worker`. Multi-purpose Python backend. Runs [Werkzeug server](https://werkzeug.palletsprojects.com/en/2.0.x/) with [gunicorn](https://gunicorn.org), queues (via `bench worker`), or schedule (via `bench schedule`). + +> `nginx`, `socketio` and `worker` images — everything we need to be able to run all processes that Frappe framework requires (take a look at [Bench Procfile reference](https://frappeframework.com/docs/v13/user/en/bench/resources/bench-procfile)). We follow [Docker best practices](https://docs.docker.com/develop/develop-images/dockerfile_best-practices/#decouple-applications) and split these processes to different containers. + +> ERPNext images don't have their own Dockerfiles. We use [multi-stage builds](https://docs.docker.com/develop/develop-images/multistage-build/) and [Docker Buildx](https://docs.docker.com/engine/reference/commandline/buildx/) to reuse as much things as possible and make our builds more efficient. + +# Compose files + +After building the images we have to run the containers. The best and simplest way to do this is to use [compose files](https://docs.docker.com/compose/compose-file/). + +We have one main compose file, `compose.yaml`. Services described, networking, volumes are also handled there. + +## Services + +All services are described in `compose.yaml` + +- `configurator`. Updates `common_site_config.json` so Frappe knows how to access db and redis. It is executed on every `docker-compose up` (and exited immediately). Other services start after this container exits successfully. +- `backend`. [Werkzeug server](https://werkzeug.palletsprojects.com/en/2.0.x/). +- `db`. Optional service that runs [MariaDB](https://mariadb.com) if you also use `overrides/compose.mariadb.yaml` or [Postgres](https://www.postgresql.org) if you also use `overrides/compose.postgres.yaml`. +- `redis`. Optional service that runs [Redis](https://redis.io) server with cache, [Socket.IO](https://socket.io) and queues data. +- `frontend`. [nginx](https://www.nginx.com) server that serves JS/CSS assets and routes incoming requests. +- `proxy`. [Traefik](https://traefik.io/traefik/) proxy. It is here for complicated setups or HTTPS override (with `overrides/compose.https.yaml`). +- `websocket`. Node server that runs [Socket.IO](https://socket.io). +- `queue-short`, `queue-default`, `queue-long`. Python servers that run job queues using [rq](https://python-rq.org). +- `scheduler`. Python server that runs tasks on schedule using [schedule](https://schedule.readthedocs.io/en/stable/). + +## Overrides + +We have several [overrides](https://docs.docker.com/compose/extends/): + +- `overrides/compose.proxy.yaml`. Adds traefik proxy to setup. +- `overrides/compose.noproxy.yaml`. Publishes `frontend` ports directly without any proxy. +- `overrides/compose.erpnext.yaml`. Replaces all Frappe images with ERPNext ones. ERPNext images are built on top of Frappe ones, so it is safe to replace them. +- `overrides/compose.https.yaml`. Automatically sets up Let's Encrypt certificate and redirects all requests to directed to http, to https. +- `overrides/compose.mariadb.yaml`. Adds `db` service and sets its image to MariaDB. +- `overrides/compose.postgres.yaml`. Adds `db` service and sets its image to Postgres. Note that ERPNext currently doesn't support Postgres. +- `overrides/compose.redis.yaml`. Adds `redis` service and sets its image to `redis`. +- `overrides/compose.swarm.yaml`. Workaround override for generating swarm stack. + +It is quite simple to run overrides. All we need to do is to specify compose files that should be used by docker-compose. For example, we want ERPNext: + +```bash +# Point to main compose file (compose.yaml) and add one more. +docker-compose -f compose.yaml -f overrides/compose.erpnext.yaml config +``` + +That's it! Of course, we also have to setup `.env` before all of that, but that's not the point. + +## Configuration + +We use environment variables to configure our setup. docker-compose uses variables from `.env` file. To get started, copy `example.env` to `.env`. + +### `FRAPPE_VERSION` + +Frappe framework release. You can find all releases [here](https://github.com/frappe/frappe/releases). + +### `DB_PASSWORD` + +Password for MariaDB (or Postgres) database. + +### `DB_HOST` + +Hostname for MariaDB (or Postgres) database. Set only if external service for database is used. + +### `DB_PORT` + +Port for MariaDB (3306) or Postgres (5432) database. Set only if external service for database is used. + +### `REDIS_CACHE` + +Hostname for redis server to store cache. Set only if external service for redis is used. + +### `REDIS_QUEUE` + +Hostname for redis server to store queue data. Set only if external service for redis is used. + +### `REDIS_SOCKETIO` + +Hostname for redis server to store socketio data. Set only if external service for redis is used. + +### `ERPNEXT_VERSION` + +ERPNext [release](https://github.com/frappe/frappe/releases). This variable is required if you use ERPNext override. + +### `LETSENCRYPT_EMAIL` + +Email that used to register https certificate. This one is required only if you use HTTPS override. + +### `FRAPPE_SITE_NAME_HEADER` + +This environment variable is not required. Default value is `$$host` which resolves site by host. For example, if your host is `example.com`, site's name should be `example.com`, or if host is `127.0.0.1` (local debugging), it should be `127.0.0.1` This variable allows to override described behavior. Let's say you create site named `mysite` and do want to access it by `127.0.0.1` host. Than you would set this variable to `mysite`. + +There is other variables not mentioned here. They're somewhat internal and you don't have to worry about them except you want to change main compose file. diff --git a/docs/multi-bench.md b/docs/multi-bench.md deleted file mode 100644 index 831ec71c..00000000 --- a/docs/multi-bench.md +++ /dev/null @@ -1,197 +0,0 @@ -# Multi bench - -This setup separates all services such that only required ones can be deployed. - -This is suitable when multiple services are installed on cluster with shared proxy/router, database, cache etc. - -Make sure you've cloned this repository and switch to the directory before executing following commands. - -## Setup Environment Variables - -Copy the example docker environment file to `.env`: - -```sh -cp env-example .env -``` - -To get started, copy the existing `env-example` file to `.env`. By default, the file will contain the following variables: - -- `VERSION=edge` - - In this case, `edge` corresponds to `develop`. To setup any other version, you may use the branch name or version specific tags. (eg. v13.0.0, version-12, v11.1.15, v11) -- `MYSQL_ROOT_PASSWORD=admin` - - Bootstraps a MariaDB container with this value set as the root password. If a managed MariaDB instance is used, there is no need to set the password here. -- `MARIADB_HOST=mariadb` - - Sets the hostname to `mariadb`. This is required if the database is managed by the containerized MariaDB instance. - - In case of a separately managed database setups, set the value to the database's hostname/IP/domain. -- `SITES=site1.domain.com,site2.domain.com` - - List of sites that are part of the deployment "bench" Each site is separated by a comma(,). - - If LetsEncrypt is being setup, make sure that the DNS for all the site's domains correctly point to the current instance. -- `LETSENCRYPT_EMAIL=your.email@your.domain.com` - - Email for LetsEncrypt expiry notification. This is only required if you are setting up LetsEncrypt. - -Notes: - -- docker-compose-erpnext.yml and docker-compose-frappe.yml set `AUTO_MIGRATE` environment variable to `1`. -- `AUTO_MIGRATE` checks if there is semver bump or git hash change in case of develop branch and automatically migrates the sites on container start up. -- It is good practice to use image tag for specific version instead of latest. e.g `frappe-socketio:v12.5.1`, `erpnext-nginx:v12.7.1`. - -## Local deployment for testing - -For trying out locally or to develop apps using ERPNext REST API port 80 must be published. -Following command will start the needed containers and expose ports. - -For Erpnext: - -```sh -docker-compose \ - --project-name \ - -f installation/docker-compose-common.yml \ - -f installation/docker-compose-erpnext.yml \ - -f installation/erpnext-publish.yml \ - up -d -``` - -For Frappe: - -```sh -docker-compose \ - --project-name \ - -f installation/docker-compose-common.yml \ - -f installation/docker-compose-frappe.yml \ - -f installation/frappe-publish.yml \ - up -d -``` - -Make sure to replace `` with the desired name you wish to set for the project. - -Notes: - -- New site (first site) needs to be added after starting the services. -- The local deployment is for testing and REST API development purpose only -- A complete development environment is available [here](../development) -- The site names are limited to patterns matching \*.localhost by default -- Additional site name patterns can be added by editing /etc/hosts of your host machine - -## Deployment for production - -### Setup Letsencrypt Nginx Proxy Companion - -Letsencrypt Nginx Proxy Companion can optionally be setup to provide SSL. This is recommended for instances accessed over the internet. - -Your DNS will need to be configured correctly for Letsencrypt to verify your domain. - -To setup the proxy companion, run the following commands: - -```sh -cd $HOME -git clone https://github.com/evertramos/docker-compose-letsencrypt-nginx-proxy-companion.git -cd docker-compose-letsencrypt-nginx-proxy-companion -cp .env.sample .env -./start.sh -``` - -It will create the required network and configure containers for Letencrypt ACME. - -For more details, see the [Letsencrypt Nginx Proxy Companion github repo](https://github.com/evertramos/docker-compose-letsencrypt-nginx-proxy-companion). Letsencrypt Nginx Proxy Companion github repo works by automatically proxying to containers with the `VIRTUAL_HOST` environmental variable. - -Notes: - -- `SITES` variables from `env-example` is set as `VIRTUAL_HOST` -- `LETSENCRYPT_EMAIL` variables from `env-example` is used as it is. -- This is simple nginx + letsencrypt solution. Any other solution can be setup. Above two variables can be re-used or removed in case any other reverse-proxy is used. - -### Start Frappe/ERPNext Services - -To start the Frappe/ERPNext services for production, run the following command: - -```sh -docker-compose \ - --project-name \ - -f installation/docker-compose-common.yml \ - -f installation/docker-compose-erpnext.yml \ - -f installation/docker-compose-networks.yml \ - up -d -``` - -Make sure to replace `` with any desired name you wish to set for the project. - -Notes: - -- Use `docker-compose-frappe.yml` in case you need only Frappe without ERPNext. -- New site (first site) needs to be added after starting the services. - -## Docker containers - -This repository contains the following docker-compose files, each one containing the described images: - -- docker-compose-common.yml - - redis-cache - - volume: redis-cache-vol - - redis-queue - - volume: redis-queue-vol - - redis-socketio - - volume: redis-socketio-vol - - mariadb: main database - - volume: mariadb-vol -- docker-compose-erpnext.yml - - - erpnext-nginx: serves static assets and proxies web request to the appropriate container, allowing to offer all services on the same port. - - volume: assets-vol - - erpnext-python: main application code - - frappe-socketio: enables realtime communication to the user interface through websockets - - frappe-worker-default: background runner - - frappe-worker-short: background runner for short-running jobs - - frappe-worker-long: background runner for long-running jobs - - frappe-schedule - -- docker-compose-frappe.yml - - - frappe-nginx: serves static assets and proxies web request to the appropriate container, allowing to offer all services on the same port. - - volume: assets-vol, sites-vol - - erpnext-python: main application code - - volume: sites-vol - - frappe-socketio: enables realtime communication to the user interface through websockets - - volume: sites-vol - - frappe-worker-default: background runner - - volume: sites-vol - - frappe-worker-short: background runner for short-running jobs - - volume: sites-vol - - frappe-worker-long: background runner for long-running jobs - - volume: sites-vol - - frappe-schedule - - volume: sites-vol - -- docker-compose-networks.yml: this yaml define the network to communicate with _Letsencrypt Nginx Proxy Companion_. - -- erpnext-publish.yml: this yml extends erpnext-nginx service to publish port 80, can only be used with docker-compose-erpnext.yml - -- frappe-publish.yml: this yml extends frappe-nginx service to publish port 80, can only be used with docker-compose-frappe.yml - -## Updating and Migrating Sites - -Switch to the root of the `frappe_docker` directory before running the following commands: - -```sh -# Update environment variable VERSION -nano .env - -# Pull new images -docker-compose \ - -f installation/docker-compose-common.yml \ - -f installation/docker-compose-erpnext.yml \ - pull - -# Restart containers -docker-compose \ - --project-name \ - -f installation/docker-compose-common.yml \ - -f installation/docker-compose-erpnext.yml \ - -f installation/docker-compose-networks.yml \ - up -d - -docker run \ - -e "MAINTENANCE_MODE=1" \ - -v _sites-vol:/home/frappe/frappe-bench/sites \ - --network _default \ - frappe/erpnext-worker:$VERSION migrate -``` diff --git a/docs/patch-code-from-images.md b/docs/patch-code-from-images.md new file mode 100644 index 00000000..deeff975 --- /dev/null +++ b/docs/patch-code-from-images.md @@ -0,0 +1,10 @@ +Example: https://discuss.erpnext.com/t/sms-two-factor-authentication-otp-msg-change/47835 + +Above example needs following Dockerfile based patch + +```Dockerfile +FROM frappe/erpnext-worker:v12.17.0 + +RUN /home/frappe/frappe-bench/env/bin/pip -e /home/frappe/frappe-bench/apps/custom_app +RUN sed -i -e "s/Your verification code is/আপনার লগইন কোড/g" /home/frappe/frappe-bench/apps/frappe/frappe/twofactor.py +``` diff --git a/docs/port-based-multi-tenancy.md b/docs/port-based-multi-tenancy.md new file mode 100644 index 00000000..d54d0b46 --- /dev/null +++ b/docs/port-based-multi-tenancy.md @@ -0,0 +1,47 @@ +WARNING: Do not use this in production if the site is going to be served over plain http. + +### Step 1 + +Remove the traefik service from docker-compose.yml + +### Step 2 + +Create nginx config file `/opt/nginx/conf/serve-8001.conf`: + +``` +server { + listen 8001; + server_name $http_host; + + location / { + + rewrite ^(.+)/$ $1 permanent; + rewrite ^(.+)/index\.html$ $1 permanent; + rewrite ^(.+)\.html$ $1 permanent; + + proxy_set_header X-Frappe-Site-Name mysite.localhost; + proxy_set_header Host mysite.localhost; + proxy_pass http://erpnext-nginx; + } +} +``` + +Notes: + +- Replace the port with any port of choice e.g. `listen 4200;` +- Change `mysite.localhost` to site name +- Repeat the server blocks for multiple ports and site names to get the effect of port based multi tenancy + +### Step 3 + +Run the docker container + +```shell +docker run --network=_default \ + -p 8001:8001 \ + --volume=/opt/nginx/conf/serve-8001.conf:/etc/nginx/conf.d/default.conf -d nginx +``` + +Note: Change the volumes, network and ports as needed + +With the above example configured site will be accessible on `http://localhost:8001` diff --git a/docs/setup-options.md b/docs/setup-options.md new file mode 100644 index 00000000..74718e99 --- /dev/null +++ b/docs/setup-options.md @@ -0,0 +1,96 @@ +# Containerized Production Setup + +Make sure you've cloned this repository and switch to the directory before executing following commands. + +Commands will generate YAML as per the environment for setup. + +## Setup Environment Variables + +Copy the example docker environment file to `.env`: + +```sh +cp example.env .env +``` + +Note: To know more about environment variable [read here](./images-and-compose-files#configuration). Set the necessary variables in the `.env` file. + +## Generate docker-compose.yml for variety of setups + +### Setup Frappe without proxy and external MariaDB and Redis + +```sh +# Generate YAML +docker-compose -f compose.yaml -f overrides/compose.noproxy.yaml config > ~/gitops/docker-compose.yml + +# Start containers +docker-compose --project-name -f ~/gitops/docker-compose.yml up -d +``` + +### Setup ERPNext with proxy and external MariaDB and Redis + +```sh +# Generate YAML +docker-compose -f compose.yaml \ + -f overrides/compose.proxy.yaml \ + -f overrides/compose.erpnext.yaml \ + config > ~/gitops/docker-compose.yml + +# Start containers +docker-compose --project-name -f ~/gitops/docker-compose.yml up -d +``` + +### Setup Frappe using containerized MariaDB and Redis with Letsencrypt certificates. + +```sh +# Generate YAML +docker-compose -f compose.yaml \ + -f overrides/compose.mariadb.yaml \ + -f overrides/compose.redis.yaml \ + -f overrides/compose.https.yaml \ + config > ~/gitops/docker-compose.yml + +# Start containers +docker-compose --project-name -f ~/gitops/docker-compose.yml up -d +``` + +### Setup ERPNext using containerized MariaDB and Redis with Letsencrypt certificates. + +```sh +# Generate YAML +docker-compose -f compose.yaml \ + -f overrides/compose.erpnext.yaml \ + -f overrides/compose.mariadb.yaml \ + -f overrides/compose.redis.yaml \ + -f overrides/compose.https.yaml \ + config > ~/gitops/docker-compose.yml + +# Start containers +docker-compose --project-name -f ~/gitops/docker-compose.yml up -d +``` + +Notes: + +- Make sure to replace `` with the desired name you wish to set for the project. +- This setup is not to be used for development. A complete development environment is available [here](../development) + +## Updating Images + +Switch to the root of the `frappe_docker` directory before running the following commands: + +```sh +# Update environment variables ERPNEXT_VERSION and FRAPPE_VERSION +nano .env + +# Pull new images +docker-compose -f compose.yaml \ + -f overrides/erpnext.yaml \ + # ... your other overrides + config > ~/gitops/docker-compose.yml + +docker-compose --project-name -f ~/gitops/docker-compose.yml pull + +# Restart containers +docker-compose --project-name -f ~/gitops/docker-compose.yml up -d +``` + +To migrate sites refer [site operations](./site-operations.md#migrate-site) diff --git a/docs/single-bench.md b/docs/single-bench.md deleted file mode 100644 index b1487516..00000000 --- a/docs/single-bench.md +++ /dev/null @@ -1,129 +0,0 @@ -# Single Bench - -This setup starts traefik service as part of single docker-compose project. It is quick to get started locally or on production for a single server with single deployment. - -This is not suitable when multiple services are installed on cluster with shared proxy/router, database, cache etc. - -Make sure you've cloned this repository and switch to the directory before executing following commands. - -## Setup Environment Variables - -Copy the example docker environment file to `.env`: - -For local setup - -```sh -cp env-local .env -``` - -For production - -```sh -cp env-production .env - -``` - -To get started, copy the existing `env-local` or `env-production` file to `.env`. By default, the file will contain the following variables: - -- `ERPNEXT_VERSION=edge` - - In this case, `edge` corresponds to `develop`. To setup any other version, you may use the branch name or version specific tags. (eg. v13.0.0, version-12, v11.1.15, v11). -- `FRAPPE_VERSION=edge` - - In this case, `edge` corresponds to `develop`. To setup any other version, you may use the branch name or version specific tags. (eg. v13.0.0, version-12, v11.1.15, v11). -- `MARIADB_HOST=mariadb` - - Sets the hostname to `mariadb`. This is required if the database is managed by the containerized MariaDB instance. -- `MYSQL_ROOT_PASSWORD=admin` - - Bootstraps a MariaDB container with this value set as the root password. If a managed MariaDB instance is used, there is no need to set the password here. - - In case of a separately managed database setups, set the value to the database's hostname/IP/domain. -- `SITE_NAME=erp.example.com` - - Creates this site after starting all services and installs ERPNext. Site name must be resolvable by users machines and the ERPNext components. e.g. `erp.example.com` or `mysite.localhost`. -- `` SITES=`erp.example.com` `` - - List of sites that are part of the deployment "bench" Each site is separated by a comma(,) and quoted in backtick (`). By default site created by `SITE_NAME` variable is added here. - - If LetsEncrypt is being setup, make sure that the DNS for all the site's domains correctly point to the current instance. -- `DB_ROOT_USER=root` - - MariaDB root username -- `ADMIN_PASSWORD=admin` - - Password for the `Administrator` user, credentials after install `Administrator:$ADMIN_PASSWORD`. -- `INSTALL_APPS=erpnext` - - Apps to install, the app must be already in the container image, to install other application read the [instructions on installing custom apps](./custom-apps-for-production.md). -- `LETSENCRYPT_EMAIL=email@example.com` - - Email for LetsEncrypt expiry notification. This is only required if you are setting up LetsEncrypt. -- `ENTRYPOINT_LABEL=traefik.http.routers.erpnext-nginx.entrypoints=websecure` - - Related to the traefik configuration, says all traffic from outside should come from HTTP or HTTPS, for local development should be web, for production websecure. if redirection is needed, read below. -- `CERT_RESOLVER_LABEL=traefik.http.routers.erpnext-nginx.tls.certresolver=myresolver` - - Which traefik resolver to use to get TLS certificate, sets `erpnext.local.no-cert-resolver` for local setup. -- `` HTTPS_REDIRECT_RULE_LABEL=traefik.http.routers.http-catchall.rule=hostregexp(`{host:.+}`) `` - - Related to the traefik https redirection configuration, sets `erpnext.local.no-redirect-rule` for local setup. -- `HTTPS_REDIRECT_ENTRYPOINT_LABEL=traefik.http.routers.http-catchall.entrypoints=web` - - Related to the traefik https redirection configuration, sets `erpnext.local.no-entrypoint` for local setup. -- `HTTPS_REDIRECT_MIDDLEWARE_LABEL=traefik.http.routers.http-catchall.middlewares=redirect-to-https` - - Related to the traefik https redirection configuration, sets `erpnext.local.no-middleware` for local setup. -- `HTTPS_USE_REDIRECT_MIDDLEWARE_LABEL=traefik.http.middlewares.redirect-to-https.redirectscheme.scheme=https` - - Related to the traefik https redirection configuration, sets `erpnext.local-no-redirect-middleware` for local setup. - -Notes: - -- `AUTO_MIGRATE` variable is set to `1` by default. It checks if there is semver bump or git hash change in case of develop branch and automatically migrates the sites on container start up. -- It is good practice to use image tag for specific version instead of latest. e.g `frappe-socketio:v12.5.1`, `erpnext-nginx:v12.7.1`. - -## Start containers - -Execute the following command: - -```sh -docker-compose --project-name up -d -``` - -Make sure to replace `` with the desired name you wish to set for the project. - -Notes: - -- If it is the first time running and site is being initialized, _it can take multiple minutes for the site to be up_. Monitor `site-creator` container logs to check progress. Use command `docker logs _site-creator_1 -f` -- After the site is ready the username is `Administrator` and the password is `$ADMIN_PASSWORD` -- The local deployment is for testing and REST API development purpose only -- A complete development environment is available [here](../development) - -## Docker containers - -The docker-compose file contains following services: - -- traefik: manages letsencrypt - - volume: cert-vol -- redis-cache: cache store - - volume: redis-cache-vol -- redis-queue: used by workers - - volume: redis-queue-vol -- redis-socketio: used by socketio service - - volume: redis-socketio-vol -- mariadb: main database - - volume: mariadb-vol -- erpnext-nginx: serves static assets and proxies web request to the appropriate container, allowing to offer all services on the same port. - - volume: assets-vol and sites-vol -- erpnext-python: main application code - - volume: assets-vol and sites-vol -- frappe-socketio: enables realtime communication to the user interface through websockets - - volume: sites-vol -- erpnext-worker-default: background runner - - volume: sites-vol -- erpnext-worker-short: background runner for short-running jobs - - volume: sites-vol -- erpnext-worker-long: background runner for long-running jobs - - volume: sites-vol -- erpnext-schedule - - volume: sites-vol -- site-creator: run once container to create new site. - - volume: sites-vol - -## Updating and Migrating Sites - -Switch to the root of the `frappe_docker` directory before running the following commands: - -```sh -# Update environment variables ERPNEXT_VERSION and FRAPPE_VERSION -nano .env - -# Pull new images -docker-compose pull - -# Restart containers -docker-compose --project-name up -d -``` diff --git a/docs/site-operations.md b/docs/site-operations.md index a8b6e220..f012350c 100644 --- a/docs/site-operations.md +++ b/docs/site-operations.md @@ -1,177 +1,41 @@ # Site operations -Create and use env file to pass environment variables to containers, +> 💡 You should setup `--project-name` option in `docker-compose` commands if you have non-standard project name. -```sh -source .env -``` - -Or specify environment variables instead of passing secrets as command arguments. Refer notes section for environment variables required - -## Setup New Site +## Setup new site Note: -- Wait for the database service to start before trying to create a new site. - - If new site creation fails, retry after the MariaDB container is up and running. - - If you're using a managed database instance, make sure that the database is running before setting up a new site. - -#### MariaDB Site +- Wait for the `db` service to start and `configurator` to exit before trying to create a new site. Usually this takes up to 10 seconds. ```sh -# Create ERPNext site -docker run \ - -e "SITE_NAME=$SITE_NAME" \ - -e "DB_ROOT_USER=$DB_ROOT_USER" \ - -e "MYSQL_ROOT_PASSWORD=$MYSQL_ROOT_PASSWORD" \ - -e "ADMIN_PASSWORD=$ADMIN_PASSWORD" \ - -e "INSTALL_APPS=erpnext" \ - -v _sites-vol:/home/frappe/frappe-bench/sites \ - --network _default \ - frappe/erpnext-worker:$ERPNEXT_VERSION new +docker-compose exec backend bench new-site --mariadb-root-password --admin-password ``` -#### PostgreSQL Site +If you need to install some app, specify `--install-app`. To see all options, just run `bench new-site --help`. -PostgreSQL is only available v12 onwards. It is NOT available for ERPNext. -It is available as part of `frappe/erpnext-worker`. It inherits from `frappe/frappe-worker`. +To create Postgres site (assuming you already use [Postgres compose override](images-and-compose-files.md#overrides)) you need have to do set `root_login` and `root_password` in common config before that: ```sh -# Create ERPNext site -docker run \ - -e "SITE_NAME=$SITE_NAME" \ - -e "DB_ROOT_USER=$DB_ROOT_USER" \ - -e "POSTGRES_HOST=$POSTGRES_HOST" \ - -e "POSTGRES_PASSWORD=$POSTGRES_PASSWORD" \ - -e "ADMIN_PASSWORD=$ADMIN_PASSWORD" \ - -v _sites-vol:/home/frappe/frappe-bench/sites \ - --network _default \ - frappe/erpnext-worker:$ERPNEXT_VERSION new +docker-compose exec backend bench set-config -g root_login +docker-compose exec backend bench set-config -g root_password ``` -Environment Variables needed: - -- `SITE_NAME`: name of the new site to create. Site name is domain name that resolves. e.g. `erp.example.com` or `erp.localhost`. -- `DB_ROOT_USER`: MariaDB/PostgreSQL Root user. -- `MYSQL_ROOT_PASSWORD`: In case of the MariaDB docker container use the one set in `MYSQL_ROOT_PASSWORD` in previous steps. In case of a managed database use the appropriate password. -- `MYSQL_ROOT_PASSWORD_FILE` - When the MariaDB root password is stored using docker secrets. -- `ADMIN_PASSWORD`: set the administrator password for the new site. -- `ADMIN_PASSWORD_FILE`: set the administrator password for the new site using docker secrets. -- `INSTALL_APPS=erpnext`: available only in erpnext-worker and erpnext containers (or other containers with custom apps). Installs ERPNext (and/or the specified apps, comma-delinieated) on this new site. -- `FORCE=1`: optional variable which force installation of the same site. - -Environment Variables for PostgreSQL only: - -- `POSTGRES_HOST`: host for PostgreSQL server -- `POSTGRES_PASSWORD`: Password for `postgres`. The database root user. - -Notes: - -- To setup existing frappe-bench deployment with default database as PostgreSQL edit the common_site_config.json and set `db_host` to PostgreSQL hostname and `db_port` to PostgreSQL port. -- To create new frappe-bench deployment with default database as PostgreSQL use `POSTGRES_HOST` and `DB_PORT` environment variables in `erpnext-python` service instead of `MARIADB_HOST` - -## Add sites to proxy - -Change `SITES` variable to the list of sites created encapsulated in backtick and separated by comma with no space. e.g. `` SITES=`site1.example.com`,`site2.example.com` ``. - -Reload variables with following command. +Also command is slightly different: ```sh -docker-compose --project-name up -d +docker-compose exec backend bench new-site --db-type postgres --admin-password ``` -## Backup Sites +## Push backup to S3 storage -Environment Variables - -- `SITES` is list of sites separated by `:` colon to migrate. e.g. `SITES=site1.domain.com` or `SITES=site1.domain.com:site2.domain.com` By default all sites in bench will be backed up. -- `WITH_FILES` if set to 1, it will backup user-uploaded files. -- By default `backup` takes mariadb dump and gzips it. Example file, `20200325_221230-test_localhost-database.sql.gz` -- If `WITH_FILES` is set then it will also backup public and private files of each site as uncompressed tarball. Example files, `20200325_221230-test_localhost-files.tar` and `20200325_221230-test_localhost-private-files.tar` -- All the files generated by backup are placed at volume location `sites-vol:/{site-name}/private/backups/*` +We have the script that helps to push latest backup to S3. ```sh -docker run \ - -e "SITES=site1.domain.com:site2.domain.com" \ - -e "WITH_FILES=1" \ - -v _sites-vol:/home/frappe/frappe-bench/sites \ - --network _default \ - frappe/erpnext-worker:$ERPNEXT_VERSION backup +docker-compose exec backend push_backup.py --site-name --bucket --region-name --endpoint-url --aws-access-key-id --aws-secret-access-key ``` -The backup will be available in the `sites-vol` volume. - -## Push backup to s3 compatible storage - -Environment Variables - -- `BUCKET_NAME`, Required to set bucket created on S3 compatible storage. -- `REGION`, Required to set region for S3 compatible storage. -- `ACCESS_KEY_ID`, Required to set access key. -- `SECRET_ACCESS_KEY`, Required to set secret access key. -- `ENDPOINT_URL`, Required to set URL of S3 compatible storage. -- `BUCKET_DIR`, Required to set directory in bucket where sites from this deployment will be backed up. -- `BACKUP_LIMIT`, Optionally set this to limit number of backups in bucket directory. Defaults to 3. - -```sh - docker run \ - -e "BUCKET_NAME=backups" \ - -e "REGION=region" \ - -e "ACCESS_KEY_ID=access_id_from_provider" \ - -e "SECRET_ACCESS_KEY=secret_access_from_provider" \ - -e "ENDPOINT_URL=https://region.storage-provider.com" \ - -e "BUCKET_DIR=frappe-bench" \ - -v _sites-vol:/home/frappe/frappe-bench/sites \ - --network _default \ - frappe/frappe-worker:$FRAPPE_VERSION push-backup -``` - -Note: - -- Above example will backup files in bucket called `backup` at location `frappe-bench-v13/site.name.com/DATE_TIME/DATE_TIME-site_name_com-{filetype}.{extension}`, -- example DATE_TIME: 20200325_042020. -- example filetype: database, files or private-files -- example extension: sql.gz or tar - -## Restore backups - -Environment Variables - -- `MYSQL_ROOT_PASSWORD` or `MYSQL_ROOT_PASSWORD_FILE`(when using docker secrets), Required to restore mariadb backups. -- `BUCKET_NAME`, Required to set bucket created on S3 compatible storage. -- `ACCESS_KEY_ID`, Required to set access key. -- `SECRET_ACCESS_KEY`, Required to set secret access key. -- `ENDPOINT_URL`, Required to set URL of S3 compatible storage. -- `REGION`, Required to set region for s3 compatible storage. -- `BUCKET_DIR`, Required to set directory in bucket where sites from this deployment will be backed up. - -```sh -docker run \ - -e "MYSQL_ROOT_PASSWORD=admin" \ - -e "BUCKET_NAME=backups" \ - -e "REGION=region" \ - -e "ACCESS_KEY_ID=access_id_from_provider" \ - -e "SECRET_ACCESS_KEY=secret_access_from_provider" \ - -e "ENDPOINT_URL=https://region.storage-provider.com" \ - -e "BUCKET_DIR=frappe-bench" \ - -v _sites-vol:/home/frappe/frappe-bench/sites \ - -v ./backups:/home/frappe/backups \ - --network _default \ - frappe/frappe-worker:$FRAPPE_VERSION restore-backup -``` - -Note: - -- Volume must be mounted at location `/home/frappe/backups` for restoring sites -- If no backup files are found in volume, it will use s3 credentials to pull backups -- Backup structure for mounted volume or downloaded from s3: - - /home/frappe/backups - - site1.domain.com - - 20200420_162000 - - 20200420_162000-site1_domain_com-\* - - site2.domain.com - - 20200420_162000 - - 20200420_162000-site2_domain_com-\* +Note that you can restore backup only manually. ## Edit configs @@ -179,12 +43,11 @@ Editing config manually might be required in some cases, one such case is to use Amazon RDS (or any other DBaaS). For full instructions, refer to the [wiki](). Common question can be found in Issues and on forum. -`common_site_config.json` or `site_config.json` from `sites-vol` volume has to be edited using following command: +`common_site_config.json` or `site_config.json` from `sites` volume has to be edited using following command: ```sh -docker run \ - -it \ - -v _sites-vol:/sites \ +docker run --rm -it \ + -v _sites:/sites \ alpine vi /sites/common_site_config.json ``` @@ -195,8 +58,7 @@ Instead of `alpine` use any image of your choice. For socketio and gunicorn service ping the hostname:port and that will be sufficient. For workers and scheduler, there is a command that needs to be executed. ```shell -docker exec -it _erpnext-worker-d \ -docker-entrypoint.sh doctor -p postgresql:5432 --ping-service mongodb:27017 +docker-compose exec backend healthcheck.sh --ping-service mongodb:27017 ``` Additional services can be pinged as part of health check with option `-p` or `--ping-service`. @@ -204,86 +66,20 @@ Additional services can be pinged as part of health check with option `-p` or `- This check ensures that given service should be connected along with services in common_site_config.json. If connection to service(s) fails, the command fails with exit code 1. -## Frappe internal commands using bench helper +--- -To execute commands using bench helper. - -```shell - docker run \ - -v _sites-vol:/home/frappe/frappe-bench/sites \ - --network _default \ - --user frappe \ - frappe/frappe-worker:$FRAPPE_VERSION bench --help -``` - -Example command to clear cache - -```shell - docker run \ - -v _sites-vol:/home/frappe/frappe-bench/sites \ - --network _default \ - --user frappe \ - frappe/frappe-worker:$FRAPPE_VERSION bench --site erp.mysite.com clear-cache -``` - -Notes: - -- Use it to install/uninstall custom apps, add system manager user, etc. -- To run the command as non root user add the command option `--user frappe`. - -## Delete/Drop Site - -#### MariaDB Site +For reference of commands like `backup`, `drop-site` or `migrate` check [official guide](https://frappeframework.com/docs/v13/user/en/bench/frappe-commands) or run: ```sh -# Delete/Drop ERPNext site -docker run \ - -e "SITE_NAME=$SITE_NAME" \ - -e "DB_ROOT_USER=$DB_ROOT_USER" \ - -e "MYSQL_ROOT_PASSWORD=$MYSQL_ROOT_PASSWORD" \ - -v _sites-vol:/home/frappe/frappe-bench/sites \ - --network _default \ - frappe/erpnext-worker:$ERPNEXT_VERSION drop +docker-compose exec backend bench --help ``` -#### PostgreSQL Site +## Migrate site + +Note: + +- Wait for the `db` service to start and `configurator` to exit before trying to migrate a site. Usually this takes up to 10 seconds. ```sh -# Delete/Drop ERPNext site -docker run \ - -e "SITE_NAME=$SITE_NAME" \ - -e "DB_ROOT_USER=$DB_ROOT_USER" \ - -e "POSTGRES_PASSWORD=$POSTGRES_PASSWORD" \ - -v _sites-vol:/home/frappe/frappe-bench/sites \ - --network _default \ - frappe/erpnext-worker:$ERPNEXT_VERSION drop +docker-compose exec backend bench --site migrate ``` - -Environment Variables needed: - -- `SITE_NAME`: name of the site to be deleted. Site name is domain name that resolves. e.g. `erp.example.com` or `erp.localhost`. -- `DB_ROOT_USER`: MariaDB/PostgreSQL Root user. -- `MYSQL_ROOT_PASSWORD`: Root User password for MariaDB. -- `FORCE=1`: optional variable which force deletion of the same site. -- `NO_BACKUP=1`: option variable to skip the process of taking backup before deleting the site. - -Environment Variables for PostgreSQL only: - -- `POSTGRES_PASSWORD`: Password for `postgres`. The database root user. - -## Migrate Site - -```sh -# Migrate ERPNext site -docker run \ - -e "MAINTENANCE_MODE=1" \ - -v _sites-vol:/home/frappe/frappe-bench/sites \ - -v _assets-vol:/home/frappe/frappe-bench/sites/assets \ - --network _default \ - frappe/erpnext-worker:$ERPNEXT_VERSION migrate -``` - -Environment Variables needed: - -- `MAINTENANCE_MODE`: If set to `1`, this will ensure the bench is switched to maintenance mode during migration. -- `SITES`: Optional list of sites to be migrated, separated by colon (`:`). e.g. `erp.site1.com:erp.site2.com`. If not used, all sites will be migrated by default. diff --git a/docs/tips-for-moving-deployments.md b/docs/tips-for-moving-deployments.md deleted file mode 100644 index 8687c5a4..00000000 --- a/docs/tips-for-moving-deployments.md +++ /dev/null @@ -1,15 +0,0 @@ -# Tips for moving deployments - -- Take regular automatic backups and push the files to S3 compatible cloud. Setup backup and push with cronjobs - - Use regular cron for single machine installs - - Use [swarm-cronjob](https://github.com/crazy-max/swarm-cronjob) for docker swarm - - Use Kubernetes CronJob -- It makes it easy to transfer data from cloud to any new deployment. -- They are just [site operations](site-operations.md) that can be manually pipelined as per need. -- Remember to restore encryption keys and other custom configuration from `site_config.json`. -- Steps to move deployment: - - [Take backup](site-operations.md#backup-sites) - - [Push backup to cloud](site-operations.md#push-backup-to-s3-compatible-storage) - - Create new deployment type anywhere - - [Restore backup from cloud](site-operations.md#restore-backups) - - [Restore `site_config.json` from cloud](site-operations.md#edit-configs) diff --git a/docs/troubleshoot.md b/docs/troubleshoot.md new file mode 100644 index 00000000..56a1d89f --- /dev/null +++ b/docs/troubleshoot.md @@ -0,0 +1,55 @@ +1. [Fixing MariaDB issues after rebuilding the container](#fixing-mariadb-issues-after-rebuilding-the-container) +1. [Letsencrypt companion not working](#letsencrypt-companion-not-working) +1. [docker-compose does not recognize variables from `.env` file](#docker-compose-does-not-recognize-variables-from-env-file) +1. [Windows Based Installation](#windows-based-installation) + +### Fixing MariaDB issues after rebuilding the container + +For any reason after rebuilding the container if you are not be able to access MariaDB correctly with the previous configuration. Follow these instructions. + +The parameter `'db_name'@'%'` needs to be set in MariaDB and permission to the site database suitably assigned to the user. + +This step has to be repeated for all sites available under the current bench. +Example shows the queries to be executed for site `localhost` + +Open sites/localhost/site_config.json: + +```shell +code sites/localhost/site_config.json +``` + +and take note of the parameters `db_name` and `db_password`. + +Enter MariaDB Interactive shell: + +```shell +mysql -uroot -p123 -hmariadb +``` + +Execute following queries replacing `db_name` and `db_password` with the values found in site_config.json. + +```sql +UPDATE mysql.user SET Host = '%' where User = 'db_name'; FLUSH PRIVILEGES; +SET PASSWORD FOR 'db_name'@'%' = PASSWORD('db_password'); FLUSH PRIVILEGES; +GRANT ALL PRIVILEGES ON `db_name`.* TO 'db_name'@'%'; FLUSH PRIVILEGES; +EXIT; +``` + +### Letsencrypt companion not working + +- Nginx Letsencrypt Companion needs to be setup before starting ERPNext services. +- Are domain names in `SITES` variable correct? +- Is DNS record configured? `A Name` record needs to point to Public IP of server. +- Try Restarting containers. + +### docker-compose does not recognize variables from `.env` file + +If you are using old version of `docker-compose` the .env file needs to be located in directory from where the docker-compose command is executed. There may also be difference in official `docker-compose` and the one packaged by distro. + +### Windows Based Installation + +- Set environment variable `COMPOSE_CONVERT_WINDOWS_PATHS` e.g. `set COMPOSE_CONVERT_WINDOWS_PATHS=1` +- Make the `frappe-mariadb.cnf` read-only for mariadb container to pick it up. +- While using docker machine, port-forward the port 80 of VM to port 80 of host machine +- Name all the sites ending with `.localhost`. and access it via browser locally. e.g. `http://site1.localhost` +- related issue comment https://github.com/frappe/frappe_docker/issues/448#issuecomment-851723912 diff --git a/env-example b/env-example deleted file mode 100644 index a9c5414b..00000000 --- a/env-example +++ /dev/null @@ -1,7 +0,0 @@ -ERPNEXT_VERSION=edge -FRAPPE_VERSION=edge -MARIADB_HOST=mariadb -MYSQL_ROOT_PASSWORD=admin -SITES=your.domain.com -LETSENCRYPT_EMAIL=your.email@your.domain.com -SKIP_NGINX_TEMPLATE_GENERATION=0 diff --git a/env-local b/env-local deleted file mode 100644 index 821c23ea..00000000 --- a/env-local +++ /dev/null @@ -1,18 +0,0 @@ -LETSENCRYPT_EMAIL=email@example.com -ERPNEXT_VERSION=edge -FRAPPE_VERSION=edge -MARIADB_HOST=mariadb -MYSQL_ROOT_PASSWORD=admin -SITE_NAME=mysite.localhost -SITES=`mysite.localhost` -DB_ROOT_USER=root -ADMIN_PASSWORD=admin -INSTALL_APPS=erpnext -ENTRYPOINT_LABEL=traefik.http.routers.erpnext-nginx.entrypoints=web -CERT_RESOLVER_LABEL=erpnext.local.no-cert-resolver -HTTPS_REDIRECT_RULE_LABEL=erpnext.local.no-redirect-rule -HTTPS_REDIRECT_ENTRYPOINT_LABEL=erpnext.local.no-entrypoint -HTTPS_REDIRECT_MIDDLEWARE_LABEL=erpnext.local.no-middleware -HTTPS_USE_REDIRECT_MIDDLEWARE_LABEL=erpnext.local-no-redirect-middleware -SKIP_NGINX_TEMPLATE_GENERATION=0 -WORKER_CLASS=gthread diff --git a/env-production b/env-production deleted file mode 100644 index b6de88bc..00000000 --- a/env-production +++ /dev/null @@ -1,18 +0,0 @@ -LETSENCRYPT_EMAIL=email@example.com -ERPNEXT_VERSION=edge -FRAPPE_VERSION=edge -MARIADB_HOST=mariadb -MYSQL_ROOT_PASSWORD=admin -SITE_NAME=erp.example.com -SITES=`erp.example.com` -DB_ROOT_USER=root -ADMIN_PASSWORD=admin -INSTALL_APPS=erpnext -ENTRYPOINT_LABEL=traefik.http.routers.erpnext-nginx.entrypoints=websecure -CERT_RESOLVER_LABEL=traefik.http.routers.erpnext-nginx.tls.certresolver=myresolver -HTTPS_REDIRECT_RULE_LABEL=traefik.http.routers.http-catchall.rule=hostregexp(`{host:.+}`) -HTTPS_REDIRECT_ENTRYPOINT_LABEL=traefik.http.routers.http-catchall.entrypoints=web -HTTPS_REDIRECT_MIDDLEWARE_LABEL=traefik.http.routers.http-catchall.middlewares=redirect-to-https -HTTPS_USE_REDIRECT_MIDDLEWARE_LABEL=traefik.http.middlewares.redirect-to-https.redirectscheme.scheme=https -SKIP_NGINX_TEMPLATE_GENERATION=0 -WORKER_CLASS=gthread diff --git a/example.env b/example.env new file mode 100644 index 00000000..81e77a5d --- /dev/null +++ b/example.env @@ -0,0 +1,40 @@ +# Reference: https://github.com/frappe/frappe_docker/blob/main/docs/images-and-compose-files.md + +FRAPPE_VERSION=v13.22.1 + +# Only with ERPNext override +ERPNEXT_VERSION=v13.22.0 + +DB_PASSWORD=123 + +# Only if you use external database +DB_HOST= +DB_PORT= + +# Only if you use external Redis +REDIS_CACHE= +REDIS_QUEUE= +REDIS_SOCKETIO= + +# Only with HTTPS override +LETSENCRYPT_EMAIL=mail@example.com + +# These environment variables are not required. + +# Default value is `$$host` which resolves site by host. For example, if your host is `example.com`, +# site's name should be `example.com`, or if host is `127.0.0.1` (local debugging), it should be `127.0.0.1`. +# This variable allows to override described behavior. Let's say you create site named `mysite` +# and do want to access it by `127.0.0.1` host. Than you would set this variable to `mysite`. +FRAPPE_SITE_NAME_HEADER= + +# Default value is `127.0.0.1`. Set IP address as our trusted upstream address. +UPSTREAM_REAL_IP_ADDRESS= + +# Default value is `X-Forwarded-For`. Set request header field whose value will be used to replace the client address +UPSTREAM_REAL_IP_HEADER= + +# Allowed values are on|off. Default value is `off`. If recursive search is disabled, +# the original client address that matches one of the trusted addresses +# is replaced by the last address sent in the request header field defined by the real_ip_header directive. +# If recursive search is enabled, the original client address that matches one of the trusted addresses is replaced by the last non-trusted address sent in the request header field. +UPSTREAM_REAL_IP_RECURSIVE= diff --git a/build/bench/Dockerfile b/images/bench/Dockerfile similarity index 100% rename from build/bench/Dockerfile rename to images/bench/Dockerfile diff --git a/images/nginx/Dockerfile b/images/nginx/Dockerfile new file mode 100644 index 00000000..e23453ea --- /dev/null +++ b/images/nginx/Dockerfile @@ -0,0 +1,88 @@ +FROM node:14-bullseye-slim as base + +RUN apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y \ + git \ + build-essential \ + python \ + ca-certificates \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /root/frappe-bench +RUN mkdir -p sites/assets + +ARG FRAPPE_VERSION +RUN git clone --depth 1 -b ${FRAPPE_VERSION} https://github.com/frappe/frappe apps/frappe + + + +FROM base as frappe_prod_node_modules + +# Install production node modules +RUN yarn --cwd apps/frappe --prod + + + +FROM frappe_prod_node_modules as frappe_assets + +# Install development node modules +RUN yarn --cwd apps/frappe + +# Build assets they're stored in frappe-bench/sites/assets +RUN echo "frappe" >sites/apps.txt \ + && yarn --cwd apps/frappe run production \ + && rm sites/apps.txt \ + # TODO: Currently `yarn run production` doesn't create .build on develop branch: https://github.com/frappe/frappe/issues/15396 + && if [ ! -f sites/.build ]; then touch sites/.build; fi + + + +FROM base as erpnext_prod_node_modules + +ARG ERPNEXT_VERSION +RUN git clone --depth 1 -b ${ERPNEXT_VERSION} https://github.com/frappe/erpnext apps/erpnext + +RUN yarn --cwd apps/erpnext --prod + + + +FROM erpnext_prod_node_modules as erpnext_assets + +RUN yarn --cwd apps/erpnext + +COPY --from=frappe_assets /root/frappe-bench/apps/frappe/node_modules /root/frappe-bench/apps/frappe/node_modules +COPY --from=frappe_assets /root/frappe-bench/apps/frappe/package.json /root/frappe-bench/apps/frappe/yarn.lock /root/frappe-bench/apps/frappe/ + +RUN echo "frappe\nerpnext" >sites/apps.txt \ + && yarn --cwd apps/frappe run production --app erpnext \ + && rm sites/apps.txt + + + +FROM base as error_pages + +RUN git clone --depth 1 https://github.com/frappe/bench /root/bench + + + +FROM nginxinc/nginx-unprivileged:1.20-alpine as frappe + +COPY --from=error_pages /root/bench/bench/config/templates/502.html /usr/share/nginx/html +COPY --from=base /root/frappe-bench/apps/frappe/frappe/public /usr/share/nginx/html/assets/frappe +COPY --from=frappe_prod_node_modules /root/frappe-bench/apps/frappe/node_modules /usr/share/nginx/html/assets/frappe/node_modules +COPY --from=frappe_assets /root/frappe-bench/sites /usr/share/nginx/html + +# https://github.com/nginxinc/docker-nginx-unprivileged/blob/main/stable/alpine/20-envsubst-on-templates.sh +COPY nginx-template.conf /etc/nginx/templates/default.conf.template +# https://github.com/nginxinc/docker-nginx-unprivileged/blob/main/stable/alpine/docker-entrypoint.sh +COPY entrypoint.sh /docker-entrypoint.d/frappe-entrypoint.sh + +USER 1000 + + + +FROM frappe as erpnext + +COPY --from=erpnext_prod_node_modules /root/frappe-bench/apps/erpnext/erpnext/public /usr/share/nginx/html/assets/erpnext +COPY --from=erpnext_prod_node_modules /root/frappe-bench/apps/erpnext/node_modules /usr/share/nginx/html/assets/erpnext/node_modules +COPY --from=erpnext_assets /root/frappe-bench/sites /usr/share/nginx/html diff --git a/images/nginx/entrypoint.sh b/images/nginx/entrypoint.sh new file mode 100755 index 00000000..78c15c9b --- /dev/null +++ b/images/nginx/entrypoint.sh @@ -0,0 +1,9 @@ +#!/bin/sh + +set -e + +# Update timestamp for ".build" file to enable caching assets: +# https://github.com/frappe/frappe/blob/52d8e6d952130eea64a9990b9fd5b1f6877be1b7/frappe/utils/__init__.py#L799-L805 +if [ -d /usr/share/nginx/html/sites ]; then + touch /usr/share/nginx/html/sites/.build -r /usr/share/nginx/html/.build +fi diff --git a/images/nginx/nginx-template.conf b/images/nginx/nginx-template.conf new file mode 100644 index 00000000..be55ac47 --- /dev/null +++ b/images/nginx/nginx-template.conf @@ -0,0 +1,120 @@ +upstream backend-server { + server ${BACKEND} fail_timeout=0; +} + +upstream socketio-server { + server ${SOCKETIO} fail_timeout=0; +} + +# Parse the X-Forwarded-Proto header - if set - defaulting to $scheme. +map $http_x_forwarded_proto $proxy_x_forwarded_proto { + default $scheme; + https https; +} + +server { + listen 8080; + server_name $http_host; + root /usr/share/nginx/html; + + proxy_buffer_size 128k; + proxy_buffers 4 256k; + proxy_busy_buffers_size 256k; + + add_header X-Frame-Options "SAMEORIGIN"; + add_header Strict-Transport-Security "max-age=63072000; includeSubDomains; preload"; + add_header X-Content-Type-Options nosniff; + add_header X-XSS-Protection "1; mode=block"; + add_header Referrer-Policy "same-origin, strict-origin-when-cross-origin"; + + set_real_ip_from ${UPSTREAM_REAL_IP_ADDRESS}; + real_ip_header ${UPSTREAM_REAL_IP_HEADER}; + real_ip_recursive ${UPSTREAM_REAL_IP_RECURSIVE}; + + location /assets { + try_files $uri =404; + } + + location ~ ^/protected/(.*) { + internal; + try_files /sites/$http_host/$1 =404; + } + + location /socket.io { + proxy_http_version 1.1; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $proxy_x_forwarded_proto; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + proxy_set_header X-Frappe-Site-Name ${FRAPPE_SITE_NAME_HEADER}; + proxy_set_header Origin $scheme://$http_host; + proxy_set_header Host $host; + + proxy_pass http://socketio-server; + } + + location / { + rewrite ^(.+)/$ $1 permanent; + rewrite ^(.+)/index\.html$ $1 permanent; + rewrite ^(.+)\.html$ $1 permanent; + + location ~ ^/files/.*.(htm|html|svg|xml) { + add_header Content-disposition "attachment"; + try_files /sites/$http_host/public/$uri @webserver; + } + + try_files /sites/$http_host/public/$uri @webserver; + } + + location @webserver { + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $proxy_x_forwarded_proto; + proxy_set_header X-Frappe-Site-Name ${FRAPPE_SITE_NAME_HEADER}; + proxy_set_header Host $host; + proxy_set_header X-Use-X-Accel-Redirect True; + proxy_read_timeout 120; + proxy_redirect off; + + proxy_pass http://backend-server; + } + + # error pages + error_page 502 /502.html; + location /502.html { + internal; + } + + # optimizations + sendfile on; + keepalive_timeout 15; + client_max_body_size 50m; + client_body_buffer_size 16K; + client_header_buffer_size 1k; + + # enable gzip compression + # based on https://mattstauffer.co/blog/enabling-gzip-on-nginx-servers-including-laravel-forge + gzip on; + gzip_http_version 1.1; + gzip_comp_level 5; + gzip_min_length 256; + gzip_proxied any; + gzip_vary on; + gzip_types + application/atom+xml + application/javascript + application/json + application/rss+xml + application/vnd.ms-fontobject + application/x-font-ttf + application/font-woff + application/x-web-app-manifest+json + application/xhtml+xml + application/xml + font/opentype + image/svg+xml + image/x-icon + text/css + text/plain + text/x-component; + # text/html is always compressed by HttpGzipModule +} diff --git a/images/socketio/Dockerfile b/images/socketio/Dockerfile new file mode 100644 index 00000000..b32ad878 --- /dev/null +++ b/images/socketio/Dockerfile @@ -0,0 +1,24 @@ +FROM alpine/git as builder + +ARG FRAPPE_VERSION +RUN git clone --depth 1 -b ${FRAPPE_VERSION} https://github.com/frappe/frappe /opt/frappe + + +FROM node:17-alpine + +RUN addgroup -S frappe \ + && adduser -S frappe -G frappe +USER frappe + +WORKDIR /home/frappe/frappe-bench +RUN mkdir -p sites apps/frappe + +COPY --from=builder /opt/frappe/socketio.js /opt/frappe/node_utils.js apps/frappe/ +COPY package.json apps/frappe/ + +RUN cd apps/frappe \ + && npm install + +WORKDIR /home/frappe/frappe-bench/sites + +CMD [ "node", "/home/frappe/frappe-bench/apps/frappe/socketio.js" ] diff --git a/build/frappe-socketio/package.json b/images/socketio/package.json similarity index 78% rename from build/frappe-socketio/package.json rename to images/socketio/package.json index 0c969142..04539034 100644 --- a/build/frappe-socketio/package.json +++ b/images/socketio/package.json @@ -2,10 +2,6 @@ "name": "frappe-socketio", "version": "1.0.1", "description": "Frappe SocketIO Server", - "main": "socketio.js", - "scripts": { - "start": "node socketio.js" - }, "author": "Revant Nandgaonkar", "license": "MIT", "dependencies": { diff --git a/images/worker/Dockerfile b/images/worker/Dockerfile new file mode 100644 index 00000000..e46d1ef0 --- /dev/null +++ b/images/worker/Dockerfile @@ -0,0 +1,123 @@ +# syntax=docker/dockerfile:1.3 + +ARG PYTHON_VERSION +FROM python:${PYTHON_VERSION}-slim-bullseye as base + +RUN apt-get update \ + && apt-get install --no-install-recommends -y \ + # Postgres + libpq-dev \ + && rm -rf /var/lib/apt/lists/* + +RUN useradd -ms /bin/bash frappe +USER frappe +RUN mkdir -p /home/frappe/frappe-bench/apps /home/frappe/frappe-bench/logs /home/frappe/frappe-bench/sites +WORKDIR /home/frappe/frappe-bench + +RUN --mount=type=cache,target=/home/frappe/.cache/pip \ + pip install -U pip wheel \ + && python -m venv env \ + && env/bin/pip install -U pip wheel +USER root + + + +FROM base as build_deps + +RUN apt-get update \ + && apt-get install --no-install-recommends -y \ + # Install git here because it is not required in production + git \ + # gcc and g++ are required for building different packages across different versions + # of Frappe and ERPNext and also on different platforms (for example, linux/arm64). + # It is safe to install build deps even if they are not required + # because they won't be included in final images. + gcc \ + g++ \ + # Make is required to build wheels of ERPNext deps in develop branch for linux/arm64 + make \ + && rm -rf /var/lib/apt/lists/* +USER frappe + + + +FROM build_deps as frappe_builder + +ARG FRAPPE_VERSION +RUN --mount=type=cache,target=/home/frappe/.cache/pip \ + git clone --depth 1 -b ${FRAPPE_VERSION} https://github.com/frappe/frappe apps/frappe \ + && env/bin/pip install -e apps/frappe \ + && env/bin/pip install -U gevent \ + && rm -r apps/frappe/.git + + + +# We split ERPNext wheels build in separate stage to achieve concurrency with Frappe build +FROM build_deps as erpnext_wheels + +ARG ERPNEXT_VERSION +RUN git clone --depth 1 -b ${ERPNEXT_VERSION} https://github.com/frappe/erpnext apps/erpnext \ + && rm -r apps/erpnext/.git + +RUN --mount=type=cache,target=/home/frappe/.cache/pip \ + pip wheel --wheel-dir /home/frappe/erpnext-wheels -r apps/erpnext/requirements.txt + + + +FROM frappe_builder as erpnext_builder + +COPY --from=erpnext_wheels --chown=frappe /home/frappe/frappe-bench/apps/erpnext /home/frappe/frappe-bench/apps/erpnext +RUN --mount=type=bind,target=/home/frappe/erpnext-wheels,source=/home/frappe/erpnext-wheels,from=erpnext_wheels \ + --mount=type=cache,target=/home/frappe/.cache/pip \ + --mount=type=cache,target=/home/frappe/.cache/pip,source=/home/frappe/.cache/pip,from=erpnext_wheels \ + env/bin/pip install --find-links=/home/frappe/erpnext-wheels -e apps/erpnext + + + +FROM base as configured_base + +RUN apt-get update \ + && apt-get install --no-install-recommends -y curl \ + && curl -sL https://deb.nodesource.com/setup_14.x | bash - \ + && apt-get purge -y --auto-remove curl \ + && apt-get update \ + && apt-get install --no-install-recommends -y \ + # MariaDB + mariadb-client \ + # Postgres + postgresql-client \ + # wkhtmltopdf + xvfb \ + libfontconfig \ + wkhtmltopdf \ + # For healthcheck + wait-for-it \ + jq \ + # other + nodejs \ + && rm -rf /var/lib/apt/lists/* +USER frappe + +COPY pretend-bench.sh /usr/local/bin/bench +COPY push_backup.py /usr/local/bin/push-backup +COPY configure.py patched_bench_helper.py /usr/local/bin/ +COPY gevent_patch.py /opt/patches/ + +WORKDIR /home/frappe/frappe-bench/sites + +CMD [ "/home/frappe/frappe-bench/env/bin/gunicorn", "-b", "0.0.0.0:8000", "frappe.app:application" ] + + +FROM configured_base as frappe + +RUN echo "frappe" >/home/frappe/frappe-bench/sites/apps.txt +COPY --from=frappe_builder /home/frappe/frappe-bench/apps/frappe /home/frappe/frappe-bench/apps/frappe +COPY --from=frappe_builder /home/frappe/frappe-bench/env /home/frappe/frappe-bench/env + + +FROM configured_base as erpnext + +RUN echo "frappe\nerpnext" >/home/frappe/frappe-bench/sites/apps.txt +COPY --from=frappe_builder /home/frappe/frappe-bench/apps/frappe /home/frappe/frappe-bench/apps/frappe +COPY --from=erpnext_builder /home/frappe/frappe-bench/apps/erpnext /home/frappe/frappe-bench/apps/erpnext +COPY --from=erpnext_builder /home/frappe/frappe-bench/env /home/frappe/frappe-bench/env diff --git a/images/worker/configure.py b/images/worker/configure.py new file mode 100755 index 00000000..76ba6d61 --- /dev/null +++ b/images/worker/configure.py @@ -0,0 +1,55 @@ +#!/usr/local/bin/python + +import json +import os +from typing import Any, Type, TypeVar + + +def update_config(**values: Any): + fname = "common_site_config.json" + if not os.path.exists(fname): + with open(fname, "a") as f: + json.dump({}, f) + + with open(fname, "r+") as f: + config: dict[str, Any] = json.load(f) + config.update(values) + f.seek(0) + f.truncate() + json.dump(config, f) + + +_T = TypeVar("_T") + + +def env(name: str, type_: Type[_T] = str) -> _T: + value = os.getenv(name) + if not value: + raise RuntimeError(f'Required environment variable "{name}" not set') + try: + value = type_(value) + except Exception: + raise RuntimeError( + f'Cannot convert environment variable "{name}" to type "{type_}"' + ) + return value + + +def generate_redis_url(url: str): + return f"redis://{url}" + + +def main() -> int: + update_config( + db_host=env("DB_HOST"), + db_port=env("DB_PORT", int), + redis_cache=generate_redis_url(env("REDIS_CACHE")), + redis_queue=generate_redis_url(env("REDIS_QUEUE")), + redis_socketio=generate_redis_url(env("REDIS_SOCKETIO")), + socketio_port=env("SOCKETIO_PORT", int), + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/build/frappe-worker/commands/gevent_patch.py b/images/worker/gevent_patch.py similarity index 100% rename from build/frappe-worker/commands/gevent_patch.py rename to images/worker/gevent_patch.py diff --git a/images/worker/patched_bench_helper.py b/images/worker/patched_bench_helper.py new file mode 100644 index 00000000..e4631118 --- /dev/null +++ b/images/worker/patched_bench_helper.py @@ -0,0 +1,46 @@ +import click +import click.exceptions +import frappe.app +import frappe.database.db_manager +import frappe.utils.bench_helper + + +def patch_database_creator(): + """ + We need to interrupt Frappe site database creation to monkeypatch + functions that resolve host for user that owns site database. + In frappe_docker this was implemented in "new" command: + https://github.com/frappe/frappe_docker/blob/c808ad1767feaf793a2d14541ac0f4d9cbab45b3/build/frappe-worker/commands/new.py#L87 + """ + + frappe.database.db_manager.DbManager.get_current_host = lambda self: "%" + + +def patch_click_usage_error(): + bits: tuple[str, ...] = ( + click.style( + "Only Frappe framework bench commands are available in container setup.", + fg="yellow", + bold=True, + ), + "https://frappeframework.com/docs/v13/user/en/bench/frappe-commands", + ) + notice = "\n".join(bits) + + def format_message(self: click.exceptions.UsageError): + if "No such command" in self.message: + return f"{notice}\n\n{self.message}" + return self.message + + click.exceptions.UsageError.format_message = format_message + + +def main() -> int: + patch_database_creator() + patch_click_usage_error() + frappe.utils.bench_helper.main() + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/images/worker/pretend-bench.sh b/images/worker/pretend-bench.sh new file mode 100755 index 00000000..fe39f0a3 --- /dev/null +++ b/images/worker/pretend-bench.sh @@ -0,0 +1,5 @@ +#!/bin/bash +set -e + +# shellcheck disable=SC2068 +~/frappe-bench/env/bin/python /usr/local/bin/patched_bench_helper.py frappe $@ diff --git a/images/worker/push_backup.py b/images/worker/push_backup.py new file mode 100755 index 00000000..9ceace52 --- /dev/null +++ b/images/worker/push_backup.py @@ -0,0 +1,105 @@ +#!/home/frappe/frappe-bench/env/bin/python + +import argparse +import os +import sys +from pathlib import Path +from typing import TYPE_CHECKING, Any, List, cast + +import boto3 +import frappe +from frappe.utils.backups import BackupGenerator + +if TYPE_CHECKING: + from mypy_boto3_s3.service_resource import _Bucket + + +class Arguments(argparse.Namespace): + site: str + bucket: str + region_name: str + endpoint_url: str + aws_access_key_id: str + aws_secret_access_key: str + + +def _get_files_from_previous_backup(site_name: str) -> list[Path]: + frappe.connect(site_name) + + conf = cast(Any, frappe.conf) + backup_generator = BackupGenerator( + db_name=conf.db_name, + user=conf.db_name, + password=conf.db_password, + db_host=frappe.db.host, + db_port=frappe.db.port, + db_type=conf.db_type, + ) + recent_backup_files = backup_generator.get_recent_backup(24) + + frappe.destroy() + return [Path(f) for f in recent_backup_files if f] + + +def get_files_from_previous_backup(site_name: str) -> list[Path]: + files = _get_files_from_previous_backup(site_name) + if not files: + print("No backup found that was taken <24 hours ago.") + return files + + +def get_bucket(args: Arguments) -> "_Bucket": + return boto3.resource( + service_name="s3", + endpoint_url=args.endpoint_url, + region_name=args.region_name, + aws_access_key_id=args.aws_access_key_id, + aws_secret_access_key=args.aws_secret_access_key, + ).Bucket(args.bucket) + + +def upload_file(path: Path, site_name: str, bucket: "_Bucket") -> None: + filename = str(path.absolute()) + key = str(Path(site_name) / path.name) + print(f"Uploading {key}") + bucket.upload_file(Filename=filename, Key=key) + os.remove(path) + + +def push_backup(args: Arguments) -> None: + """Get latest backup files using Frappe utils, push them to S3 and remove local copy""" + + files = get_files_from_previous_backup(args.site) + bucket = get_bucket(args) + + for path in files: + upload_file(path=path, site_name=args.site, bucket=bucket) + + print("Done!") + + +def parse_args(args: List[str]) -> Arguments: + parser = argparse.ArgumentParser() + parser.add_argument("--site", required=True) + parser.add_argument("--bucket", required=True) + parser.add_argument("--region-name", required=True) + parser.add_argument("--endpoint-url", required=True) + # Looking for default AWS credentials variables + parser.add_argument( + "--aws-access-key-id", required=True, default=os.getenv("AWS_ACCESS_KEY_ID") + ) + parser.add_argument( + "--aws-secret-access-key", + required=True, + default=os.getenv("AWS_SECRET_ACCESS_KEY"), + ) + return parser.parse_args(args, namespace=Arguments()) + + +def main(args: List[str]) -> int: + push_backup(parse_args(args)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main(sys.argv[1:])) diff --git a/installation/docker-compose-common.yml b/installation/docker-compose-common.yml deleted file mode 100644 index 3786a32e..00000000 --- a/installation/docker-compose-common.yml +++ /dev/null @@ -1,42 +0,0 @@ -version: "3" - -services: - redis-cache: - image: redis:latest - restart: on-failure - volumes: - - redis-cache-vol:/data - - redis-queue: - image: redis:latest - restart: on-failure - volumes: - - redis-queue-vol:/data - - redis-socketio: - image: redis:latest - restart: on-failure - volumes: - - redis-socketio-vol:/data - - mariadb: - image: mariadb:10.6 - restart: on-failure - command: - - --character-set-server=utf8mb4 - - --collation-server=utf8mb4_unicode_ci - - --skip-character-set-client-handshake - - --skip-innodb-read-only-compressed # Temporary fix for MariaDB 10.6 - environment: - - MYSQL_ROOT_PASSWORD=${MYSQL_ROOT_PASSWORD} - # Sometimes db initialization takes longer than 10 seconds and site-creator goes away. - # Frappe doesn't use CONVERT_TZ() function that requires time zone info, so we can just skip it. - - MYSQL_INITDB_SKIP_TZINFO=1 - volumes: - - mariadb-vol:/var/lib/mysql - -volumes: - mariadb-vol: - redis-cache-vol: - redis-queue-vol: - redis-socketio-vol: diff --git a/installation/docker-compose-custom.yml b/installation/docker-compose-custom.yml deleted file mode 100644 index 9dacfd91..00000000 --- a/installation/docker-compose-custom.yml +++ /dev/null @@ -1,123 +0,0 @@ -version: '3' - -services: - [app]-assets: - image: [app]-assets - build: - context: ../build/[app]-nginx - restart: on-failure - environment: - - FRAPPE_PY=[app]-python - - FRAPPE_PY_PORT=8000 - - FRAPPE_SOCKETIO=frappe-socketio - - SOCKETIO_PORT=9000 - - LETSENCRYPT_HOST=${SITES} - - VIRTUAL_HOST=${SITES} - - LETSENCRYPT_EMAIL=${LETSENCRYPT_EMAIL} - depends_on: - - [app]-python - - frappe-socketio - - frappe-worker-default - - frappe-worker-long - - frappe-worker-short - links: - - [app]-python - - frappe-socketio - - frappe-worker-default - - frappe-worker-long - - frappe-worker-short - volumes: - - sites-vol:/var/www/html/sites:rw - - assets-vol:/assets:rw - - [app]-python: - image: [app]-worker - build: - context: ../build/[app]-worker - restart: on-failure - environment: - - MARIADB_HOST=${MARIADB_HOST} - - REDIS_CACHE=redis-cache:6379 - - REDIS_QUEUE=redis-queue:6379 - - REDIS_SOCKETIO=redis-socketio:6379 - - SOCKETIO_PORT=9000 - volumes: - - sites-vol:/home/frappe/frappe-bench/sites:rw - - logs-vol:/home/frappe/frappe-bench/logs:rw - - assets-vol:/home/frappe/frappe-bench/sites/assets:rw - - frappe-socketio: - image: frappe/frappe-socketio:${FRAPPE_VERSION} - restart: on-failure - depends_on: - - redis-socketio - links: - - redis-socketio - volumes: - - sites-vol:/home/frappe/frappe-bench/sites:rw - - logs-vol:/home/frappe/frappe-bench/logs:rw - - frappe-worker-default: - image: [app]-worker - restart: on-failure - command: worker - depends_on: - - redis-queue - - redis-cache - links: - - redis-queue - - redis-cache - volumes: - - sites-vol:/home/frappe/frappe-bench/sites:rw - - logs-vol:/home/frappe/frappe-bench/logs:rw - - frappe-worker-short: - image: [app]-worker - restart: on-failure - command: worker - environment: - - WORKER_TYPE=short - depends_on: - - redis-queue - - redis-cache - links: - - redis-queue - - redis-cache - volumes: - - sites-vol:/home/frappe/frappe-bench/sites:rw - - logs-vol:/home/frappe/frappe-bench/logs:rw - - frappe-worker-long: - image: [app]-worker - restart: on-failure - command: worker - environment: - - WORKER_TYPE=long - depends_on: - - redis-queue - - redis-cache - links: - - redis-queue - - redis-cache - volumes: - - sites-vol:/home/frappe/frappe-bench/sites:rw - - logs-vol:/home/frappe/frappe-bench/logs:rw - - frappe-schedule: - image: [app]-worker - restart: on-failure - command: schedule - depends_on: - - redis-queue - - redis-cache - links: - - redis-queue - - redis-cache - volumes: - - sites-vol:/home/frappe/frappe-bench/sites:rw - - logs-vol:/home/frappe/frappe-bench/logs:rw - -volumes: - assets-vol: - sites-vol: - logs-vol: diff --git a/installation/docker-compose-erpnext.yml b/installation/docker-compose-erpnext.yml deleted file mode 100644 index 6593e18c..00000000 --- a/installation/docker-compose-erpnext.yml +++ /dev/null @@ -1,120 +0,0 @@ -version: "3" - -services: - erpnext-nginx: - image: frappe/erpnext-nginx:${ERPNEXT_VERSION} - restart: on-failure - environment: - - FRAPPE_PY=erpnext-python - - FRAPPE_PY_PORT=8000 - - FRAPPE_SOCKETIO=frappe-socketio - - SOCKETIO_PORT=9000 - - LETSENCRYPT_HOST=${SITES} - - VIRTUAL_HOST=${SITES} - - LETSENCRYPT_EMAIL=${LETSENCRYPT_EMAIL} - depends_on: - - erpnext-python - - frappe-socketio - - frappe-worker-default - - frappe-worker-long - - frappe-worker-short - links: - - erpnext-python - - frappe-socketio - - frappe-worker-default - - frappe-worker-long - - frappe-worker-short - volumes: - - sites-vol:/var/www/html/sites:rw - - assets-vol:/assets:rw - - erpnext-python: - image: frappe/erpnext-worker:${ERPNEXT_VERSION} - restart: on-failure - environment: - - MARIADB_HOST=${MARIADB_HOST} - - REDIS_CACHE=redis-cache:6379 - - REDIS_QUEUE=redis-queue:6379 - - REDIS_SOCKETIO=redis-socketio:6379 - - SOCKETIO_PORT=9000 - - AUTO_MIGRATE=1 - volumes: - - sites-vol:/home/frappe/frappe-bench/sites:rw - - assets-vol:/home/frappe/frappe-bench/sites/assets:rw - - logs-vol:/home/frappe/frappe-bench/logs:rw - - frappe-socketio: - image: frappe/frappe-socketio:${FRAPPE_VERSION} - restart: on-failure - depends_on: - - redis-socketio - links: - - redis-socketio - volumes: - - sites-vol:/home/frappe/frappe-bench/sites:rw - - logs-vol:/home/frappe/frappe-bench/logs:rw - - frappe-worker-default: - image: frappe/erpnext-worker:${ERPNEXT_VERSION} - restart: on-failure - command: worker - depends_on: - - redis-queue - - redis-cache - links: - - redis-queue - - redis-cache - volumes: - - sites-vol:/home/frappe/frappe-bench/sites:rw - - logs-vol:/home/frappe/frappe-bench/logs:rw - - frappe-worker-short: - image: frappe/erpnext-worker:${ERPNEXT_VERSION} - restart: on-failure - command: worker - environment: - - WORKER_TYPE=short - depends_on: - - redis-queue - - redis-cache - links: - - redis-queue - - redis-cache - volumes: - - sites-vol:/home/frappe/frappe-bench/sites:rw - - logs-vol:/home/frappe/frappe-bench/logs:rw - - frappe-worker-long: - image: frappe/erpnext-worker:${ERPNEXT_VERSION} - restart: on-failure - command: worker - environment: - - WORKER_TYPE=long - depends_on: - - redis-queue - - redis-cache - links: - - redis-queue - - redis-cache - volumes: - - sites-vol:/home/frappe/frappe-bench/sites:rw - - logs-vol:/home/frappe/frappe-bench/logs:rw - - frappe-schedule: - image: frappe/erpnext-worker:${ERPNEXT_VERSION} - restart: on-failure - command: schedule - depends_on: - - redis-queue - - redis-cache - links: - - redis-queue - - redis-cache - volumes: - - sites-vol:/home/frappe/frappe-bench/sites:rw - - logs-vol:/home/frappe/frappe-bench/logs:rw - -volumes: - assets-vol: - sites-vol: - logs-vol: diff --git a/installation/docker-compose-frappe.yml b/installation/docker-compose-frappe.yml deleted file mode 100644 index 0ffacbeb..00000000 --- a/installation/docker-compose-frappe.yml +++ /dev/null @@ -1,120 +0,0 @@ -version: "3" - -services: - frappe-nginx: - image: frappe/frappe-nginx:${FRAPPE_VERSION} - restart: on-failure - environment: - - FRAPPE_PY=frappe-python - - FRAPPE_PY_PORT=8000 - - FRAPPE_SOCKETIO=frappe-socketio - - SOCKETIO_PORT=9000 - - LETSENCRYPT_HOST=${SITES} - - VIRTUAL_HOST=${SITES} - - LETSENCRYPT_EMAIL=${LETSENCRYPT_EMAIL} - depends_on: - - frappe-python - - frappe-socketio - - frappe-worker-default - - frappe-worker-long - - frappe-worker-short - links: - - frappe-python - - frappe-socketio - - frappe-worker-default - - frappe-worker-long - - frappe-worker-short - volumes: - - sites-vol:/var/www/html/sites:rw - - assets-vol:/assets:rw - - frappe-python: - image: frappe/frappe-worker:${FRAPPE_VERSION} - restart: on-failure - environment: - - MARIADB_HOST=${MARIADB_HOST} - - REDIS_CACHE=redis-cache:6379 - - REDIS_QUEUE=redis-queue:6379 - - REDIS_SOCKETIO=redis-socketio:6379 - - SOCKETIO_PORT=9000 - - AUTO_MIGRATE=1 - volumes: - - sites-vol:/home/frappe/frappe-bench/sites:rw - - logs-vol:/home/frappe/frappe-bench/logs:rw - - assets-vol:/home/frappe/frappe-bench/sites/assets:rw - - frappe-socketio: - image: frappe/frappe-socketio:${FRAPPE_VERSION} - restart: on-failure - depends_on: - - redis-socketio - links: - - redis-socketio - volumes: - - sites-vol:/home/frappe/frappe-bench/sites:rw - - logs-vol:/home/frappe/frappe-bench/logs:rw - - frappe-worker-default: - image: frappe/frappe-worker:${FRAPPE_VERSION} - restart: on-failure - command: worker - depends_on: - - redis-queue - - redis-cache - links: - - redis-queue - - redis-cache - volumes: - - sites-vol:/home/frappe/frappe-bench/sites:rw - - logs-vol:/home/frappe/frappe-bench/logs:rw - - frappe-worker-short: - image: frappe/frappe-worker:${FRAPPE_VERSION} - restart: on-failure - command: worker - environment: - - WORKER_TYPE=short - depends_on: - - redis-queue - - redis-cache - links: - - redis-queue - - redis-cache - volumes: - - sites-vol:/home/frappe/frappe-bench/sites:rw - - logs-vol:/home/frappe/frappe-bench/logs:rw - - frappe-worker-long: - image: frappe/frappe-worker:${FRAPPE_VERSION} - restart: on-failure - command: worker - environment: - - WORKER_TYPE=long - depends_on: - - redis-queue - - redis-cache - links: - - redis-queue - - redis-cache - volumes: - - sites-vol:/home/frappe/frappe-bench/sites:rw - - logs-vol:/home/frappe/frappe-bench/logs:rw - - frappe-schedule: - image: frappe/frappe-worker:${FRAPPE_VERSION} - restart: on-failure - command: schedule - depends_on: - - redis-queue - - redis-cache - links: - - redis-queue - - redis-cache - volumes: - - sites-vol:/home/frappe/frappe-bench/sites:rw - - logs-vol:/home/frappe/frappe-bench/logs:rw - -volumes: - assets-vol: - sites-vol: - logs-vol: diff --git a/installation/docker-compose-networks.yml b/installation/docker-compose-networks.yml deleted file mode 100644 index 4daa6684..00000000 --- a/installation/docker-compose-networks.yml +++ /dev/null @@ -1,6 +0,0 @@ -version: "3" - -networks: - default: - external: - name: webproxy diff --git a/installation/erpnext-publish.yml b/installation/erpnext-publish.yml deleted file mode 100644 index 130c52b6..00000000 --- a/installation/erpnext-publish.yml +++ /dev/null @@ -1,6 +0,0 @@ -version: "3" - -services: - erpnext-nginx: - ports: - - "80:8080" diff --git a/installation/frappe-postgresql/docker-compose.yml b/installation/frappe-postgresql/docker-compose.yml deleted file mode 100644 index a3fd1665..00000000 --- a/installation/frappe-postgresql/docker-compose.yml +++ /dev/null @@ -1,165 +0,0 @@ -version: "3" - -services: - traefik: - image: "traefik:v2.2" - command: - - "--providers.docker=true" - - "--providers.docker.exposedbydefault=false" - - "--entrypoints.web.address=:80" - - "--entrypoints.websecure.address=:443" - - "--certificatesresolvers.myresolver.acme.httpchallenge=true" - - "--certificatesresolvers.myresolver.acme.httpchallenge.entrypoint=web" - - "--certificatesresolvers.myresolver.acme.email=${LETSENCRYPT_EMAIL}" - - "--certificatesresolvers.myresolver.acme.storage=/letsencrypt/acme.json" - labels: - # enable traefik - - "traefik.enable=true" - # global redirect to https for production only - - "${HTTPS_REDIRECT_RULE_LABEL}" - - "${HTTPS_REDIRECT_ENTRYPOINT_LABEL}" - - "${HTTPS_REDIRECT_MIDDLEWARE_LABEL}" - # middleware redirect for production only - - "${HTTPS_USE_REDIRECT_MIDDLEWARE_LABEL}" - ports: - - "80:80" - - "443:443" - volumes: - - cert-vol:/letsencrypt - - /var/run/docker.sock:/var/run/docker.sock:ro - - frappe-nginx: - image: frappe/frappe-nginx:${FRAPPE_VERSION} - restart: on-failure - environment: - - FRAPPE_PY=frappe-python - - FRAPPE_PY_PORT=8000 - - FRAPPE_SOCKETIO=frappe-socketio - - SOCKETIO_PORT=9000 - labels: - - "traefik.enable=true" - - "traefik.http.routers.frappe-nginx.rule=Host(${SITES})" - - "${ENTRYPOINT_LABEL}" - - "${CERT_RESOLVER_LABEL}" - - "traefik.http.services.frappe-nginx.loadbalancer.server.port=8080" - volumes: - - sites-vol:/var/www/html/sites:rw - - assets-vol:/assets:rw - - frappe-python: - image: frappe/frappe-worker:${FRAPPE_VERSION} - restart: on-failure - environment: - - POSTGRES_HOST=${POSTGRES_HOST} - - DB_PORT=5432 - - REDIS_CACHE=redis-cache:6379 - - REDIS_QUEUE=redis-queue:6379 - - REDIS_SOCKETIO=redis-socketio:6379 - - SOCKETIO_PORT=9000 - - AUTO_MIGRATE=1 - volumes: - - sites-vol:/home/frappe/frappe-bench/sites:rw - - assets-vol:/home/frappe/frappe-bench/sites/assets:rw - - frappe-socketio: - image: frappe/frappe-socketio:${FRAPPE_VERSION} - restart: on-failure - depends_on: - - redis-socketio - volumes: - - sites-vol:/home/frappe/frappe-bench/sites:rw - - frappe-worker-default: - image: frappe/frappe-worker:${FRAPPE_VERSION} - restart: on-failure - command: worker - depends_on: - - redis-queue - - redis-cache - volumes: - - sites-vol:/home/frappe/frappe-bench/sites:rw - - frappe-worker-short: - image: frappe/frappe-worker:${FRAPPE_VERSION} - restart: on-failure - command: worker - environment: - - WORKER_TYPE=short - depends_on: - - redis-queue - - redis-cache - volumes: - - sites-vol:/home/frappe/frappe-bench/sites:rw - - frappe-worker-long: - image: frappe/frappe-worker:${FRAPPE_VERSION} - restart: on-failure - command: worker - environment: - - WORKER_TYPE=long - depends_on: - - redis-queue - - redis-cache - volumes: - - sites-vol:/home/frappe/frappe-bench/sites:rw - - frappe-schedule: - image: frappe/frappe-worker:${FRAPPE_VERSION} - restart: on-failure - command: schedule - depends_on: - - redis-queue - - redis-cache - volumes: - - sites-vol:/home/frappe/frappe-bench/sites:rw - - redis-cache: - image: redis:latest - restart: on-failure - volumes: - - redis-cache-vol:/data - - redis-queue: - image: redis:latest - restart: on-failure - volumes: - - redis-queue-vol:/data - - redis-socketio: - image: redis:latest - restart: on-failure - volumes: - - redis-socketio-vol:/data - - postgresql: - image: postgres:11.8 - restart: on-failure - environment: - - POSTGRES_PASSWORD=${POSTGRES_PASSWORD} - volumes: - - postgresql-vol:/var/lib/postgresql/data - - site-creator: - image: frappe/frappe-worker:${FRAPPE_VERSION} - restart: "no" - command: new - depends_on: - - frappe-python - environment: - - POSTGRES_HOST=${POSTGRES_HOST} - - SITE_NAME=${SITE_NAME} - - DB_ROOT_USER=${DB_ROOT_USER} - - POSTGRES_PASSWORD=${POSTGRES_PASSWORD} - - ADMIN_PASSWORD=${ADMIN_PASSWORD} - - INSTALL_APPS=${INSTALL_APPS} - volumes: - - sites-vol:/home/frappe/frappe-bench/sites:rw - -volumes: - postgresql-vol: - redis-cache-vol: - redis-queue-vol: - redis-socketio-vol: - assets-vol: - sites-vol: - cert-vol: diff --git a/installation/frappe-postgresql/env-local b/installation/frappe-postgresql/env-local deleted file mode 100644 index 14a0d12f..00000000 --- a/installation/frappe-postgresql/env-local +++ /dev/null @@ -1,14 +0,0 @@ -LETSENCRYPT_EMAIL=email@example.com -FRAPPE_VERSION=edge -POSTGRES_HOST=postgresql -POSTGRES_PASSWORD=admin -SITE_NAME=mysite.localhost -SITES=`mysite.localhost` -DB_ROOT_USER=postgres -ADMIN_PASSWORD=admin -ENTRYPOINT_LABEL=traefik.http.routers.erpnext-nginx.entrypoints=web -CERT_RESOLVER_LABEL=erpnext.local.no-cert-resolver -HTTPS_REDIRECT_RULE_LABEL=erpnext.local.no-redirect-rule -HTTPS_REDIRECT_ENTRYPOINT_LABEL=erpnext.local.no-entrypoint -HTTPS_REDIRECT_MIDDLEWARE_LABEL=erpnext.local.no-middleware -HTTPS_USE_REDIRECT_MIDDLEWARE_LABEL=erpnext.local-no-redirect-middleware diff --git a/installation/frappe-postgresql/env-production b/installation/frappe-postgresql/env-production deleted file mode 100644 index 689f0339..00000000 --- a/installation/frappe-postgresql/env-production +++ /dev/null @@ -1,14 +0,0 @@ -LETSENCRYPT_EMAIL=email@example.com -FRAPPE_VERSION=edge -POSTGRES_HOST=postgresql -POSTGRES_PASSWORD=admin -SITE_NAME=erp.example.com -SITES=`erp.example.com` -DB_ROOT_USER=postgres -ADMIN_PASSWORD=admin -ENTRYPOINT_LABEL=traefik.http.routers.erpnext-nginx.entrypoints=websecure -CERT_RESOLVER_LABEL=traefik.http.routers.erpnext-nginx.tls.certresolver=myresolver -HTTPS_REDIRECT_RULE_LABEL=traefik.http.routers.http-catchall.rule=hostregexp(`{host:.+}`) -HTTPS_REDIRECT_ENTRYPOINT_LABEL=traefik.http.routers.http-catchall.entrypoints=web -HTTPS_REDIRECT_MIDDLEWARE_LABEL=traefik.http.routers.http-catchall.middlewares=redirect-to-https -HTTPS_USE_REDIRECT_MIDDLEWARE_LABEL=traefik.http.middlewares.redirect-to-https.redirectscheme.scheme=https diff --git a/installation/frappe-publish.yml b/installation/frappe-publish.yml deleted file mode 100644 index f573cb79..00000000 --- a/installation/frappe-publish.yml +++ /dev/null @@ -1,6 +0,0 @@ -version: "3" - -services: - frappe-nginx: - ports: - - "80:8080" diff --git a/overrides/compose.erpnext.yaml b/overrides/compose.erpnext.yaml new file mode 100644 index 00000000..36f10ad6 --- /dev/null +++ b/overrides/compose.erpnext.yaml @@ -0,0 +1,24 @@ +x-erpnext-backend-image: &erpnext_backend_image + image: frappe/erpnext-worker:${ERPNEXT_VERSION:?No ERPNext version set} + +services: + configurator: + <<: *erpnext_backend_image + + backend: + <<: *erpnext_backend_image + + frontend: + image: frappe/erpnext-nginx:${ERPNEXT_VERSION} + + queue-short: + <<: *erpnext_backend_image + + queue-default: + <<: *erpnext_backend_image + + queue-long: + <<: *erpnext_backend_image + + scheduler: + <<: *erpnext_backend_image diff --git a/overrides/compose.https.yaml b/overrides/compose.https.yaml new file mode 100644 index 00000000..3ac68dec --- /dev/null +++ b/overrides/compose.https.yaml @@ -0,0 +1,30 @@ +services: + frontend: + labels: + - traefik.enable=true + - traefik.http.services.frontend.loadbalancer.server.port=8080 + - traefik.http.routers.frontend-http.entrypoints=websecure + - traefik.http.routers.frontend-http.tls.certresolver=main-resolver + + proxy: + image: traefik:2.5 + command: + - --providers.docker=true + - --providers.docker.exposedbydefault=false + - --entrypoints.web.address=:80 + - --entrypoints.web.http.redirections.entrypoint.to=websecure + - --entrypoints.web.http.redirections.entrypoint.scheme=https + - --entrypoints.websecure.address=:443 + - --certificatesResolvers.main-resolver.acme.httpChallenge=true + - --certificatesResolvers.main-resolver.acme.httpChallenge.entrypoint=web + - --certificatesResolvers.main-resolver.acme.email=${LETSENCRYPT_EMAIL:?No Let's Encrypt email set} + - --certificatesResolvers.main-resolver.acme.storage=/letsencrypt/acme.json + ports: + - 80:80 + - 443:443 + volumes: + - cert-data:/letsencrypt + - /var/run/docker.sock:/var/run/docker.sock:ro + +volumes: + cert-data: diff --git a/overrides/compose.mariadb.yaml b/overrides/compose.mariadb.yaml new file mode 100644 index 00000000..cd719b62 --- /dev/null +++ b/overrides/compose.mariadb.yaml @@ -0,0 +1,27 @@ +services: + configurator: + environment: + DB_HOST: db + DB_PORT: 3306 + depends_on: + db: + condition: service_healthy + + db: + image: mariadb:10.6 + healthcheck: + test: mysqladmin ping -h localhost --password=${DB_PASSWORD} + interval: 1s + retries: 15 + command: + - --character-set-server=utf8mb4 + - --collation-server=utf8mb4_unicode_ci + - --skip-character-set-client-handshake + - --skip-innodb-read-only-compressed # Temporary fix for MariaDB 10.6 + environment: + MYSQL_ROOT_PASSWORD: ${DB_PASSWORD:?No db password set} + volumes: + - db-data:/var/lib/mysql + +volumes: + db-data: diff --git a/overrides/compose.noproxy.yaml b/overrides/compose.noproxy.yaml new file mode 100644 index 00000000..5c4f83c0 --- /dev/null +++ b/overrides/compose.noproxy.yaml @@ -0,0 +1,4 @@ +services: + frontend: + ports: + - 8080:8080 diff --git a/overrides/compose.postgres.yaml b/overrides/compose.postgres.yaml new file mode 100644 index 00000000..f126fa31 --- /dev/null +++ b/overrides/compose.postgres.yaml @@ -0,0 +1,18 @@ +services: + configurator: + environment: + DB_HOST: db + DB_PORT: 5432 + depends_on: + - db + + db: + image: postgres:13.5 + command: [] + environment: + POSTGRES_PASSWORD: ${DB_PASSWORD:?No db password set} + volumes: + - db-data:/var/lib/postgresql + +volumes: + db-data: diff --git a/overrides/compose.proxy.yaml b/overrides/compose.proxy.yaml new file mode 100644 index 00000000..fca651b1 --- /dev/null +++ b/overrides/compose.proxy.yaml @@ -0,0 +1,12 @@ +services: + proxy: + image: traefik:2.5 + command: + - --providers.docker + - --providers.docker.exposedbydefault=false + - --entrypoints.web.address=:80 + ports: + - 80:80 + volumes: + - /var/run/docker.sock:/var/run/docker.sock:ro + userns_mode: host diff --git a/overrides/compose.redis.yaml b/overrides/compose.redis.yaml new file mode 100644 index 00000000..5bd3a0aa --- /dev/null +++ b/overrides/compose.redis.yaml @@ -0,0 +1,16 @@ +services: + configurator: + environment: + REDIS_CACHE: redis:6379/0 + REDIS_QUEUE: redis:6379/1 + REDIS_SOCKETIO: redis:6379/2 + depends_on: + - redis + + redis: + image: redis:6.2-alpine + volumes: + - redis-data:/data + +volumes: + redis-data: diff --git a/overrides/compose.swarm.yaml b/overrides/compose.swarm.yaml new file mode 100644 index 00000000..cc3da5ec --- /dev/null +++ b/overrides/compose.swarm.yaml @@ -0,0 +1,10 @@ +services: + frontend: + deploy: + restart_policy: + condition: on-failure + labels: + traefik.enable: true + traefik.http.services.frontend.loadbalancer.server.port: 8080 + traefik.http.routers.frontend-http.entrypoints: websecure + traefik.http.routers.frontend-http.tls.certresolver: main-resolver diff --git a/pwd.yml b/pwd.yml deleted file mode 100644 index f6105550..00000000 --- a/pwd.yml +++ /dev/null @@ -1,194 +0,0 @@ -version: "3" - -services: - traefik: - image: "traefik:v2.2" - ports: - - "80:80" - deploy: - restart_policy: - condition: on-failure - labels: - - "traefik.enable=true" - command: - - "--providers.docker" - - "--providers.docker.exposedbydefault=false" - - "--entrypoints.web.address=:80" - - "--providers.docker.swarmmode" - - "--accesslog" - - "--log" - volumes: - - /var/run/docker.sock:/var/run/docker.sock:ro - - erpnext-nginx: - image: frappe/erpnext-nginx:edge - environment: - - FRAPPE_PY=erpnext-python - - FRAPPE_PY_PORT=8000 - - FRAPPE_SOCKETIO=frappe-socketio - - SOCKETIO_PORT=9000 - deploy: - restart_policy: - condition: on-failure - labels: - - "traefik.enable=true" - - "traefik.http.routers.erpnext-nginx.rule=HostRegexp(`{catchall:.*}`)" - - "traefik.http.middlewares.erpnext-nginx.headers.customrequestheaders.Host=erpnext-nginx" - - "traefik.http.routers.erpnext-nginx.middlewares=erpnext-nginx" - - "traefik.http.routers.erpnext-nginx.entrypoints=web" - - "traefik.http.services.erpnext-nginx.loadbalancer.server.port=8080" - volumes: - - sites-vol:/var/www/html/sites:rw - - assets-vol:/assets:rw - - erpnext-python: - image: frappe/erpnext-worker:edge - deploy: - restart_policy: - condition: on-failure - environment: - - MARIADB_HOST=mariadb - - REDIS_CACHE=redis-cache:6379 - - REDIS_QUEUE=redis-queue:6379 - - REDIS_SOCKETIO=redis-socketio:6379 - - SOCKETIO_PORT=9000 - - AUTO_MIGRATE=1 - volumes: - - sites-vol:/home/frappe/frappe-bench/sites:rw - - assets-vol:/home/frappe/frappe-bench/sites/assets:rw - - frappe-socketio: - image: frappe/frappe-socketio:edge - deploy: - restart_policy: - condition: on-failure - volumes: - - sites-vol:/home/frappe/frappe-bench/sites:rw - - erpnext-worker-default: - image: frappe/erpnext-worker:edge - deploy: - restart_policy: - condition: on-failure - command: worker - depends_on: - - redis-queue - - redis-cache - volumes: - - sites-vol:/home/frappe/frappe-bench/sites:rw - - erpnext-worker-short: - image: frappe/erpnext-worker:edge - deploy: - restart_policy: - condition: on-failure - command: worker - environment: - - WORKER_TYPE=short - depends_on: - - redis-queue - - redis-cache - volumes: - - sites-vol:/home/frappe/frappe-bench/sites:rw - - erpnext-worker-long: - image: frappe/erpnext-worker:edge - deploy: - restart_policy: - condition: on-failure - command: worker - environment: - - WORKER_TYPE=long - depends_on: - - redis-queue - - redis-cache - volumes: - - sites-vol:/home/frappe/frappe-bench/sites:rw - - erpnext-schedule: - image: frappe/erpnext-worker:edge - deploy: - restart_policy: - condition: on-failure - command: schedule - depends_on: - - redis-queue - - redis-cache - volumes: - - sites-vol:/home/frappe/frappe-bench/sites:rw - - redis-cache: - image: redis:latest - deploy: - restart_policy: - condition: on-failure - volumes: - - redis-cache-vol:/data - - redis-queue: - image: redis:latest - deploy: - restart_policy: - condition: on-failure - volumes: - - redis-queue-vol:/data - - redis-socketio: - image: redis:latest - deploy: - restart_policy: - condition: on-failure - volumes: - - redis-socketio-vol:/data - - site-configurator: - image: frappe/erpnext-worker:edge - deploy: - restart_policy: - condition: none - command: ["bash", "-c", "echo erpnext-nginx > /sites/currentsite.txt"] - volumes: - - sites-vol:/sites:rw - - mariadb: - image: mariadb:10.6 - deploy: - restart_policy: - condition: on-failure - command: - - --character-set-server=utf8mb4 - - --collation-server=utf8mb4_unicode_ci - - --skip-character-set-client-handshake - - --skip-innodb-read-only-compressed # Temporary fix for MariaDB 10.6 - environment: - - MYSQL_ROOT_PASSWORD=admin - # Sometimes db initialization takes longer than 10 seconds and site-creator goes away. - # Frappe doesn't use CONVERT_TZ() function that requires time zone info, so we can just skip it. - - MYSQL_INITDB_SKIP_TZINFO=1 - volumes: - - mariadb-conf-vol:/etc/mysql/conf.d - - mariadb-vol:/var/lib/mysql - - site-creator: - image: frappe/erpnext-worker:edge - deploy: - restart_policy: - condition: none - command: new - environment: - - SITE_NAME=erpnext-nginx - - DB_ROOT_USER=root - - MYSQL_ROOT_PASSWORD=admin - - ADMIN_PASSWORD=admin - - INSTALL_APPS=erpnext - volumes: - - sites-vol:/home/frappe/frappe-bench/sites:rw - -volumes: - mariadb-vol: - mariadb-conf-vol: - redis-cache-vol: - redis-queue-vol: - redis-socketio-vol: - assets-vol: - sites-vol: diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 00000000..e1982a58 --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,2 @@ +frappe @ git+git://github.com/frappe/frappe.git +boto3-stubs[s3] diff --git a/setup.cfg b/setup.cfg index 2a22823c..9b30c812 100644 --- a/setup.cfg +++ b/setup.cfg @@ -6,4 +6,4 @@ profile = black known_third_party = frappe [codespell] -skip = build/bench/Dockerfile +skip = images/bench/Dockerfile diff --git a/tests/_check_backup_files.py b/tests/_check_backup_files.py new file mode 100644 index 00000000..6a18bc55 --- /dev/null +++ b/tests/_check_backup_files.py @@ -0,0 +1,69 @@ +import os +import re +from typing import TYPE_CHECKING + +import boto3 + +if TYPE_CHECKING: + from mypy_boto3_s3.service_resource import BucketObjectsCollection, _Bucket + + +def get_bucket() -> "_Bucket": + return boto3.resource( + service_name="s3", + endpoint_url="http://minio:9000", + region_name="us-east-1", + aws_access_key_id=os.getenv("S3_ACCESS_KEY"), + aws_secret_access_key=os.getenv("S3_SECRET_KEY"), + ).Bucket("frappe") + + +def get_key_builder(): + site_name = os.getenv("SITE_NAME") + assert site_name + + def builder(key: str, suffix: str) -> bool: + return bool(re.match(rf"{site_name}.*{suffix}$", key)) + + return builder + + +def check_keys(objects: "BucketObjectsCollection"): + check_key = get_key_builder() + + db = False + config = False + private_files = False + public_files = False + + for obj in objects: + if check_key(obj.key, "database.sql.gz"): + db = True + elif check_key(obj.key, "site_config_backup.json"): + config = True + elif check_key(obj.key, "private-files.tar"): + private_files = True + elif check_key(obj.key, "files.tar"): + public_files = True + + exc = lambda type_: Exception(f"Didn't push {type_} backup") + if not db: + raise exc("database") + if not config: + raise exc("site config") + if not private_files: + raise exc("private files") + if not public_files: + raise exc("public files") + + print("All files were pushed to S3!") + + +def main() -> int: + bucket = get_bucket() + check_keys(bucket.objects.all()) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/_create_bucket.py b/tests/_create_bucket.py new file mode 100644 index 00000000..e0b55cbf --- /dev/null +++ b/tests/_create_bucket.py @@ -0,0 +1,19 @@ +import os + +import boto3 + + +def main() -> int: + resource = boto3.resource( + service_name="s3", + endpoint_url="http://minio:9000", + region_name="us-east-1", + aws_access_key_id=os.getenv("S3_ACCESS_KEY"), + aws_secret_access_key=os.getenv("S3_SECRET_KEY"), + ) + resource.create_bucket(Bucket="frappe") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/_ping_frappe_connections.py b/tests/_ping_frappe_connections.py new file mode 100644 index 00000000..da08301f --- /dev/null +++ b/tests/_ping_frappe_connections.py @@ -0,0 +1,26 @@ +import frappe + + +def check_db(): + doc = frappe.get_single("System Settings") + assert any(v is None for v in doc.as_dict().values()), "Database test didn't pass" + print("Database works!") + + +def check_cache(): + key_and_name = "mytestkey", "mytestname" + frappe.cache().hset(*key_and_name, "mytestvalue") + assert frappe.cache().hget(*key_and_name) == "mytestvalue", "Cache test didn't pass" + frappe.cache().hdel(*key_and_name) + print("Cache works!") + + +def main() -> int: + frappe.connect(site="tests") + check_db() + check_cache() + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/compose.ci-erpnext.yaml b/tests/compose.ci-erpnext.yaml new file mode 100644 index 00000000..c767b9da --- /dev/null +++ b/tests/compose.ci-erpnext.yaml @@ -0,0 +1,24 @@ +services: + configurator: + image: localhost:5000/frappe/erpnext-worker:${ERPNEXT_VERSION} + + backend: + image: localhost:5000/frappe/erpnext-worker:${ERPNEXT_VERSION} + + frontend: + image: localhost:5000/frappe/erpnext-nginx:${ERPNEXT_VERSION} + + websocket: + image: localhost:5000/frappe/frappe-socketio:${FRAPPE_VERSION} + + queue-short: + image: localhost:5000/frappe/erpnext-worker:${ERPNEXT_VERSION} + + queue-default: + image: localhost:5000/frappe/erpnext-worker:${ERPNEXT_VERSION} + + queue-long: + image: localhost:5000/frappe/erpnext-worker:${ERPNEXT_VERSION} + + scheduler: + image: localhost:5000/frappe/erpnext-worker:${ERPNEXT_VERSION} diff --git a/tests/compose.ci.yaml b/tests/compose.ci.yaml new file mode 100644 index 00000000..c7e168f9 --- /dev/null +++ b/tests/compose.ci.yaml @@ -0,0 +1,24 @@ +services: + configurator: + image: localhost:5000/frappe/frappe-worker:${FRAPPE_VERSION} + + backend: + image: localhost:5000/frappe/frappe-worker:${FRAPPE_VERSION} + + frontend: + image: localhost:5000/frappe/frappe-nginx:${FRAPPE_VERSION} + + websocket: + image: localhost:5000/frappe/frappe-socketio:${FRAPPE_VERSION} + + queue-short: + image: localhost:5000/frappe/frappe-worker:${FRAPPE_VERSION} + + queue-default: + image: localhost:5000/frappe/frappe-worker:${FRAPPE_VERSION} + + queue-long: + image: localhost:5000/frappe/frappe-worker:${FRAPPE_VERSION} + + scheduler: + image: localhost:5000/frappe/frappe-worker:${FRAPPE_VERSION} diff --git a/tests/functions.sh b/tests/functions.sh deleted file mode 100644 index e077dbd9..00000000 --- a/tests/functions.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/bash - -print_group() { - echo ::endgroup:: - echo "::group::$*" -} - -ping_site() { - print_group "Ping site $SITE_NAME" - - echo Ping version - ping_res=$(curl -sS "http://$SITE_NAME/api/method/version") - echo "$ping_res" - if [[ -z $(echo "$ping_res" | grep "message" || echo "") ]]; then - echo "Ping failed" - exit 1 - fi - - echo Check index - index_res=$(curl -sS "http://$SITE_NAME") - if [[ -n $(echo "$index_res" | grep "Internal Server Error" || echo "") ]]; then - echo "Index check failed" - echo "$index_res" - exit 1 - fi -} diff --git a/tests/healthcheck.sh b/tests/healthcheck.sh new file mode 100755 index 00000000..a3fffac1 --- /dev/null +++ b/tests/healthcheck.sh @@ -0,0 +1,28 @@ +#!/bin/bash +set -e + +get_key() { + jq -r ".$1" /home/frappe/frappe-bench/sites/common_site_config.json +} + +get_redis_url() { + URL=$(get_key "$1" | sed 's|redis://||g') + if [[ ${URL} == *"/"* ]]; then + URL=$(echo "${URL}" | cut -f1 -d"/") + fi + echo "$URL" +} + +check_connection() { + echo "Check $1" + wait-for-it "$1" -t 1 +} + +check_connection "$(get_key db_host):$(get_key db_port)" +check_connection "$(get_redis_url redis_cache)" +check_connection "$(get_redis_url redis_queue)" +check_connection "$(get_redis_url redis_socketio)" + +if [[ "$1" = -p || "$1" = --ping-service ]]; then + check_connection "$2" +fi diff --git a/tests/integration-test.sh b/tests/integration-test.sh deleted file mode 100755 index a76a5479..00000000 --- a/tests/integration-test.sh +++ /dev/null @@ -1,283 +0,0 @@ -#!/bin/bash - -set -e - -source tests/functions.sh - -project_name=frappe_bench_00 - -docker_compose_with_args() { - # shellcheck disable=SC2068 - docker-compose \ - -p $project_name \ - -f installation/docker-compose-common.yml \ - -f installation/docker-compose-frappe.yml \ - -f installation/frappe-publish.yml \ - $@ -} - -check_migration_complete() { - print_group Check migration - - container_id=$(docker_compose_with_args ps -q frappe-python) - cmd="docker logs ${container_id} 2>&1 | grep 'Starting gunicorn' || echo ''" - worker_log=$(eval "$cmd") - INCREMENT=0 - - while [[ ${worker_log} != *"Starting gunicorn"* && ${INCREMENT} -lt 120 ]]; do - sleep 3 - ((INCREMENT = INCREMENT + 1)) - echo "Wait for migration to complete..." - worker_log=$(eval "$cmd") - if [[ ${worker_log} != *"Starting gunicorn"* && ${INCREMENT} -eq 120 ]]; then - echo Migration timeout - docker logs "${container_id}" - exit 1 - fi - done - - echo Migration Log - docker logs "${container_id}" -} - -check_health() { - print_group Loop health check - - docker run --name frappe_doctor \ - -v "${project_name}_sites-vol:/home/frappe/frappe-bench/sites" \ - --network "${project_name}_default" \ - frappe/frappe-worker:edge doctor || true - - cmd='docker logs frappe_doctor | grep "Health check successful" || echo ""' - doctor_log=$(eval "$cmd") - INCREMENT=0 - - while [[ -z "${doctor_log}" && ${INCREMENT} -lt 60 ]]; do - sleep 1 - ((INCREMENT = INCREMENT + 1)) - container=$(docker start frappe_doctor) - echo "Restarting ${container}..." - doctor_log=$(eval "$cmd") - - if [[ ${INCREMENT} -eq 60 ]]; then - docker logs "${container}" - exit 1 - fi - done -} - -# Initial group -echo ::group::Setup .env -cp env-example .env -sed -i -e "s/edge/v13/g" .env -cat .env -# shellcheck disable=SC2046 -export $(cat .env) - -print_group Start services -echo Start main services -docker_compose_with_args up -d --quiet-pull - -echo Start postgres -docker pull postgres:11.8 -q -docker run \ - --name postgresql \ - -d \ - -e POSTGRES_PASSWORD=admin \ - -v ${project_name}_sites-vol:/home/frappe/frappe-bench/sites \ - --network ${project_name}_default \ - postgres:11.8 - -check_health - -print_group "Create new site " -SITE_NAME=test.localhost -docker run \ - --rm \ - -e SITE_NAME=$SITE_NAME \ - -v ${project_name}_sites-vol:/home/frappe/frappe-bench/sites \ - --network ${project_name}_default \ - frappe/frappe-worker:v13 new - -ping_site - -print_group "Update .env (v13 -> edge)" -sed -i -e "s/v13/edge/g" .env -cat .env -# shellcheck disable=SC2046 -export $(cat .env) - -print_group Restart containers -docker_compose_with_args stop -docker_compose_with_args up -d - -check_migration_complete -sleep 5 -ping_site - -PG_SITE_NAME=pgsql.localhost -print_group "Create new site (Postgres)" -docker run \ - --rm \ - -e SITE_NAME=$PG_SITE_NAME \ - -e POSTGRES_HOST=postgresql \ - -e DB_ROOT_USER=postgres \ - -e POSTGRES_PASSWORD=admin \ - -v ${project_name}_sites-vol:/home/frappe/frappe-bench/sites \ - --network ${project_name}_default \ - frappe/frappe-worker:edge new - -check_migration_complete -SITE_NAME=$PG_SITE_NAME ping_site - -print_group Backup site -docker run \ - --rm \ - -e WITH_FILES=1 \ - -v ${project_name}_sites-vol:/home/frappe/frappe-bench/sites \ - --network ${project_name}_default \ - frappe/frappe-worker:edge backup - -MINIO_ACCESS_KEY="AKIAIOSFODNN7EXAMPLE" -MINIO_SECRET_KEY="wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" - -print_group Prepare S3 server -echo Start S3 server -docker run \ - --name minio \ - -d \ - -e "MINIO_ACCESS_KEY=$MINIO_ACCESS_KEY" \ - -e "MINIO_SECRET_KEY=$MINIO_SECRET_KEY" \ - --network ${project_name}_default \ - minio/minio server /data - -echo Create bucket -docker run \ - --rm \ - --network ${project_name}_default \ - vltgroup/s3cmd:latest \ - s3cmd \ - --access_key=$MINIO_ACCESS_KEY \ - --secret_key=$MINIO_SECRET_KEY \ - --region=us-east-1 \ - --no-ssl \ - --host=minio:9000 \ - --host-bucket=minio:9000 \ - mb s3://frappe - -print_group Push backup -docker run \ - --rm \ - -e BUCKET_NAME=frappe \ - -e REGION=us-east-1 \ - -e BUCKET_DIR=local \ - -e ACCESS_KEY_ID=$MINIO_ACCESS_KEY \ - -e SECRET_ACCESS_KEY=$MINIO_SECRET_KEY \ - -e ENDPOINT_URL=http://minio:9000 \ - -v ${project_name}_sites-vol:/home/frappe/frappe-bench/sites \ - --network ${project_name}_default \ - frappe/frappe-worker:edge push-backup - -print_group Prune and restart services -docker_compose_with_args stop -docker container prune -f && docker volume prune -f -docker_compose_with_args up -d - -check_health - -print_group Restore backup from S3 -docker run \ - --rm \ - -e MYSQL_ROOT_PASSWORD=admin \ - -e BUCKET_NAME=frappe \ - -e BUCKET_DIR=local \ - -e ACCESS_KEY_ID=$MINIO_ACCESS_KEY \ - -e SECRET_ACCESS_KEY=$MINIO_SECRET_KEY \ - -e ENDPOINT_URL=http://minio:9000 \ - -e REGION=us-east-1 \ - -v ${project_name}_sites-vol:/home/frappe/frappe-bench/sites \ - --network ${project_name}_default \ - frappe/frappe-worker:edge restore-backup - -check_health -ping_site -SITE_NAME=$PG_SITE_NAME ping_site - -EDGE_SITE_NAME=edge.localhost -print_group "Create new site (edge)" -docker run \ - --rm \ - -e SITE_NAME=$EDGE_SITE_NAME \ - -e INSTALL_APPS=frappe \ - -v ${project_name}_sites-vol:/home/frappe/frappe-bench/sites \ - --network ${project_name}_default \ - frappe/frappe-worker:edge new - -check_health -SITE_NAME=$EDGE_SITE_NAME ping_site - -print_group Migrate edge site -docker run \ - --rm \ - -e MAINTENANCE_MODE=1 \ - -v ${project_name}_sites-vol:/home/frappe/frappe-bench/sites \ - -v ${project_name}_assets-vol:/home/frappe/frappe-bench/sites/assets \ - --network ${project_name}_default \ - frappe/frappe-worker:edge migrate - -check_migration_complete - -print_group "Restore backup S3 (overwrite)" -docker run \ - --rm \ - -e MYSQL_ROOT_PASSWORD=admin \ - -e BUCKET_NAME=frappe \ - -e BUCKET_DIR=local \ - -e ACCESS_KEY_ID=$MINIO_ACCESS_KEY \ - -e SECRET_ACCESS_KEY=$MINIO_SECRET_KEY \ - -e ENDPOINT_URL=http://minio:9000 \ - -e REGION=us-east-1 \ - -v ${project_name}_sites-vol:/home/frappe/frappe-bench/sites \ - --network ${project_name}_default \ - frappe/frappe-worker:edge restore-backup - -check_migration_complete -ping_site - -print_group "Check console for $SITE_NAME" -docker run \ - --rm \ - -v ${project_name}_sites-vol:/home/frappe/frappe-bench/sites \ - --network ${project_name}_default \ - frappe/frappe-worker:edge console $SITE_NAME - -print_group "Check console for $PG_SITE_NAME" -docker run \ - --rm \ - -v ${project_name}_sites-vol:/home/frappe/frappe-bench/sites \ - --network ${project_name}_default \ - frappe/frappe-worker:edge console $PG_SITE_NAME - -print_group "Check drop site for $SITE_NAME (MariaDB)" -docker run \ - --rm \ - -e SITE_NAME=$SITE_NAME \ - -v ${project_name}_sites-vol:/home/frappe/frappe-bench/sites \ - --network ${project_name}_default \ - frappe/frappe-worker:edge drop - -print_group "Check drop site for $PG_SITE_NAME (Postgres)" -docker run \ - --rm \ - -e SITE_NAME=$PG_SITE_NAME \ - -v ${project_name}_sites-vol:/home/frappe/frappe-bench/sites \ - --network ${project_name}_default \ - frappe/frappe-worker:edge drop - -print_group Check bench --help -docker run \ - --rm \ - -v ${project_name}_sites-vol:/home/frappe/frappe-bench/sites \ - --network ${project_name}_default \ - --user frappe \ - frappe/frappe-worker:edge bench --help diff --git a/tests/main.py b/tests/main.py new file mode 100644 index 00000000..1910fb4d --- /dev/null +++ b/tests/main.py @@ -0,0 +1,502 @@ +import os +import shlex +import shutil +import ssl +import subprocess +import sys +from enum import Enum +from functools import wraps +from textwrap import dedent +from time import sleep +from typing import Any, Callable, Optional +from urllib.error import HTTPError, URLError +from urllib.request import Request, urlopen + +CI = os.getenv("CI") +SITE_NAME = "tests" +BACKEND_SERVICES = ( + "backend", + "queue-short", + "queue-default", + "queue-long", + "scheduler", +) +S3_ACCESS_KEY = "AKIAIOSFODNN7EXAMPLE" +S3_SECRET_KEY = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" +TTY = sys.stdout.isatty() + + +def patch_print(): + # Patch `print()` builtin to have nice logs when running GitHub Actions + if not CI: + return + global print + _old_print = print + + def print( + *values: Any, + sep: Optional[str] = None, + end: Optional[str] = None, + file: Any = None, + flush: bool = False, + ): + return _old_print(*values, sep=sep, end=end, file=file, flush=True) + + +class Color(Enum): + GREY = 30 + RED = 31 + GREEN = 32 + YELLOW = 33 + BLUE = 34 + MAGENTA = 35 + CYAN = 36 + WHITE = 37 + + +def colored(text: str, color: Color): + return f"\033[{color.value}m{text}\033[0m" + + +def log(text: str): + def decorator(f: Callable[..., Any]): + @wraps(f) + def wrapper(*args: Any, **kwargs: Any): + if CI: + print(f"::group::{text}") + else: + output = ( + f"\n{f' {text} '.center(os.get_terminal_size().columns, '=')}\n" + ) + print(colored(output, Color.YELLOW)) + + ret = f(*args, **kwargs) + + if CI: + print("::endgroup::") + return ret + + return wrapper + + return decorator + + +def run(*cmd: str): + print(colored(f"> {shlex.join(cmd)}", Color.GREEN)) + return subprocess.check_call(cmd) + + +def docker_compose(*cmd: str): + args = [ + "docker", + "compose", + "-p", + "test", + "--env-file", + "tests/.env", + "-f", + "compose.yaml", + "-f", + "overrides/compose.proxy.yaml", + "-f", + "overrides/compose.mariadb.yaml", + "-f", + "overrides/compose.redis.yaml", + ] + if CI: + args.extend(("-f", "tests/compose.ci.yaml")) + return run(*args, *cmd) + + +def docker_compose_exec(*cmd: str): + if TTY: + return docker_compose("exec", *cmd) + else: + return docker_compose("exec", "-T", *cmd) + + +@log("Setup .env") +def setup_env(): + shutil.copy("example.env", "tests/.env") + + if not CI: + return + + for var in ("FRAPPE_VERSION", "ERPNEXT_VERSION"): + if os.environ[var] == "develop": + with open(os.environ["GITHUB_ENV"], "a") as f: + f.write(f"\n{var}=latest") + os.environ[var] = "latest" + + with open("tests/.env", "a") as f: + f.write( + dedent( + f""" + FRAPPE_VERSION={os.environ['FRAPPE_VERSION']} + ERPNEXT_VERSION={os.environ['ERPNEXT_VERSION']} + """ + ) + ) + + with open("tests/.env") as f: + print(f.read()) + + +@log("Print configuration") +def print_compose_configuration(): + docker_compose("config") + + +@log("Create containers") +def create_containers(): + docker_compose("up", "-d", "--quiet-pull") + + +@log("Check if Python services have connections") +def ping_links_in_backends(): + for service in BACKEND_SERVICES: + docker_compose("cp", "tests/healthcheck.sh", f"{service}:/tmp/") + for _ in range(10): + try: + docker_compose_exec(service, "bash", "/tmp/healthcheck.sh") + break + except subprocess.CalledProcessError: + sleep(1) + else: + raise RuntimeError(f"Connections healthcheck failed for service {service}") + + +@log("Create site") +def create_site(): + docker_compose_exec( + "backend", + "bench", + "new-site", + SITE_NAME, + "--mariadb-root-password", + "123", + "--admin-password", + "admin", + ) + docker_compose("restart", "backend") + + +# This is needed to check https override +_ssl_ctx = ssl.create_default_context() +_ssl_ctx.check_hostname = False +_ssl_ctx.verify_mode = ssl.CERT_NONE + + +def ping_and_check_content(url: str, callback: Callable[[str], Optional[str]]): + request = Request(url, headers={"Host": SITE_NAME}) + print(f"Checking {url}") + + for _ in range(100): + try: + response = urlopen(request, context=_ssl_ctx) + + except HTTPError as exc: + if exc.code not in (404, 502): + raise + + except URLError: + pass + + else: + text: str = response.read().decode() + ret = callback(text) + if ret: + print(ret) + return + + sleep(0.1) + + raise RuntimeError(f"Couldn't ping {url}") + + +def check_index_callback(text: str): + if "404 page not found" not in text: + return text[:200] + + +@log("Check /") +def check_index(): + ping_and_check_content("http://127.0.0.1", check_index_callback) + + +@log("Check /api/method/version") +def check_api(): + ping_and_check_content( + "http://127.0.0.1/api/method/version", + lambda text: text if '"message"' in text else None, + ) + + +@log("Check if Frappe can connect to services in Python services") +def ping_frappe_connections_in_backends(): + for service in BACKEND_SERVICES: + docker_compose("cp", "tests/_ping_frappe_connections.py", f"{service}:/tmp/") + docker_compose_exec( + service, + "/home/frappe/frappe-bench/env/bin/python", + f"/tmp/_ping_frappe_connections.py", + ) + + +@log("Check /assets") +def check_assets(): + ping_and_check_content( + "http://127.0.0.1/assets/frappe/images/frappe-framework-logo.svg", + lambda text: text[:200] if text else None, + ) + + +@log("Check /files") +def check_files(): + file_name = "testfile.txt" + docker_compose( + "cp", + f"tests/{file_name}", + f"backend:/home/frappe/frappe-bench/sites/{SITE_NAME}/public/files/", + ) + ping_and_check_content( + f"http://127.0.0.1/files/{file_name}", + lambda text: text if text == "lalala\n" else None, + ) + + +@log("Prepare S3 server") +def prepare_s3_server(): + run( + "docker", + "run", + "--name", + "minio", + "-d", + "-e", + f"MINIO_ACCESS_KEY={S3_ACCESS_KEY}", + "-e", + f"MINIO_SECRET_KEY={S3_SECRET_KEY}", + "--network", + "test_default", + "minio/minio", + "server", + "/data", + ) + docker_compose("cp", "tests/_create_bucket.py", "backend:/tmp") + docker_compose_exec( + "-e", + f"S3_ACCESS_KEY={S3_ACCESS_KEY}", + "-e", + f"S3_SECRET_KEY={S3_SECRET_KEY}", + "backend", + "/home/frappe/frappe-bench/env/bin/python", + "/tmp/_create_bucket.py", + ) + + +@log("Push backup to S3") +def push_backup_to_s3(): + docker_compose_exec( + "backend", "bench", "--site", SITE_NAME, "backup", "--with-files" + ) + docker_compose_exec( + "backend", + "push-backup", + "--site", + SITE_NAME, + "--bucket", + "frappe", + "--region-name", + "us-east-1", + "--endpoint-url", + "http://minio:9000", + "--aws-access-key-id", + S3_ACCESS_KEY, + "--aws-secret-access-key", + S3_SECRET_KEY, + ) + + +@log("Check backup files in S3") +def check_backup_in_s3(): + docker_compose("cp", "tests/_check_backup_files.py", "backend:/tmp") + docker_compose_exec( + "-e", + f"S3_ACCESS_KEY={S3_ACCESS_KEY}", + "-e", + f"S3_SECRET_KEY={S3_SECRET_KEY}", + "-e", + f"SITE_NAME={SITE_NAME}", + "backend", + "/home/frappe/frappe-bench/env/bin/python", + "/tmp/_check_backup_files.py", + ) + + +@log("Stop S3 container") +def stop_s3_container(): + run("docker", "rm", "minio", "-f") + + +@log("Recreate with HTTPS override") +def recreate_with_https_override(): + docker_compose("-f", "overrides/compose.https.yaml", "up", "-d") + + +@log("Check / (HTTPS)") +def check_index_https(): + ping_and_check_content("https://127.0.0.1", check_index_callback) + + +@log("Stop containers") +def stop_containers(): + docker_compose("down", "-v", "--remove-orphans") + + +@log("Recreate with ERPNext override") +def create_containers_with_erpnext_override(): + args = ["-f", "overrides/compose.erpnext.yaml"] + if CI: + args += ("-f", "tests/compose.ci-erpnext.yaml") + + docker_compose(*args, "up", "-d", "--quiet-pull") + + +@log("Create ERPNext site") +def create_erpnext_site(): + docker_compose_exec( + "backend", + "bench", + "new-site", + SITE_NAME, + "--mariadb-root-password", + "123", + "--admin-password", + "admin", + "--install-app", + "erpnext", + ) + docker_compose("restart", "backend") + + +@log("Check /api/method") +def check_erpnext_api(): + ping_and_check_content( + "http://127.0.0.1/api/method/erpnext.templates.pages.product_search.get_product_list", + lambda text: text if '"message"' in text else None, + ) + + +@log("Check /assets") +def check_erpnext_assets(): + ping_and_check_content( + "http://127.0.0.1/assets/erpnext/js/setup_wizard.js", + lambda text: text[:200] if text else None, + ) + + +@log("Create containers with Postgres override") +def create_containers_with_postgres_override(): + docker_compose("-f", "overrides/compose.postgres.yaml", "up", "-d", "--quiet-pull") + + +@log("Create Postgres site") +def create_postgres_site(): + docker_compose_exec( + "backend", "bench", "set-config", "-g", "root_login", "postgres" + ) + docker_compose_exec("backend", "bench", "set-config", "-g", "root_password", "123") + docker_compose_exec( + "backend", + "bench", + "new-site", + SITE_NAME, + "--db-type", + "postgres", + "--admin-password", + "admin", + ) + docker_compose("restart", "backend") + + +@log("Delete .env") +def delete_env(): + os.remove("tests/.env") + + +@log("Show logs") +def show_docker_compose_logs(): + docker_compose("logs") + + +def start(): + setup_env() + print_compose_configuration() + create_containers() + ping_links_in_backends() + + +def create_frappe_site_and_check_availability(): + create_site() + check_index() + check_api() + ping_frappe_connections_in_backends() + check_assets() + check_files() + + +def check_s3(): + prepare_s3_server() + + try: + push_backup_to_s3() + check_backup_in_s3() + finally: + stop_s3_container() + + +def check_https(): + print_compose_configuration() + recreate_with_https_override() + check_index_https() + stop_containers() + + +def check_erpnext(): + print_compose_configuration() + create_containers_with_erpnext_override() + create_erpnext_site() + check_erpnext_api() + check_erpnext_assets() + stop_containers() + + +def check_postgres(): + print_compose_configuration() + create_containers_with_postgres_override() + create_postgres_site() + ping_links_in_backends() + + +def main() -> int: + try: + patch_print() + start() + create_frappe_site_and_check_availability() + check_s3() + check_https() + check_erpnext() + check_postgres() + + finally: + delete_env() + show_docker_compose_logs() + stop_containers() + + print(colored("\nTests successfully passed!", Color.YELLOW)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test-erpnext.sh b/tests/test-erpnext.sh deleted file mode 100755 index c83151a9..00000000 --- a/tests/test-erpnext.sh +++ /dev/null @@ -1,44 +0,0 @@ -#!/bin/bash - -set -e - -source tests/functions.sh - -project_name="test_erpnext" -SITE_NAME="test_erpnext.localhost" - -echo ::group::Setup env -cp env-example .env -sed -i -e "s/edge/test/g" .env -# shellcheck disable=SC2046 -export $(cat .env) -cat .env - -print_group Start services -docker-compose \ - -p $project_name \ - -f installation/docker-compose-common.yml \ - -f installation/docker-compose-erpnext.yml \ - -f installation/erpnext-publish.yml \ - up -d - -print_group Fix permissions -docker run \ - --rm \ - --user root \ - -v ${project_name}_sites-vol:/sites \ - -v ${project_name}_assets-vol:/assets \ - -v ${project_name}_logs-vol:/logs \ - frappe/erpnext-worker:test chown -R 1000:1000 /logs /sites /assets - -print_group Create site -docker run \ - --rm \ - -e "SITE_NAME=$SITE_NAME" \ - -e "INSTALL_APPS=erpnext" \ - -v ${project_name}_sites-vol:/home/frappe/frappe-bench/sites \ - --network ${project_name}_default \ - frappe/erpnext-worker:test new - -ping_site -rm .env diff --git a/tests/test-frappe.sh b/tests/test-frappe.sh deleted file mode 100755 index 12131839..00000000 --- a/tests/test-frappe.sh +++ /dev/null @@ -1,43 +0,0 @@ -#!/bin/bash - -set -e - -source tests/functions.sh - -project_name="test_frappe" -SITE_NAME="test_frappe.localhost" - -docker_compose_with_args() { - # shellcheck disable=SC2068 - docker-compose \ - -p $project_name \ - -f installation/docker-compose-common.yml \ - -f installation/docker-compose-frappe.yml \ - -f installation/frappe-publish.yml \ - $@ -} - -echo ::group::Setup env -cp env-example .env -sed -i -e "s/edge/test/g" .env -# shellcheck disable=SC2046 -export $(cat .env) -cat .env - -print_group Start services -docker_compose_with_args up -d - -print_group Create site -docker run \ - --rm \ - -e "SITE_NAME=$SITE_NAME" \ - -v ${project_name}_sites-vol:/home/frappe/frappe-bench/sites \ - --network ${project_name}_default \ - frappe/frappe-worker:test new - -ping_site - -print_group Stop and remove containers -docker_compose_with_args down - -rm .env diff --git a/tests/testfile.txt b/tests/testfile.txt new file mode 100644 index 00000000..f22355f5 --- /dev/null +++ b/tests/testfile.txt @@ -0,0 +1 @@ +lalala