From 25aa226d8f69b4f14f564faad6274422dba424bf Mon Sep 17 00:00:00 2001 From: Revant Nandgaonkar Date: Sun, 26 Dec 2021 13:38:51 +0530 Subject: [PATCH 1/8] docs: README and docs --- README.md | 2 +- compose.yaml | 49 +---- docs/docker-swarm.md | 323 ++++--------------------------- docs/images-and-compose-files.md | 35 +++- docs/multi-bench.md | 197 ------------------- docs/setup-options.md | 95 +++++++++ docs/single-bench.md | 129 ------------ docs/site-operations.md | 10 + example.env | 5 + overrides/compose.https.yaml | 2 + overrides/compose.mariadb.yaml | 27 +++ overrides/compose.noproxy.yaml | 5 + overrides/compose.postgres.yaml | 7 +- overrides/compose.proxy.yaml | 12 ++ overrides/compose.redis.yaml | 17 ++ overrides/compose.swarm.yaml | 10 + 16 files changed, 265 insertions(+), 660 deletions(-) delete mode 100644 docs/multi-bench.md create mode 100644 docs/setup-options.md delete mode 100644 docs/single-bench.md create mode 100644 overrides/compose.mariadb.yaml create mode 100644 overrides/compose.noproxy.yaml create mode 100644 overrides/compose.proxy.yaml create mode 100644 overrides/compose.redis.yaml create mode 100644 overrides/compose.swarm.yaml diff --git a/README.md b/README.md index 964d2e9c..26f5ef25 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,7 @@ We provide simple and intuitive production setup with prebuilt Frappe and ERPNex Also, there's docs to help with deployment: -- [on single server](docs/single-server.md), +- [setup options](docs/setup-options.md), - in cluster: - [Docker Swarm](docs/docker-swarm.md), - [Kubernetes (frappe/helm)](https://helm.erpnext.com), diff --git a/compose.yaml b/compose.yaml index 6a4e7a25..a37286f6 100644 --- a/compose.yaml +++ b/compose.yaml @@ -14,15 +14,13 @@ services: <<: *backend_defaults command: configure.py environment: - DB_HOST: db - DB_PORT: 3306 - REDIS_CACHE: redis:6379/0 - REDIS_QUEUE: redis:6379/1 - REDIS_SOCKETIO: redis:6379/2 + DB_HOST: ${DB_HOST} + DB_PORT: ${DB_PORT} + REDIS_CACHE: ${REDIS_CACHE} + REDIS_QUEUE: ${REDIS_QUEUE} + REDIS_SOCKETIO: ${REDIS_SOCKETIO} SOCKETIO_PORT: 9000 - depends_on: - db: - condition: service_healthy + depends_on: {} backend: <<: *backend_defaults @@ -30,27 +28,6 @@ services: - sites:/home/frappe/frappe-bench/sites - assets:/home/frappe/frappe-bench/sites/assets:ro - db: - image: mariadb:10.6 - healthcheck: - test: mysqladmin ping -h localhost --password=${DB_PASSWORD} - interval: 1s - retries: 15 - command: - - --character-set-server=utf8mb4 - - --collation-server=utf8mb4_unicode_ci - - --skip-character-set-client-handshake - - --skip-innodb-read-only-compressed # Temporary fix for MariaDB 10.6 - environment: - MYSQL_ROOT_PASSWORD: ${DB_PASSWORD:?No db password set} - volumes: - - db-data:/var/lib/mysql - - redis: - image: redis:6.2-alpine - volumes: - - redis-data:/data - frontend: image: frappe/frappe-nginx:${FRAPPE_VERSION} environment: @@ -69,18 +46,6 @@ services: - traefik.http.routers.frontend-http.entrypoints=web - traefik.http.routers.frontend-http.rule=HostRegexp(`{any:.+}`) - proxy: - image: traefik:2.5 - command: - - --providers.docker - - --providers.docker.exposedbydefault=false - - --entrypoints.web.address=:80 - ports: - - 80:80 - volumes: - - /var/run/docker.sock:/var/run/docker.sock:ro - userns_mode: host - websocket: <<: *depends_on_configurator image: frappe/frappe-socketio:${FRAPPE_VERSION} @@ -107,5 +72,3 @@ services: volumes: sites: assets: - db-data: - redis-data: diff --git a/docs/docker-swarm.md b/docs/docker-swarm.md index 7bf2a412..d2394f05 100644 --- a/docs/docker-swarm.md +++ b/docs/docker-swarm.md @@ -1,295 +1,50 @@ -### Prerequisites +## Prerequisites -IMPORTANT: All commands are executed on live server with public IP and DNS Configured. +- [yq](https://mikefarah.gitbook.io/yq) +- [docker-compose](https://docs.docker.com/compose/) +- [docker swarm](https://docs.docker.com/engine/swarm/) -#### Setup docker swarm +#### Generate setup for docker swarm -Follow [dockerswarm.rocks](https://dockerswarm.rocks) guide to setup Docker swarm, Traefik and Portainer. +Generate the swarm compatible YAML, -Use Portainer for rest of the guide - -### Create Config - -Configs > Add Config > `frappe-mariadb-config` - -``` -[mysqld] -character-set-client-handshake = FALSE -character-set-server = utf8mb4 -collation-server = utf8mb4_unicode_ci - -[mysql] -default-character-set = utf8mb4 +```bash +docker-compose -f compose.yaml \ + -f overrides/compose.erpnext.yaml \ + -f overrides/compose.swarm.yaml \ + -f overrides/compose.https.yaml \ + config \ + | yq eval 'del(.services.*.depends_on) | del(.services.frontend.labels)' - \ + | yq eval '.services.proxy.command += "--providers.docker.swarmmode"' - > \ + ~/gitops/compose.yaml ``` -### Create Secret +In case you need to generate config for multiple benches. Install the proxy separately only once and generate stacks for each bench as follows: -Secret > Add Secret > `frappe-mariadb-root-password` - -``` -longsecretpassword +```bash +# Setup Bench $BENCH_SUFFIX +export BENCH_SUFFIX=one +docker-compose -f compose.yaml \ + -f overrides/compose.erpnext.yaml \ + -f overrides/compose.swarm.yaml \ + config \ + | yq eval 'del(.services.*.depends_on) | del(.services.frontend.labels)' - \ + | sed "s|frontend|frontend-${BENCH_SUFFIX}|g" \ + | yq eval ".services.frontend-${BENCH_SUFFIX}.\"networks\"=[\"traefik-public\",\"default\"]" - \ + | yq eval ".\"networks\"={\"traefik-public\":{\"external\":true}}" - > \ + ~/gitops/compose-${BENCH_SUFFIX}.yaml ``` -Note down this password. -It is only available in mariadb containers at location `/run/secrets/frappe-mariadb-root-password` later +Commands explained: -### Deploy MariaDB Replication +- `docker-compose -f ... -f ... config`, this command generates the YAML based on the overrides +- `yq eval 'del(.services.*.depends_on) | del(.services.frontend.labels)'`, this command removes the `depends_on` from all services and `labels` from frontend generated from previous command. +- `yq eval '.services.proxy.command += "--providers.docker.swarmmode"'`, this command enables swarmmode for traefik proxy. +- `sed "s|frontend|frontend-${BENCH_SUFFIX}|g"`, this command replaces the service name `frontend` with `frontend-` and `BENCH_SUFFIX` provided. +- `yq eval ".services.frontend-${BENCH_SUFFIX}.\"networks\"=[\"traefik-public\",\"default\"]"`, this command attaches `traefik-public` and `default` network to frontend service. +- `yq eval ".\"networks\"={\"traefik-public\":{\"external\":true}}"`, this commands adds external network `traefik-public` to the stack -Stacks > Add Stacks > `frappe-mariadb` - -```yaml -version: "3.7" - -services: - mariadb-master: - image: "bitnami/mariadb:10.3" - deploy: - restart_policy: - condition: on-failure - configs: - - source: frappe-mariadb-config - target: /opt/bitnami/mariadb/conf/bitnami/my_custom.cnf - networks: - - frappe-network - secrets: - - frappe-mariadb-root-password - volumes: - - "mariadb_master_data:/bitnami/mariadb" - environment: - - MARIADB_REPLICATION_MODE=master - - MARIADB_REPLICATION_USER=repl_user - - MARIADB_REPLICATION_PASSWORD_FILE=/run/secrets/frappe-mariadb-root-password - - MARIADB_ROOT_PASSWORD_FILE=/run/secrets/frappe-mariadb-root-password - - mariadb-slave: - image: "bitnami/mariadb:10.3" - deploy: - restart_policy: - condition: on-failure - configs: - - source: frappe-mariadb-config - target: /opt/bitnami/mariadb/conf/bitnami/my_custom.cnf - networks: - - frappe-network - secrets: - - frappe-mariadb-root-password - volumes: - - "mariadb_slave_data:/bitnami/mariadb" - environment: - - MARIADB_REPLICATION_MODE=slave - - MARIADB_REPLICATION_USER=repl_user - - MARIADB_REPLICATION_PASSWORD_FILE=/run/secrets/frappe-mariadb-root-password - - MARIADB_MASTER_HOST=mariadb-master - - MARIADB_MASTER_PORT_NUMBER=3306 - - MARIADB_MASTER_ROOT_PASSWORD_FILE=/run/secrets/frappe-mariadb-root-password - -volumes: - mariadb_master_data: - mariadb_slave_data: - -configs: - frappe-mariadb-config: - external: true - -secrets: - frappe-mariadb-root-password: - external: true - -networks: - frappe-network: - name: frappe-network - attachable: true -``` - -### Deploy Frappe/ERPNext - -Stacks > Add Stacks > `frappe-bench-v13` - -```yaml -version: "3.7" - -services: - redis-cache: - image: redis:latest - volumes: - - redis-cache-vol:/data - deploy: - restart_policy: - condition: on-failure - networks: - - frappe-network - - redis-queue: - image: redis:latest - volumes: - - redis-queue-vol:/data - deploy: - restart_policy: - condition: on-failure - networks: - - frappe-network - - redis-socketio: - image: redis:latest - volumes: - - redis-socketio-vol:/data - deploy: - restart_policy: - condition: on-failure - networks: - - frappe-network - - erpnext-nginx: - image: frappe/erpnext-nginx:${ERPNEXT_VERSION?Variable ERPNEXT_VERSION not set} - environment: - - UPSTREAM_REAL_IP_ADDRESS=10.0.0.0/8 - - FRAPPE_PY=erpnext-python - - FRAPPE_PY_PORT=8000 - - FRAPPE_SOCKETIO=frappe-socketio - - SOCKETIO_PORT=9000 - volumes: - - sites-vol:/var/www/html/sites:rw - - assets-vol:/assets:rw - networks: - - frappe-network - - traefik-public - deploy: - restart_policy: - condition: on-failure - labels: - - "traefik.docker.network=traefik-public" - - "traefik.enable=true" - - "traefik.constraint-label=traefik-public" - - "traefik.http.routers.erpnext-nginx.rule=Host(${SITES?Variable SITES not set})" - - "traefik.http.routers.erpnext-nginx.entrypoints=http" - - "traefik.http.routers.erpnext-nginx.middlewares=https-redirect" - - "traefik.http.routers.erpnext-nginx-https.rule=Host(${SITES?Variable SITES not set})" - - "traefik.http.routers.erpnext-nginx-https.entrypoints=https" - - "traefik.http.routers.erpnext-nginx-https.tls=true" - - "traefik.http.routers.erpnext-nginx-https.tls.certresolver=le" - - "traefik.http.services.erpnext-nginx.loadbalancer.server.port=8080" - - erpnext-python: - image: frappe/erpnext-worker:${ERPNEXT_VERSION?Variable ERPNEXT_VERSION not set} - deploy: - restart_policy: - condition: on-failure - environment: - - MARIADB_HOST=${MARIADB_HOST?Variable MARIADB_HOST not set} - - REDIS_CACHE=redis-cache:6379 - - REDIS_QUEUE=redis-queue:6379 - - REDIS_SOCKETIO=redis-socketio:6379 - - SOCKETIO_PORT=9000 - - AUTO_MIGRATE=1 - volumes: - - sites-vol:/home/frappe/frappe-bench/sites:rw - - assets-vol:/home/frappe/frappe-bench/sites/assets:rw - networks: - - frappe-network - - frappe-socketio: - image: frappe/frappe-socketio:${FRAPPE_VERSION?Variable FRAPPE_VERSION not set} - deploy: - restart_policy: - condition: on-failure - volumes: - - sites-vol:/home/frappe/frappe-bench/sites:rw - networks: - - frappe-network - - erpnext-worker-default: - image: frappe/erpnext-worker:${ERPNEXT_VERSION?Variable ERPNEXT_VERSION not set} - deploy: - restart_policy: - condition: on-failure - command: worker - volumes: - - sites-vol:/home/frappe/frappe-bench/sites:rw - networks: - - frappe-network - - erpnext-worker-short: - image: frappe/erpnext-worker:${ERPNEXT_VERSION?Variable ERPNEXT_VERSION not set} - deploy: - restart_policy: - condition: on-failure - command: worker - environment: - - WORKER_TYPE=short - volumes: - - sites-vol:/home/frappe/frappe-bench/sites:rw - networks: - - frappe-network - - erpnext-worker-long: - image: frappe/erpnext-worker:${ERPNEXT_VERSION?Variable ERPNEXT_VERSION not set} - deploy: - restart_policy: - condition: on-failure - command: worker - environment: - - WORKER_TYPE=long - volumes: - - sites-vol:/home/frappe/frappe-bench/sites:rw - networks: - - frappe-network - - frappe-schedule: - image: frappe/erpnext-worker:${ERPNEXT_VERSION?Variable ERPNEXT_VERSION not set} - deploy: - restart_policy: - condition: on-failure - command: schedule - volumes: - - sites-vol:/home/frappe/frappe-bench/sites:rw - networks: - - frappe-network - -volumes: - redis-cache-vol: - redis-queue-vol: - redis-socketio-vol: - assets-vol: - sites-vol: - -networks: - traefik-public: - external: true - frappe-network: - external: true -``` - -Use environment variables: - -- `ERPNEXT_VERSION` variable to be set to desired version of ERPNext. e.g. 12.10.0 -- `FRAPPE_VERSION` variable to be set to desired version of Frappe Framework. e.g. 12.7.0 -- `MARIADB_HOST=frappe-mariadb_mariadb-master` -- `SITES` variable is list of sites in back tick and separated by comma - -``` -SITES=`site1.example.com`,`site2.example.com` -``` - -### Create new site job - -1. Containers > Add Container > `add-site1-example-com` -2. Select Image frappe/erpnext-worker:v13 -3. Set command as `new` -4. Select network `frappe-network` -5. Select Volume `frappe-bench-v13_sites-vol` and mount in container `/home/frappe/frappe-bench/sites` -6. Env variables: - - MYSQL_ROOT_PASSWORD=longsecretpassword - - SITE_NAME=site1.example.com - - INSTALL_APPS=erpnext -7. Start container - -### Migrate Sites job - -1. Containers > Add Container > `migrate-sites` -2. Select Image frappe/erpnext-worker:v13 -3. Set command as `migrate` -4. Select network `frappe-network` -5. Select Volume `frappe-bench-v13_sites-vol` and mount in container `/home/frappe/frappe-bench/sites` -6. Env variables: - - MAINTENANCE_MODE=1 -7. Start container +Notes: +- Set `BENCH_SUFFIX` to the stack name. the stack will be located at `~/gitops/compose-${BENCH_SUFFIX}.yaml`. +- `traefik-public` is assumed to be the network for traefik loadbalancer for swarm. +- Once the stack YAML is generated, you can edit it further for advance setup and commit it to your gitops diff --git a/docs/images-and-compose-files.md b/docs/images-and-compose-files.md index fa0f11e4..35dad2e8 100644 --- a/docs/images-and-compose-files.md +++ b/docs/images-and-compose-files.md @@ -9,7 +9,7 @@ There's 4 images that you can find in `/build` directory: > `nginx`, `socketio` and `worker` images — everything we need to be able to run all processes that Frappe framework requires (take a look at [Bench Procfile reference](https://frappeframework.com/docs/v13/user/en/bench/resources/bench-procfile)). We follow [Docker best practices](https://docs.docker.com/develop/develop-images/dockerfile_best-practices/#decouple-applications) and split these processes to different containers. -> ERPNext images don't have their own Dockerfiles. We use [multi-stage builds](https://docs.docker.com/develop/develop-images/multistage-build/) and [Docker Buildx](https://docs.docker.com/engine/reference/commandline/buildx/) to reuse as much things as possible and make are builds more efficient. +> ERPNext images don't have their own Dockerfiles. We use [multi-stage builds](https://docs.docker.com/develop/develop-images/multistage-build/) and [Docker Buildx](https://docs.docker.com/engine/reference/commandline/buildx/) to reuse as much things as possible and make our builds more efficient. # Compose files @@ -23,8 +23,8 @@ All services are described in `compose.yaml` - `configurator`. Updates `common_site_config.json` so Frappe knows how to access db and redis. It is executed on every `docker-compose up` (and exited immediately). Other services start after this container exits successfully. - `backend`. [Werkzeug server](https://werkzeug.palletsprojects.com/en/2.0.x/). -- `db`. [MariaDB](https://mariadb.com), can be overwritten with [Postgres](https://www.postgresql.org) if you also use `overrides/compose.postgres.yaml`. -- `redis`. [Redis](https://redis.io) server with cache, [Socket.IO](https://socket.io) and queues data. +- `db`. Optional service that runs [MariaDB](https://mariadb.com) if you also use `overrides/compose.mariadb.yaml` or [Postgres](https://www.postgresql.org) if you also use `overrides/compose.postgres.yaml`. +- `redis`. Optional service that runs [Redis](https://redis.io) server with cache, [Socket.IO](https://socket.io) and queues data. - `frontend`. [nginx](https://www.nginx.com) server that serves JS/CSS assets and routes incoming requests. - `proxy`. [Traefik](https://traefik.io/traefik/) proxy. It is here for complicated setups or HTTPS override (with `overrides/compose.https.yaml`). - `websocket`. Node server that runs [Socket.IO](https://socket.io). @@ -35,15 +35,20 @@ All services are described in `compose.yaml` We have several [overrides](https://docs.docker.com/compose/extends/): +- `overrides/compose.proxy.yaml`. Adds traefik proxy to setup. +- `overrides/compose.noproxy.yaml`. Publishes `frontend` ports directly without any proxy. - `overrides/compose.erpnext.yaml`. Replaces all Frappe images with ERPNext ones. ERPNext images are built on top of Frappe ones, so it is safe to replace them. - `overrides/compose.https.yaml`. Automatically sets up Let's Encrypt certificate and redirects all requests to directed to http, to https. -- `overrides/compose.postgres.yaml`. Replaces `db` service's image from MariaDB to Postgres. Note that ERPNext currently doesn't support Postgres. +- `overrides/compose.mariadb.yaml`. Adds `db` service and sets its image to MariaDB. +- `overrides/compose.postgres.yaml`. Adds `db` service and sets its image to Postgres. Note that ERPNext currently doesn't support Postgres. +- `overrides/compose.redis.yaml`. Adds `redis` service and sets its image to `redis`. +- `overrides/compose.swarm.yaml`. Workaround override for generating swarm stack. It is quite simple to run overrides. All we need to do is to specify compose files that should be used by docker-compose. For example, we want ERPNext: ```bash # Point to main compose file (compose.yaml) and add one more. -docker-compose -f compose.yaml -f overrides/compose.erpnext.yaml +docker-compose -f compose.yaml -f overrides/compose.erpnext.yaml config ``` That's it! Of course, we also have to setup `.env` before all of that, but that's not the point. @@ -60,6 +65,26 @@ Frappe framework release. You can find all releases [here](https://github.com/fr Password for MariaDB (or Postgres) database. +### `DB_HOST` + +Hostname for MariaDB (or Postgres) database. Set only if external service for database is used. + +### `DB_PORT` + +Port for MariaDB (3306) or Postgres (5432) database. Set only if external service for database is used. + +### `REDIS_CACHE` + +Hostname for redis server to store cache. Set only if external service for redis is used. + +### `REDIS_QUEUE` + +Hostname for redis server to store queue data. Set only if external service for redis is used. + +### `REDIS_SOCKETIO` + +Hostname for redis server to store socketio data. Set only if external service for redis is used. + ### `ERPNEXT_VERSION` ERPNext [release](https://github.com/frappe/frappe/releases). This variable is required if you use ERPNext override. diff --git a/docs/multi-bench.md b/docs/multi-bench.md deleted file mode 100644 index 831ec71c..00000000 --- a/docs/multi-bench.md +++ /dev/null @@ -1,197 +0,0 @@ -# Multi bench - -This setup separates all services such that only required ones can be deployed. - -This is suitable when multiple services are installed on cluster with shared proxy/router, database, cache etc. - -Make sure you've cloned this repository and switch to the directory before executing following commands. - -## Setup Environment Variables - -Copy the example docker environment file to `.env`: - -```sh -cp env-example .env -``` - -To get started, copy the existing `env-example` file to `.env`. By default, the file will contain the following variables: - -- `VERSION=edge` - - In this case, `edge` corresponds to `develop`. To setup any other version, you may use the branch name or version specific tags. (eg. v13.0.0, version-12, v11.1.15, v11) -- `MYSQL_ROOT_PASSWORD=admin` - - Bootstraps a MariaDB container with this value set as the root password. If a managed MariaDB instance is used, there is no need to set the password here. -- `MARIADB_HOST=mariadb` - - Sets the hostname to `mariadb`. This is required if the database is managed by the containerized MariaDB instance. - - In case of a separately managed database setups, set the value to the database's hostname/IP/domain. -- `SITES=site1.domain.com,site2.domain.com` - - List of sites that are part of the deployment "bench" Each site is separated by a comma(,). - - If LetsEncrypt is being setup, make sure that the DNS for all the site's domains correctly point to the current instance. -- `LETSENCRYPT_EMAIL=your.email@your.domain.com` - - Email for LetsEncrypt expiry notification. This is only required if you are setting up LetsEncrypt. - -Notes: - -- docker-compose-erpnext.yml and docker-compose-frappe.yml set `AUTO_MIGRATE` environment variable to `1`. -- `AUTO_MIGRATE` checks if there is semver bump or git hash change in case of develop branch and automatically migrates the sites on container start up. -- It is good practice to use image tag for specific version instead of latest. e.g `frappe-socketio:v12.5.1`, `erpnext-nginx:v12.7.1`. - -## Local deployment for testing - -For trying out locally or to develop apps using ERPNext REST API port 80 must be published. -Following command will start the needed containers and expose ports. - -For Erpnext: - -```sh -docker-compose \ - --project-name \ - -f installation/docker-compose-common.yml \ - -f installation/docker-compose-erpnext.yml \ - -f installation/erpnext-publish.yml \ - up -d -``` - -For Frappe: - -```sh -docker-compose \ - --project-name \ - -f installation/docker-compose-common.yml \ - -f installation/docker-compose-frappe.yml \ - -f installation/frappe-publish.yml \ - up -d -``` - -Make sure to replace `` with the desired name you wish to set for the project. - -Notes: - -- New site (first site) needs to be added after starting the services. -- The local deployment is for testing and REST API development purpose only -- A complete development environment is available [here](../development) -- The site names are limited to patterns matching \*.localhost by default -- Additional site name patterns can be added by editing /etc/hosts of your host machine - -## Deployment for production - -### Setup Letsencrypt Nginx Proxy Companion - -Letsencrypt Nginx Proxy Companion can optionally be setup to provide SSL. This is recommended for instances accessed over the internet. - -Your DNS will need to be configured correctly for Letsencrypt to verify your domain. - -To setup the proxy companion, run the following commands: - -```sh -cd $HOME -git clone https://github.com/evertramos/docker-compose-letsencrypt-nginx-proxy-companion.git -cd docker-compose-letsencrypt-nginx-proxy-companion -cp .env.sample .env -./start.sh -``` - -It will create the required network and configure containers for Letencrypt ACME. - -For more details, see the [Letsencrypt Nginx Proxy Companion github repo](https://github.com/evertramos/docker-compose-letsencrypt-nginx-proxy-companion). Letsencrypt Nginx Proxy Companion github repo works by automatically proxying to containers with the `VIRTUAL_HOST` environmental variable. - -Notes: - -- `SITES` variables from `env-example` is set as `VIRTUAL_HOST` -- `LETSENCRYPT_EMAIL` variables from `env-example` is used as it is. -- This is simple nginx + letsencrypt solution. Any other solution can be setup. Above two variables can be re-used or removed in case any other reverse-proxy is used. - -### Start Frappe/ERPNext Services - -To start the Frappe/ERPNext services for production, run the following command: - -```sh -docker-compose \ - --project-name \ - -f installation/docker-compose-common.yml \ - -f installation/docker-compose-erpnext.yml \ - -f installation/docker-compose-networks.yml \ - up -d -``` - -Make sure to replace `` with any desired name you wish to set for the project. - -Notes: - -- Use `docker-compose-frappe.yml` in case you need only Frappe without ERPNext. -- New site (first site) needs to be added after starting the services. - -## Docker containers - -This repository contains the following docker-compose files, each one containing the described images: - -- docker-compose-common.yml - - redis-cache - - volume: redis-cache-vol - - redis-queue - - volume: redis-queue-vol - - redis-socketio - - volume: redis-socketio-vol - - mariadb: main database - - volume: mariadb-vol -- docker-compose-erpnext.yml - - - erpnext-nginx: serves static assets and proxies web request to the appropriate container, allowing to offer all services on the same port. - - volume: assets-vol - - erpnext-python: main application code - - frappe-socketio: enables realtime communication to the user interface through websockets - - frappe-worker-default: background runner - - frappe-worker-short: background runner for short-running jobs - - frappe-worker-long: background runner for long-running jobs - - frappe-schedule - -- docker-compose-frappe.yml - - - frappe-nginx: serves static assets and proxies web request to the appropriate container, allowing to offer all services on the same port. - - volume: assets-vol, sites-vol - - erpnext-python: main application code - - volume: sites-vol - - frappe-socketio: enables realtime communication to the user interface through websockets - - volume: sites-vol - - frappe-worker-default: background runner - - volume: sites-vol - - frappe-worker-short: background runner for short-running jobs - - volume: sites-vol - - frappe-worker-long: background runner for long-running jobs - - volume: sites-vol - - frappe-schedule - - volume: sites-vol - -- docker-compose-networks.yml: this yaml define the network to communicate with _Letsencrypt Nginx Proxy Companion_. - -- erpnext-publish.yml: this yml extends erpnext-nginx service to publish port 80, can only be used with docker-compose-erpnext.yml - -- frappe-publish.yml: this yml extends frappe-nginx service to publish port 80, can only be used with docker-compose-frappe.yml - -## Updating and Migrating Sites - -Switch to the root of the `frappe_docker` directory before running the following commands: - -```sh -# Update environment variable VERSION -nano .env - -# Pull new images -docker-compose \ - -f installation/docker-compose-common.yml \ - -f installation/docker-compose-erpnext.yml \ - pull - -# Restart containers -docker-compose \ - --project-name \ - -f installation/docker-compose-common.yml \ - -f installation/docker-compose-erpnext.yml \ - -f installation/docker-compose-networks.yml \ - up -d - -docker run \ - -e "MAINTENANCE_MODE=1" \ - -v _sites-vol:/home/frappe/frappe-bench/sites \ - --network _default \ - frappe/erpnext-worker:$VERSION migrate -``` diff --git a/docs/setup-options.md b/docs/setup-options.md new file mode 100644 index 00000000..f2fcb8ec --- /dev/null +++ b/docs/setup-options.md @@ -0,0 +1,95 @@ +# Containerized Production Setup + +Make sure you've cloned this repository and switch to the directory before executing following commands. + +Commands will generate YAML as per the environment for setup. + +## Setup Environment Variables + +Copy the example docker environment file to `.env`: + +```sh +cp example.env .env +``` + +Note: To know more about environment variable [read here](./images-and-compose-files#configuration). Set the necessary variables in the `.env` file. + +## Generate docker-compose.yml for variety of setups + +### Setup Frappe without proxy and external MariaDB and Redis + +```sh +# Generate YAML +docker-compose -f compose.yaml -f overrides/compose.noproxy.yaml config > ~/gitops/docker-compose.yml + +# Start containers +docker-compose --project -f ~/gitops/docker-compose.yml up -d +``` + +### Setup ERPNext with proxy and external MariaDB and Redis + +```sh +# Generate YAML +docker-compose -f compose.yaml \ + -f overrides/compose.proxy.yaml \ + -f overrides/compose.erpnext.yaml \ + config > ~/gitops/docker-compose.yml + +# Start containers +docker-compose --project -f ~/gitops/docker-compose.yml up -d +``` + +### Setup Frappe using containerized MariaDB and Redis with Letsencrypt certificates. + +```sh +# Generate YAML +docker-compose -f compose.yaml \ + -f overrides/compose.mariadb.yaml \ + -f overrides/compose.redis.yaml \ + -f overrides/compose.https.yaml \ + config > ~/gitops/docker-compose.yml + +# Start containers +docker-compose --project -f ~/gitops/docker-compose.yml up -d +``` +### Setup ERPNext using containerized MariaDB and Redis with Letsencrypt certificates. + +```sh +# Generate YAML +docker-compose -f compose.yaml \ + -f overrides/compose.erpnext.yaml \ + -f overrides/compose.mariadb.yaml \ + -f overrides/compose.redis.yaml \ + -f overrides/compose.https.yaml \ + config > ~/gitops/docker-compose.yml + +# Start containers +docker-compose --project -f ~/gitops/docker-compose.yml up -d +``` + +Notes: + +- Make sure to replace `` with the desired name you wish to set for the project. +- This setup is not to be used for development. A complete development environment is available [here](../development) + +## Updating Images + +Switch to the root of the `frappe_docker` directory before running the following commands: + +```sh +# Update environment variables ERPNEXT_VERSION and FRAPPE_VERSION +nano .env + +# Pull new images +docker-compose -f compose.yaml \ + -f overrides/erpnext.yaml \ + # ... your other overrides + config > ~/gitops/docker-compose.yml + +docker-compose --project -f ~/gitops/docker-compose.yml pull + +# Restart containers +docker-compose --project -f ~/gitops/docker-compose.yml up -d +``` + +To migrate sites refer [site operations](./site-operations.md#migrate-site) diff --git a/docs/single-bench.md b/docs/single-bench.md deleted file mode 100644 index da13e893..00000000 --- a/docs/single-bench.md +++ /dev/null @@ -1,129 +0,0 @@ -# Single Bench - -This setup starts traefik service as part of single docker-compose project. It is quick to get started locally or on production for a single server with single deployment. - -This is not suitable when multiple services are installed on cluster with shared proxy/router, database, cache etc. - -Make sure you've cloned this repository and switch to the directory before executing following commands. - -## Setup Environment Variables - -Copy the example docker environment file to `.env`: - -For local setup - -```sh -cp env-local .env -``` - -For production - -```sh -cp env-production .env - -``` - -To get started, copy the existing `env-local` or `env-production` file to `.env`. By default, the file will contain the following variables: - -- `ERPNEXT_VERSION=edge` - - In this case, `edge` corresponds to `develop`. To setup any other version, you may use the branch name or version specific tags. (eg. v13.0.0, version-12, v11.1.15, v11). -- `FRAPPE_VERSION=edge` - - In this case, `edge` corresponds to `develop`. To setup any other version, you may use the branch name or version specific tags. (eg. v13.0.0, version-12, v11.1.15, v11). -- `MARIADB_HOST=mariadb` - - Sets the hostname to `mariadb`. This is required if the database is managed by the containerized MariaDB instance. -- `MYSQL_ROOT_PASSWORD=admin` - - Bootstraps a MariaDB container with this value set as the root password. If a managed MariaDB instance is used, there is no need to set the password here. - - In case of a separately managed database setups, set the value to the database's hostname/IP/domain. -- `SITE_NAME=erp.example.com` - - Creates this site after starting all services and installs ERPNext. Site name must be resolvable by users machines and the ERPNext components. e.g. `erp.example.com` or `mysite.localhost`. -- `` SITES=`erp.example.com` `` - - List of sites that are part of the deployment "bench" Each site is separated by a comma(,) and quoted in backtick (`). By default site created by `SITE_NAME` variable is added here. - - If LetsEncrypt is being setup, make sure that the DNS for all the site's domains correctly point to the current instance. -- `DB_ROOT_USER=root` - - MariaDB root username -- `ADMIN_PASSWORD=admin` - - Password for the `Administrator` user, credentials after install `Administrator:$ADMIN_PASSWORD`. -- `INSTALL_APPS=erpnext` - - Apps to install, the app must be already in the container image, to install other application read the [instructions on installing custom apps](../custom_app/README.md). -- `LETSENCRYPT_EMAIL=email@example.com` - - Email for LetsEncrypt expiry notification. This is only required if you are setting up LetsEncrypt. -- `ENTRYPOINT_LABEL=traefik.http.routers.erpnext-nginx.entrypoints=websecure` - - Related to the traefik configuration, says all traffic from outside should come from HTTP or HTTPS, for local development should be web, for production websecure. if redirection is needed, read below. -- `CERT_RESOLVER_LABEL=traefik.http.routers.erpnext-nginx.tls.certresolver=myresolver` - - Which traefik resolver to use to get TLS certificate, sets `erpnext.local.no-cert-resolver` for local setup. -- `` HTTPS_REDIRECT_RULE_LABEL=traefik.http.routers.http-catchall.rule=hostregexp(`{host:.+}`) `` - - Related to the traefik https redirection configuration, sets `erpnext.local.no-redirect-rule` for local setup. -- `HTTPS_REDIRECT_ENTRYPOINT_LABEL=traefik.http.routers.http-catchall.entrypoints=web` - - Related to the traefik https redirection configuration, sets `erpnext.local.no-entrypoint` for local setup. -- `HTTPS_REDIRECT_MIDDLEWARE_LABEL=traefik.http.routers.http-catchall.middlewares=redirect-to-https` - - Related to the traefik https redirection configuration, sets `erpnext.local.no-middleware` for local setup. -- `HTTPS_USE_REDIRECT_MIDDLEWARE_LABEL=traefik.http.middlewares.redirect-to-https.redirectscheme.scheme=https` - - Related to the traefik https redirection configuration, sets `erpnext.local-no-redirect-middleware` for local setup. - -Notes: - -- `AUTO_MIGRATE` variable is set to `1` by default. It checks if there is semver bump or git hash change in case of develop branch and automatically migrates the sites on container start up. -- It is good practice to use image tag for specific version instead of latest. e.g `frappe-socketio:v12.5.1`, `erpnext-nginx:v12.7.1`. - -## Start containers - -Execute the following command: - -```sh -docker-compose --project-name up -d -``` - -Make sure to replace `` with the desired name you wish to set for the project. - -Notes: - -- If it is the first time running and site is being initialized, _it can take multiple minutes for the site to be up_. Monitor `site-creator` container logs to check progress. Use command `docker logs _site-creator_1 -f` -- After the site is ready the username is `Administrator` and the password is `$ADMIN_PASSWORD` -- The local deployment is for testing and REST API development purpose only -- A complete development environment is available [here](../development) - -## Docker containers - -The docker-compose file contains following services: - -- traefik: manages letsencrypt - - volume: cert-vol -- redis-cache: cache store - - volume: redis-cache-vol -- redis-queue: used by workers - - volume: redis-queue-vol -- redis-socketio: used by socketio service - - volume: redis-socketio-vol -- mariadb: main database - - volume: mariadb-vol -- erpnext-nginx: serves static assets and proxies web request to the appropriate container, allowing to offer all services on the same port. - - volume: assets-vol and sites-vol -- erpnext-python: main application code - - volume: sites-vol and sites-vol -- frappe-socketio: enables realtime communication to the user interface through websockets - - volume: sites-vol -- erpnext-worker-default: background runner - - volume: sites-vol -- erpnext-worker-short: background runner for short-running jobs - - volume: sites-vol -- erpnext-worker-long: background runner for long-running jobs - - volume: sites-vol -- erpnext-schedule - - volume: sites-vol -- site-creator: run once container to create new site. - - volume: sites-vol - -## Updating and Migrating Sites - -Switch to the root of the `frappe_docker` directory before running the following commands: - -```sh -# Update environment variables ERPNEXT_VERSION and FRAPPE_VERSION -nano .env - -# Pull new images -docker-compose pull - -# Restart containers -docker-compose --project-name up -d -``` diff --git a/docs/site-operations.md b/docs/site-operations.md index cc73036c..4a5ce63e 100644 --- a/docs/site-operations.md +++ b/docs/site-operations.md @@ -71,3 +71,13 @@ For reference of commands like `backup`, `drop-site` or `migrate` check [officia ```sh docker-compose exec backend bench --help ``` + +## Migrate site + +Note: + +- Wait for the `db` service to start and `configurator` to exit before trying to migrate a site. Usually this takes up to 10 seconds. + +```sh +docker-compose exec backend bench --site migrate +``` diff --git a/example.env b/example.env index a6ccb157..587c6c81 100644 --- a/example.env +++ b/example.env @@ -2,6 +2,11 @@ FRAPPE_VERSION=v13.17.0 DB_PASSWORD=123 +DB_HOST= +DB_PORT= +REDIS_CACHE= +REDIS_QUEUE= +REDIS_SOCKETIO= # Only with ERPNext override ERPNEXT_VERSION=v13.17.0 diff --git a/overrides/compose.https.yaml b/overrides/compose.https.yaml index 594eaba0..04a5cf48 100644 --- a/overrides/compose.https.yaml +++ b/overrides/compose.https.yaml @@ -7,6 +7,7 @@ services: - traefik.http.routers.frontend-http.tls.certresolver=main-resolver proxy: + image: traefik:2.5 command: - --providers.docker=true - --providers.docker.exposedbydefault=false @@ -19,6 +20,7 @@ services: - --certificatesResolvers.main-resolver.acme.email=${LETSENCRYPT_EMAIL:?No Let's Encrypt email set} - --certificatesResolvers.main-resolver.acme.storage=/letsencrypt/acme.json ports: + - 80:80 - 443:443 volumes: - cert-data:/letsencrypt diff --git a/overrides/compose.mariadb.yaml b/overrides/compose.mariadb.yaml new file mode 100644 index 00000000..cd719b62 --- /dev/null +++ b/overrides/compose.mariadb.yaml @@ -0,0 +1,27 @@ +services: + configurator: + environment: + DB_HOST: db + DB_PORT: 3306 + depends_on: + db: + condition: service_healthy + + db: + image: mariadb:10.6 + healthcheck: + test: mysqladmin ping -h localhost --password=${DB_PASSWORD} + interval: 1s + retries: 15 + command: + - --character-set-server=utf8mb4 + - --collation-server=utf8mb4_unicode_ci + - --skip-character-set-client-handshake + - --skip-innodb-read-only-compressed # Temporary fix for MariaDB 10.6 + environment: + MYSQL_ROOT_PASSWORD: ${DB_PASSWORD:?No db password set} + volumes: + - db-data:/var/lib/mysql + +volumes: + db-data: diff --git a/overrides/compose.noproxy.yaml b/overrides/compose.noproxy.yaml new file mode 100644 index 00000000..b704f2df --- /dev/null +++ b/overrides/compose.noproxy.yaml @@ -0,0 +1,5 @@ +services: + frontend: + ports: + - 8080:8080 + - 80:80 # For old images diff --git a/overrides/compose.postgres.yaml b/overrides/compose.postgres.yaml index f0b4baf6..fec8c97a 100644 --- a/overrides/compose.postgres.yaml +++ b/overrides/compose.postgres.yaml @@ -1,9 +1,11 @@ services: configurator: environment: + DB_HOST: db DB_PORT: 5432 depends_on: - - db + db: + condition: service_healthy db: image: postgres:14.1 @@ -16,3 +18,6 @@ services: POSTGRES_PASSWORD: ${DB_PASSWORD:?No db password set} volumes: - db-data:/var/lib/postgresql + +volumes: + db-data: diff --git a/overrides/compose.proxy.yaml b/overrides/compose.proxy.yaml new file mode 100644 index 00000000..fca651b1 --- /dev/null +++ b/overrides/compose.proxy.yaml @@ -0,0 +1,12 @@ +services: + proxy: + image: traefik:2.5 + command: + - --providers.docker + - --providers.docker.exposedbydefault=false + - --entrypoints.web.address=:80 + ports: + - 80:80 + volumes: + - /var/run/docker.sock:/var/run/docker.sock:ro + userns_mode: host diff --git a/overrides/compose.redis.yaml b/overrides/compose.redis.yaml new file mode 100644 index 00000000..d3d3a863 --- /dev/null +++ b/overrides/compose.redis.yaml @@ -0,0 +1,17 @@ +services: + configurator: + environment: + REDIS_CACHE: redis:6379/0 + REDIS_QUEUE: redis:6379/1 + REDIS_SOCKETIO: redis:6379/2 + depends_on: + redis: + condition: service_healthy + + redis: + image: redis:6.2-alpine + volumes: + - redis-data:/data + +volumes: + redis-data: diff --git a/overrides/compose.swarm.yaml b/overrides/compose.swarm.yaml new file mode 100644 index 00000000..cc3da5ec --- /dev/null +++ b/overrides/compose.swarm.yaml @@ -0,0 +1,10 @@ +services: + frontend: + deploy: + restart_policy: + condition: on-failure + labels: + traefik.enable: true + traefik.http.services.frontend.loadbalancer.server.port: 8080 + traefik.http.routers.frontend-http.entrypoints: websecure + traefik.http.routers.frontend-http.tls.certresolver: main-resolver From e8b1c137fb30ff781a75a51384e5e06400af3703 Mon Sep 17 00:00:00 2001 From: Revant Nandgaonkar Date: Sun, 26 Dec 2021 13:52:09 +0530 Subject: [PATCH 2/8] docs: add link to site operations from docker swarm --- docs/docker-swarm.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/docker-swarm.md b/docs/docker-swarm.md index d2394f05..e1c47177 100644 --- a/docs/docker-swarm.md +++ b/docs/docker-swarm.md @@ -48,3 +48,7 @@ Notes: - Set `BENCH_SUFFIX` to the stack name. the stack will be located at `~/gitops/compose-${BENCH_SUFFIX}.yaml`. - `traefik-public` is assumed to be the network for traefik loadbalancer for swarm. - Once the stack YAML is generated, you can edit it further for advance setup and commit it to your gitops + +#### Site Operations + +Refer [site operations documentation](./site-operations) to create new site, migrate site, drop site and perform other site operations. From 73f3d453c407b1028b929a4b8bd1b4e2103327f4 Mon Sep 17 00:00:00 2001 From: Revant Nandgaonkar Date: Tue, 28 Dec 2021 11:46:53 +0530 Subject: [PATCH 3/8] ci: fix tests as per changes to compose.yaml --- tests/main.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/main.py b/tests/main.py index 3560da1f..4ea5c742 100644 --- a/tests/main.py +++ b/tests/main.py @@ -87,6 +87,12 @@ def docker_compose(*cmd: str): "tests/.env", "-f", "compose.yaml", + "-f", + "overrides/compose.proxy.yaml", + "-f", + "overrides/compose.mariadb.yaml", + "-f", + "overrides/compose.redis.yaml", ] if CI: args.extend(("-f", "tests/compose.ci.yaml")) From 8f814e5f5dc48f9d5cab8334650974ebaf9085ee Mon Sep 17 00:00:00 2001 From: Revant Nandgaonkar Date: Mon, 10 Jan 2022 14:10:25 +0530 Subject: [PATCH 4/8] docs: move wiki articles to docs --- README.md | 9 ++ development/README.md | 32 ----- docs/add-custom-domain-using-traefik.md | 42 ++++++ docs/backup-and-push-cronjob.md | 121 ++++++++++++++++++ docs/bench-console-and-vscode-debugger.md | 18 +++ docs/build-version-10-images.md | 16 +++ ...om-containers-for-local-app-development.md | 10 ++ docs/patch-code-from-images.md | 10 ++ docs/port-based-multi-tenancy.md | 47 +++++++ docs/troubleshoot.md | 55 ++++++++ 10 files changed, 328 insertions(+), 32 deletions(-) create mode 100644 docs/add-custom-domain-using-traefik.md create mode 100644 docs/backup-and-push-cronjob.md create mode 100644 docs/bench-console-and-vscode-debugger.md create mode 100644 docs/build-version-10-images.md create mode 100644 docs/connect-to-localhost-services-from-containers-for-local-app-development.md create mode 100644 docs/patch-code-from-images.md create mode 100644 docs/port-based-multi-tenancy.md create mode 100644 docs/troubleshoot.md diff --git a/README.md b/README.md index 26f5ef25..4d7666e9 100644 --- a/README.md +++ b/README.md @@ -28,6 +28,15 @@ Also, there's docs to help with deployment: - [Docker Swarm](docs/docker-swarm.md), - [Kubernetes (frappe/helm)](https://helm.erpnext.com), - [site operations](docs/site-operations.md). +- Other + - [add custom domain using traefik](docs/add-custom-domain-using-traefik.md) + - [backup and push cron jobs](docs/backup-and-push-cronjob.md) + - [bench console and vscode debugger](docs/bench-console-and-vscode-debugger.md) + - [build version 10](docs/build-version-10-images.md) + - [connect to localhost services from containers for local app development](docs/connect-to-localhost-services-from-containers-for-local-app-development.md) + - [patch code from images](docs/patch-code-from-images.md) + - [port based multi tenancy](docs/port-based-multi-tenancy.md) +- [Troubleshoot](docs/troubleshoot.md) # Custom app diff --git a/development/README.md b/development/README.md index 4f81d353..46fe1afa 100644 --- a/development/README.md +++ b/development/README.md @@ -251,38 +251,6 @@ frappe.db.connect() The first command can take a few seconds to be executed, this is to be expected. -### Fixing MariaDB issues after rebuilding the container - -For any reason after rebuilding the container if you are not be able to access MariaDB correctly with the previous configuration. Follow these instructions. - -The parameter `'db_name'@'%'` needs to be set in MariaDB and permission to the site database suitably assigned to the user. - -This step has to be repeated for all sites available under the current bench. -Example shows the queries to be executed for site `localhost` - -Open sites/localhost/site_config.json: - -```shell -code sites/localhost/site_config.json -``` - -and take note of the parameters `db_name` and `db_password`. - -Enter MariaDB Interactive shell: - -```shell -mysql -uroot -p123 -hmariadb -``` - -Execute following queries replacing `db_name` and `db_password` with the values found in site_config.json. - -```sql -UPDATE mysql.user SET Host = '%' where User = 'db_name'; FLUSH PRIVILEGES; -SET PASSWORD FOR 'db_name'@'%' = PASSWORD('db_password'); FLUSH PRIVILEGES; -GRANT ALL PRIVILEGES ON `db_name`.* TO 'db_name'@'%'; FLUSH PRIVILEGES; -EXIT; -``` - ## Manually start containers In case you don't use VSCode, you may start the containers manually with the following command: diff --git a/docs/add-custom-domain-using-traefik.md b/docs/add-custom-domain-using-traefik.md new file mode 100644 index 00000000..54edd47e --- /dev/null +++ b/docs/add-custom-domain-using-traefik.md @@ -0,0 +1,42 @@ +Add following labels to *-nginx service + +```yaml + - "traefik.http.routers.custom-domain.rule=Host(`custom.localhost`)" + # Comment the entrypoints label if traefik already has default entrypoint set + - "traefik.http.routers.custom-domain.entrypoints=web" + - "traefik.http.middlewares.custom-domain.headers.customrequestheaders.Host=mysite.localhost" + - "traefik.http.routers.custom-domain.middlewares=custom-domain" + # Add following header only if TLS is needed in case of live server, use one of below + - "traefik.http.routers.custom-domain.tls.certresolver=myresolver" # For Single Bench + - "traefik.http.routers.custom-domain.tls.certresolver=le" # For Docker Swarm +``` + +Example: + +```yaml +frontend: + image: frappe/erpnext-nginx:${ERPNEXT_VERSION} + restart: on-failure + environment: + - FRAPPE_PY=erpnext-python + - FRAPPE_PY_PORT=8000 + - FRAPPE_SOCKETIO=frappe-socketio + - SOCKETIO_PORT=9000 + labels: + - "traefik.enable=true" + - "traefik.http.routers.frontend.rule=Host(${SITES})" + - "traefik.http.routers.custom-domain.rule=Host(`custom.localhost`)" + - "traefik.http.routers.custom-domain.entrypoints=web" + - "traefik.http.middlewares.custom-domain.headers.customrequestheaders.Host=mysite.localhost" + - "traefik.http.routers.custom-domain.middlewares=custom-domain" + # Add following header only if TLS is needed in case of live server + - "traefik.http.routers.custom-domain.tls.certresolver=myresolver" + - "${ENTRYPOINT_LABEL}" + - "${CERT_RESOLVER_LABEL}" + - "traefik.http.services.frontend.loadbalancer.server.port=80" + volumes: + - sites-vol:/var/www/html/sites:rw + - assets-vol:/assets:rw +``` + +This will add `custom.localhost` as custom domain for `mysite.localhost` diff --git a/docs/backup-and-push-cronjob.md b/docs/backup-and-push-cronjob.md new file mode 100644 index 00000000..bcb63b7b --- /dev/null +++ b/docs/backup-and-push-cronjob.md @@ -0,0 +1,121 @@ +Install [crazy-max/swarm-cronjob](https://github.com/crazy-max/swarm-cronjob) and then deploy following stack. + +```yaml +version: "3.7" + +services: + backup: + image: frappe/erpnext-worker:version-13 + entrypoint: ["bash", "-c"] + command: ["docker-entrypoint.sh backup; docker-entrypoint.sh push-backup"] + environment: + - WITH_FILES=1 + - BUCKET_NAME=backups + - REGION=region + - ACCESS_KEY_ID=access_id_from_provider + - SECRET_ACCESS_KEY=secret_access_from_provider + - ENDPOINT_URL=https://region.storage-provider.com + - BUCKET_DIR=frappe-bench + volumes: + - "sites-vol:/home/frappe/frappe-bench/sites" + deploy: + labels: + - "swarm.cronjob.enable=true" + - "swarm.cronjob.schedule=0 */3 * * *" + - "swarm.cronjob.skip-running=true" + replicas: 0 + restart_policy: + condition: none + networks: + - frappe-network + +volumes: + sites-vol: + external: true + name: frappe-bench-v12_sites-vol + +networks: + frappe-network: + external: true + +``` + +Note: +- In Above stack, `backup` runs every 3 hours. +- Change image and tag version as per need. +- Change environment variables as per the bucket credentials. +- Change cron string(s) as per need. + +### For docker-compose based installation not using docker swarm + +Add minio + +```yaml +version: "3.7" +services: + minio: + image: minio/minio + command: ["server", "/data"] + environment: + - MINIO_ACCESS_KEY=RANDOMACCESSKEY + - MINIO_SECRET_KEY=RANDOMSECRETKEY + volumes: + - "minio-vol:/data" + networks: + - erpnext-network + # Do not enable, check how to secure minio, out of scope of this project. + #labels: + # - "traefik.enable=true" + # - "traefik.http.routers.minio.rule=Host(`backup.example.com`)" + # - "traefik.http.routers.minio.entrypoints=websecure" + # - "traefik.http.routers.minio.tls.certresolver=myresolver" + # - "traefik.http.services.minio.loadbalancer.server.port=9000" + +networks: + erpnext-network: + external: true + name: _default + +volumes: + minio-vol: +``` + +Create backup service. Create file `backup-job.yml` + +```yaml +version: "3.7" +services: + push-backup: + image: frappe/erpnext-worker:v13 + entrypoint: ["bash", "-c"] + command: ["docker-entrypoint.sh backup; docker-entrypoint.sh push-backup"] + environment: + - WITH_FILES=1 + - BUCKET_NAME=erpnext + - REGION=us-east-1 + - ACCESS_KEY_ID=RANDOMACCESSKEY + - SECRET_ACCESS_KEY=RANDOMSECRETKEY + - ENDPOINT_URL=http://minio:9000 + - BUCKET_DIR=backups + - BACKUP_LIMIT=8 + volumes: + - "sites-vol:/home/frappe/frappe-bench/sites" + networks: + - erpnext-network + +networks: + erpnext-network: + external: true + name: _default + +volumes: + sites-vol: + external: true + name: _sites-vol +``` + +Add crontab entry for backup every 6 hours + +``` +0 */6 * * * /usr/local/bin/docker-compose -f /path/to/backup-job.yml up -d > /dev/null +``` diff --git a/docs/bench-console-and-vscode-debugger.md b/docs/bench-console-and-vscode-debugger.md new file mode 100644 index 00000000..f5d38864 --- /dev/null +++ b/docs/bench-console-and-vscode-debugger.md @@ -0,0 +1,18 @@ +Add the following configuration to `launch.json` `configurations` array to start bench console and use debugger. Replace `mysite.localhost` with appropriate site. Also replace `frappe-bench` with name of the bench directory. + +```json + { + "name": "Bench Console", + "type": "python", + "request": "launch", + "program": "${workspaceFolder}/frappe-bench/apps/frappe/frappe/utils/bench_helper.py", + "args": [ + "frappe", "--site", "mysite.localhost", "console" + ], + "pythonPath": "${workspaceFolder}/frappe-bench/env/bin/python", + "cwd": "${workspaceFolder}/frappe-bench/sites", + "env": { + "DEV_SERVER": "1" + } + } +``` diff --git a/docs/build-version-10-images.md b/docs/build-version-10-images.md new file mode 100644 index 00000000..613f06ee --- /dev/null +++ b/docs/build-version-10-images.md @@ -0,0 +1,16 @@ +Clone the version-10 branch of this repo + +```shell +git clone https://github.com/frappe/frappe_docker.git -b version-10 && cd frappe_docker +``` + +Build the images + +```shell +export DOCKER_REGISTRY_PREFIX=frappe +docker build -t ${DOCKER_REGISTRY_PREFIX}/frappe-socketio:v10 -f build/frappe-socketio/Dockerfile . +docker build -t ${DOCKER_REGISTRY_PREFIX}/frappe-nginx:v10 -f build/frappe-nginx/Dockerfile . +docker build -t ${DOCKER_REGISTRY_PREFIX}/erpnext-nginx:v10 -f build/erpnext-nginx/Dockerfile . +docker build -t ${DOCKER_REGISTRY_PREFIX}/frappe-worker:v10 -f build/frappe-worker/Dockerfile . +docker build -t ${DOCKER_REGISTRY_PREFIX}/erpnext-worker:v10 -f build/erpnext-worker/Dockerfile . +``` diff --git a/docs/connect-to-localhost-services-from-containers-for-local-app-development.md b/docs/connect-to-localhost-services-from-containers-for-local-app-development.md new file mode 100644 index 00000000..73ee06ae --- /dev/null +++ b/docs/connect-to-localhost-services-from-containers-for-local-app-development.md @@ -0,0 +1,10 @@ +Not using separate container +Add following to frappe container from the `.devcontainer/docker-compose.yml`: + +```yaml + extra_hosts: + app1.localhost: 172.17.0.1 + app2.localhost: 172.17.0.1 +``` + +This is makes the domain names `app1.localhost` and `app2.localhost` connect to docker host and connect to services running on `localhost`. diff --git a/docs/patch-code-from-images.md b/docs/patch-code-from-images.md new file mode 100644 index 00000000..57e30454 --- /dev/null +++ b/docs/patch-code-from-images.md @@ -0,0 +1,10 @@ +Example: https://discuss.erpnext.com/t/sms-two-factor-authentication-otp-msg-change/47835 + +Above example needs following Dockerfile based patch + +```Dockerfile +FROM frappe/erpnext-worker:v12.17.0 + +RUN install_app custom_app https://github.com/username/custom_app version-12 +RUN sed -i -e "s/Your verification code is/আপনার লগইন কোড/g" /home/frappe/frappe-bench/apps/frappe/frappe/twofactor.py +``` diff --git a/docs/port-based-multi-tenancy.md b/docs/port-based-multi-tenancy.md new file mode 100644 index 00000000..d54d0b46 --- /dev/null +++ b/docs/port-based-multi-tenancy.md @@ -0,0 +1,47 @@ +WARNING: Do not use this in production if the site is going to be served over plain http. + +### Step 1 + +Remove the traefik service from docker-compose.yml + +### Step 2 + +Create nginx config file `/opt/nginx/conf/serve-8001.conf`: + +``` +server { + listen 8001; + server_name $http_host; + + location / { + + rewrite ^(.+)/$ $1 permanent; + rewrite ^(.+)/index\.html$ $1 permanent; + rewrite ^(.+)\.html$ $1 permanent; + + proxy_set_header X-Frappe-Site-Name mysite.localhost; + proxy_set_header Host mysite.localhost; + proxy_pass http://erpnext-nginx; + } +} +``` + +Notes: + +- Replace the port with any port of choice e.g. `listen 4200;` +- Change `mysite.localhost` to site name +- Repeat the server blocks for multiple ports and site names to get the effect of port based multi tenancy + +### Step 3 + +Run the docker container + +```shell +docker run --network=_default \ + -p 8001:8001 \ + --volume=/opt/nginx/conf/serve-8001.conf:/etc/nginx/conf.d/default.conf -d nginx +``` + +Note: Change the volumes, network and ports as needed + +With the above example configured site will be accessible on `http://localhost:8001` diff --git a/docs/troubleshoot.md b/docs/troubleshoot.md new file mode 100644 index 00000000..56a1d89f --- /dev/null +++ b/docs/troubleshoot.md @@ -0,0 +1,55 @@ +1. [Fixing MariaDB issues after rebuilding the container](#fixing-mariadb-issues-after-rebuilding-the-container) +1. [Letsencrypt companion not working](#letsencrypt-companion-not-working) +1. [docker-compose does not recognize variables from `.env` file](#docker-compose-does-not-recognize-variables-from-env-file) +1. [Windows Based Installation](#windows-based-installation) + +### Fixing MariaDB issues after rebuilding the container + +For any reason after rebuilding the container if you are not be able to access MariaDB correctly with the previous configuration. Follow these instructions. + +The parameter `'db_name'@'%'` needs to be set in MariaDB and permission to the site database suitably assigned to the user. + +This step has to be repeated for all sites available under the current bench. +Example shows the queries to be executed for site `localhost` + +Open sites/localhost/site_config.json: + +```shell +code sites/localhost/site_config.json +``` + +and take note of the parameters `db_name` and `db_password`. + +Enter MariaDB Interactive shell: + +```shell +mysql -uroot -p123 -hmariadb +``` + +Execute following queries replacing `db_name` and `db_password` with the values found in site_config.json. + +```sql +UPDATE mysql.user SET Host = '%' where User = 'db_name'; FLUSH PRIVILEGES; +SET PASSWORD FOR 'db_name'@'%' = PASSWORD('db_password'); FLUSH PRIVILEGES; +GRANT ALL PRIVILEGES ON `db_name`.* TO 'db_name'@'%'; FLUSH PRIVILEGES; +EXIT; +``` + +### Letsencrypt companion not working + +- Nginx Letsencrypt Companion needs to be setup before starting ERPNext services. +- Are domain names in `SITES` variable correct? +- Is DNS record configured? `A Name` record needs to point to Public IP of server. +- Try Restarting containers. + +### docker-compose does not recognize variables from `.env` file + +If you are using old version of `docker-compose` the .env file needs to be located in directory from where the docker-compose command is executed. There may also be difference in official `docker-compose` and the one packaged by distro. + +### Windows Based Installation + +- Set environment variable `COMPOSE_CONVERT_WINDOWS_PATHS` e.g. `set COMPOSE_CONVERT_WINDOWS_PATHS=1` +- Make the `frappe-mariadb.cnf` read-only for mariadb container to pick it up. +- While using docker machine, port-forward the port 80 of VM to port 80 of host machine +- Name all the sites ending with `.localhost`. and access it via browser locally. e.g. `http://site1.localhost` +- related issue comment https://github.com/frappe/frappe_docker/issues/448#issuecomment-851723912 From 1365662cbb7c32183460c0d1d0f85839fb8ca387 Mon Sep 17 00:00:00 2001 From: Revant Nandgaonkar Date: Fri, 14 Jan 2022 10:55:23 +0530 Subject: [PATCH 5/8] docs: fix add custom domain --- docs/add-custom-domain-using-traefik.md | 20 ++++---------------- 1 file changed, 4 insertions(+), 16 deletions(-) diff --git a/docs/add-custom-domain-using-traefik.md b/docs/add-custom-domain-using-traefik.md index 54edd47e..569f18a4 100644 --- a/docs/add-custom-domain-using-traefik.md +++ b/docs/add-custom-domain-using-traefik.md @@ -1,4 +1,4 @@ -Add following labels to *-nginx service +Add following labels to `frontend` service ```yaml - "traefik.http.routers.custom-domain.rule=Host(`custom.localhost`)" @@ -15,28 +15,16 @@ Example: ```yaml frontend: - image: frappe/erpnext-nginx:${ERPNEXT_VERSION} - restart: on-failure - environment: - - FRAPPE_PY=erpnext-python - - FRAPPE_PY_PORT=8000 - - FRAPPE_SOCKETIO=frappe-socketio - - SOCKETIO_PORT=9000 + ... labels: - - "traefik.enable=true" - - "traefik.http.routers.frontend.rule=Host(${SITES})" + ... - "traefik.http.routers.custom-domain.rule=Host(`custom.localhost`)" - "traefik.http.routers.custom-domain.entrypoints=web" - "traefik.http.middlewares.custom-domain.headers.customrequestheaders.Host=mysite.localhost" - "traefik.http.routers.custom-domain.middlewares=custom-domain" # Add following header only if TLS is needed in case of live server - "traefik.http.routers.custom-domain.tls.certresolver=myresolver" - - "${ENTRYPOINT_LABEL}" - - "${CERT_RESOLVER_LABEL}" - - "traefik.http.services.frontend.loadbalancer.server.port=80" - volumes: - - sites-vol:/var/www/html/sites:rw - - assets-vol:/assets:rw + ... ``` This will add `custom.localhost` as custom domain for `mysite.localhost` From b8b703572ba67e7c3500828c5f5534d93a3f371b Mon Sep 17 00:00:00 2001 From: Revant Nandgaonkar Date: Fri, 14 Jan 2022 11:49:23 +0530 Subject: [PATCH 6/8] docs: fix patch code from images --- docs/patch-code-from-images.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/patch-code-from-images.md b/docs/patch-code-from-images.md index 57e30454..deeff975 100644 --- a/docs/patch-code-from-images.md +++ b/docs/patch-code-from-images.md @@ -5,6 +5,6 @@ Above example needs following Dockerfile based patch ```Dockerfile FROM frappe/erpnext-worker:v12.17.0 -RUN install_app custom_app https://github.com/username/custom_app version-12 +RUN /home/frappe/frappe-bench/env/bin/pip -e /home/frappe/frappe-bench/apps/custom_app RUN sed -i -e "s/Your verification code is/আপনার লগইন কোড/g" /home/frappe/frappe-bench/apps/frappe/frappe/twofactor.py ``` From c615e4a41bf66235b785a107a262c59fabd3acc7 Mon Sep 17 00:00:00 2001 From: Revant Nandgaonkar Date: Sun, 6 Feb 2022 21:57:53 +0530 Subject: [PATCH 7/8] fix: do not expose port 80 for old images --- overrides/compose.noproxy.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/overrides/compose.noproxy.yaml b/overrides/compose.noproxy.yaml index b704f2df..5c4f83c0 100644 --- a/overrides/compose.noproxy.yaml +++ b/overrides/compose.noproxy.yaml @@ -2,4 +2,3 @@ services: frontend: ports: - 8080:8080 - - 80:80 # For old images From e0a5b37d561c79e93e692f690f158445b123b06d Mon Sep 17 00:00:00 2001 From: Revant Nandgaonkar Date: Sun, 6 Feb 2022 22:10:15 +0530 Subject: [PATCH 8/8] fix: custom domain labels to frontend container/service --- docs/add-custom-domain-using-traefik.md | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/docs/add-custom-domain-using-traefik.md b/docs/add-custom-domain-using-traefik.md index 569f18a4..fbea49a0 100644 --- a/docs/add-custom-domain-using-traefik.md +++ b/docs/add-custom-domain-using-traefik.md @@ -1,14 +1,13 @@ Add following labels to `frontend` service ```yaml - - "traefik.http.routers.custom-domain.rule=Host(`custom.localhost`)" + traefik.http.routers.custom-domain.rule: Host(`custom.localhost`) # Comment the entrypoints label if traefik already has default entrypoint set - - "traefik.http.routers.custom-domain.entrypoints=web" - - "traefik.http.middlewares.custom-domain.headers.customrequestheaders.Host=mysite.localhost" - - "traefik.http.routers.custom-domain.middlewares=custom-domain" - # Add following header only if TLS is needed in case of live server, use one of below - - "traefik.http.routers.custom-domain.tls.certresolver=myresolver" # For Single Bench - - "traefik.http.routers.custom-domain.tls.certresolver=le" # For Docker Swarm + traefik.http.routers.custom-domain.entrypoints: web + traefik.http.middlewares.custom-domain.headers.customrequestheaders.Host: mysite.localhost + traefik.http.routers.custom-domain.middlewares: custom-domain + # Add following header only if TLS is needed in case of live server + traefik.http.routers.custom-domain.tls.certresolver: main-resolver ``` Example: @@ -18,12 +17,11 @@ frontend: ... labels: ... - - "traefik.http.routers.custom-domain.rule=Host(`custom.localhost`)" - - "traefik.http.routers.custom-domain.entrypoints=web" - - "traefik.http.middlewares.custom-domain.headers.customrequestheaders.Host=mysite.localhost" - - "traefik.http.routers.custom-domain.middlewares=custom-domain" - # Add following header only if TLS is needed in case of live server - - "traefik.http.routers.custom-domain.tls.certresolver=myresolver" + traefik.http.routers.custom-domain.rule: Host(`custom.localhost`) + traefik.http.routers.custom-domain.entrypoints: web + traefik.http.middlewares.custom-domain.headers.customrequestheaders.Host: mysite.localhost + traefik.http.routers.custom-domain.middlewares: custom-domain + traefik.http.routers.custom-domain.tls.certresolver: main-resolver ... ```