Merge remote-tracking branch 'upstream/develop' into install_app

.devcontainer/devcontainer.json

@@ -0,0 +1,12 @@
+{
+  "name": "Frappe Bench",
+  "appPort": [8000, 9000, 6787],
+  "remoteUser": "frappe",
+  "settings": {
+    "terminal.integrated.shell.linux": "/bin/bash"
+  },
+  "dockerComposeFile": "./docker-compose.yml",
+  "service": "frappe",
+  "workspaceFolder": "/workspace/development",
+  "shutdownAction": "stopCompose"
+}

.devcontainer/docker-compose.yml

@@ -0,0 +1,30 @@
+version: "3.7"
+services:
+  mariadb:
+    image: mariadb:10.3
+    environment:
+      - MYSQL_ROOT_PASSWORD=123
+      - MYSQL_USER=root
+    volumes:
+      - ../installation/frappe-mariadb.cnf:/etc/mysql/conf.d/frappe.cnf
+      - mariadb-vol:/var/lib/mysql
+
+  redis-cache:
+    image: redis:alpine
+
+  redis-queue:
+    image: redis:alpine
+
+  redis-socketio:
+    image: redis:alpine
+
+  frappe:
+    image: frappe/bench:latest
+    command: sleep infinity
+    volumes:
+      - ..:/workspace:cached
+    ports:
+      - "8000:8000"
+
+volumes:
+  mariadb-vol:

.gitignore

@@ -5,3 +5,5 @@
 
 # mounted volume
 sites
+
+development

.travis.yml

@@ -20,12 +20,17 @@ after_success:
 
 jobs:
   include:
-    - stage: "Build Frappe python environment (edge)"
+    - name: "Build Frappe bench development environment (latest)"
       if: branch = develop AND type != pull_request
       script:
-        - ./travis.py frappe --worker --tag edge
-        - ./travis.py frappe --worker --tag develop --tag-only
-    - stage: "Build Frappe nginx + static assets (edge)"
+        - docker build -t frappe/bench:latest -f build/bench/Dockerfile .
+        - docker push frappe/bench:latest
+    - name: "Build Frappe python environment (edge)"
+      if: branch = develop AND type != pull_request
+      script:
+        - ./travis.py frappe --nginx --tag edge
+        - ./travis.py frappe --nginx --tag develop--tag-only
+    - name: "Build Frappe nginx + static assets (edge)"
       if: branch = develop AND type != pull_request
       script:
         - ./travis.py frappe --nginx --tag edge

README.md

@@ -1,4 +1,4 @@
-### Getting Started
+## Getting Started
 
 The templates in this repository will help deploy Frappe/ERPNext docker in a production environment.
 
@@ -8,31 +8,15 @@ This docker installation takes care of the following:
 * Setting up all the system requirements: eg. MariaDB, Node, Redis.
 * [OPTIONAL] Configuring networking for remote access and setting up LetsEncrypt
 
-### Installation Process
+For docker based development refer this [README](development/README.md)
 
-#### Setting up Pre-requisites
+## Deployment
+
+### Setting up Pre-requisites
 
 This repository requires Docker and Git to be setup on the instance to be used.
 
-#### Setup Letsencrypt Nginx Proxy Companion
-
-This is an optional first step. This step is only required if you want to have SSL setup on your docker instance.
-
-This step also assumes that the DNS is preconfigured since it automatically handles setup and renewal of SSL certificates.
-
-For more details, see: https://github.com/evertramos/docker-compose-letsencrypt-nginx-proxy-companion
-
-To setup the proxy companion, run the following steps:
-
-```sh
-cd $HOME
-git clone https://github.com/evertramos/docker-compose-letsencrypt-nginx-proxy-companion.git
-cd docker-compose-letsencrypt-nginx-proxy-companion
-cp .env.sample .env
-./start.sh
-```
-
-#### Setting up Frappe/ERPNext Docker
+### Cloning the repository and preliminary steps
 
 Clone this repository somewhere in your system:
 
@@ -43,7 +27,7 @@ cd frappe_docker
 
 Copy the example docker environment file to `.env`:
 
-```
+```sh
 cp installation/env-example installation/.env
 ```
 
@@ -53,9 +37,7 @@ Make a directory for sites:
 mkdir installation/sites
 ```
 
-#### Setup Environment Variables
-
-Docker allows passing an environment file to aide in setting up containers, which is used by this repository to pass secret and variable data.
+### Setup Environment Variables
 
 To get started, copy the existing `env-example` file to `.env` inside the `installation` directory. By default, the file will contain the following variables:
 
@@ -68,13 +50,86 @@ To get started, copy the existing `env-example` file to `.env` inside the `insta
     - In case of a separately managed database setup, set the value to the database's hostname/IP/domain.
 - `SITES=site1.domain.com,site2.domain.com`
     - List of sites that are part of the deployment "bench". Each site is separated by a comma(,).
-    - If LetsEncrypt is being setup, make sure that the DNS for all the site domains are pointing to the current instance.
+    - If LetsEncrypt is being setup, make sure that the DNS for all the site's domains are pointing to the current instance.
 - `LETSENCRYPT_EMAIL=your.email@your.domain.com`
-    - Email for LetsEncrypt expiry notification. This is only required if you are setting up the nginx proxy companion.
+    - Email for LetsEncrypt expiry notification. This is only required if you are setting up LetsEncrypt.
+
+
+### Local deployment
+
+For trying out locally or to develop apps using ERPNext ReST API port 80 must be published.
+First start the containers and then run an additional command to publish port of *-nginx container.
+
+To start and publish Frappe/ERPNext services as local api, run the following commands:
+
+For Erpnext:
+
+```sh
+# Start services
+docker-compose \
+    --project-name <project-name> \
+    -f installation/docker-compose-common.yml \
+    -f installation/docker-compose-erpnext.yml \
+    --project-directory installation up -d
+
+# Publish port
+docker-compose \
+    --project-name <project-name> \
+    -f installation/docker-compose-common.yml \
+    -f installation/docker-compose-erpnext.yml \
+    --project-directory installation run --publish 80:80 -d erpnext-nginx
+```
+
+For Frappe:
+
+```sh
+# Start services
+docker-compose \
+    --project-name <project-name> \
+    -f installation/docker-compose-common.yml \
+    -f installation/docker-compose-frappe.yml \
+    --project-directory installation up -d
+
+# Publish port
+docker-compose \
+    --project-name <project-name> \
+    -f installation/docker-compose-common.yml \
+    -f installation/docker-compose-frappe.yml \
+    --project-directory installation run --publish 80:80 -d frappe-nginx
+```
+
+Make sure to replace `<project-name>` with any desired name you wish to set for the project.
+
+Note:
+ - This command adds an additional container for frappe-nginx with published ports.
+ - The local deployment is for testing and REST API development purpose only.
+ - The site names are limited to patterns matching \*.localhost by default
+ - Additional site name patterns can be added to /etc/hosts of desired container or host
+
+### Deployment for production
+
+#### Setup Letsencrypt Nginx Proxy Companion
+
+Letsencrypt Nginx Proxy Companion can optionally be setup to provide SSL. This is recommended for instances accessed over the internet.
+
+Your DNS will need to be configured correctly in order for Letsencrypt to verify your domain.
+
+To setup the proxy companion, run the following commands:
+
+```sh
+cd $HOME
+git clone https://github.com/evertramos/docker-compose-letsencrypt-nginx-proxy-companion.git
+cd docker-compose-letsencrypt-nginx-proxy-companion
+cp .env.sample .env
+./start.sh
+```
+
+For more details, see: https://github.com/evertramos/docker-compose-letsencrypt-nginx-proxy-companion
+Letsencrypt Nginx Proxy Companion works by automatically proxying to containers with the `VIRTUAL_HOST` environmental variable.
 
 #### Start Frappe/ERPNext Services
 
-To start the Frappe/ERPNext services, run the following command:
+To start the Frappe/ERPNext services for production, run the following command:
 
 ```sh
 docker-compose \
@@ -86,8 +141,44 @@ docker-compose \
 ```
 
 Make sure to replace `<project-name>` with any desired name you wish to set for the project.
+Note: use `docker-compose-frappe.yml` in case you need only Frappe without ERPNext.
 
-Note: use `docker-compose-frappe.yml` in case you need bench with just frappe installed.
+### Docker containers
+
+This repository contains the following docker-compose files each one containing the described images:
+* docker-compose-common.yml
+    * redis-cache
+        * volume: redis-cache-vol
+    * redis-queue
+        * volume: redis-queue-vol
+    * redis-socketio
+        * volume: redis-socketio-vol
+    * mariadb: main database
+        * volume: mariadb-vol
+* docker-compose-erpnext.yml
+    * erpnext-nginx: serves static assets and proxies web request to the appropriate container, allowing to offer all services on the same port.
+        * volume: assets
+    * erpnext-python: main application code
+    * frappe-socketio: enables realtime communication to the user interface through websockets
+    * frappe-worker-default: background runner
+    * frappe-worker-short: background runner for short-running jobs
+    * frappe-worker-long: background runner for long-running jobs
+    * frappe-schedule
+
+* docker-compose-frappe.yml
+    * frappe-nginx: serves static assets and proxies web request to the appropriate container, allowing to offer all services on the same port.
+        * volume: assets
+    * erpnext-python: main application code
+    * frappe-socketio: enables realtime communication to the user interface through websockets
+    * frappe-worker-default: background runner
+    * frappe-worker-short: background runner for short-running jobs
+    * frappe-worker-long: background runner for long-running jobs
+    * frappe-schedule
+
+* docker-compose-networks.yml: this yml define the network to communicate with *Letsencrypt Nginx Proxy Companion*.
+
+
+### Site operations
 
 #### Setup New Sites
 
@@ -218,7 +309,7 @@ To add your own apps to the image, we'll need to create a custom image with the
 
 6. Install like usuall, except that when you set the `INSTALL_APPS` variable set it to `erpnext,[custom]`.
 
-### Troubleshoot
+## Troubleshoot
 
 1. Remove containers and volumes, and clear redis cache:
 

build/bench/Dockerfile

@@ -0,0 +1,59 @@
+# Frappe Bench Dockerfile
+
+FROM debian:9.6-slim
+LABEL author=frappé
+
+RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends locales \
+  && sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen \
+  && dpkg-reconfigure --frontend=noninteractive locales \
+  && apt-get clean && rm -rf /var/lib/apt/lists/*
+
+# Set locale en_us.UTF-8 for mariadb and general locale data
+ENV PYTHONIOENCODING=utf-8
+ENV LANGUAGE=en_US.UTF-8
+ENV LANG=en_US.UTF-8
+ENV LC_ALL=en_US.UTF-8
+
+# Install all neccesary packages
+RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-suggests --no-install-recommends \
+  build-essential cron curl git libffi-dev liblcms2-dev libldap2-dev libmariadbclient-dev libsasl2-dev libssl1.0-dev libtiff5-dev \
+  libwebp-dev mariadb-client iputils-ping python3-dev python3-pip python3-setuptools python3-tk redis-tools rlwrap \
+  software-properties-common sudo tk8.6-dev vim xfonts-75dpi xfonts-base wget wkhtmltopdf fonts-cantarell \
+  && apt-get clean && rm -rf /var/lib/apt/lists/*
+
+# Add frappe user and setup sudo
+RUN groupadd -g 1000 frappe \
+  && useradd -ms /bin/bash -u 1000 -g 1000 -G sudo frappe \
+  && chown -R 1000:1000 /home/frappe \
+  && echo '. "$NVM_DIR/nvm.sh"' >> /home/frappe/.bashrc
+
+# Install nvm with node
+ENV NVM_DIR /home/frappe/.nvm
+ENV NODE_VERSION 12.16.1
+RUN mkdir /home/frappe/.nvm \
+    && wget -qO- https://raw.githubusercontent.com/creationix/nvm/v0.35.2/install.sh | bash \
+    && . $NVM_DIR/nvm.sh \
+    && nvm install $NODE_VERSION \
+    && nvm install 10.19.0 \
+    && npm install yarn -g \
+    && nvm alias default $NODE_VERSION \
+    && nvm use default \
+    && npm install yarn -g \
+    && chown -R frappe:frappe /home/frappe
+ENV NODE_PATH $NVM_DIR/v$NODE_VERSION/lib/node_modules
+ENV PATH $NVM_DIR/v$NODE_VERSION/bin:$PATH
+
+# Install wkhtmltox correctly
+RUN wget https://github.com/wkhtmltopdf/wkhtmltopdf/releases/download/0.12.5/wkhtmltox_0.12.5-1.stretch_amd64.deb
+RUN dpkg -i wkhtmltox_0.12.5-1.stretch_amd64.deb && rm wkhtmltox_0.12.5-1.stretch_amd64.deb
+
+# Install bench
+RUN pip3 install -e git+https://github.com/frappe/bench.git#egg=bench --no-cache
+
+USER frappe
+
+WORKDIR /home/frappe/frappe-bench
+
+EXPOSE 8000 9000 6787
+
+VOLUME [ "/home/frappe/frappe-bench" ]

build/erpnext-assets/v11.Dockerfile

@@ -1,17 +0,0 @@
-FROM bitnami/node:10-prod
-
-COPY build/erpnext-assets/install_app.sh /install_app
-
-RUN /install_app erpnext https://github.com/frappe/erpnext version-11
-
-FROM frappe/frappe-assets:v11
-RUN cp /home/frappe/frappe-bench/sites/apps.txt /home/frappe/frappe-bench/sites/apps.bak
-COPY --from=0 /home/frappe/frappe-bench/sites/ /var/www/html/
-COPY --from=0 /rsync /rsync
-RUN mv /home/frappe/frappe-bench/sites/apps.bak /home/frappe/frappe-bench/sites/apps.txt \
-  && echo -n "\nerpnext" >> /home/frappe/frappe-bench/sites/apps.txt
-
-VOLUME [ "/assets" ]
-
-ENTRYPOINT ["/docker-entrypoint.sh"]
-CMD ["nginx", "-g", "daemon off;"]

build/erpnext-assets/v12.Dockerfile

@@ -1,16 +0,0 @@
-FROM bitnami/node:12-prod
-
-COPY build/erpnext-assets/install_app.sh /install_app
-
-RUN /install_app erpnext https://github.com/frappe/erpnext version-12
-
-FROM frappe/frappe-assets:v12
-RUN cp /home/frappe/frappe-bench/sites/apps.txt /home/frappe/frappe-bench/sites/apps.bak
-COPY --from=0 /home/frappe/frappe-bench/sites/ /var/www/html/
-COPY --from=0 /rsync /rsync
-RUN mv /home/frappe/frappe-bench/sites/apps.bak /home/frappe/frappe-bench/sites/apps.txt \
-  && echo -n "\nerpnext" >> /home/frappe/frappe-bench/sites/apps.txt
-VOLUME [ "/assets" ]
-
-ENTRYPOINT ["/docker-entrypoint.sh"]
-CMD ["nginx", "-g", "daemon off;"]

build/erpnext-nginx/Dockerfile

@@ -1,6 +1,6 @@
 FROM bitnami/node:12-prod
 
-COPY build/erpnext-assets/install_app.sh /install_app
+COPY build/erpnext-nginx/install_app.sh /install_app
 
 RUN /install_app erpnext https://github.com/frappe/erpnext
 

build/erpnext-nginx/v11.Dockerfile

@@ -0,0 +1,16 @@
+FROM bitnami/node:10-prod
+
+COPY build/erpnext-nginx/install_app.sh /install_app
+
+RUN /install_app erpnext https://github.com/frappe/erpnext version-11
+
+FROM frappe/frappe-nginx:v11
+
+COPY --from=0 /home/frappe/frappe-bench/sites/ /var/www/html/
+COPY --from=0 /rsync /rsync
+RUN echo -n "\nerpnext" >> /home/frappe/frappe-bench/sites/apps.txt
+
+VOLUME [ "/assets" ]
+
+ENTRYPOINT ["/docker-entrypoint.sh"]
+CMD ["nginx", "-g", "daemon off;"]

build/erpnext-nginx/v12.Dockerfile

@@ -0,0 +1,16 @@
+FROM bitnami/node:12-prod
+
+COPY build/erpnext-nginx/install_app.sh /install_app
+
+RUN /install_app erpnext https://github.com/frappe/erpnext version-12
+
+FROM frappe/frappe-nginx:v12
+
+COPY --from=0 /home/frappe/frappe-bench/sites/ /var/www/html/
+COPY --from=0 /rsync /rsync
+RUN echo -n "\nerpnext" >> /home/frappe/frappe-bench/sites/apps.txt
+
+VOLUME [ "/assets" ]
+
+ENTRYPOINT ["/docker-entrypoint.sh"]
+CMD ["nginx", "-g", "daemon off;"]

build/frappe-nginx/Dockerfile

@@ -27,7 +27,7 @@ FROM nginx:latest
 COPY --from=0 /home/frappe/frappe-bench/sites /var/www/html/
 COPY --from=0 /var/www/error_pages /var/www/
 COPY build/common/nginx-default.conf.template /etc/nginx/conf.d/default.conf.template
-COPY build/frappe-assets/docker-entrypoint.sh /
+COPY build/frappe-nginx/docker-entrypoint.sh /
 
 RUN apt-get update && apt-get install -y rsync && apt-get clean
 

build/frappe-nginx/v11.Dockerfile

@@ -27,7 +27,7 @@ FROM nginx:latest
 COPY --from=0 /home/frappe/frappe-bench/sites /var/www/html/
 COPY --from=0 /var/www/error_pages /var/www/
 COPY build/common/nginx-default.conf.template /etc/nginx/conf.d/default.conf.template
-COPY build/frappe-assets/docker-entrypoint.sh /
+COPY build/frappe-nginx/docker-entrypoint.sh /
 
 RUN apt-get update && apt-get install -y rsync && apt-get clean
 

build/frappe-nginx/v12.Dockerfile

@@ -27,7 +27,7 @@ FROM nginx:latest
 COPY --from=0 /home/frappe/frappe-bench/sites /var/www/html/
 COPY --from=0 /var/www/error_pages /var/www/
 COPY build/common/nginx-default.conf.template /etc/nginx/conf.d/default.conf.template
-COPY build/frappe-assets/docker-entrypoint.sh /
+COPY build/frappe-nginx/docker-entrypoint.sh /
 
 RUN apt-get update && apt-get install -y rsync && apt-get clean
 

development/.vscode/launch.json

@@ -0,0 +1,22 @@
+{
+    // Use IntelliSense to learn about possible attributes.
+    // Hover to view descriptions of existing attributes.
+    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+    "version": "0.2.0",
+    "configurations": [
+        {
+            "name": "Bench",
+            "type": "python",
+            "request": "launch",
+            "program": "${workspaceFolder}/frappe-bench/apps/frappe/frappe/utils/bench_helper.py",
+            "args": [
+                "frappe", "serve", "--port", "8000", "--noreload", "--nothreading"
+            ],
+            "pythonPath": "${workspaceFolder}/frappe-bench/env/bin/python",
+            "cwd": "${workspaceFolder}/frappe-bench/sites",
+            "env": {
+                "DEV_SERVER": "1"
+            }
+        }
+    ]
+}

development/.vscode/settings.json

@@ -0,0 +1,4 @@
+{
+    "python.pythonPath": "frappe-bench/env/bin/python",
+    "debug.node.autoAttach": "on"
+}

development/README.md

@@ -0,0 +1,123 @@
+# Getting Started
+
+## Prerequisites
+
+- Docker
+- docker-compose
+- user added to docker group
+
+### Bootstrap Containers for development
+
+Clone and change to frappe_docker directory
+
+```shell
+git clone https://github.com/frappe/frappe_docker.git
+cd frappe_docker
+```
+
+#### Manually start containers
+
+In case VS Code is not used follow these steps.
+
+```shell
+docker-compose -f .devcontainer/docker-compose.yml up -d
+```
+
+Enter the bench container at location `/workspace/development`
+
+```shell
+docker exec -e "TERM=xterm-256color" -w /workspace/development -it devcontainer_frappe_1 bash
+```
+
+#### Use VS Code Remote Containers extension
+
+Follow these in case VS Code is used. Do not start containers manually.
+
+- Install Remote Development Pack / Remote Containers extension.
+- Install VS Code Python extension.
+- Open frappe_docker in VS Code.
+- From Command Palette (Ctrl + Shift + P) Execute Remote Containers : Reopen in Container.
+
+### Setup Docker specific bench environment
+
+Notes:
+
+- `development` directory is ignored by git. It is mounted and available in container. Create all your benches inside this directory.
+- Execute these commands from container.
+- nvm with node v12 and v10 is installed. Check with `nvm ls`. Node v12 is default.
+
+#### Setup first bench
+
+```shell
+bench init --skip-redis-config-generation --frappe-branch version-12 frappe-bench
+cd frappe-bench
+```
+
+#### Set hosts
+
+```shell
+bench set-mariadb-host mariadb
+bench set-redis-cache-host redis-cache:6379
+bench set-redis-queue-host redis-queue:6379
+bench set-redis-socketio-host redis-socketio:6379
+```
+
+#### Changes related to bench start / honcho / Procfile
+
+- honcho/Procfile starts processes required for development.
+- By default Procfile has 3 redis processes that it starts. Comment (`#`) or remove these lines and then run `bench start`.
+- Another option is to run following command :
+
+```shell
+honcho start \
+    web \
+    socketio \
+    watch \
+    schedule \
+    worker_short \
+    worker_long \
+    worker_default
+```
+
+#### Changes related to MariaDB
+
+Notes:
+
+- `bench new-site` command creates a user in mariadb with container IP as host
+- After rebuilding container there is a chance that new bench container will not be able to access mariadb
+- `'db_name'@'%'` needs to be set in mariadb and permission to the site database be given to the user
+- Replace `db_name` and `db_password` from site's `site_config.json`
+- MariaDB root password is 123
+
+Enter mariadb shell
+
+```shell
+mysql -uroot -p123 -hmariadb
+```
+
+Execute following queries
+
+```sql
+UPDATE mysql.user SET Host = '%' where User = 'db_name';
+SET PASSWORD FOR 'db_name'@'%' = PASSWORD('db_password');
+GRANT ALL PRIVILEGES ON `db_name`.* TO 'db_name'@'%';
+FLUSH PRIVILEGES;
+```
+
+### Visual Studio Code Python Debugging
+
+- Install VS Code Python Extension once in remote container
+- Reload VS Code
+- Do not start `web` process with honcho
+
+```shell
+honcho start \
+    socketio \
+    watch \
+    schedule \
+    worker_short \
+    worker_long \
+    worker_default
+```
+
+- On debugger tab, Connect debugger. This will start the web process with debugger connected

installation/docker-compose-erpnext.yml

@@ -1,8 +1,8 @@
 version: '3'
 
 services:
-  erpnext-assets:
-    image: frappe/erpnext-assets:${VERSION}
+  erpnext-nginx:
+    image: frappe/erpnext-nginx:${VERSION}
     restart: on-failure
     environment:
       - FRAPPE_PY=erpnext-python

installation/docker-compose-frappe.yml

@@ -1,8 +1,8 @@
 version: '3'
 
 services:
-  frappe-assets:
-    image: frappe/frappe-assets:${VERSION}
+  frappe-nginx:
+    image: frappe/frappe-nginx:${VERSION}
     restart: on-failure
     environment:
       - FRAPPE_PY=frappe-python