Merge commit '78b2db1af10c0f94a50395f06f6021934921fe17' into develop

README.md

@@ -101,6 +101,7 @@ Make sure to replace `<project-name>` with the desired name you wish to set for
 
 Notes:
 
+- New site (first site) needs to be added after starting the services.
 - The local deployment is for testing and REST API development purpose only
 - A complete development environment is available [here](Development/README.md)
 - The site names are limited to patterns matching \*.localhost by default
@@ -140,7 +141,11 @@ docker-compose \
 ```
 
 Make sure to replace `<project-name>` with any desired name you wish to set for the project.
-Note: use `docker-compose-frappe.yml` in case you need only Frappe without ERPNext.
+
+Notes:
+
+- Use `docker-compose-frappe.yml` in case you need only Frappe without ERPNext.
+- New site (first site) needs to be added after starting the services.
 
 ### Docker containers
 
@@ -204,7 +209,9 @@ Environment Variables needed:
 - `SITE_NAME`: name of the new site to create.
 - `DB_ROOT_USER`: MariaDB Root user.
 - `MYSQL_ROOT_PASSWORD`: In case of the MariaDB docker container use the one set in `MYSQL_ROOT_PASSWORD` in previous steps. In case of a managed database use the appropriate password.
+- `MYSQL_ROOT_PASSWORD_FILE` - When the MariaDB root password is stored using docker secrets.
 - `ADMIN_PASSWORD`: set the administrator password for the new site.
+- `ADMIN_PASSWORD_FILE`: set the administrator password for the new site using docker secrets.
 - `INSTALL_APPS=erpnext`: available only in erpnext-worker and erpnext containers (or other containers with custom apps). Installs ERPNext (and/or the specified apps, comma-delinieated) on this new site.
 - `FORCE=1`: optional variable which force installation of the same site.
 
@@ -214,6 +221,9 @@ Environment Variables
 
 - `SITES` is list of sites separated by `:` colon to migrate. e.g. `SITES=site1.domain.com` or `SITES=site1.domain.com:site2.domain.com` By default all sites in bench will be backed up.
 - `WITH_FILES` if set to 1, it will backup user-uploaded files.
+- By default `backup` takes mariadb dump and gzips it. Example file, `20200325_221230-test_localhost-database.sql.gz`
+- If `WITH_FILES` is set then it will also backup public and private files of each site as uncompressed tarball. Example files, `20200325_221230-test_localhost-files.tar` and `20200325_221230-test_localhost-private-files.tar`
+- All the files generated by backup are placed at volume location `sites/{site-name}/private/backups/*`
 
 ```sh
 docker exec -it \
@@ -285,7 +295,7 @@ docker exec -it \
 
 Environment Variables
 
-- `MYSQL_ROOT_PASSWORD`, Required to restore mariadb backups.
+- `MYSQL_ROOT_PASSWORD` or `MYSQL_ROOT_PASSWORD_FILE`(when using docker secrets), Required to restore mariadb backups.
 - `BUCKET_NAME`, Required to set bucket created on S3 compatible storage.
 - `ACCESS_KEY_ID`, Required to set access key.
 - `SECRET_ACCESS_KEY`, Required to set secret access key.
@@ -456,4 +466,4 @@ This repository includes a complete setup to develop with Frappe/ERPNext and Ben
 - VSCode Python debugger
 - Pre-configured Docker containers for an easy start
 
-A complete Readme is available in [development/README.md](development/README.md)
+A complete Readme is available in [development/README.md](development/README.md)

build/common/commands/new.py

@@ -3,10 +3,32 @@ import os, frappe, json
 from frappe.commands.site import _new_site
 from check_connection import get_config, get_site_config
 
+def get_password(env_var, default=None):
+    return os.environ.get(env_var) or _get_password_from_secret(f"{env_var}_FILE") or default
+
+
+def _get_password_from_secret(env_var):
+    """Fetches the secret value from the docker secret file 
+    usually located inside /run/secrets/
+    Arguments:
+        env_var {str} -- Name of the environment variable 
+        containing the path to the secret file.
+    Returns:
+        [str] -- Secret value
+    """
+    passwd = None
+    secret_file_path = os.environ.get(env_var)
+    if secret_file_path:
+        with open(secret_file_path) as secret_file:
+            passwd = secret_file.read().strip()
+
+    return passwd
+
+
 def main():
     site_name = os.environ.get("SITE_NAME", 'site1.localhost')
     mariadb_root_username = os.environ.get("DB_ROOT_USER", 'root')
-    mariadb_root_password = os.environ.get("MYSQL_ROOT_PASSWORD", 'admin')
+    mariadb_root_password = get_password("MYSQL_ROOT_PASSWORD", 'admin')
     force = True if os.environ.get("FORCE", None) else False
     install_apps = os.environ.get("INSTALL_APPS", None)
     install_apps = install_apps.split(',') if install_apps else []
@@ -17,7 +39,7 @@ def main():
         site_name,
         mariadb_root_username=mariadb_root_username,
         mariadb_root_password=mariadb_root_password,
-        admin_password=os.environ.get("ADMIN_PASSWORD", 'admin'),
+        admin_password=get_password("ADMIN_PASSWORD", 'admin'),
         verbose=True,
         install_apps=install_apps,
         source_sql=None,

build/common/commands/restore_backup.py

@@ -5,6 +5,7 @@ import hashlib
 import frappe
 import boto3
 
+from new import get_password
 from push_backup import DATE_FORMAT, check_environment_variables
 from frappe.utils import get_sites, random_string
 from frappe.commands.site import _new_site
@@ -38,7 +39,7 @@ def decompress_db(files_base, site):
     os.system(command)
 
 def restore_database(files_base, site):
-    db_root_password = os.environ.get('MYSQL_ROOT_PASSWORD')
+    db_root_password = get_password('MYSQL_ROOT_PASSWORD')
     if not db_root_password:
         print('Variable MYSQL_ROOT_PASSWORD not set')
         exit(1)
@@ -158,7 +159,7 @@ def main():
             restore_private_files(files_base)
             restore_files(files_base)
         else:
-            mariadb_root_password = os.environ.get('MYSQL_ROOT_PASSWORD')
+            mariadb_root_password = get_password('MYSQL_ROOT_PASSWORD')
             if not mariadb_root_password:
                 print('Variable MYSQL_ROOT_PASSWORD not set')
                 exit(1)

build/common/nginx-default.conf.template

@@ -29,9 +29,9 @@ server {
         proxy_http_version 1.1;
         proxy_set_header Upgrade $http_upgrade;
         proxy_set_header Connection "upgrade";
-        proxy_set_header X-Frappe-Site-Name $http_host;
+        proxy_set_header X-Frappe-Site-Name $host;
         proxy_set_header Origin $scheme://$http_host;
-        proxy_set_header Host $host;
+        proxy_set_header Host $http_host;
 
         proxy_pass http://socketio-server;
     }
@@ -52,8 +52,8 @@ server {
     location @webserver {
         proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
         proxy_set_header X-Forwarded-Proto $scheme;
-        proxy_set_header X-Frappe-Site-Name $http_host;
+        proxy_set_header X-Frappe-Site-Name $host;
-        proxy_set_header Host $host;
+        proxy_set_header Host $http_host;
         proxy_set_header X-Use-X-Accel-Redirect True;
         proxy_read_timeout 120;
         proxy_redirect off;

build/erpnext-nginx/docker-entrypoint.sh

@@ -40,8 +40,8 @@ envsubst '${API_HOST}
 echo "Waiting for frappe-python to be available on $FRAPPE_PY port $FRAPPE_PY_PORT"
 timeout 10 bash -c 'until printf "" 2>>/dev/null >>/dev/tcp/$0/$1; do sleep 1; done' $FRAPPE_PY $FRAPPE_PY_PORT
 echo "Frappe-python available on $FRAPPE_PY port $FRAPPE_PY_PORT"
-echo "Waiting for frappe-socketio to be available on $FRAPPE_PY port $FRAPPE_PY_PORT"
+echo "Waiting for frappe-socketio to be available on $FRAPPE_SOCKETIO port $SOCKETIO_PORT"
 timeout 10 bash -c 'until printf "" 2>>/dev/null >>/dev/tcp/$0/$1; do sleep 1; done' $FRAPPE_SOCKETIO $SOCKETIO_PORT
-echo "Frappe-socketio available on $FRAPPE_PY port $FRAPPE_PY_PORT"
+echo "Frappe-socketio available on $FRAPPE_SOCKETIO port $SOCKETIO_PORT"
 
 exec "$@"

development/README.md

@@ -122,6 +122,16 @@ bench get-app myapp https://github.com/myusername/myapp.git
 bench --site my.site install-app myapp
 ```
 
+### Start development
+
+Execute following command from the `frappe-bench` directory.
+
+```shell
+bench start
+```
+
+Note: To start bench with debugger refer section for debugging.
+
 ### Fixing MariaDB issues after rebuilding the container
 
 The `bench new-site` command creates a user in MariaDB with container IP as host, for this reason after rebuilding the container there is a chance that you will not be able to access MariaDB correctly with the previous configuration