From 830d3959b71250dfef6a9379b74a57798d5ff712 Mon Sep 17 00:00:00 2001 From: Revant Nandgaonkar Date: Mon, 30 Mar 2020 13:08:55 +0530 Subject: [PATCH 1/4] docs: add info about backup command in readme --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index 2a9e26d0..860ceaac 100644 --- a/README.md +++ b/README.md @@ -214,6 +214,9 @@ Environment Variables - `SITES` is list of sites separated by `:` colon to migrate. e.g. `SITES=site1.domain.com` or `SITES=site1.domain.com:site2.domain.com` By default all sites in bench will be backed up. - `WITH_FILES` if set to 1, it will backup user-uploaded files. +- By default `backup` takes mariadb dump and gzips it. Example file, `20200325_221230-test_localhost-database.sql.gz` +- If `WITH_FILES` is set then it will also backup public and private files of each site as uncompressed tarball. Example files, `20200325_221230-test_localhost-files.tar` and `20200325_221230-test_localhost-private-files.tar` +- All the files generated by backup are placed at volume location `sites/{site-name}/private/backups/*` ```sh docker exec -it \ From 87c251a464ef56de9be9d99710305e14f4a7e671 Mon Sep 17 00:00:00 2001 From: girish pasupathy Date: Thu, 9 Apr 2020 18:58:34 +0530 Subject: [PATCH 2/4] Changed X-Frappe-Site-Name header to use value from `$host` instead of `$http_host` in nginx configuration ISSUE ----- `$http_host` is used for setting header 'X-Frappe-Site-Name' which adds port number to the header along with the host value. Frappe source app.py expects the header value to contain only the host name and not the port number. So `$host` should be used instead of `$http_host` to set the 'X-Frappe-Site-Name' header `$http_host` vs `$host` in nginx -------------------------------- `$http_host` contains the host name along with port number whereas `$host` contains only the host name in lowercase without the port number. > `$host` - This variable is equal to line Host in the header of request or > name of the server processing the request if the Host header is not available. > This variable may have a different value from $http_host in such cases: > * when the Host input header is absent or has an empty value, > `$host` equals to the value of server_name directive; > * when the value of Host contains port number, `$host` doesn't include > that port number. $host's value is always lowercase since 0.8.17. > - [$host vs $http_host stackoverflow](https://stackoverflow.com/questions/15414810/whats-the-difference-of-host-and-http-host-in-nginx) From the frappe source file [app.py](https://github.com/frappe/frappe/blob/develop/frappe/app.py#L107), X-Frappe-Site-Name is used if its set. ```Python site = _site or request.headers.get('X-Frappe-Site-Name') or get_site_name(request.host) ``` Since `$host` variable will never contain port number which is not the case with `$http_host`, `$host` should be used for setting the header 'X-Frappe-Site-Name'. Otherwise we have issues with site serving. Tested the above changes in compose as well as in swarm environment. In compose, tested the site with host mapping of 80 and 8000. Works with both the host port mapping. Tested with erpnext version - v12.5.2 Changes to be committed: modified: build/common/nginx-default.conf.template --- build/common/nginx-default.conf.template | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/build/common/nginx-default.conf.template b/build/common/nginx-default.conf.template index f19ccee2..076b2c42 100644 --- a/build/common/nginx-default.conf.template +++ b/build/common/nginx-default.conf.template @@ -29,9 +29,9 @@ server { proxy_http_version 1.1; proxy_set_header Upgrade $http_upgrade; proxy_set_header Connection "upgrade"; - proxy_set_header X-Frappe-Site-Name $http_host; + proxy_set_header X-Frappe-Site-Name $host; proxy_set_header Origin $scheme://$http_host; - proxy_set_header Host $host; + proxy_set_header Host $http_host; proxy_pass http://socketio-server; } @@ -52,8 +52,8 @@ server { location @webserver { proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto $scheme; - proxy_set_header X-Frappe-Site-Name $http_host; - proxy_set_header Host $host; + proxy_set_header X-Frappe-Site-Name $host; + proxy_set_header Host $http_host; proxy_set_header X-Use-X-Accel-Redirect True; proxy_read_timeout 120; proxy_redirect off; From 362075a2bc5cd99a18a1ee742b30dfa0c3d611c0 Mon Sep 17 00:00:00 2001 From: Revant Nandgaonkar Date: Fri, 10 Apr 2020 14:26:00 +0530 Subject: [PATCH 3/4] fix: improve readme mention to add site for production deployment mention to start bench for development deployment fixes #181 --- README.md | 7 ++++++- development/README.md | 10 ++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 860ceaac..0f090c8f 100644 --- a/README.md +++ b/README.md @@ -101,6 +101,7 @@ Make sure to replace `` with the desired name you wish to set for Notes: +- New site (first site) needs to be added after starting the services. - The local deployment is for testing and REST API development purpose only - A complete development environment is available [here](Development/README.md) - The site names are limited to patterns matching \*.localhost by default @@ -140,7 +141,11 @@ docker-compose \ ``` Make sure to replace `` with any desired name you wish to set for the project. -Note: use `docker-compose-frappe.yml` in case you need only Frappe without ERPNext. + +Notes: + +- Use `docker-compose-frappe.yml` in case you need only Frappe without ERPNext. +- New site (first site) needs to be added after starting the services. ### Docker containers diff --git a/development/README.md b/development/README.md index 2f08d3d1..00fbd1a5 100644 --- a/development/README.md +++ b/development/README.md @@ -103,6 +103,16 @@ bench set-config developer_mode 1 bench clear-cache ``` +### Start development + +Execute following command from the `frappe-bench` directory. + +```shell +bench start +``` + +Note: To start bench with debugger refer section for debugging. + ### Fixing MariaDB issues after rebuilding the container The `bench new-site` command creates a user in MariaDB with container IP as host, for this reason after rebuilding the container there is a chance that you will not be able to access MariaDB correctly with the previous configuration From 7498d5439a266e39fa84799623390de09f103453 Mon Sep 17 00:00:00 2001 From: girish pasupathy Date: Thu, 16 Apr 2020 11:49:40 +0530 Subject: [PATCH 4/4] Support for reading Mariadb and Admin password from file when using docker secrets With this PR, password can be read from docker secrets in both compose as well as swarm environment. ```YAML secrets: mariadb-root-password: file: mariadb-root-password.txt erpnext-admin-password: file: erpnext-admin-password.txt services: erpnext: image: frappe/erpnext-worker:${ERPNEXT_VERSION:-v12.5.2} environment: - SITE_NAME=example.com - DB_ROOT_USER=root - MARIADB_HOST=mariadb - INSTALL_APPS=erpnext - FORCE=1 - REDIS_CACHE=redis-cache:6379 - REDIS_QUEUE=redis-queue:6379 - REDIS_SOCKETIO=redis-socketio:6379 - SOCKETIO_PORT=9000 - AUTO_MIGRATE=1 - ADMIN_PASSWORD_FILE=/run/secrets/erpnext-admin-password - MYSQL_ROOT_PASSWORD_FILE=/run/secrets/mariadb-root-password secrets: - erpnext-admin-password - mariadb-root-password restart: on-failure volumes: - erpnext-data:/home/frappe/frappe-bench/sites - assets-data:/home/frappe/frappe-bench/sites/assets links: - redis-cache - redis-queue - redis-socketio - mariadb depends_on: - mariadb - redis-cache - redis-queue - redis-socketio networks: - erpnext-net ``` Reference: [Addind docker secrets in to your images](https://docs.docker.com/engine/swarm/secrets/#build-support-for-docker-secrets-into-your-images) Changes to be committed: modified: ../../README.md modified: ../common/commands/new.py modified: ../common/commands/restore_backup.py modified: ../erpnext-nginx/docker-entrypoint.sh --- README.md | 6 ++++-- build/common/commands/new.py | 26 ++++++++++++++++++++++-- build/common/commands/restore_backup.py | 5 +++-- build/erpnext-nginx/docker-entrypoint.sh | 4 ++-- 4 files changed, 33 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 0f090c8f..a71ff7dd 100644 --- a/README.md +++ b/README.md @@ -209,7 +209,9 @@ Environment Variables needed: - `SITE_NAME`: name of the new site to create. - `DB_ROOT_USER`: MariaDB Root user. - `MYSQL_ROOT_PASSWORD`: In case of the MariaDB docker container use the one set in `MYSQL_ROOT_PASSWORD` in previous steps. In case of a managed database use the appropriate password. +- `MYSQL_ROOT_PASSWORD_FILE` - When the MariaDB root password is stored using docker secrets. - `ADMIN_PASSWORD`: set the administrator password for the new site. +- `ADMIN_PASSWORD_FILE`: set the administrator password for the new site using docker secrets. - `INSTALL_APPS=erpnext`: available only in erpnext-worker and erpnext containers (or other containers with custom apps). Installs ERPNext (and/or the specified apps, comma-delinieated) on this new site. - `FORCE=1`: optional variable which force installation of the same site. @@ -293,7 +295,7 @@ docker exec -it \ Environment Variables -- `MYSQL_ROOT_PASSWORD`, Required to restore mariadb backups. +- `MYSQL_ROOT_PASSWORD` or `MYSQL_ROOT_PASSWORD_FILE`(when using docker secrets), Required to restore mariadb backups. - `BUCKET_NAME`, Required to set bucket created on S3 compatible storage. - `ACCESS_KEY_ID`, Required to set access key. - `SECRET_ACCESS_KEY`, Required to set secret access key. @@ -464,4 +466,4 @@ This repository includes a complete setup to develop with Frappe/ERPNext and Ben - VSCode Python debugger - Pre-configured Docker containers for an easy start -A complete Readme is available in [development/README.md](development/README.md) \ No newline at end of file +A complete Readme is available in [development/README.md](development/README.md) diff --git a/build/common/commands/new.py b/build/common/commands/new.py index 5fbe2cb9..48ea44fb 100644 --- a/build/common/commands/new.py +++ b/build/common/commands/new.py @@ -3,10 +3,32 @@ import os, frappe, json from frappe.commands.site import _new_site from check_connection import get_config, get_site_config +def get_password(env_var, default=None): + return os.environ.get(env_var) or _get_password_from_secret(f"{env_var}_FILE") or default + + +def _get_password_from_secret(env_var): + """Fetches the secret value from the docker secret file + usually located inside /run/secrets/ + Arguments: + env_var {str} -- Name of the environment variable + containing the path to the secret file. + Returns: + [str] -- Secret value + """ + passwd = None + secret_file_path = os.environ.get(env_var) + if secret_file_path: + with open(secret_file_path) as secret_file: + passwd = secret_file.read().strip() + + return passwd + + def main(): site_name = os.environ.get("SITE_NAME", 'site1.localhost') mariadb_root_username = os.environ.get("DB_ROOT_USER", 'root') - mariadb_root_password = os.environ.get("MYSQL_ROOT_PASSWORD", 'admin') + mariadb_root_password = get_password("MYSQL_ROOT_PASSWORD", 'admin') force = True if os.environ.get("FORCE", None) else False install_apps = os.environ.get("INSTALL_APPS", None) install_apps = install_apps.split(',') if install_apps else [] @@ -17,7 +39,7 @@ def main(): site_name, mariadb_root_username=mariadb_root_username, mariadb_root_password=mariadb_root_password, - admin_password=os.environ.get("ADMIN_PASSWORD", 'admin'), + admin_password=get_password("ADMIN_PASSWORD", 'admin'), verbose=True, install_apps=install_apps, source_sql=None, diff --git a/build/common/commands/restore_backup.py b/build/common/commands/restore_backup.py index 82854e85..f248bbf7 100644 --- a/build/common/commands/restore_backup.py +++ b/build/common/commands/restore_backup.py @@ -5,6 +5,7 @@ import hashlib import frappe import boto3 +from new import get_password from push_backup import DATE_FORMAT, check_environment_variables from frappe.utils import get_sites, random_string from frappe.commands.site import _new_site @@ -38,7 +39,7 @@ def decompress_db(files_base, site): os.system(command) def restore_database(files_base, site): - db_root_password = os.environ.get('MYSQL_ROOT_PASSWORD') + db_root_password = get_password('MYSQL_ROOT_PASSWORD') if not db_root_password: print('Variable MYSQL_ROOT_PASSWORD not set') exit(1) @@ -158,7 +159,7 @@ def main(): restore_private_files(files_base) restore_files(files_base) else: - mariadb_root_password = os.environ.get('MYSQL_ROOT_PASSWORD') + mariadb_root_password = get_password('MYSQL_ROOT_PASSWORD') if not mariadb_root_password: print('Variable MYSQL_ROOT_PASSWORD not set') exit(1) diff --git a/build/erpnext-nginx/docker-entrypoint.sh b/build/erpnext-nginx/docker-entrypoint.sh index aec5b0aa..c1caad54 100755 --- a/build/erpnext-nginx/docker-entrypoint.sh +++ b/build/erpnext-nginx/docker-entrypoint.sh @@ -40,8 +40,8 @@ envsubst '${API_HOST} echo "Waiting for frappe-python to be available on $FRAPPE_PY port $FRAPPE_PY_PORT" timeout 10 bash -c 'until printf "" 2>>/dev/null >>/dev/tcp/$0/$1; do sleep 1; done' $FRAPPE_PY $FRAPPE_PY_PORT echo "Frappe-python available on $FRAPPE_PY port $FRAPPE_PY_PORT" -echo "Waiting for frappe-socketio to be available on $FRAPPE_PY port $FRAPPE_PY_PORT" +echo "Waiting for frappe-socketio to be available on $FRAPPE_SOCKETIO port $SOCKETIO_PORT" timeout 10 bash -c 'until printf "" 2>>/dev/null >>/dev/tcp/$0/$1; do sleep 1; done' $FRAPPE_SOCKETIO $SOCKETIO_PORT -echo "Frappe-socketio available on $FRAPPE_PY port $FRAPPE_PY_PORT" +echo "Frappe-socketio available on $FRAPPE_SOCKETIO port $SOCKETIO_PORT" exec "$@"