Merge pull request #225 from frappe/develop

Frappe & ERPNext Version 13.0.0 Beta.1 Released

.travis.yml

@@ -50,6 +50,36 @@ jobs:
       script:
         - ./travis.py erpnext --nginx --tag edge
         - ./travis.py erpnext --nginx --tag develop --tag-only
+    - stage: "Frappe (v13-beta)"
+      if: branch = master AND type != pull_request
+      script:
+        - ./travis.py frappe --worker --git-version 13-beta
+        - ./travis.py frappe --worker --tag v13-beta --tag-only
+        - ./travis.py frappe --worker --tag version-13-beta --tag-only
+    - stage: "Frappe (v13-beta)"
+      if: branch = master AND type != pull_request
+      script:
+        - ./travis.py frappe --nginx --git-version 13-beta
+        - ./travis.py frappe --nginx --tag v13-beta --tag-only
+        - ./travis.py frappe --nginx --tag version-13-beta --tag-only
+    - stage: "Frappe (v13-beta)"
+      if: branch = master AND type != pull_request
+      script:
+        - ./travis.py frappe --socketio --git-version 13-beta
+        - ./travis.py frappe --socketio --tag v13-beta --tag-only
+        - ./travis.py frappe --socketio --tag version-13-beta --tag-only
+    - stage: "ERPNext (v13-beta)"
+      if: branch = master AND type != pull_request
+      script:
+        - ./travis.py erpnext --worker --git-version 13-beta
+        - ./travis.py erpnext --worker --tag v13-beta --tag-only
+        - ./travis.py erpnext --worker --tag version-13-beta --tag-only
+    - stage: "ERPNext (v13-beta)"
+      if: branch = master AND type != pull_request
+      script:
+        - ./travis.py erpnext --nginx --git-version 13-beta
+        - ./travis.py erpnext --nginx --tag v13-beta --tag-only
+        - ./travis.py erpnext --nginx --tag version-13-beta --tag-only
     - stage: "Frappe (v12)"
       if: branch = master AND type != pull_request
       script:
@@ -110,3 +140,20 @@ jobs:
         - ./travis.py erpnext --nginx --git-version 11
         - ./travis.py erpnext --nginx --tag v11 --tag-only
         - ./travis.py erpnext --nginx --tag version-11 --tag-only
+    - stage: "Build and test edge images"
+      if: type = pull_request
+      before_install:
+        - sudo apt-get update && sudo apt-get -y install docker-compose
+      script:
+        - docker build -t frappe/frappe-socketio:edge -f build/frappe-socketio/Dockerfile .
+        - docker build -t frappe/frappe-worker:develop -f build/frappe-worker/Dockerfile .
+        - docker build -t frappe/erpnext-worker:edge -f build/erpnext-worker/Dockerfile .
+        - docker build -t frappe/frappe-nginx:develop -f build/frappe-nginx/Dockerfile .
+        - docker build -t frappe/erpnext-nginx:edge -f build/erpnext-nginx/Dockerfile .
+        - ./tests/docker-test.sh
+    - stage: "Pull and test edge images"
+      if: branch = develop AND type != pull_request
+      before_install:
+        - sudo apt-get update && sudo apt-get -y install docker-compose
+      script:
+        - ./tests/docker-test.sh

README.md

@@ -19,7 +19,7 @@ For docker based development refer to this [README](development/README.md)
 
 ### Setting up Pre-requisites
 
-This repository requires Docker and Git to be setup on the instance to be used.
+This repository requires Docker, docker-compose and Git to be setup on the instance to be used.
 
 ### Cloning the repository and preliminary steps
 
@@ -33,7 +33,7 @@ cd frappe_docker
 Copy the example docker environment file to `.env`:
 
 ```sh
-cp installation/env-example installation/.env
+cp env-example .env
 ```
 
 ### Setup Environment Variables
@@ -63,37 +63,23 @@ The first command will start the containers; the second command will publish the
 For Erpnext:
 
 ```sh
-# Start services
 docker-compose \
     --project-name <project-name> \
     -f installation/docker-compose-common.yml \
     -f installation/docker-compose-erpnext.yml \
-    --project-directory installation up -d
+    -f installation/erpnext-publish.yml \
-
+    up -d
-# Publish port
-docker-compose \
-    --project-name <project-name> \
-    -f installation/docker-compose-common.yml \
-    -f installation/docker-compose-erpnext.yml \
-    --project-directory installation run --publish 80:80 -d erpnext-nginx
 ```
 
 For Frappe:
 
 ```sh
-# Start services
 docker-compose \
     --project-name <project-name> \
     -f installation/docker-compose-common.yml \
     -f installation/docker-compose-frappe.yml \
-    --project-directory installation up -d
+    -f installation/frappe-publish.yml \
-
+    up -d
-# Publish port
-docker-compose \
-    --project-name <project-name> \
-    -f installation/docker-compose-common.yml \
-    -f installation/docker-compose-frappe.yml \
-    --project-directory installation run --publish 80:80 -d frappe-nginx
 ```
 
 Make sure to replace `<project-name>` with the desired name you wish to set for the project.
@@ -124,8 +110,16 @@ cp .env.sample .env
 ./start.sh
 ```
 
+It will create the required network and configure containers for Letencrypt ACME.
+
 For more details, see the [Letsencrypt Nginx Proxy Companion github repo](https://github.com/evertramos/docker-compose-letsencrypt-nginx-proxy-companion). Letsencrypt Nginx Proxy Companion github repo works by automatically proxying to containers with the `VIRTUAL_HOST` environmental variable.
 
+Notes:
+
+- `SITES` variables from `env-example` is set as `VIRTUAL_HOST`
+- `LETSENCRYPT_EMAIL` variables from `env-example` is used as it is.
+- This is simple nginx + letsencrypt solution. Any other solution can be setup. Above two variables can be re-used or removed in case any other reverse-proxy is used.
+
 #### Start Frappe/ERPNext Services
 
 To start the Frappe/ERPNext services for production, run the following command:
@@ -136,7 +130,7 @@ docker-compose \
     -f installation/docker-compose-common.yml \
     -f installation/docker-compose-erpnext.yml \
     -f installation/docker-compose-networks.yml \
-    --project-directory installation up -d
+    up -d
 ```
 
 Make sure to replace `<project-name>` with any desired name you wish to set for the project.
@@ -159,7 +153,7 @@ docker run \
 
 Instead of `alpine` you can use any image you like.
 
-For full instructions, refer to the [wiki](https://github.com/frappe/frappe/wiki/Using-Frappe-with-Amazon-RDS-(or-any-other-DBaaS). Common question can be found in Issues and on forum.
+For full instructions, refer to the [wiki](https://github.com/frappe/frappe/wiki/Using-Frappe-with-Amazon-RDS-(or-any-other-DBaaS)). Common question can be found in Issues and on forum.
 
 ### Docker containers
 
@@ -201,9 +195,20 @@ This repository contains the following docker-compose files, each one containing
 
 * docker-compose-networks.yml: this yaml define the network to communicate with *Letsencrypt Nginx Proxy Companion*.
 
+* erpnext-publish.yml: this yml extends erpnext-nginx service to publish port 80, can only be used with docker-compose-erpnext.yml
+
+* frappe-publish.yml: this yml extends frappe-nginx service to publish port 80, can only be used with docker-compose-frappe.yml
 
 ### Site operations
 
+Use env file,
+
+```sh
+source .env
+```
+
+Or specify environment variables instead of passing secrets as command arguments. Refer notes section for environment variables required
+
 #### Setup New Sites
 
 Note:
@@ -211,17 +216,18 @@ Note:
 - Wait for the MariaDB service to start before trying to create a new site.
     - If new site creation fails, retry after the MariaDB container is up and running.
     - If you're using a managed database instance, make sure that the database is running before setting up a new site.
-- Use `.env` file or environment variables instead of passing secrets as command arguments.
 
 ```sh
 # Create ERPNext site
-docker exec -it \
+docker run \
     -e "SITE_NAME=$SITE_NAME" \
     -e "DB_ROOT_USER=$DB_ROOT_USER" \
     -e "MYSQL_ROOT_PASSWORD=$MYSQL_ROOT_PASSWORD" \
     -e "ADMIN_PASSWORD=$ADMIN_PASSWORD" \
     -e "INSTALL_APPS=erpnext" \
-    <project-name>_erpnext-python_1 docker-entrypoint.sh new
+    -v <project-name>_sites-vol:/home/frappe/frappe-bench/sites \
+    --network <project-name>_default \
+    frappe/erpnext-worker:$VERSION new
 ```
 
 Environment Variables needed:
@@ -246,10 +252,12 @@ Environment Variables
 - All the files generated by backup are placed at volume location `sites-vol:/{site-name}/private/backups/*`
 
 ```sh
-docker exec -it \
+docker run \
     -e "SITES=site1.domain.com:site2.domain.com" \
     -e "WITH_FILES=1" \
-    <project-name>_erpnext-python_1 docker-entrypoint.sh backup
+    -v <project-name>_sites-vol:/home/frappe/frappe-bench/sites \
+    --network <project-name>_default \
+    frappe/erpnext-worker:$VERSION backup
 ```
 
 The backup will be available in the `sites-vol` volume.
@@ -259,6 +267,7 @@ The backup will be available in the `sites-vol` volume.
 Environment Variables
 
 - `BUCKET_NAME`, Required to set bucket created on S3 compatible storage.
+- `REGION`, Required to set region for S3 compatible storage.
 - `ACCESS_KEY_ID`, Required to set access key.
 - `SECRET_ACCESS_KEY`, Required to set secret access key.
 - `ENDPOINT_URL`, Required to set URL of S3 compatible storage.
@@ -268,13 +277,14 @@ Environment Variables
 ```sh
  docker run \
     -e "BUCKET_NAME=backups" \
+    -e "REGION=region" \
     -e "ACCESS_KEY_ID=access_id_from_provider" \
     -e "SECRET_ACCESS_KEY=secret_access_from_provider" \
     -e "ENDPOINT_URL=https://region.storage-provider.com" \
-    -e "BUCKET_DIR=frappe-bench-v12" \
+    -e "BUCKET_DIR=frappe-bench" \
     -v <project-name>_sites-vol:/home/frappe/frappe-bench/sites \
     --network <project-name>_default \
-    frappe/frappe-worker:v12 push-backup
+    frappe/frappe-worker:$VERSION push-backup
 ```
 
 Note:
@@ -304,11 +314,13 @@ docker-compose \
     -f installation/docker-compose-common.yml \
     -f installation/docker-compose-erpnext.yml \
     -f installation/docker-compose-networks.yml \
-    --project-directory installation up -d
+    up -d
 
-docker exec -it \
+docker run \
     -e "MAINTENANCE_MODE=1" \
-    <project-name>_erpnext-python_1 docker-entrypoint.sh migrate
+    -v <project-name>_sites-vol:/home/frappe/frappe-bench/sites \
+    --network <project-name>_default \
+    frappe/erpnext-worker:$VERSION migrate
 ```
 
 #### Restore backups
@@ -320,20 +332,22 @@ Environment Variables
 - `ACCESS_KEY_ID`, Required to set access key.
 - `SECRET_ACCESS_KEY`, Required to set secret access key.
 - `ENDPOINT_URL`, Required to set URL of S3 compatible storage.
+- `REGION`, Required to set region for s3 compatible storage.
 - `BUCKET_DIR`, Required to set directory in bucket where sites from this deployment will be backed up.
 
 ```sh
 docker run \
     -e "MYSQL_ROOT_PASSWORD=admin" \
     -e "BUCKET_NAME=backups" \
+    -e "REGION=region" \
     -e "ACCESS_KEY_ID=access_id_from_provider" \
     -e "SECRET_ACCESS_KEY=secret_access_from_provider" \
     -e "ENDPOINT_URL=https://region.storage-provider.com" \
-    -e "BUCKET_DIR=frappe-bench-v12" \
+    -e "BUCKET_DIR=frappe-bench" \
     -v <project-name>_sites-vol:/home/frappe/frappe-bench/sites \
     -v ./backups:/home/frappe/backups \
     --network <project-name>_default \
-    frappe/frappe-worker:v12 restore-backup
+    frappe/frappe-worker:$VERSION restore-backup
 ```
 
 Note:
@@ -406,78 +420,6 @@ sed -i "s#\[app\]#[custom]#" ./installation/docker-compose-custom.yml
 
 Install like usual, except that when you set the `INSTALL_APPS` variable to `erpnext,[custom]`.
 
-## Troubleshoot
-
-### Failed migration after image upgrade
-
-Issue: After upgrade of the containers, the automatic migration fails.
-Solution: Remove containers and volumes, and clear redis cache:
-
-```shell
-# change to repo root
-cd $HOME/frappe_docker
-
-# Stop all bench containers
-docker-compose \
-    --project-name <project-name> \
-    -f installation/docker-compose-common.yml \
-    -f installation/docker-compose-erpnext.yml \
-    -f installation/docker-compose-networks.yml \
-    --project-directory installation stop
-
-# Remove redis containers
-docker-compose \
-    --project-name <project-name> \
-    -f installation/docker-compose-common.yml \
-    -f installation/docker-compose-erpnext.yml \
-    -f installation/docker-compose-networks.yml \
-    --project-directory installation rm redis-cache redis-queue redis-socketio
-
-# Clean redis volumes
-docker volume rm \
-    <project-name>_redis-cache-vol \
-    <project-name>_redis-queue-vol \
-    <project-name>_redis-socketio-vol
-
-# Restart project
-docker-compose \
-    --project-name <project-name> \
-    -f installation/docker-compose-common.yml \
-    -f installation/docker-compose-erpnext.yml \
-    -f installation/docker-compose-networks.yml \
-    --project-directory installation up -d
-```
-
-### ValueError: There exists an active worker named XXX already
-
-Issue: You have the following error during container restart
-
-
-```
-frappe-worker-short_1    | Traceback (most recent call last):
-frappe-worker-short_1    |   File "/home/frappe/frappe-bench/commands/worker.py", line 5, in <module>
-frappe-worker-short_1    |     start_worker(queue, False)
-frappe-worker-short_1    |   File "/home/frappe/frappe-bench/apps/frappe/frappe/utils/background_jobs.py", line 147, in start_worker
-frappe-worker-short_1    |     Worker(queues, name=get_worker_name(queue)).work(logging_level = logging_level)
-frappe-worker-short_1    |   File "/home/frappe/frappe-bench/env/lib/python3.7/site-packages/rq/worker.py", line 474, in work
-frappe-worker-short_1    |     self.register_birth()
-frappe-worker-short_1    |   File "/home/frappe/frappe-bench/env/lib/python3.7/site-packages/rq/worker.py", line 261, in register_birth
-frappe-worker-short_1    |     raise ValueError(msg.format(self.name))
-frappe-worker-short_1    | ValueError: There exists an active worker named '8dfe5c234085.10.short' already
-```
-
-Solution: Clear redis cache using `docker exec` command (take care of replacing `<project-name>` accordingly):
-
-```sh
-# Clear the cache which is causing problem.
-
-docker exec -it <project-name>_redis-cache_1 redis-cli FLUSHALL
-docker exec -it <project-name>_redis-queue_1 redis-cli FLUSHALL
-docker exec -it <project-name>_redis-socketio_1 redis-cli FLUSHALL
-```
-
-Note: Environment variables from `.env` file located at the current working directory will be used.
-
 ## Development
 
 This repository includes a complete setup to develop with Frappe/ERPNext and Bench, Including the following features:

build/common/commands/auto_migrate.py

@@ -9,10 +9,12 @@ from check_connection import get_config
 APP_VERSIONS_JSON_FILE = 'app_versions.json'
 APPS_TXT_FILE = 'apps.txt'
 
+
 def save_version_file(versions):
     with open(APP_VERSIONS_JSON_FILE, 'w') as f:
         return json.dump(versions, f, indent=1, sort_keys=True)
 
+
 def get_apps():
     apps = []
     try:
@@ -24,19 +26,20 @@ def get_apps():
     except FileNotFoundError as exception:
         print(exception)
         exit(1)
-    except:
+    except Exception:
         print(APPS_TXT_FILE+" is not valid")
         exit(1)
 
     return apps
 
+
 def get_container_versions(apps):
     versions = {}
     for app in apps:
         try:
             version = __import__(app).__version__
             versions.update({app: version})
-        except:
+        except Exception:
             pass
 
         try:
@@ -44,20 +47,22 @@ def get_container_versions(apps):
             repo = git.Repo(path)
             commit_hash = repo.head.object.hexsha
             versions.update({app+'_git_hash': commit_hash})
-        except:
+        except Exception:
             pass
 
     return versions
 
+
 def get_version_file():
     versions = None
     try:
         with open(APP_VERSIONS_JSON_FILE) as versions_file:
             versions = json.load(versions_file)
-    except:
+    except Exception:
         pass
     return versions
 
+
 def main():
     is_ready = False
     apps = get_apps()
@@ -105,5 +110,6 @@ def main():
         version_file = container_versions
         save_version_file(version_file)
 
+
 if __name__ == "__main__":
     main()

build/common/commands/background.py

@@ -1,10 +1,11 @@
-import frappe
 from frappe.utils.scheduler import start_scheduler
 
+
 def main():
     print("Starting background scheduler . . .")
     start_scheduler()
     exit(0)
 
+
 if __name__ == "__main__":
     main()

build/common/commands/backup.py

@@ -1,7 +1,8 @@
-import os, frappe, compileall, re
+import os
+import frappe
 from frappe.utils.backups import scheduled_backup
-from frappe.utils import now
+from frappe.utils import cint, get_sites, now
-from frappe.utils import get_sites
+
 
 def backup(sites, with_files=False):
     for site in sites:
@@ -20,13 +21,15 @@ def backup(sites, with_files=False):
             print("private files backup taken -", odb.backup_path_private_files, "- on", now())
         frappe.destroy()
 
+
 def main():
     installed_sites = ":".join(get_sites())
     sites = os.environ.get("SITES", installed_sites).split(":")
-    with_files=True if os.environ.get("WITH_FILES") else False
+    with_files = cint(os.environ.get("WITH_FILES"))
 
     backup(sites, with_files)
     exit(0)
 
+
 if __name__ == "__main__":
     main()

build/common/commands/check_connection.py

@@ -1,4 +1,6 @@
-import socket, os, json, time
+import socket
+import json
+import time
 from six.moves.urllib.parse import urlparse
 
 COMMON_SITE_CONFIG_FILE = 'common_site_config.json'
@@ -8,6 +10,7 @@ REDIS_SOCKETIO_KEY = 'redis_socketio'
 DB_HOST_KEY = 'db_host'
 DB_PORT = 3306
 
+
 def is_open(ip, port, timeout=30):
     s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
     s.settimeout(timeout)
@@ -15,11 +18,12 @@ def is_open(ip, port, timeout=30):
         s.connect((ip, int(port)))
         s.shutdown(socket.SHUT_RDWR)
         return True
-    except:
+    except Exception:
         return False
     finally:
         s.close()
 
+
 def check_host(ip, port, retry=10, delay=3, print_attempt=True):
     ipup = False
     for i in range(retry):
@@ -32,6 +36,7 @@ def check_host(ip, port, retry=10, delay=3, print_attempt=True):
             time.sleep(delay)
     return ipup
 
+
 # Check connection to servers
 def get_config():
     config = None
@@ -41,11 +46,12 @@ def get_config():
     except FileNotFoundError as exception:
         print(exception)
         exit(1)
-    except:
+    except Exception:
         print(COMMON_SITE_CONFIG_FILE+" is not valid")
         exit(1)
     return config
 
+
 # Check mariadb
 def check_mariadb(retry=10, delay=3, print_attempt=True):
     config = get_config()
@@ -60,6 +66,7 @@ def check_mariadb(retry=10, delay=3, print_attempt=True):
         print("Connection to MariaDB timed out")
         exit(1)
 
+
 # Check redis queue
 def check_redis_queue(retry=10, delay=3, print_attempt=True):
     check_redis_queue = False
@@ -76,6 +83,7 @@ def check_redis_queue(retry=10, delay=3, print_attempt=True):
         print("Connection to redis queue timed out")
         exit(1)
 
+
 # Check redis cache
 def check_redis_cache(retry=10, delay=3, print_attempt=True):
     check_redis_cache = False
@@ -92,6 +100,7 @@ def check_redis_cache(retry=10, delay=3, print_attempt=True):
         print("Connection to redis cache timed out")
         exit(1)
 
+
 # Check redis socketio
 def check_redis_socketio(retry=10, delay=3, print_attempt=True):
     check_redis_socketio = False
@@ -108,6 +117,7 @@ def check_redis_socketio(retry=10, delay=3, print_attempt=True):
         print("Connection to redis socketio timed out")
         exit(1)
 
+
 # Get site_config.json
 def get_site_config(site_name):
     site_config = None
@@ -115,6 +125,7 @@ def get_site_config(site_name):
         site_config = json.load(site_config_file)
     return site_config
 
+
 def main():
     check_mariadb()
     check_redis_queue()
@@ -122,5 +133,6 @@ def main():
     check_redis_socketio()
     print('Connections OK')
 
+
 if __name__ == "__main__":
     main()

build/common/commands/console.py

@@ -20,6 +20,11 @@ def console(site):
     print("Apps in this namespace:\n{}".format(", ".join(all_apps)))
     IPython.embed(display_banner="", header="")
 
+
 def main():
     site = sys.argv[-1]
     console(site)
+
+
+if __name__ == "__main__":
+    main()

build/common/commands/doctor.py

@@ -1,7 +1,3 @@
-import frappe
-import json
-import redis
-from rq import Worker
 from check_connection import (
     check_mariadb,
     check_redis_cache,
@@ -9,6 +5,7 @@ from check_connection import (
     check_redis_socketio,
 )
 
+
 def main():
     check_mariadb(retry=1, delay=0, print_attempt=False)
     print("MariaDB Connected")
@@ -22,5 +19,6 @@ def main():
     print("Health check successful")
     exit(0)
 
+
 if __name__ == "__main__":
     main()

build/common/commands/migrate.py

@@ -1,13 +1,17 @@
-import os, frappe, compileall, re, json
+import os
+import frappe
+import json
 
 from frappe.migrate import migrate
-from frappe.utils import get_sites
+from frappe.utils import cint, get_sites
 from check_connection import get_config
 
+
 def save_config(config):
     with open('common_site_config.json', 'w') as f:
         return json.dump(config, f, indent=1, sort_keys=True)
 
+
 def set_maintenance_mode(enable=True):
     conf = get_config()
 
@@ -19,11 +23,12 @@ def set_maintenance_mode(enable=True):
         conf.update({"maintenance_mode": 0, "pause_scheduler": 0})
         save_config(conf)
 
+
 def migrate_sites(maintenance_mode=False):
     installed_sites = ":".join(get_sites())
     sites = os.environ.get("SITES", installed_sites).split(":")
     if not maintenance_mode:
-        maintenance_mode = True if os.environ.get("MAINTENANCE_MODE") else False
+        maintenance_mode = cint(os.environ.get("MAINTENANCE_MODE"))
 
     if maintenance_mode:
         set_maintenance_mode(True)
@@ -37,12 +42,14 @@ def migrate_sites(maintenance_mode=False):
         finally:
             frappe.destroy()
 
-    if maintenance_mode:
+    # Disable maintenance mode after migration
     set_maintenance_mode(False)
 
+
 def main():
     migrate_sites()
     exit(0)
 
+
 if __name__ == "__main__":
     main()

build/common/commands/new.py

@@ -1,8 +1,10 @@
-import os, frappe, json
+import os
+import frappe
 
 from frappe.commands.site import _new_site
 from check_connection import get_config, get_site_config
 
+
 def get_password(env_var, default=None):
     return os.environ.get(env_var) or _get_password_from_secret(f"{env_var}_FILE") or default
 
@@ -64,7 +66,7 @@ def main():
     os.system(command)
 
     # Set db password
-    command = mysql_command + "\"UPDATE mysql.user SET authentication_string = PASSWORD('{db_password}') WHERE User = \'{db_name}\' AND Host = \'%\';\"".format(
+    command = mysql_command + "\"ALTER USER '{db_name}'@'%' IDENTIFIED BY '{db_password}'; FLUSH PRIVILEGES;\"".format(
         db_name=site_config.get('db_name'),
         db_password=site_config.get('db_password')
     )
@@ -77,5 +79,6 @@ def main():
     os.system(command)
     exit(0)
 
+
 if __name__ == "__main__":
     main()

build/common/commands/push_backup.py

@@ -8,6 +8,7 @@ from frappe.utils import get_sites
 
 DATE_FORMAT = "%Y%m%d_%H%M%S"
 
+
 def get_file_ext():
     return {
         "database": "-database.sql.gz",
@@ -15,6 +16,7 @@ def get_file_ext():
         "public_files": "-files.tar"
     }
 
+
 def get_backup_details(sitename):
     backup_details = dict()
     file_ext = get_file_ext()
@@ -41,12 +43,14 @@ def get_backup_details(sitename):
 
     return backup_details
 
+
 def get_s3_config():
     check_environment_variables()
     bucket = os.environ.get('BUCKET_NAME')
 
     conn = boto3.client(
         's3',
+        region_name=os.environ.get('REGION'),
         aws_access_key_id=os.environ.get('ACCESS_KEY_ID'),
         aws_secret_access_key=os.environ.get('SECRET_ACCESS_KEY'),
         endpoint_url=os.environ.get('ENDPOINT_URL')
@@ -54,27 +58,33 @@ def get_s3_config():
 
     return conn, bucket
 
+
 def check_environment_variables():
-    if not 'BUCKET_NAME' in os.environ:
+    if 'BUCKET_NAME' not in os.environ:
         print('Variable BUCKET_NAME not set')
         exit(1)
 
-    if not 'ACCESS_KEY_ID' in os.environ:
+    if 'ACCESS_KEY_ID' not in os.environ:
         print('Variable ACCESS_KEY_ID not set')
         exit(1)
 
-    if not 'SECRET_ACCESS_KEY' in os.environ:
+    if 'SECRET_ACCESS_KEY' not in os.environ:
         print('Variable SECRET_ACCESS_KEY not set')
         exit(1)
 
-    if not 'ENDPOINT_URL' in os.environ:
+    if 'ENDPOINT_URL' not in os.environ:
         print('Variable ENDPOINT_URL not set')
         exit(1)
 
-    if not 'BUCKET_DIR' in os.environ:
+    if 'BUCKET_DIR' not in os.environ:
         print('Variable BUCKET_DIR not set')
         exit(1)
 
+    if 'REGION' not in os.environ:
+        print('Variable REGION not set')
+        exit(1)
+
+
 def upload_file_to_s3(filename, folder, conn, bucket):
 
     destpath = os.path.join(folder, os.path.basename(filename))
@@ -86,6 +96,7 @@ def upload_file_to_s3(filename, folder, conn, bucket):
         print("Error uploading: %s" % (e))
         exit(1)
 
+
 def delete_old_backups(limit, bucket, site_name):
     all_backups = list()
     all_backup_dates = list()
@@ -96,6 +107,7 @@ def delete_old_backups(limit, bucket, site_name):
 
     s3 = boto3.resource(
         's3',
+        region_name=os.environ.get('REGION'),
         aws_access_key_id=os.environ.get('ACCESS_KEY_ID'),
         aws_secret_access_key=os.environ.get('SECRET_ACCESS_KEY'),
         endpoint_url=os.environ.get('ENDPOINT_URL')
@@ -150,6 +162,7 @@ def delete_old_backups(limit, bucket, site_name):
                         print('Deleteing ' + obj.key)
                         s3.Object(bucket.name, obj.key).delete()
 
+
 def main():
     details = dict()
     sites = get_sites()
@@ -178,5 +191,6 @@ def main():
     print('push-backup complete')
     exit(0)
 
+
 if __name__ == "__main__":
     main()

build/common/commands/restore_backup.py

@@ -8,10 +8,10 @@ import boto3
 from new import get_password
 from push_backup import DATE_FORMAT, check_environment_variables
 from frappe.utils import get_sites, random_string
-from frappe.commands.site import _new_site
 from frappe.installer import make_conf, get_conf_params, make_site_dirs
 from check_connection import get_site_config, get_config
 
+
 def list_directories(path):
     directories = []
     for name in os.listdir(path):
@@ -19,17 +19,16 @@ def list_directories(path):
             directories.append(name)
     return directories
 
+
 def get_backup_dir():
     return os.path.join(
         os.path.expanduser('~'),
         'backups'
     )
 
+
 def decompress_db(files_base, site):
     database_file = files_base + '-database.sql.gz'
-    config = get_config()
-    site_config = get_site_config(site)
-    db_root_user = os.environ.get('DB_ROOT_USER', 'root')
     command = 'gunzip -c {database_file} > {database_extract}'.format(
         database_file=database_file,
         database_extract=database_file.replace('.gz', '')
@@ -38,6 +37,7 @@ def decompress_db(files_base, site):
     print('Extract Database GZip for site {}'.format(site))
     os.system(command)
 
+
 def restore_database(files_base, site):
     db_root_password = get_password('MYSQL_ROOT_PASSWORD')
     if not db_root_password:
@@ -78,17 +78,11 @@ def restore_database(files_base, site):
     )
     os.system(create_user)
 
-    # create user password
+    # grant db privileges to user
-    set_user_password = mysql_command + "\"UPDATE mysql.user SET authentication_string = PASSWORD('{db_password}') WHERE User = \'{db_name}\' AND Host = \'%\';\"".format(
+    grant_privileges = mysql_command + "\"GRANT ALL PRIVILEGES ON \`{db_name}\`.* TO '{db_name}'@'%' IDENTIFIED BY '{db_password}'; FLUSH PRIVILEGES;\"".format(
         db_name=site_config.get('db_name'),
         db_password=site_config.get('db_password')
     )
-    os.system(set_user_password)
-
-    # grant db privileges to user
-    grant_privileges = mysql_command + "\"GRANT ALL PRIVILEGES ON \`{db_name}\`.* TO '{db_name}'@'%'; FLUSH PRIVILEGES;\"".format(
-        db_name=site_config.get('db_name')
-    )
     os.system(grant_privileges)
 
     command = "mysql -u{db_root_user} -h{db_host} -p{db_password} '{db_name}' < {database_file}".format(
@@ -102,6 +96,7 @@ def restore_database(files_base, site):
     print('Restoring database for site: {}'.format(site))
     os.system(command)
 
+
 def restore_files(files_base):
     public_files = files_base + '-files.tar'
     # extract tar
@@ -109,18 +104,21 @@ def restore_files(files_base):
     print('Extracting {}'.format(public_files))
     public_tar.extractall()
 
+
 def restore_private_files(files_base):
     private_files = files_base + '-private-files.tar'
     private_tar = tarfile.open(private_files)
     print('Extracting {}'.format(private_files))
     private_tar.extractall()
 
+
 def pull_backup_from_s3():
     check_environment_variables()
 
     # https://stackoverflow.com/a/54672690
     s3 = boto3.resource(
         's3',
+        region_name=os.environ.get('REGION'),
         aws_access_key_id=os.environ.get('ACCESS_KEY_ID'),
         aws_secret_access_key=os.environ.get('SECRET_ACCESS_KEY'),
         endpoint_url=os.environ.get('ENDPOINT_URL')
@@ -142,6 +140,7 @@ def pull_backup_from_s3():
 
     os.chdir(os.path.join(os.path.expanduser('~'), 'frappe-bench', 'sites'))
 
+
 def main():
     backup_dir = get_backup_dir()
 
@@ -163,8 +162,6 @@ def main():
             if not mariadb_root_password:
                 print('Variable MYSQL_ROOT_PASSWORD not set')
                 exit(1)
-            mariadb_root_username = os.environ.get('DB_ROOT_USER', 'root')
-            database_file = files_base + '-database.sql.gz'
 
             site_config = get_conf_params(
                 db_name='_' + hashlib.sha1(site.encode()).hexdigest()[:16],
@@ -185,5 +182,6 @@ def main():
 
     exit(0)
 
+
 if __name__ == "__main__":
     main()

build/common/commands/worker.py

@@ -1,10 +1,12 @@
-import os, frappe
+import os
 from frappe.utils.background_jobs import start_worker
 
+
 def main():
     queue = os.environ.get("WORKER_TYPE", "default")
     start_worker(queue, False)
     exit(0)
 
+
 if __name__ == "__main__":
     main()

build/erpnext-nginx/v13-beta.Dockerfile

@@ -0,0 +1,16 @@
+FROM bitnami/node:12-prod
+
+COPY build/erpnext-nginx/install_app.sh /install_app
+
+RUN /install_app erpnext https://github.com/frappe/erpnext version-13-beta
+
+FROM frappe/frappe-nginx:v13-beta
+
+COPY --from=0 /home/frappe/frappe-bench/sites/ /var/www/html/
+COPY --from=0 /rsync /rsync
+RUN echo -n "\nerpnext" >> /var/www/html/apps.txt
+
+VOLUME [ "/assets" ]
+
+ENTRYPOINT ["/docker-entrypoint.sh"]
+CMD ["nginx", "-g", "daemon off;"]

build/erpnext-worker/v13-beta.Dockerfile

@@ -0,0 +1,3 @@
+FROM frappe/frappe-worker:v13-beta
+
+RUN install_app erpnext https://github.com/frappe/erpnext version-13-beta

build/frappe-nginx/v13-beta.Dockerfile

@@ -0,0 +1,52 @@
+FROM bitnami/node:12-prod
+
+WORKDIR /home/frappe/frappe-bench
+RUN mkdir -p /home/frappe/frappe-bench/sites \
+    && echo "frappe" > /home/frappe/frappe-bench/sites/apps.txt
+
+RUN install_packages git
+
+RUN mkdir -p apps sites/assets/css  \
+    && cd apps \
+    && git clone --depth 1 https://github.com/frappe/frappe  --branch version-13-beta
+
+COPY build/frappe-nginx/generate_standard_style_css.js \
+    /home/frappe/frappe-bench/apps/frappe/generate_standard_style_css.js
+
+RUN cd /home/frappe/frappe-bench/apps/frappe \
+    && yarn \
+    && yarn run production \
+    && yarn add nunjucks -D \
+    && node generate_standard_style_css.js \
+        frappe/website/doctype/website_theme/website_theme_template.scss > \
+        /home/frappe/standard_templates_string \
+    && node generate_bootstrap_theme.js \
+        /home/frappe/frappe-bench/sites/assets/css/standard_style.css \
+        "$(cat /home/frappe/standard_templates_string)" \
+    && rm -fr node_modules \
+    && yarn install --production=true \
+    && node --version \
+    && npm --version \
+    && yarn --version
+
+RUN git clone --depth 1 https://github.com/frappe/bench /tmp/bench \
+    && mkdir -p /var/www/error_pages \
+    && cp -r /tmp/bench/bench/config/templates /var/www/error_pages
+
+RUN cp -R /home/frappe/frappe-bench/apps/frappe/frappe/public/* /home/frappe/frappe-bench/sites/assets/frappe \
+    && cp -R /home/frappe/frappe-bench/apps/frappe/node_modules /home/frappe/frappe-bench/sites/assets/frappe/
+
+FROM nginx:latest
+COPY --from=0 /home/frappe/frappe-bench/sites /var/www/html/
+COPY --from=0 /var/www/error_pages /var/www/
+COPY build/common/nginx-default.conf.template /etc/nginx/conf.d/default.conf.template
+COPY build/frappe-nginx/docker-entrypoint.sh /
+
+RUN apt-get update && apt-get install -y rsync && apt-get clean \
+    && echo "#!/bin/bash" > /rsync \
+    && chmod +x /rsync
+
+VOLUME [ "/assets" ]
+
+ENTRYPOINT ["/docker-entrypoint.sh"]
+CMD ["nginx", "-g", "daemon off;"]

build/frappe-socketio/v13-beta.Dockerfile

@@ -0,0 +1,36 @@
+FROM node:slim
+
+# Install needed packages
+RUN apt-get update && apt-get install -y curl && apt-get clean
+
+RUN useradd -ms /bin/bash frappe
+
+# Make bench directories
+RUN mkdir -p /home/frappe/frappe-bench/sites /home/frappe/frappe-bench/apps/frappe
+
+COPY build/frappe-socketio/package.json /home/frappe/frappe-bench/apps/frappe
+
+
+# get socketio
+RUN cd /home/frappe/frappe-bench/apps/frappe \
+    && curl https://raw.githubusercontent.com/frappe/frappe/version-13-beta/socketio.js \
+        --output /home/frappe/frappe-bench/apps/frappe/socketio.js \
+    && curl https://raw.githubusercontent.com/frappe/frappe/version-13-beta/node_utils.js \
+        --output /home/frappe/frappe-bench/apps/frappe/node_utils.js
+
+RUN cd /home/frappe/frappe-bench/apps/frappe \
+    && npm install --only=production \
+    && node --version \
+    && npm --version
+
+COPY build/frappe-socketio/health.js /home/frappe/frappe-bench/apps/frappe/health.js
+RUN chown -R frappe:frappe /home/frappe
+
+# Setup docker-entrypoint
+COPY build/frappe-socketio/docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh
+RUN ln -s /usr/local/bin/docker-entrypoint.sh / # backwards compat
+
+WORKDIR /home/frappe/frappe-bench/sites
+
+ENTRYPOINT ["docker-entrypoint.sh"]
+CMD ["start"]

build/frappe-worker/v13-beta.Dockerfile

@@ -0,0 +1,48 @@
+FROM bitnami/python:latest-prod
+
+RUN useradd -ms /bin/bash frappe
+WORKDIR /home/frappe/frappe-bench
+RUN install_packages \
+    git \
+    wkhtmltopdf \
+    mariadb-client \
+    gettext-base \
+    wget \
+    # for PDF
+    libssl-dev \
+    fonts-cantarell \
+    xfonts-75dpi \
+    xfonts-base \
+    # For psycopg2
+    libpq-dev \
+    build-essential
+
+# Install wkhtmltox correctly
+RUN wget https://github.com/wkhtmltopdf/wkhtmltopdf/releases/download/0.12.5/wkhtmltox_0.12.5-1.stretch_amd64.deb
+RUN dpkg -i wkhtmltox_0.12.5-1.stretch_amd64.deb && rm wkhtmltox_0.12.5-1.stretch_amd64.deb
+
+RUN mkdir -p apps logs commands /home/frappe/backups
+
+RUN virtualenv env \
+    && . env/bin/activate \
+    && cd apps \
+    && git clone --depth 1 -o upstream https://github.com/frappe/frappe  --branch version-13-beta \
+    && pip3 install --no-cache-dir -e /home/frappe/frappe-bench/apps/frappe
+
+COPY build/common/commands/* /home/frappe/frappe-bench/commands/
+COPY build/common/common_site_config.json.template /opt/frappe/common_site_config.json.template
+
+# Setup docker-entrypoint
+COPY build/common/worker/docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh
+RUN ln -s /usr/local/bin/docker-entrypoint.sh / # backwards compat
+
+COPY build/common/worker/install_app.sh /usr/local/bin/install_app
+
+WORKDIR /home/frappe/frappe-bench/sites
+
+RUN chown -R frappe:frappe /home/frappe/frappe-bench/sites /home/frappe/backups
+
+VOLUME [ "/home/frappe/frappe-bench/sites", "/home/frappe/backups" ]
+
+ENTRYPOINT ["docker-entrypoint.sh"]
+CMD ["start"]

installation/erpnext-publish.yml

@@ -0,0 +1,6 @@
+version: "3"
+
+services:
+  erpnext-nginx:
+    ports:
+      - "80:80"

installation/frappe-publish.yml

@@ -0,0 +1,6 @@
+version: "3"
+
+services:
+  frappe-nginx:
+    ports:
+      - "80:80"

tests/docker-test.sh

@@ -0,0 +1,216 @@
+#!/bin/bash
+
+function checkMigrationComplete() {
+    echo "Check Auto Migration"
+    CONTAINER_ID=$(docker-compose \
+        --project-name frappebench00 \
+        -f installation/docker-compose-common.yml \
+        -f installation/docker-compose-erpnext.yml \
+        -f installation/erpnext-publish.yml \
+        ps -q erpnext-python)
+
+    DOCKER_LOG=$(docker logs $CONTAINER_ID 2>&1 | grep "Starting gunicorn")
+    INCREMENT=0
+    while [[ $DOCKER_LOG != *"Starting gunicorn"* && $INCREMENT -lt 60 ]]; do
+        sleep 3
+        echo "Wait for migration to complete ..."
+        ((INCREMENT=INCREMENT+1))
+        DOCKER_LOG=$(docker logs $CONTAINER_ID 2>&1 | grep "Starting gunicorn")
+        if [[ $DOCKER_LOG != *"Starting gunicorn"* && $INCREMENT -eq 60  ]]; then
+            docker logs $CONTAINER_ID
+            exit 1
+        fi
+    done
+}
+
+function loopHealthCheck() {
+    echo "Create Container to Check MariaDB"
+    docker run --name frappe_doctor \
+        -v frappebench00_sites-vol:/home/frappe/frappe-bench/sites \
+        --network frappebench00_default \
+        frappe/erpnext-worker:edge doctor || true
+
+    echo "Loop Health Check"
+    FRAPPE_LOG=$(docker logs frappe_doctor | grep "Health check successful" || echo "")
+    while [[ -z "$FRAPPE_LOG" ]]; do
+        sleep 1
+        CONTAINER=$(docker start frappe_doctor)
+        echo "Restarting $CONTAINER ..."
+        FRAPPE_LOG=$(docker logs frappe_doctor | grep "Health check successful" || echo "")
+    done
+    echo "Health check successful"
+}
+
+echo "Copy env-example file"
+cp env-example .env
+
+echo "Set version to v12"
+sed -i -e "s/edge/v12/g" .env
+
+echo "Start Services"
+docker-compose \
+    --project-name frappebench00 \
+    -f installation/docker-compose-common.yml \
+    -f installation/docker-compose-erpnext.yml \
+    -f installation/erpnext-publish.yml \
+    up -d
+
+loopHealthCheck
+
+echo "Create new site (v12)"
+docker run -it \
+    -e "SITE_NAME=test.localhost" \
+    -e "INSTALL_APPS=erpnext" \
+    -v frappebench00_sites-vol:/home/frappe/frappe-bench/sites \
+    --network frappebench00_default \
+    frappe/erpnext-worker:v12 new
+
+echo "Ping created site"
+curl -S http://test.localhost/api/method/version
+echo ""
+
+echo "Set version to edge"
+sed -i -e "s/v12/edge/g" .env
+
+echo "Restart containers with edge image"
+docker-compose \
+    --project-name frappebench00 \
+    -f installation/docker-compose-common.yml \
+    -f installation/docker-compose-erpnext.yml \
+    -f installation/erpnext-publish.yml \
+    stop
+docker-compose \
+    --project-name frappebench00 \
+    -f installation/docker-compose-common.yml \
+    -f installation/docker-compose-erpnext.yml \
+    -f installation/erpnext-publish.yml \
+    up -d
+
+checkMigrationComplete
+
+echo "Ping migrated site"
+sleep 3
+curl -S http://test.localhost/api/method/version
+echo ""
+
+echo "Backup site"
+docker run -it \
+    -e "WITH_FILES=1" \
+    -v frappebench00_sites-vol:/home/frappe/frappe-bench/sites \
+    --network frappebench00_default \
+    frappe/erpnext-worker:edge backup
+
+export MINIO_ACCESS_KEY="AKIAIOSFODNN7EXAMPLE"
+export MINIO_SECRET_KEY="wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"
+
+echo "Start MinIO container for s3 compatible storage"
+docker run -d --name minio \
+  -e "MINIO_ACCESS_KEY=$MINIO_ACCESS_KEY" \
+  -e "MINIO_SECRET_KEY=$MINIO_SECRET_KEY" \
+  --network frappebench00_default \
+  minio/minio server /data
+
+echo "Create bucket named erpnext after 3 sec."
+sleep 3
+docker run \
+    --network frappebench00_default \
+    vltgroup/s3cmd:latest s3cmd --access_key=$MINIO_ACCESS_KEY \
+    --secret_key=$MINIO_SECRET_KEY \
+    --region=us-east-1 \
+    --no-ssl \
+    --host=minio:9000 \
+    --host-bucket=minio:9000 \
+    mb s3://erpnext
+
+echo "Push backup to MinIO s3"
+docker run \
+    -e BUCKET_NAME=erpnext \
+    -e REGION=us-east-1 \
+    -e BUCKET_DIR=local \
+    -e ACCESS_KEY_ID=$MINIO_ACCESS_KEY \
+    -e SECRET_ACCESS_KEY=$MINIO_SECRET_KEY \
+    -e ENDPOINT_URL=http://minio:9000 \
+    -v frappebench00_sites-vol:/home/frappe/frappe-bench/sites \
+    --network frappebench00_default \
+    frappe/erpnext-worker:edge push-backup
+
+echo "Stop Services"
+docker-compose \
+    --project-name frappebench00 \
+    -f installation/docker-compose-common.yml \
+    -f installation/docker-compose-erpnext.yml \
+    -f installation/erpnext-publish.yml \
+    stop
+
+echo "Prune Containers"
+docker container prune -f && docker volume prune -f
+
+echo "Start Services"
+docker-compose \
+    --project-name frappebench00 \
+    -f installation/docker-compose-common.yml \
+    -f installation/docker-compose-erpnext.yml \
+    -f installation/erpnext-publish.yml \
+    up -d
+
+loopHealthCheck
+
+echo "Restore backup from MinIO / S3"
+docker run \
+    -e MYSQL_ROOT_PASSWORD=admin \
+    -e BUCKET_NAME=erpnext \
+    -e BUCKET_DIR=local \
+    -e ACCESS_KEY_ID=$MINIO_ACCESS_KEY \
+    -e SECRET_ACCESS_KEY=$MINIO_SECRET_KEY \
+    -e ENDPOINT_URL=http://minio:9000 \
+    -e REGION=us-east-1 \
+    -v frappebench00_sites-vol:/home/frappe/frappe-bench/sites \
+    --network frappebench00_default \
+    frappe/erpnext-worker:edge restore-backup
+
+echo "Check Restored Site"
+sleep 3
+RESTORE_STATUS=$(curl -S http://test.localhost/api/method/version || echo "")
+INCREMENT=0
+while [[ -z "$RESTORE_STATUS" && $INCREMENT -lt 60 ]]; do
+    sleep 1
+    echo "Wait for restoration to complete ..."
+    RESTORE_STATUS=$(curl -S http://test.localhost/api/method/version || echo "")
+    ((INCREMENT=INCREMENT+1))
+    if [[ -z "$RESTORE_STATUS" && $INCREMENT -eq 60 ]]; then
+        CONTAINER_ID=$(docker-compose \
+            --project-name frappebench00 \
+            -f installation/docker-compose-common.yml \
+            -f installation/docker-compose-erpnext.yml \
+            -f installation/erpnext-publish.yml \
+            ps -q erpnext-python)
+        docker logs $CONTAINER_ID
+        exit 1
+    fi
+done
+
+echo "Ping restored site"
+echo $RESTORE_STATUS
+
+echo "Migrate command in edge container"
+docker run -it \
+    -e "MAINTENANCE_MODE=1" \
+    -v frappebench00_sites-vol:/home/frappe/frappe-bench/sites \
+    --network frappebench00_default \
+    frappe/erpnext-worker:edge migrate
+
+checkMigrationComplete
+
+echo "Create new site (edge)"
+docker run -it \
+    -e "SITE_NAME=edge.localhost" \
+    -e "INSTALL_APPS=erpnext" \
+    -v frappebench00_sites-vol:/home/frappe/frappe-bench/sites \
+    --network frappebench00_default \
+    frappe/erpnext-worker:edge new
+
+echo "Check console command for site test.localhost"
+docker run \
+    -v frappebench00_sites-vol:/home/frappe/frappe-bench/sites \
+    --network frappebench00_default \
+    frappe/erpnext-worker:edge console test.localhost

travis.py

@@ -15,7 +15,7 @@ image_type.add_argument('-s', '--socketio', action='store_const', dest='image_ty
 image_type.add_argument('-w', '--worker', action='store_const', dest='image_type', const='worker', help='Build the python environment image')
 
 tag_type = parser.add_mutually_exclusive_group(required=True)
-tag_type.add_argument('-g', '--git-version', action='store', type=int, dest='version', help='The version number of --service (i.e. "11", "12", etc.)')
+tag_type.add_argument('-g', '--git-version', action='store', type=str, dest='version', help='The version number of --service (i.e. "11", "12", etc.)')
 tag_type.add_argument('-t', '--tag', action='store', type=str, dest='tag', help='The image tag (i.e. erpnext-worker:$TAG )')
 
 args = parser.parse_args()
@@ -26,6 +26,10 @@ def git_version(service, version):
   cd = os.getcwd()
   os.chdir(os.getcwd() + f'/{service}')
   subprocess.run('git fetch --tags', shell=True)
+
+  # XX-beta becomes XX for tags search
+  version = version.split('-')[0]
+
   version_tag = subprocess.check_output(f'git tag --list --sort=-version:refname "v{version}*" | sed -n 1p | sed -e \'s#.*@\(\)#\\1#\'', shell=True).strip().decode('ascii')
   os.chdir(cd)
   return version_tag