Merge pull request #288 from revant/feat-frappe-postgres

.devcontainer/docker-compose.yml

@@ -9,6 +9,14 @@ services:
       - ../installation/frappe-mariadb.cnf:/etc/mysql/conf.d/frappe.cnf
       - mariadb-vol:/var/lib/mysql
 
+  postgresql:
+    image: postgres:11.8
+    restart: on-failure
+    environment:
+      - POSTGRES_PASSWORD=123
+    volumes:
+      - postgresql-vol:/var/lib/postgresql/data
+
   redis-cache:
     image: redis:alpine
 
@@ -30,3 +38,4 @@ services:
 
 volumes:
   mariadb-vol:
+  postgresql-vol:

build/bench/Dockerfile

@@ -6,6 +6,7 @@ RUN install_packages \
     git \
     wkhtmltopdf \
     mariadb-client \
+    postgresql-client \
     gettext-base \
     wget \
     # for PDF
@@ -84,8 +85,8 @@ RUN bash -c "bench --version"
 # https://nodejs.org/download/release/latest-v10.x/
 # https://nodejs.org/download/release/latest-v12.x/
 # https://nodejs.org/download/release/latest-v13.x/
-ENV NODE_VERSION=12.16.3
+ENV NODE_VERSION=12
-ENV NODE_VERSION_FRAPPEV11=10.20.1
+ENV NODE_VERSION_FRAPPEV11=10
 
 # Install nvm with node
 RUN wget https://raw.githubusercontent.com/nvm-sh/nvm/v0.35.3/install.sh

build/common/commands/backup.py

@@ -28,6 +28,10 @@ def main():
     with_files = cint(os.environ.get("WITH_FILES"))
 
     backup(sites, with_files)
+
+    if frappe.redis_server:
+        frappe.redis_server.connection_pool.disconnect()
+
     exit(0)
 
 

build/common/commands/check_connection.py

@@ -8,6 +8,7 @@ REDIS_QUEUE_KEY = 'redis_queue'
 REDIS_CACHE_KEY = 'redis_cache'
 REDIS_SOCKETIO_KEY = 'redis_socketio'
 DB_HOST_KEY = 'db_host'
+DB_PORT_KEY = 'db_port'
 DB_PORT = 3306
 
 
@@ -55,14 +56,14 @@ def get_config():
 # Check mariadb
 def check_mariadb(retry=10, delay=3, print_attempt=True):
     config = get_config()
-    check_mariadb = False
+    is_db_connected = False
-    check_mariadb = check_host(
+    is_db_connected = check_host(
         config.get(DB_HOST_KEY, 'mariadb'),
-        DB_PORT,
+        config.get(DB_PORT_KEY, DB_PORT),
         retry,
         delay,
         print_attempt)
-    if not check_mariadb:
+    if not is_db_connected:
         print("Connection to MariaDB timed out")
         exit(1)
 

build/common/commands/console.py

@@ -24,6 +24,8 @@ def console(site):
 def main():
     site = sys.argv[-1]
     console(site)
+    if frappe.redis_server:
+        frappe.redis_server.connection_pool.disconnect()
 
 
 if __name__ == "__main__":

build/common/commands/migrate.py

@@ -3,11 +3,11 @@ import frappe
 import json
 
 from frappe.utils import cint, get_sites
-from check_connection import get_config
+from check_connection import get_config, COMMON_SITE_CONFIG_FILE
 
 
 def save_config(config):
-    with open('common_site_config.json', 'w') as f:
+    with open(COMMON_SITE_CONFIG_FILE, 'w') as f:
         return json.dump(config, f, indent=1, sort_keys=True)
 
 
@@ -48,6 +48,8 @@ def migrate_sites(maintenance_mode=False):
 
 def main():
     migrate_sites()
+    if frappe.redis_server:
+        frappe.redis_server.connection_pool.disconnect()
     exit(0)
 
 

build/common/commands/new.py

@@ -1,8 +1,10 @@
 import os
 import frappe
+import semantic_version
 
 from frappe.commands.site import _new_site
-from check_connection import get_config, get_site_config
+from frappe.installer import update_site_config
+from check_connection import get_config, get_site_config, COMMON_SITE_CONFIG_FILE
 
 
 def get_password(env_var, default=None):
@@ -28,55 +30,96 @@ def _get_password_from_secret(env_var):
 
 
 def main():
+    config = get_config()
+    db_type = 'mariadb'
+    db_port = config.get('db_port', 3306)
+    db_host = config.get('db_host')
     site_name = os.environ.get("SITE_NAME", 'site1.localhost')
     mariadb_root_username = os.environ.get("DB_ROOT_USER", 'root')
     mariadb_root_password = get_password("MYSQL_ROOT_PASSWORD", 'admin')
+    postgres_root_password = get_password("POSTGRES_PASSWORD")
+
+    if postgres_root_password:
+        db_type = 'postgres'
+        db_host = os.environ.get("POSTGRES_HOST")
+        db_port = 5432
+        if not db_host:
+            db_host = config.get('db_host')
+            print('Environment variable POSTGRES_HOST not found.')
+            print('Using db_host from common_site_config.json')
+
+        sites_path = os.getcwd()
+        common_site_config_path = os.path.join(sites_path, COMMON_SITE_CONFIG_FILE)
+        update_site_config("root_login", mariadb_root_username, validate = False, site_config_path = common_site_config_path)
+        update_site_config("root_password", postgres_root_password, validate = False, site_config_path = common_site_config_path)
+
     force = True if os.environ.get("FORCE", None) else False
     install_apps = os.environ.get("INSTALL_APPS", None)
     install_apps = install_apps.split(',') if install_apps else []
     frappe.init(site_name, new_site=True)
 
-    _new_site(
+    if semantic_version.Version(frappe.__version__).major > 11:
-        None,
+        _new_site(
-        site_name,
+            None,
-        mariadb_root_username=mariadb_root_username,
+            site_name,
-        mariadb_root_password=mariadb_root_password,
+            mariadb_root_username=mariadb_root_username,
-        admin_password=get_password("ADMIN_PASSWORD", 'admin'),
+            mariadb_root_password=mariadb_root_password,
-        verbose=True,
+            admin_password=get_password("ADMIN_PASSWORD", 'admin'),
-        install_apps=install_apps,
+            verbose=True,
-        source_sql=None,
+            install_apps=install_apps,
-        force=force,
+            source_sql=None,
-        reinstall=False,
+            force=force,
-    )
+            db_type=db_type,
+            reinstall=False,
+            db_host=db_host,
+            db_port=db_port,
+        )
+    else:
+        _new_site(
+            None,
+            site_name,
+            mariadb_root_username=mariadb_root_username,
+            mariadb_root_password=mariadb_root_password,
+            admin_password=get_password("ADMIN_PASSWORD", 'admin'),
+            verbose=True,
+            install_apps=install_apps,
+            source_sql=None,
+            force=force,
+            reinstall=False,
+        )
 
-    config = get_config()
 
-    site_config = get_site_config(site_name)
+    if db_type == "mariadb":
+        site_config = get_site_config(site_name)
 
-    mysql_command = 'mysql -h{db_host} -u{mariadb_root_username} -p{mariadb_root_password} -e '.format(
+        mysql_command = 'mysql -h{db_host} -u{mariadb_root_username} -p{mariadb_root_password} -e '.format(
-        db_host=config.get('db_host'),
+            db_host=config.get('db_host'),
-        mariadb_root_username=mariadb_root_username,
+            mariadb_root_username=mariadb_root_username,
-        mariadb_root_password=mariadb_root_password
+            mariadb_root_password=mariadb_root_password
-    )
+        )
 
-    # update User's host to '%' required to connect from any container
+        # update User's host to '%' required to connect from any container
-    command = mysql_command + "\"UPDATE mysql.user SET Host = '%' where User = '{db_name}'; FLUSH PRIVILEGES;\"".format(
+        command = mysql_command + "\"UPDATE mysql.user SET Host = '%' where User = '{db_name}'; FLUSH PRIVILEGES;\"".format(
-        db_name=site_config.get('db_name')
+            db_name=site_config.get('db_name')
-    )
+        )
-    os.system(command)
+        os.system(command)
 
-    # Set db password
+        # Set db password
-    command = mysql_command + "\"ALTER USER '{db_name}'@'%' IDENTIFIED BY '{db_password}'; FLUSH PRIVILEGES;\"".format(
+        command = mysql_command + "\"ALTER USER '{db_name}'@'%' IDENTIFIED BY '{db_password}'; FLUSH PRIVILEGES;\"".format(
-        db_name=site_config.get('db_name'),
+            db_name=site_config.get('db_name'),
-        db_password=site_config.get('db_password')
+            db_password=site_config.get('db_password')
-    )
+        )
-    os.system(command)
+        os.system(command)
+
+        # Grant permission to database
+        command = mysql_command + "\"GRANT ALL PRIVILEGES ON \`{db_name}\`.* TO '{db_name}'@'%'; FLUSH PRIVILEGES;\"".format(
+            db_name=site_config.get('db_name')
+        )
+        os.system(command)
+
+    if frappe.redis_server:
+        frappe.redis_server.connection_pool.disconnect()
 
-    # Grant permission to database
-    command = mysql_command + "\"GRANT ALL PRIVILEGES ON \`{db_name}\`.* TO '{db_name}'@'%'; FLUSH PRIVILEGES;\"".format(
-        db_name=site_config.get('db_name')
-    )
-    os.system(command)
     exit(0)
 
 

build/common/commands/restore_backup.py

@@ -10,7 +10,7 @@ from new import get_password
 from push_backup import DATE_FORMAT, check_environment_variables
 from frappe.utils import get_sites, random_string
 from frappe.installer import make_conf, get_conf_params, make_site_dirs, update_site_config
-from check_connection import get_site_config, get_config
+from check_connection import get_site_config, get_config, COMMON_SITE_CONFIG_FILE
 
 
 def list_directories(path):
@@ -40,71 +40,58 @@ def decompress_db(files_base, site):
 
 
 def restore_database(files_base, site_config_path, site):
-    db_root_password = get_password('MYSQL_ROOT_PASSWORD')
-    if not db_root_password:
-        print('Variable MYSQL_ROOT_PASSWORD not set')
-        exit(1)
-
-    db_root_user = os.environ.get("DB_ROOT_USER", 'root')
-
     # restore database
     database_file = files_base + '-database.sql.gz'
     decompress_db(files_base, site)
     config = get_config()
+
+    # Set db_type if it exists in backup site_config.json
+    set_key_in_site_config('db_type', site, site_config_path)
+    # Set db_host if it exists in backup site_config.json
+    set_key_in_site_config('db_host', site, site_config_path)
+    # Set db_port if it exists in backup site_config.json
+    set_key_in_site_config('db_port', site, site_config_path)
+
+    # get updated site_config
     site_config = get_site_config(site)
 
-    # mysql command prefix
+    # if no db_type exists, default to mariadb
-    mysql_command = 'mysql -u{db_root_user} -h{db_host} -p{db_password} -e '.format(
+    db_type = site_config.get('db_type', 'mariadb')
-        db_root_user=db_root_user,
+    is_database_restored = False
-        db_host=config.get('db_host'),
-        db_password=db_root_password
-    )
 
-    # drop db if exists for clean restore
+    if db_type == 'mariadb':
-    drop_database = mysql_command + "\"DROP DATABASE IF EXISTS \`{db_name}\`;\"".format(
+        restore_mariadb(
-        db_name=site_config.get('db_name')
+            config=config,
-    )
+            site_config=site_config,
-    os.system(drop_database)
+            database_file=database_file)
+        is_database_restored = True
+    elif db_type == 'postgres':
+        restore_postgres(
+            config=config,
+            site_config=site_config,
+            database_file=database_file)
+        is_database_restored = True
 
-    # create db
+    if is_database_restored:
-    create_database = mysql_command + "\"CREATE DATABASE IF NOT EXISTS \`{db_name}\`;\"".format(
+        # Set encryption_key if it exists in backup site_config.json
-        db_name=site_config.get('db_name')
+        set_key_in_site_config('encryption_key', site, site_config_path)
-    )
-    os.system(create_database)
 
-    # create user
-    create_user = mysql_command + "\"CREATE USER IF NOT EXISTS \'{db_name}\'@\'%\' IDENTIFIED BY \'{db_password}\'; FLUSH PRIVILEGES;\"".format(
-        db_name=site_config.get('db_name'),
-        db_password=site_config.get('db_password')
-    )
-    os.system(create_user)
 
-    # grant db privileges to user
+def set_key_in_site_config(key, site, site_config_path):
-    grant_privileges = mysql_command + "\"GRANT ALL PRIVILEGES ON \`{db_name}\`.* TO '{db_name}'@'%' IDENTIFIED BY '{db_password}'; FLUSH PRIVILEGES;\"".format(
+    site_config = get_site_config_from_path(site_config_path)
-        db_name=site_config.get('db_name'),
+    value = site_config.get(key)
-        db_password=site_config.get('db_password')
+    if value:
-    )
+        print('Set {key} in site config for site: {site}'.format(key=key, site=site))
-    os.system(grant_privileges)
+        update_site_config(key, value,
+                            site_config_path=os.path.join(os.getcwd(), site, "site_config.json"))
 
-    command = "mysql -u{db_root_user} -h{db_host} -p{db_password} '{db_name}' < {database_file}".format(
-        db_root_user=db_root_user,
-        db_host=config.get('db_host'),
-        db_password=db_root_password,
-        db_name=site_config.get('db_name'),
-        database_file=database_file.replace('.gz', ''),
-    )
-
-    print('Restoring database for site: {}'.format(site))
-    os.system(command)
 
+def get_site_config_from_path(site_config_path):
+    site_config = dict()
     if os.path.exists(site_config_path):
         with open(site_config_path, 'r') as sc:
             site_config = json.load(sc)
-        encryption_key = site_config.get("encryption_key")
+    return site_config
-        if encryption_key:
-            print('Restoring site config for site: {}'.format(site))
-            update_site_config('encryption_key', encryption_key,
-                               site_config_path=os.path.join(os.getcwd(), site, "site_config.json"))
 
 
 def restore_files(files_base):
@@ -178,6 +165,127 @@ def pull_backup_from_s3():
     os.chdir(os.path.join(os.path.expanduser('~'), 'frappe-bench', 'sites'))
 
 
+def restore_postgres(config, site_config, database_file):
+    # common config
+    common_site_config_path = os.path.join(os.getcwd(), COMMON_SITE_CONFIG_FILE)
+
+    db_root_user = config.get('root_login')
+    if not db_root_user:
+        postgres_user = os.environ.get('DB_ROOT_USER')
+        if not postgres_user:
+            print('Variable DB_ROOT_USER not set')
+            exit(1)
+
+        db_root_user = postgres_user
+        update_site_config(
+            "root_login",
+            db_root_user,
+            validate=False,
+            site_config_path=common_site_config_path)
+
+    db_root_password = config.get('root_password')
+    if not db_root_password:
+        root_password = get_password('POSTGRES_PASSWORD')
+        if not root_password:
+            print('Variable POSTGRES_PASSWORD not set')
+            exit(1)
+
+        db_root_password = root_password
+        update_site_config(
+            "root_password",
+            db_root_password,
+            validate=False,
+            site_config_path=common_site_config_path)
+
+    # site config
+    db_host = site_config.get('db_host')
+    db_port = site_config.get('db_port', 5432)
+    db_name = site_config.get('db_name')
+    db_password = site_config.get('db_password')
+
+    psql_command = "psql postgres://{root_login}:{root_password}@{db_host}:{db_port}".format(
+        root_login=db_root_user,
+        root_password=db_root_password,
+        db_host=db_host,
+        db_port=db_port
+    )
+
+    print('Restoring PostgreSQL')
+    os.system(psql_command + ' -c "DROP DATABASE IF EXISTS \"{db_name}\""'.format(db_name=db_name))
+    os.system(psql_command + ' -c "DROP USER IF EXISTS {db_name}"'.format(db_name=db_name))
+    os.system(psql_command + ' -c "CREATE DATABASE \"{db_name}\""'.format(db_name=db_name))
+    os.system(psql_command + ' -c "CREATE user {db_name} password \'{db_password}\'"'.format(
+        db_name=db_name,
+        db_password=db_password))
+    os.system(psql_command + ' -c "GRANT ALL PRIVILEGES ON DATABASE \"{db_name}\" TO {db_name}"'.format(
+        db_name=db_name))
+
+    os.system("{psql_command}/{db_name} < {database_file}".format(
+        psql_command=psql_command,
+        database_file=database_file.replace('.gz', ''),
+        db_name=db_name,
+    ))
+
+
+def restore_mariadb(config, site_config, database_file):
+    db_root_password = get_password('MYSQL_ROOT_PASSWORD')
+    if not db_root_password:
+        print('Variable MYSQL_ROOT_PASSWORD not set')
+        exit(1)
+
+    db_root_user = os.environ.get("DB_ROOT_USER", 'root')
+
+    db_host = site_config.get('db_host', config.get('db_host'))
+    db_port = site_config.get('db_port', config.get('db_port'))
+
+    # mysql command prefix
+    mysql_command = 'mysql -u{db_root_user} -h{db_host} -p{db_password}'.format(
+        db_root_user=db_root_user,
+        db_host=db_host,
+        db_port=db_port,
+        db_password=db_root_password
+    )
+
+    # drop db if exists for clean restore
+    drop_database = "{mysql_command} -e \"DROP DATABASE IF EXISTS \`{db_name}\`;\"".format(
+        mysql_command=mysql_command,
+        db_name=site_config.get('db_name'),
+    )
+    os.system(drop_database)
+
+    # create db
+    create_database = "{mysql_command} -e \"CREATE DATABASE IF NOT EXISTS \`{db_name}\`;\"".format(
+        mysql_command=mysql_command,
+        db_name=site_config.get('db_name'),
+    )
+    os.system(create_database)
+
+    # create user
+    create_user = "{mysql_command} -e \"CREATE USER IF NOT EXISTS \'{db_name}\'@\'%\' IDENTIFIED BY \'{db_password}\'; FLUSH PRIVILEGES;\"".format(
+        mysql_command=mysql_command,
+        db_name=site_config.get('db_name'),
+        db_password=site_config.get('db_password'),
+    )
+    os.system(create_user)
+
+    # grant db privileges to user
+    grant_privileges = "{mysql_command} -e \"GRANT ALL PRIVILEGES ON \`{db_name}\`.* TO '{db_name}'@'%' IDENTIFIED BY '{db_password}'; FLUSH PRIVILEGES;\"".format(
+        mysql_command=mysql_command,
+        db_name=site_config.get('db_name'),
+        db_password=site_config.get('db_password'),
+    )
+    os.system(grant_privileges)
+
+    command = "{mysql_command} '{db_name}' < {database_file}".format(
+        mysql_command=mysql_command,
+        db_name=site_config.get('db_name'),
+        database_file=database_file.replace('.gz', ''),
+    )
+
+    print('Restoring MariaDB')
+    os.system(command)
+
+
 def main():
     backup_dir = get_backup_dir()
 
@@ -194,15 +302,11 @@ def main():
         if not os.path.exists(site_config_path):
             site_config_path = os.path.join(backup_dir, site, 'site_config.json')
         if site in get_sites():
+            print('Overwrite site {}'.format(site))
             restore_database(files_base, site_config_path, site)
             restore_private_files(files_base)
             restore_files(files_base)
         else:
-            mariadb_root_password = get_password('MYSQL_ROOT_PASSWORD')
-            if not mariadb_root_password:
-                print('Variable MYSQL_ROOT_PASSWORD not set')
-                exit(1)
-
             site_config = get_conf_params(
                 db_name='_' + hashlib.sha1(site.encode()).hexdigest()[:16],
                 db_password=random_string(16)
@@ -216,10 +320,15 @@ def main():
                 db_password=site_config.get('db_password'),
             )
             make_site_dirs()
+
+            print('Create site {}'.format(site))
             restore_database(files_base, site_config_path, site)
             restore_private_files(files_base)
             restore_files(files_base)
 
+    if frappe.redis_server:
+        frappe.redis_server.connection_pool.disconnect()
+
     exit(0)
 
 

build/common/common_site_config.json.template

@@ -1,5 +1,6 @@
 {
-    "db_host": "${MARIADB_HOST}",
+    "db_host": "${DB_HOST}",
+    "db_port": ${DB_PORT},
     "redis_cache": "redis://${REDIS_CACHE}",
     "redis_queue": "redis://${REDIS_QUEUE}",
     "redis_socketio": "redis://${REDIS_SOCKETIO}",

build/common/worker/docker-entrypoint.sh

@@ -4,8 +4,10 @@ function configureEnv() {
   if [ ! -f /home/frappe/frappe-bench/sites/common_site_config.json ]; then
 
     if [[ -z "$MARIADB_HOST" ]]; then
-      echo "MARIADB_HOST is not set"
+      if [[ -z "$POSTGRES_HOST" ]]; then
-      exit 1
+        echo "MARIADB_HOST or POSTGRES_HOST is not set"
+        exit 1
+      fi
     fi
 
     if [[ -z "$REDIS_CACHE" ]]; then
@@ -28,7 +30,14 @@ function configureEnv() {
       exit 1
     fi
 
-    envsubst '${MARIADB_HOST}
+    if [[ -z "$DB_PORT" ]]; then
+      export DB_PORT=3306
+    fi
+
+    export DB_HOST="${MARIADB_HOST:-$POSTGRES_HOST}"
+
+    envsubst '${DB_HOST}
+      ${DB_PORT}
       ${REDIS_CACHE}
       ${REDIS_QUEUE}
       ${REDIS_SOCKETIO}

build/erpnext-nginx/Dockerfile

@@ -11,7 +11,7 @@ FROM frappe/frappe-nginx:${GIT_BRANCH}
 
 COPY --from=0 /home/frappe/frappe-bench/sites/ /var/www/html/
 COPY --from=0 /rsync /rsync
-RUN echo -n "\nerpnext" >> /var/www/html/apps.txt
+RUN echo "erpnext" >> /var/www/html/apps.txt
 
 VOLUME [ "/assets" ]
 

build/erpnext-nginx/install_app.sh

@@ -19,7 +19,7 @@ git clone --depth 1 ${APP_REPO} ${BRANCH}
 
 cd /home/frappe/frappe-bench/apps/frappe
 yarn
-yarn production
+yarn production --app ${APP_NAME}
 rm -fr node_modules
 yarn install --production=true
 

development/README.md

@@ -104,6 +104,23 @@ This will create a new site and a `mysite.localhost` directory under `frappe-ben
 The option `--no-mariadb-socket` will configure site's database credentials to work with docker.
 You may need to configure your system /etc/hosts if you're on Linux, Mac, or its Windows equivalent.
 
+To setup site with PostgreSQL as database use option `--db-type postgres` and `--db-host postgresql`. (Available only v12 onwards, currently NOT available for ERPNext).
+
+Example:
+
+```shell
+bench new-site mypgsql.localhost --db-type postgres --db-host postgresql
+```
+
+To avoid entering postgresql username and root password, set it in `common_site_config.json`,
+
+```shell
+bench config set-common-config -c root_login postgres
+bench config set-common-config -c root_password '"123"'
+```
+
+Note: If PostgreSQL is not required, the postgresql service / container can be stopped.
+
 ### Set bench developer mode on the new site
 
 To develop a new app, the last step will be setting the site into developer mode. Documentation is available at [this link](https://frappe.io/docs/user/en/guides/app-development/how-enable-developer-mode-in-frappe).

docs/site-operations.md

@@ -12,10 +12,12 @@ Or specify environment variables instead of passing secrets as command arguments
 
 Note:
 
-- Wait for the MariaDB service to start before trying to create a new site.
+- Wait for the database service to start before trying to create a new site.
     - If new site creation fails, retry after the MariaDB container is up and running.
     - If you're using a managed database instance, make sure that the database is running before setting up a new site.
 
+#### MariaDB Site
+
 ```sh
 # Create ERPNext site
 docker run \
@@ -29,10 +31,28 @@ docker run \
     frappe/erpnext-worker:$VERSION new
 ```
 
+#### PostgreSQL Site
+
+PostgreSQL is only available v12 onwards. It is NOT available for ERPNext.
+It is available as part of `frappe/erpnext-worker`. It inherits from `frappe/frappe-worker`.
+
+```sh
+# Create ERPNext site
+docker run \
+    -e "SITE_NAME=$SITE_NAME" \
+    -e "DB_ROOT_USER=$DB_ROOT_USER" \
+    -e "POSTGRES_HOST=$POSTGRES_HOST" \
+    -e "POSTGRES_PASSWORD=$POSTGRES_PASSWORD" \
+    -e "ADMIN_PASSWORD=$ADMIN_PASSWORD" \
+    -v <project-name>_sites-vol:/home/frappe/frappe-bench/sites \
+    --network <project-name>_default \
+    frappe/erpnext-worker:$VERSION new
+```
+
 Environment Variables needed:
 
 - `SITE_NAME`: name of the new site to create. Site name is domain name that resolves. e.g. `erp.example.com` or `erp.localhost`.
-- `DB_ROOT_USER`: MariaDB Root user.
+- `DB_ROOT_USER`: MariaDB/PostgreSQL Root user.
 - `MYSQL_ROOT_PASSWORD`: In case of the MariaDB docker container use the one set in `MYSQL_ROOT_PASSWORD` in previous steps. In case of a managed database use the appropriate password.
 - `MYSQL_ROOT_PASSWORD_FILE` - When the MariaDB root password is stored using docker secrets.
 - `ADMIN_PASSWORD`: set the administrator password for the new site.
@@ -40,6 +60,16 @@ Environment Variables needed:
 - `INSTALL_APPS=erpnext`: available only in erpnext-worker and erpnext containers (or other containers with custom apps). Installs ERPNext (and/or the specified apps, comma-delinieated) on this new site.
 - `FORCE=1`: optional variable which force installation of the same site.
 
+Environment Variables for PostgreSQL only:
+
+- `POSTGRES_HOST`: host for PostgreSQL server
+- `POSTGRES_PASSWORD`: Password for `postgres`. The database root user.
+
+Notes:
+
+- To setup existing frappe-bench deployment with default database as PostgreSQL edit the common_site_config.json and set `db_host` to PostgreSQL hostname and `db_port` to PostgreSQL port.
+- To create new frappe-bench deployment with default database as PostgreSQL use `POSTGRES_HOST` and `DB_PORT` environment variables in `erpnext-python` service instead of `MARIADB_HOST`
+
 ## Add sites to proxy
 
 Change `SITES` variable to the list of sites created encapsulated in backtick and separated by comma with no space. e.g. ``SITES=`site1.example.com`,`site2.example.com` ``.

installation/frappe-postgresql/docker-compose.yml

@@ -0,0 +1,166 @@
+version: "3"
+
+services:
+  traefik:
+    image: "traefik:v2.2"
+    command:
+      - "--log.level=DEBUG"
+      - "--providers.docker=true"
+      - "--providers.docker.exposedbydefault=false"
+      - "--entrypoints.web.address=:80"
+      - "--entrypoints.websecure.address=:443"
+      - "--certificatesresolvers.myresolver.acme.httpchallenge=true"
+      - "--certificatesresolvers.myresolver.acme.httpchallenge.entrypoint=web"
+      - "--certificatesresolvers.myresolver.acme.email=${LETSENCRYPT_EMAIL}"
+      - "--certificatesresolvers.myresolver.acme.storage=/letsencrypt/acme.json"
+    labels:
+      # enable traefik
+      - "traefik.enable=true"
+      # global redirect to https for production only
+      - "${HTTPS_REDIRECT_RULE_LABEL}"
+      - "${HTTPS_REDIRECT_ENTRYPOINT_LABEL}"
+      - "${HTTPS_REDIRECT_MIDDLEWARE_LABEL}"
+      # middleware redirect for production only
+      - "${HTTPS_USE_REDIRECT_MIDDLEWARE_LABEL}"
+    ports:
+      - "80:80"
+      - "443:443"
+    volumes:
+      - cert-vol:/letsencrypt
+      - /var/run/docker.sock:/var/run/docker.sock:ro
+
+  frappe-nginx:
+    image: frappe/frappe-nginx:${FRAPPE_VERSION}
+    restart: on-failure
+    environment:
+      - FRAPPE_PY=frappe-python
+      - FRAPPE_PY_PORT=8000
+      - FRAPPE_SOCKETIO=frappe-socketio
+      - SOCKETIO_PORT=9000
+    labels:
+      - "traefik.enable=true"
+      - "traefik.http.routers.frappe-nginx.rule=Host(${SITES})"
+      - "${ENTRYPOINT_LABEL}"
+      - "${CERT_RESOLVER_LABEL}"
+      - "traefik.http.services.frappe-nginx.loadbalancer.server.port=80"
+    volumes:
+      - sites-vol:/var/www/html/sites:rw
+      - assets-vol:/assets:rw
+
+  frappe-python:
+    image: frappe/frappe-worker:${FRAPPE_VERSION}
+    restart: on-failure
+    environment:
+      - POSTGRES_HOST=${POSTGRES_HOST}
+      - DB_PORT=5432
+      - REDIS_CACHE=redis-cache:6379
+      - REDIS_QUEUE=redis-queue:6379
+      - REDIS_SOCKETIO=redis-socketio:6379
+      - SOCKETIO_PORT=9000
+      - AUTO_MIGRATE=1
+    volumes:
+      - sites-vol:/home/frappe/frappe-bench/sites:rw
+      - assets-vol:/home/frappe/frappe-bench/sites/assets:rw
+
+  frappe-socketio:
+    image: frappe/frappe-socketio:${FRAPPE_VERSION}
+    restart: on-failure
+    depends_on:
+      - redis-socketio
+    volumes:
+      - sites-vol:/home/frappe/frappe-bench/sites:rw
+
+  frappe-worker-default:
+    image: frappe/frappe-worker:${FRAPPE_VERSION}
+    restart: on-failure
+    command: worker
+    depends_on:
+      - redis-queue
+      - redis-cache
+    volumes:
+      - sites-vol:/home/frappe/frappe-bench/sites:rw
+
+  frappe-worker-short:
+    image: frappe/frappe-worker:${FRAPPE_VERSION}
+    restart: on-failure
+    command: worker
+    environment:
+      - WORKER_TYPE=short
+    depends_on:
+      - redis-queue
+      - redis-cache
+    volumes:
+      - sites-vol:/home/frappe/frappe-bench/sites:rw
+
+  frappe-worker-long:
+    image: frappe/frappe-worker:${FRAPPE_VERSION}
+    restart: on-failure
+    command: worker
+    environment:
+      - WORKER_TYPE=long
+    depends_on:
+      - redis-queue
+      - redis-cache
+    volumes:
+      - sites-vol:/home/frappe/frappe-bench/sites:rw
+
+  frappe-schedule:
+    image: frappe/frappe-worker:${FRAPPE_VERSION}
+    restart: on-failure
+    command: schedule
+    depends_on:
+      - redis-queue
+      - redis-cache
+    volumes:
+      - sites-vol:/home/frappe/frappe-bench/sites:rw
+
+  redis-cache:
+    image: redis:latest
+    restart: on-failure
+    volumes:
+      - redis-cache-vol:/data
+
+  redis-queue:
+    image: redis:latest
+    restart: on-failure
+    volumes:
+      - redis-queue-vol:/data
+
+  redis-socketio:
+    image: redis:latest
+    restart: on-failure
+    volumes:
+      - redis-socketio-vol:/data
+
+  postgresql:
+    image: postgres:11.8
+    restart: on-failure
+    environment:
+      - POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
+    volumes:
+      - postgresql-vol:/var/lib/postgresql/data
+
+  site-creator:
+    image: frappe/frappe-worker:${FRAPPE_VERSION}
+    restart: "no"
+    command: new
+    depends_on:
+      - frappe-python
+    environment:
+      - POSTGRES_HOST=${POSTGRES_HOST}
+      - SITE_NAME=${SITE_NAME}
+      - DB_ROOT_USER=${DB_ROOT_USER}
+      - POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
+      - ADMIN_PASSWORD=${ADMIN_PASSWORD}
+      - INSTALL_APPS=${INSTALL_APPS}
+    volumes:
+      - sites-vol:/home/frappe/frappe-bench/sites:rw
+
+volumes:
+  postgresql-vol:
+  redis-cache-vol:
+  redis-queue-vol:
+  redis-socketio-vol:
+  assets-vol:
+  sites-vol:
+  cert-vol:

installation/frappe-postgresql/env-local

@@ -0,0 +1,14 @@
+LETSENCRYPT_EMAIL=email@example.com
+FRAPPE_VERSION=edge
+POSTGRES_HOST=postgresql
+POSTGRES_PASSWORD=admin
+SITE_NAME=mysite.localhost
+SITES=`mysite.localhost`
+DB_ROOT_USER=postgres
+ADMIN_PASSWORD=admin
+ENTRYPOINT_LABEL=traefik.http.routers.erpnext-nginx.entrypoints=web
+CERT_RESOLVER_LABEL=erpnext.local.no-cert-resolver
+HTTPS_REDIRECT_RULE_LABEL=erpnext.local.no-redirect-rule
+HTTPS_REDIRECT_ENTRYPOINT_LABEL=erpnext.local.no-entrypoint
+HTTPS_REDIRECT_MIDDLEWARE_LABEL=erpnext.local.no-middleware
+HTTPS_USE_REDIRECT_MIDDLEWARE_LABEL=erpnext.local-no-redirect-middleware

installation/frappe-postgresql/env-production

@@ -0,0 +1,14 @@
+LETSENCRYPT_EMAIL=email@example.com
+FRAPPE_VERSION=edge
+POSTGRES_HOST=postgresql
+POSTGRES_PASSWORD=admin
+SITE_NAME=erp.example.com
+SITES=`erp.example.com`
+DB_ROOT_USER=postgres
+ADMIN_PASSWORD=admin
+ENTRYPOINT_LABEL=traefik.http.routers.erpnext-nginx.entrypoints=websecure
+CERT_RESOLVER_LABEL=traefik.http.routers.erpnext-nginx.tls.certresolver=myresolver
+HTTPS_REDIRECT_RULE_LABEL=traefik.http.routers.http-catchall.rule=hostregexp(`{host:.+}`)
+HTTPS_REDIRECT_ENTRYPOINT_LABEL=traefik.http.routers.http-catchall.entrypoints=web
+HTTPS_REDIRECT_MIDDLEWARE_LABEL=traefik.http.routers.http-catchall.middlewares=redirect-to-https
+HTTPS_USE_REDIRECT_MIDDLEWARE_LABEL=traefik.http.middlewares.redirect-to-https.redirectscheme.scheme=https

tests/docker-test.sh

@@ -59,12 +59,19 @@ docker-compose \
     -f installation/docker-compose-erpnext.yml \
     -f installation/erpnext-publish.yml \
     pull
+docker pull postgres:11.8
 docker-compose \
     --project-name frappebench00 \
     -f installation/docker-compose-common.yml \
     -f installation/docker-compose-erpnext.yml \
     -f installation/erpnext-publish.yml \
     up -d
+# Start postgres
+docker run --name postgresql -d \
+    -e "POSTGRES_PASSWORD=admin" \
+    -v frappebench00_sites-vol:/home/frappe/frappe-bench/sites \
+    --network frappebench00_default \
+    postgres:11.8
 
 loopHealthCheck
 echo -e "\n"
@@ -116,6 +123,47 @@ echo -e "\e[1m\e[4mCheck Migrated Site Index Page\e[0m"
 curl -s http://test.localhost | w3m -T text/html -dump
 echo -e "\n"
 
+echo -e "\e[1m\e[4mCreate new site (pgsql)\e[0m"
+docker run -it \
+    -e "SITE_NAME=pgsql.localhost" \
+    -e "POSTGRES_HOST=postgresql" \
+    -e "DB_ROOT_USER=postgres" \
+    -e "POSTGRES_PASSWORD=admin" \
+    -v frappebench00_sites-vol:/home/frappe/frappe-bench/sites \
+    --network frappebench00_default \
+    frappe/erpnext-worker:edge new
+echo -e "\n"
+
+echo -e "\e[1m\e[4mCheck New PGSQL Site\e[0m"
+sleep 3
+RESTORE_STATUS=$(curl -sS http://pgsql.localhost/api/method/version || echo "")
+INCREMENT=0
+while [[ -z "$RESTORE_STATUS" && $INCREMENT -lt 60 ]]; do
+    sleep 1
+    echo -e "\e[1m\e[4mWait for restoration to complete ..."
+    RESTORE_STATUS=$(curl -sS http://pgsql.localhost/api/method/version || echo "")
+    ((INCREMENT=INCREMENT+1))
+    if [[ -z "$RESTORE_STATUS" && $INCREMENT -eq 60 ]]; then
+        CONTAINER_ID=$(docker-compose \
+            --project-name frappebench00 \
+            -f installation/docker-compose-common.yml \
+            -f installation/docker-compose-erpnext.yml \
+            -f installation/erpnext-publish.yml \
+            ps -q erpnext-python)
+        docker logs $CONTAINER_ID
+        exit 1
+    fi
+done
+echo -e "\n"
+
+echo -e "\e[1m\e[4mPing new pgsql site\e[0m"
+echo $RESTORE_STATUS
+echo -e "\n"
+
+echo -e "\e[1m\e[4mCheck New PGSQL Index Page\e[0m"
+curl -s http://pgsql.localhost | w3m -T text/html -dump
+echo -e "\n"
+
 echo -e "\e[1m\e[4mBackup site\e[0m"
 docker run -it \
     -e "WITH_FILES=1" \
@@ -198,7 +246,7 @@ docker run \
     frappe/erpnext-worker:edge restore-backup
 echo -e "\n"
 
-echo -e "\e[1m\e[4mCheck Restored Site\e[0m"
+echo -e "\e[1m\e[4mCheck Restored Site (test)\e[0m"
 sleep 3
 RESTORE_STATUS=$(curl -sS http://test.localhost/api/method/version || echo "")
 INCREMENT=0
@@ -219,14 +267,43 @@ while [[ -z "$RESTORE_STATUS" && $INCREMENT -lt 60 ]]; do
     fi
 done
 
-echo -e "\e[1m\e[4mPing restored site\e[0m"
+echo -e "\e[1m\e[4mPing restored site (test)\e[0m"
 echo $RESTORE_STATUS
 echo -e "\n"
 
-echo -e "\e[1m\e[4mCheck Restored Site Index Page\e[0m"
+echo -e "\e[1m\e[4mCheck Restored Site Index Page (test)\e[0m"
 curl -s http://test.localhost | w3m -T text/html -dump
 echo -e "\n"
 
+echo -e "\e[1m\e[4mCheck Restored Site (pgsql)\e[0m"
+sleep 3
+RESTORE_STATUS=$(curl -sS http://pgsql.localhost/api/method/version || echo "")
+INCREMENT=0
+while [[ -z "$RESTORE_STATUS" && $INCREMENT -lt 60 ]]; do
+    sleep 1
+    echo "Wait for restoration to complete ..."
+    RESTORE_STATUS=$(curl -sS http://pgsql.localhost/api/method/version || echo "")
+    ((INCREMENT=INCREMENT+1))
+    if [[ -z "$RESTORE_STATUS" && $INCREMENT -eq 60 ]]; then
+        CONTAINER_ID=$(docker-compose \
+            --project-name frappebench00 \
+            -f installation/docker-compose-common.yml \
+            -f installation/docker-compose-erpnext.yml \
+            -f installation/erpnext-publish.yml \
+            ps -q erpnext-python)
+        docker logs $CONTAINER_ID
+        exit 1
+    fi
+done
+
+echo -e "\e[1m\e[4mPing restored site (pgsql)\e[0m"
+echo $RESTORE_STATUS
+echo -e "\n"
+
+echo -e "\e[1m\e[4mCheck Restored Site Index Page (pgsql)\e[0m"
+curl -s http://pgsql.localhost | w3m -T text/html -dump
+echo -e "\n"
+
 echo -e "\e[1m\e[4mCreate new site (edge)\e[0m"
 docker run -it \
     -e "SITE_NAME=edge.localhost" \
@@ -325,3 +402,9 @@ docker run \
     -v frappebench00_sites-vol:/home/frappe/frappe-bench/sites \
     --network frappebench00_default \
     frappe/erpnext-worker:edge console test.localhost
+
+echo -e "\e[1m\e[4mCheck console command for site pgsql.localhost\e[0m"
+docker run \
+    -v frappebench00_sites-vol:/home/frappe/frappe-bench/sites \
+    --network frappebench00_default \
+    frappe/erpnext-worker:edge console pgsql.localhost