diff --git a/.devcontainer/docker-compose.yml b/.devcontainer/docker-compose.yml index b0a1e6d6..104c5442 100644 --- a/.devcontainer/docker-compose.yml +++ b/.devcontainer/docker-compose.yml @@ -9,6 +9,14 @@ services: - ../installation/frappe-mariadb.cnf:/etc/mysql/conf.d/frappe.cnf - mariadb-vol:/var/lib/mysql + postgresql: + image: postgres:11.8 + restart: on-failure + environment: + - POSTGRES_PASSWORD=123 + volumes: + - postgresql-vol:/var/lib/postgresql/data + redis-cache: image: redis:alpine @@ -30,3 +38,4 @@ services: volumes: mariadb-vol: + postgresql-vol: diff --git a/build/bench/Dockerfile b/build/bench/Dockerfile index 7a215911..e237ae3c 100644 --- a/build/bench/Dockerfile +++ b/build/bench/Dockerfile @@ -6,6 +6,7 @@ RUN install_packages \ git \ wkhtmltopdf \ mariadb-client \ + postgresql-client \ gettext-base \ wget \ # for PDF @@ -84,8 +85,8 @@ RUN bash -c "bench --version" # https://nodejs.org/download/release/latest-v10.x/ # https://nodejs.org/download/release/latest-v12.x/ # https://nodejs.org/download/release/latest-v13.x/ -ENV NODE_VERSION=12.16.3 -ENV NODE_VERSION_FRAPPEV11=10.20.1 +ENV NODE_VERSION=12 +ENV NODE_VERSION_FRAPPEV11=10 # Install nvm with node RUN wget https://raw.githubusercontent.com/nvm-sh/nvm/v0.35.3/install.sh diff --git a/build/common/commands/backup.py b/build/common/commands/backup.py index c26468dd..10ed6a92 100644 --- a/build/common/commands/backup.py +++ b/build/common/commands/backup.py @@ -28,6 +28,10 @@ def main(): with_files = cint(os.environ.get("WITH_FILES")) backup(sites, with_files) + + if frappe.redis_server: + frappe.redis_server.connection_pool.disconnect() + exit(0) diff --git a/build/common/commands/check_connection.py b/build/common/commands/check_connection.py index 35fe93ff..cd87e44b 100644 --- a/build/common/commands/check_connection.py +++ b/build/common/commands/check_connection.py @@ -8,6 +8,7 @@ REDIS_QUEUE_KEY = 'redis_queue' REDIS_CACHE_KEY = 'redis_cache' REDIS_SOCKETIO_KEY = 'redis_socketio' DB_HOST_KEY = 'db_host' +DB_PORT_KEY = 'db_port' DB_PORT = 3306 @@ -55,14 +56,14 @@ def get_config(): # Check mariadb def check_mariadb(retry=10, delay=3, print_attempt=True): config = get_config() - check_mariadb = False - check_mariadb = check_host( + is_db_connected = False + is_db_connected = check_host( config.get(DB_HOST_KEY, 'mariadb'), - DB_PORT, + config.get(DB_PORT_KEY, DB_PORT), retry, delay, print_attempt) - if not check_mariadb: + if not is_db_connected: print("Connection to MariaDB timed out") exit(1) diff --git a/build/common/commands/console.py b/build/common/commands/console.py index 369f049a..b5c06b5e 100644 --- a/build/common/commands/console.py +++ b/build/common/commands/console.py @@ -24,6 +24,8 @@ def console(site): def main(): site = sys.argv[-1] console(site) + if frappe.redis_server: + frappe.redis_server.connection_pool.disconnect() if __name__ == "__main__": diff --git a/build/common/commands/migrate.py b/build/common/commands/migrate.py index 9248a130..08550a94 100644 --- a/build/common/commands/migrate.py +++ b/build/common/commands/migrate.py @@ -3,11 +3,11 @@ import frappe import json from frappe.utils import cint, get_sites -from check_connection import get_config +from check_connection import get_config, COMMON_SITE_CONFIG_FILE def save_config(config): - with open('common_site_config.json', 'w') as f: + with open(COMMON_SITE_CONFIG_FILE, 'w') as f: return json.dump(config, f, indent=1, sort_keys=True) @@ -48,6 +48,8 @@ def migrate_sites(maintenance_mode=False): def main(): migrate_sites() + if frappe.redis_server: + frappe.redis_server.connection_pool.disconnect() exit(0) diff --git a/build/common/commands/new.py b/build/common/commands/new.py index 4771b21d..cab9bc61 100644 --- a/build/common/commands/new.py +++ b/build/common/commands/new.py @@ -1,8 +1,10 @@ import os import frappe +import semantic_version from frappe.commands.site import _new_site -from check_connection import get_config, get_site_config +from frappe.installer import update_site_config +from check_connection import get_config, get_site_config, COMMON_SITE_CONFIG_FILE def get_password(env_var, default=None): @@ -28,55 +30,96 @@ def _get_password_from_secret(env_var): def main(): + config = get_config() + db_type = 'mariadb' + db_port = config.get('db_port', 3306) + db_host = config.get('db_host') site_name = os.environ.get("SITE_NAME", 'site1.localhost') mariadb_root_username = os.environ.get("DB_ROOT_USER", 'root') mariadb_root_password = get_password("MYSQL_ROOT_PASSWORD", 'admin') + postgres_root_password = get_password("POSTGRES_PASSWORD") + + if postgres_root_password: + db_type = 'postgres' + db_host = os.environ.get("POSTGRES_HOST") + db_port = 5432 + if not db_host: + db_host = config.get('db_host') + print('Environment variable POSTGRES_HOST not found.') + print('Using db_host from common_site_config.json') + + sites_path = os.getcwd() + common_site_config_path = os.path.join(sites_path, COMMON_SITE_CONFIG_FILE) + update_site_config("root_login", mariadb_root_username, validate = False, site_config_path = common_site_config_path) + update_site_config("root_password", postgres_root_password, validate = False, site_config_path = common_site_config_path) + force = True if os.environ.get("FORCE", None) else False install_apps = os.environ.get("INSTALL_APPS", None) install_apps = install_apps.split(',') if install_apps else [] frappe.init(site_name, new_site=True) - _new_site( - None, - site_name, - mariadb_root_username=mariadb_root_username, - mariadb_root_password=mariadb_root_password, - admin_password=get_password("ADMIN_PASSWORD", 'admin'), - verbose=True, - install_apps=install_apps, - source_sql=None, - force=force, - reinstall=False, - ) + if semantic_version.Version(frappe.__version__).major > 11: + _new_site( + None, + site_name, + mariadb_root_username=mariadb_root_username, + mariadb_root_password=mariadb_root_password, + admin_password=get_password("ADMIN_PASSWORD", 'admin'), + verbose=True, + install_apps=install_apps, + source_sql=None, + force=force, + db_type=db_type, + reinstall=False, + db_host=db_host, + db_port=db_port, + ) + else: + _new_site( + None, + site_name, + mariadb_root_username=mariadb_root_username, + mariadb_root_password=mariadb_root_password, + admin_password=get_password("ADMIN_PASSWORD", 'admin'), + verbose=True, + install_apps=install_apps, + source_sql=None, + force=force, + reinstall=False, + ) - config = get_config() - site_config = get_site_config(site_name) + if db_type == "mariadb": + site_config = get_site_config(site_name) - mysql_command = 'mysql -h{db_host} -u{mariadb_root_username} -p{mariadb_root_password} -e '.format( - db_host=config.get('db_host'), - mariadb_root_username=mariadb_root_username, - mariadb_root_password=mariadb_root_password - ) + mysql_command = 'mysql -h{db_host} -u{mariadb_root_username} -p{mariadb_root_password} -e '.format( + db_host=config.get('db_host'), + mariadb_root_username=mariadb_root_username, + mariadb_root_password=mariadb_root_password + ) - # update User's host to '%' required to connect from any container - command = mysql_command + "\"UPDATE mysql.user SET Host = '%' where User = '{db_name}'; FLUSH PRIVILEGES;\"".format( - db_name=site_config.get('db_name') - ) - os.system(command) + # update User's host to '%' required to connect from any container + command = mysql_command + "\"UPDATE mysql.user SET Host = '%' where User = '{db_name}'; FLUSH PRIVILEGES;\"".format( + db_name=site_config.get('db_name') + ) + os.system(command) - # Set db password - command = mysql_command + "\"ALTER USER '{db_name}'@'%' IDENTIFIED BY '{db_password}'; FLUSH PRIVILEGES;\"".format( - db_name=site_config.get('db_name'), - db_password=site_config.get('db_password') - ) - os.system(command) + # Set db password + command = mysql_command + "\"ALTER USER '{db_name}'@'%' IDENTIFIED BY '{db_password}'; FLUSH PRIVILEGES;\"".format( + db_name=site_config.get('db_name'), + db_password=site_config.get('db_password') + ) + os.system(command) + + # Grant permission to database + command = mysql_command + "\"GRANT ALL PRIVILEGES ON \`{db_name}\`.* TO '{db_name}'@'%'; FLUSH PRIVILEGES;\"".format( + db_name=site_config.get('db_name') + ) + os.system(command) + + if frappe.redis_server: + frappe.redis_server.connection_pool.disconnect() - # Grant permission to database - command = mysql_command + "\"GRANT ALL PRIVILEGES ON \`{db_name}\`.* TO '{db_name}'@'%'; FLUSH PRIVILEGES;\"".format( - db_name=site_config.get('db_name') - ) - os.system(command) exit(0) diff --git a/build/common/commands/restore_backup.py b/build/common/commands/restore_backup.py index 131babb3..efb51545 100644 --- a/build/common/commands/restore_backup.py +++ b/build/common/commands/restore_backup.py @@ -10,7 +10,7 @@ from new import get_password from push_backup import DATE_FORMAT, check_environment_variables from frappe.utils import get_sites, random_string from frappe.installer import make_conf, get_conf_params, make_site_dirs, update_site_config -from check_connection import get_site_config, get_config +from check_connection import get_site_config, get_config, COMMON_SITE_CONFIG_FILE def list_directories(path): @@ -40,71 +40,58 @@ def decompress_db(files_base, site): def restore_database(files_base, site_config_path, site): - db_root_password = get_password('MYSQL_ROOT_PASSWORD') - if not db_root_password: - print('Variable MYSQL_ROOT_PASSWORD not set') - exit(1) - - db_root_user = os.environ.get("DB_ROOT_USER", 'root') - # restore database database_file = files_base + '-database.sql.gz' decompress_db(files_base, site) config = get_config() + + # Set db_type if it exists in backup site_config.json + set_key_in_site_config('db_type', site, site_config_path) + # Set db_host if it exists in backup site_config.json + set_key_in_site_config('db_host', site, site_config_path) + # Set db_port if it exists in backup site_config.json + set_key_in_site_config('db_port', site, site_config_path) + + # get updated site_config site_config = get_site_config(site) - # mysql command prefix - mysql_command = 'mysql -u{db_root_user} -h{db_host} -p{db_password} -e '.format( - db_root_user=db_root_user, - db_host=config.get('db_host'), - db_password=db_root_password - ) + # if no db_type exists, default to mariadb + db_type = site_config.get('db_type', 'mariadb') + is_database_restored = False - # drop db if exists for clean restore - drop_database = mysql_command + "\"DROP DATABASE IF EXISTS \`{db_name}\`;\"".format( - db_name=site_config.get('db_name') - ) - os.system(drop_database) + if db_type == 'mariadb': + restore_mariadb( + config=config, + site_config=site_config, + database_file=database_file) + is_database_restored = True + elif db_type == 'postgres': + restore_postgres( + config=config, + site_config=site_config, + database_file=database_file) + is_database_restored = True - # create db - create_database = mysql_command + "\"CREATE DATABASE IF NOT EXISTS \`{db_name}\`;\"".format( - db_name=site_config.get('db_name') - ) - os.system(create_database) + if is_database_restored: + # Set encryption_key if it exists in backup site_config.json + set_key_in_site_config('encryption_key', site, site_config_path) - # create user - create_user = mysql_command + "\"CREATE USER IF NOT EXISTS \'{db_name}\'@\'%\' IDENTIFIED BY \'{db_password}\'; FLUSH PRIVILEGES;\"".format( - db_name=site_config.get('db_name'), - db_password=site_config.get('db_password') - ) - os.system(create_user) - # grant db privileges to user - grant_privileges = mysql_command + "\"GRANT ALL PRIVILEGES ON \`{db_name}\`.* TO '{db_name}'@'%' IDENTIFIED BY '{db_password}'; FLUSH PRIVILEGES;\"".format( - db_name=site_config.get('db_name'), - db_password=site_config.get('db_password') - ) - os.system(grant_privileges) +def set_key_in_site_config(key, site, site_config_path): + site_config = get_site_config_from_path(site_config_path) + value = site_config.get(key) + if value: + print('Set {key} in site config for site: {site}'.format(key=key, site=site)) + update_site_config(key, value, + site_config_path=os.path.join(os.getcwd(), site, "site_config.json")) - command = "mysql -u{db_root_user} -h{db_host} -p{db_password} '{db_name}' < {database_file}".format( - db_root_user=db_root_user, - db_host=config.get('db_host'), - db_password=db_root_password, - db_name=site_config.get('db_name'), - database_file=database_file.replace('.gz', ''), - ) - - print('Restoring database for site: {}'.format(site)) - os.system(command) +def get_site_config_from_path(site_config_path): + site_config = dict() if os.path.exists(site_config_path): with open(site_config_path, 'r') as sc: site_config = json.load(sc) - encryption_key = site_config.get("encryption_key") - if encryption_key: - print('Restoring site config for site: {}'.format(site)) - update_site_config('encryption_key', encryption_key, - site_config_path=os.path.join(os.getcwd(), site, "site_config.json")) + return site_config def restore_files(files_base): @@ -178,6 +165,127 @@ def pull_backup_from_s3(): os.chdir(os.path.join(os.path.expanduser('~'), 'frappe-bench', 'sites')) +def restore_postgres(config, site_config, database_file): + # common config + common_site_config_path = os.path.join(os.getcwd(), COMMON_SITE_CONFIG_FILE) + + db_root_user = config.get('root_login') + if not db_root_user: + postgres_user = os.environ.get('DB_ROOT_USER') + if not postgres_user: + print('Variable DB_ROOT_USER not set') + exit(1) + + db_root_user = postgres_user + update_site_config( + "root_login", + db_root_user, + validate=False, + site_config_path=common_site_config_path) + + db_root_password = config.get('root_password') + if not db_root_password: + root_password = get_password('POSTGRES_PASSWORD') + if not root_password: + print('Variable POSTGRES_PASSWORD not set') + exit(1) + + db_root_password = root_password + update_site_config( + "root_password", + db_root_password, + validate=False, + site_config_path=common_site_config_path) + + # site config + db_host = site_config.get('db_host') + db_port = site_config.get('db_port', 5432) + db_name = site_config.get('db_name') + db_password = site_config.get('db_password') + + psql_command = "psql postgres://{root_login}:{root_password}@{db_host}:{db_port}".format( + root_login=db_root_user, + root_password=db_root_password, + db_host=db_host, + db_port=db_port + ) + + print('Restoring PostgreSQL') + os.system(psql_command + ' -c "DROP DATABASE IF EXISTS \"{db_name}\""'.format(db_name=db_name)) + os.system(psql_command + ' -c "DROP USER IF EXISTS {db_name}"'.format(db_name=db_name)) + os.system(psql_command + ' -c "CREATE DATABASE \"{db_name}\""'.format(db_name=db_name)) + os.system(psql_command + ' -c "CREATE user {db_name} password \'{db_password}\'"'.format( + db_name=db_name, + db_password=db_password)) + os.system(psql_command + ' -c "GRANT ALL PRIVILEGES ON DATABASE \"{db_name}\" TO {db_name}"'.format( + db_name=db_name)) + + os.system("{psql_command}/{db_name} < {database_file}".format( + psql_command=psql_command, + database_file=database_file.replace('.gz', ''), + db_name=db_name, + )) + + +def restore_mariadb(config, site_config, database_file): + db_root_password = get_password('MYSQL_ROOT_PASSWORD') + if not db_root_password: + print('Variable MYSQL_ROOT_PASSWORD not set') + exit(1) + + db_root_user = os.environ.get("DB_ROOT_USER", 'root') + + db_host = site_config.get('db_host', config.get('db_host')) + db_port = site_config.get('db_port', config.get('db_port')) + + # mysql command prefix + mysql_command = 'mysql -u{db_root_user} -h{db_host} -p{db_password}'.format( + db_root_user=db_root_user, + db_host=db_host, + db_port=db_port, + db_password=db_root_password + ) + + # drop db if exists for clean restore + drop_database = "{mysql_command} -e \"DROP DATABASE IF EXISTS \`{db_name}\`;\"".format( + mysql_command=mysql_command, + db_name=site_config.get('db_name'), + ) + os.system(drop_database) + + # create db + create_database = "{mysql_command} -e \"CREATE DATABASE IF NOT EXISTS \`{db_name}\`;\"".format( + mysql_command=mysql_command, + db_name=site_config.get('db_name'), + ) + os.system(create_database) + + # create user + create_user = "{mysql_command} -e \"CREATE USER IF NOT EXISTS \'{db_name}\'@\'%\' IDENTIFIED BY \'{db_password}\'; FLUSH PRIVILEGES;\"".format( + mysql_command=mysql_command, + db_name=site_config.get('db_name'), + db_password=site_config.get('db_password'), + ) + os.system(create_user) + + # grant db privileges to user + grant_privileges = "{mysql_command} -e \"GRANT ALL PRIVILEGES ON \`{db_name}\`.* TO '{db_name}'@'%' IDENTIFIED BY '{db_password}'; FLUSH PRIVILEGES;\"".format( + mysql_command=mysql_command, + db_name=site_config.get('db_name'), + db_password=site_config.get('db_password'), + ) + os.system(grant_privileges) + + command = "{mysql_command} '{db_name}' < {database_file}".format( + mysql_command=mysql_command, + db_name=site_config.get('db_name'), + database_file=database_file.replace('.gz', ''), + ) + + print('Restoring MariaDB') + os.system(command) + + def main(): backup_dir = get_backup_dir() @@ -194,15 +302,11 @@ def main(): if not os.path.exists(site_config_path): site_config_path = os.path.join(backup_dir, site, 'site_config.json') if site in get_sites(): + print('Overwrite site {}'.format(site)) restore_database(files_base, site_config_path, site) restore_private_files(files_base) restore_files(files_base) else: - mariadb_root_password = get_password('MYSQL_ROOT_PASSWORD') - if not mariadb_root_password: - print('Variable MYSQL_ROOT_PASSWORD not set') - exit(1) - site_config = get_conf_params( db_name='_' + hashlib.sha1(site.encode()).hexdigest()[:16], db_password=random_string(16) @@ -216,10 +320,15 @@ def main(): db_password=site_config.get('db_password'), ) make_site_dirs() + + print('Create site {}'.format(site)) restore_database(files_base, site_config_path, site) restore_private_files(files_base) restore_files(files_base) + if frappe.redis_server: + frappe.redis_server.connection_pool.disconnect() + exit(0) diff --git a/build/common/common_site_config.json.template b/build/common/common_site_config.json.template index 27593289..f7272fd6 100755 --- a/build/common/common_site_config.json.template +++ b/build/common/common_site_config.json.template @@ -1,5 +1,6 @@ { - "db_host": "${MARIADB_HOST}", + "db_host": "${DB_HOST}", + "db_port": ${DB_PORT}, "redis_cache": "redis://${REDIS_CACHE}", "redis_queue": "redis://${REDIS_QUEUE}", "redis_socketio": "redis://${REDIS_SOCKETIO}", diff --git a/build/common/worker/docker-entrypoint.sh b/build/common/worker/docker-entrypoint.sh index d6194c17..0a8fc0f6 100755 --- a/build/common/worker/docker-entrypoint.sh +++ b/build/common/worker/docker-entrypoint.sh @@ -4,8 +4,10 @@ function configureEnv() { if [ ! -f /home/frappe/frappe-bench/sites/common_site_config.json ]; then if [[ -z "$MARIADB_HOST" ]]; then - echo "MARIADB_HOST is not set" - exit 1 + if [[ -z "$POSTGRES_HOST" ]]; then + echo "MARIADB_HOST or POSTGRES_HOST is not set" + exit 1 + fi fi if [[ -z "$REDIS_CACHE" ]]; then @@ -28,7 +30,14 @@ function configureEnv() { exit 1 fi - envsubst '${MARIADB_HOST} + if [[ -z "$DB_PORT" ]]; then + export DB_PORT=3306 + fi + + export DB_HOST="${MARIADB_HOST:-$POSTGRES_HOST}" + + envsubst '${DB_HOST} + ${DB_PORT} ${REDIS_CACHE} ${REDIS_QUEUE} ${REDIS_SOCKETIO} diff --git a/build/erpnext-nginx/Dockerfile b/build/erpnext-nginx/Dockerfile index c814a6cd..72488ab7 100644 --- a/build/erpnext-nginx/Dockerfile +++ b/build/erpnext-nginx/Dockerfile @@ -11,7 +11,7 @@ FROM frappe/frappe-nginx:${GIT_BRANCH} COPY --from=0 /home/frappe/frappe-bench/sites/ /var/www/html/ COPY --from=0 /rsync /rsync -RUN echo -n "\nerpnext" >> /var/www/html/apps.txt +RUN echo "erpnext" >> /var/www/html/apps.txt VOLUME [ "/assets" ] diff --git a/build/erpnext-nginx/install_app.sh b/build/erpnext-nginx/install_app.sh index 324f2bd3..9042c5f4 100755 --- a/build/erpnext-nginx/install_app.sh +++ b/build/erpnext-nginx/install_app.sh @@ -19,7 +19,7 @@ git clone --depth 1 ${APP_REPO} ${BRANCH} cd /home/frappe/frappe-bench/apps/frappe yarn -yarn production +yarn production --app ${APP_NAME} rm -fr node_modules yarn install --production=true diff --git a/development/README.md b/development/README.md index 030bfe35..b4e72dce 100644 --- a/development/README.md +++ b/development/README.md @@ -104,6 +104,23 @@ This will create a new site and a `mysite.localhost` directory under `frappe-ben The option `--no-mariadb-socket` will configure site's database credentials to work with docker. You may need to configure your system /etc/hosts if you're on Linux, Mac, or its Windows equivalent. +To setup site with PostgreSQL as database use option `--db-type postgres` and `--db-host postgresql`. (Available only v12 onwards, currently NOT available for ERPNext). + +Example: + +```shell +bench new-site mypgsql.localhost --db-type postgres --db-host postgresql +``` + +To avoid entering postgresql username and root password, set it in `common_site_config.json`, + +```shell +bench config set-common-config -c root_login postgres +bench config set-common-config -c root_password '"123"' +``` + +Note: If PostgreSQL is not required, the postgresql service / container can be stopped. + ### Set bench developer mode on the new site To develop a new app, the last step will be setting the site into developer mode. Documentation is available at [this link](https://frappe.io/docs/user/en/guides/app-development/how-enable-developer-mode-in-frappe). diff --git a/docs/site-operations.md b/docs/site-operations.md index 5fa29483..f6dab293 100644 --- a/docs/site-operations.md +++ b/docs/site-operations.md @@ -12,10 +12,12 @@ Or specify environment variables instead of passing secrets as command arguments Note: -- Wait for the MariaDB service to start before trying to create a new site. +- Wait for the database service to start before trying to create a new site. - If new site creation fails, retry after the MariaDB container is up and running. - If you're using a managed database instance, make sure that the database is running before setting up a new site. +#### MariaDB Site + ```sh # Create ERPNext site docker run \ @@ -29,10 +31,28 @@ docker run \ frappe/erpnext-worker:$VERSION new ``` +#### PostgreSQL Site + +PostgreSQL is only available v12 onwards. It is NOT available for ERPNext. +It is available as part of `frappe/erpnext-worker`. It inherits from `frappe/frappe-worker`. + +```sh +# Create ERPNext site +docker run \ + -e "SITE_NAME=$SITE_NAME" \ + -e "DB_ROOT_USER=$DB_ROOT_USER" \ + -e "POSTGRES_HOST=$POSTGRES_HOST" \ + -e "POSTGRES_PASSWORD=$POSTGRES_PASSWORD" \ + -e "ADMIN_PASSWORD=$ADMIN_PASSWORD" \ + -v _sites-vol:/home/frappe/frappe-bench/sites \ + --network _default \ + frappe/erpnext-worker:$VERSION new +``` + Environment Variables needed: - `SITE_NAME`: name of the new site to create. Site name is domain name that resolves. e.g. `erp.example.com` or `erp.localhost`. -- `DB_ROOT_USER`: MariaDB Root user. +- `DB_ROOT_USER`: MariaDB/PostgreSQL Root user. - `MYSQL_ROOT_PASSWORD`: In case of the MariaDB docker container use the one set in `MYSQL_ROOT_PASSWORD` in previous steps. In case of a managed database use the appropriate password. - `MYSQL_ROOT_PASSWORD_FILE` - When the MariaDB root password is stored using docker secrets. - `ADMIN_PASSWORD`: set the administrator password for the new site. @@ -40,6 +60,16 @@ Environment Variables needed: - `INSTALL_APPS=erpnext`: available only in erpnext-worker and erpnext containers (or other containers with custom apps). Installs ERPNext (and/or the specified apps, comma-delinieated) on this new site. - `FORCE=1`: optional variable which force installation of the same site. +Environment Variables for PostgreSQL only: + +- `POSTGRES_HOST`: host for PostgreSQL server +- `POSTGRES_PASSWORD`: Password for `postgres`. The database root user. + +Notes: + +- To setup existing frappe-bench deployment with default database as PostgreSQL edit the common_site_config.json and set `db_host` to PostgreSQL hostname and `db_port` to PostgreSQL port. +- To create new frappe-bench deployment with default database as PostgreSQL use `POSTGRES_HOST` and `DB_PORT` environment variables in `erpnext-python` service instead of `MARIADB_HOST` + ## Add sites to proxy Change `SITES` variable to the list of sites created encapsulated in backtick and separated by comma with no space. e.g. ``SITES=`site1.example.com`,`site2.example.com` ``. diff --git a/installation/frappe-postgresql/docker-compose.yml b/installation/frappe-postgresql/docker-compose.yml new file mode 100644 index 00000000..8f479aa2 --- /dev/null +++ b/installation/frappe-postgresql/docker-compose.yml @@ -0,0 +1,166 @@ +version: "3" + +services: + traefik: + image: "traefik:v2.2" + command: + - "--log.level=DEBUG" + - "--providers.docker=true" + - "--providers.docker.exposedbydefault=false" + - "--entrypoints.web.address=:80" + - "--entrypoints.websecure.address=:443" + - "--certificatesresolvers.myresolver.acme.httpchallenge=true" + - "--certificatesresolvers.myresolver.acme.httpchallenge.entrypoint=web" + - "--certificatesresolvers.myresolver.acme.email=${LETSENCRYPT_EMAIL}" + - "--certificatesresolvers.myresolver.acme.storage=/letsencrypt/acme.json" + labels: + # enable traefik + - "traefik.enable=true" + # global redirect to https for production only + - "${HTTPS_REDIRECT_RULE_LABEL}" + - "${HTTPS_REDIRECT_ENTRYPOINT_LABEL}" + - "${HTTPS_REDIRECT_MIDDLEWARE_LABEL}" + # middleware redirect for production only + - "${HTTPS_USE_REDIRECT_MIDDLEWARE_LABEL}" + ports: + - "80:80" + - "443:443" + volumes: + - cert-vol:/letsencrypt + - /var/run/docker.sock:/var/run/docker.sock:ro + + frappe-nginx: + image: frappe/frappe-nginx:${FRAPPE_VERSION} + restart: on-failure + environment: + - FRAPPE_PY=frappe-python + - FRAPPE_PY_PORT=8000 + - FRAPPE_SOCKETIO=frappe-socketio + - SOCKETIO_PORT=9000 + labels: + - "traefik.enable=true" + - "traefik.http.routers.frappe-nginx.rule=Host(${SITES})" + - "${ENTRYPOINT_LABEL}" + - "${CERT_RESOLVER_LABEL}" + - "traefik.http.services.frappe-nginx.loadbalancer.server.port=80" + volumes: + - sites-vol:/var/www/html/sites:rw + - assets-vol:/assets:rw + + frappe-python: + image: frappe/frappe-worker:${FRAPPE_VERSION} + restart: on-failure + environment: + - POSTGRES_HOST=${POSTGRES_HOST} + - DB_PORT=5432 + - REDIS_CACHE=redis-cache:6379 + - REDIS_QUEUE=redis-queue:6379 + - REDIS_SOCKETIO=redis-socketio:6379 + - SOCKETIO_PORT=9000 + - AUTO_MIGRATE=1 + volumes: + - sites-vol:/home/frappe/frappe-bench/sites:rw + - assets-vol:/home/frappe/frappe-bench/sites/assets:rw + + frappe-socketio: + image: frappe/frappe-socketio:${FRAPPE_VERSION} + restart: on-failure + depends_on: + - redis-socketio + volumes: + - sites-vol:/home/frappe/frappe-bench/sites:rw + + frappe-worker-default: + image: frappe/frappe-worker:${FRAPPE_VERSION} + restart: on-failure + command: worker + depends_on: + - redis-queue + - redis-cache + volumes: + - sites-vol:/home/frappe/frappe-bench/sites:rw + + frappe-worker-short: + image: frappe/frappe-worker:${FRAPPE_VERSION} + restart: on-failure + command: worker + environment: + - WORKER_TYPE=short + depends_on: + - redis-queue + - redis-cache + volumes: + - sites-vol:/home/frappe/frappe-bench/sites:rw + + frappe-worker-long: + image: frappe/frappe-worker:${FRAPPE_VERSION} + restart: on-failure + command: worker + environment: + - WORKER_TYPE=long + depends_on: + - redis-queue + - redis-cache + volumes: + - sites-vol:/home/frappe/frappe-bench/sites:rw + + frappe-schedule: + image: frappe/frappe-worker:${FRAPPE_VERSION} + restart: on-failure + command: schedule + depends_on: + - redis-queue + - redis-cache + volumes: + - sites-vol:/home/frappe/frappe-bench/sites:rw + + redis-cache: + image: redis:latest + restart: on-failure + volumes: + - redis-cache-vol:/data + + redis-queue: + image: redis:latest + restart: on-failure + volumes: + - redis-queue-vol:/data + + redis-socketio: + image: redis:latest + restart: on-failure + volumes: + - redis-socketio-vol:/data + + postgresql: + image: postgres:11.8 + restart: on-failure + environment: + - POSTGRES_PASSWORD=${POSTGRES_PASSWORD} + volumes: + - postgresql-vol:/var/lib/postgresql/data + + site-creator: + image: frappe/frappe-worker:${FRAPPE_VERSION} + restart: "no" + command: new + depends_on: + - frappe-python + environment: + - POSTGRES_HOST=${POSTGRES_HOST} + - SITE_NAME=${SITE_NAME} + - DB_ROOT_USER=${DB_ROOT_USER} + - POSTGRES_PASSWORD=${POSTGRES_PASSWORD} + - ADMIN_PASSWORD=${ADMIN_PASSWORD} + - INSTALL_APPS=${INSTALL_APPS} + volumes: + - sites-vol:/home/frappe/frappe-bench/sites:rw + +volumes: + postgresql-vol: + redis-cache-vol: + redis-queue-vol: + redis-socketio-vol: + assets-vol: + sites-vol: + cert-vol: diff --git a/installation/frappe-postgresql/env-local b/installation/frappe-postgresql/env-local new file mode 100644 index 00000000..14a0d12f --- /dev/null +++ b/installation/frappe-postgresql/env-local @@ -0,0 +1,14 @@ +LETSENCRYPT_EMAIL=email@example.com +FRAPPE_VERSION=edge +POSTGRES_HOST=postgresql +POSTGRES_PASSWORD=admin +SITE_NAME=mysite.localhost +SITES=`mysite.localhost` +DB_ROOT_USER=postgres +ADMIN_PASSWORD=admin +ENTRYPOINT_LABEL=traefik.http.routers.erpnext-nginx.entrypoints=web +CERT_RESOLVER_LABEL=erpnext.local.no-cert-resolver +HTTPS_REDIRECT_RULE_LABEL=erpnext.local.no-redirect-rule +HTTPS_REDIRECT_ENTRYPOINT_LABEL=erpnext.local.no-entrypoint +HTTPS_REDIRECT_MIDDLEWARE_LABEL=erpnext.local.no-middleware +HTTPS_USE_REDIRECT_MIDDLEWARE_LABEL=erpnext.local-no-redirect-middleware diff --git a/installation/frappe-postgresql/env-production b/installation/frappe-postgresql/env-production new file mode 100644 index 00000000..689f0339 --- /dev/null +++ b/installation/frappe-postgresql/env-production @@ -0,0 +1,14 @@ +LETSENCRYPT_EMAIL=email@example.com +FRAPPE_VERSION=edge +POSTGRES_HOST=postgresql +POSTGRES_PASSWORD=admin +SITE_NAME=erp.example.com +SITES=`erp.example.com` +DB_ROOT_USER=postgres +ADMIN_PASSWORD=admin +ENTRYPOINT_LABEL=traefik.http.routers.erpnext-nginx.entrypoints=websecure +CERT_RESOLVER_LABEL=traefik.http.routers.erpnext-nginx.tls.certresolver=myresolver +HTTPS_REDIRECT_RULE_LABEL=traefik.http.routers.http-catchall.rule=hostregexp(`{host:.+}`) +HTTPS_REDIRECT_ENTRYPOINT_LABEL=traefik.http.routers.http-catchall.entrypoints=web +HTTPS_REDIRECT_MIDDLEWARE_LABEL=traefik.http.routers.http-catchall.middlewares=redirect-to-https +HTTPS_USE_REDIRECT_MIDDLEWARE_LABEL=traefik.http.middlewares.redirect-to-https.redirectscheme.scheme=https diff --git a/tests/docker-test.sh b/tests/docker-test.sh index d690feb0..48c16e21 100755 --- a/tests/docker-test.sh +++ b/tests/docker-test.sh @@ -59,12 +59,19 @@ docker-compose \ -f installation/docker-compose-erpnext.yml \ -f installation/erpnext-publish.yml \ pull +docker pull postgres:11.8 docker-compose \ --project-name frappebench00 \ -f installation/docker-compose-common.yml \ -f installation/docker-compose-erpnext.yml \ -f installation/erpnext-publish.yml \ up -d +# Start postgres +docker run --name postgresql -d \ + -e "POSTGRES_PASSWORD=admin" \ + -v frappebench00_sites-vol:/home/frappe/frappe-bench/sites \ + --network frappebench00_default \ + postgres:11.8 loopHealthCheck echo -e "\n" @@ -116,6 +123,47 @@ echo -e "\e[1m\e[4mCheck Migrated Site Index Page\e[0m" curl -s http://test.localhost | w3m -T text/html -dump echo -e "\n" +echo -e "\e[1m\e[4mCreate new site (pgsql)\e[0m" +docker run -it \ + -e "SITE_NAME=pgsql.localhost" \ + -e "POSTGRES_HOST=postgresql" \ + -e "DB_ROOT_USER=postgres" \ + -e "POSTGRES_PASSWORD=admin" \ + -v frappebench00_sites-vol:/home/frappe/frappe-bench/sites \ + --network frappebench00_default \ + frappe/erpnext-worker:edge new +echo -e "\n" + +echo -e "\e[1m\e[4mCheck New PGSQL Site\e[0m" +sleep 3 +RESTORE_STATUS=$(curl -sS http://pgsql.localhost/api/method/version || echo "") +INCREMENT=0 +while [[ -z "$RESTORE_STATUS" && $INCREMENT -lt 60 ]]; do + sleep 1 + echo -e "\e[1m\e[4mWait for restoration to complete ..." + RESTORE_STATUS=$(curl -sS http://pgsql.localhost/api/method/version || echo "") + ((INCREMENT=INCREMENT+1)) + if [[ -z "$RESTORE_STATUS" && $INCREMENT -eq 60 ]]; then + CONTAINER_ID=$(docker-compose \ + --project-name frappebench00 \ + -f installation/docker-compose-common.yml \ + -f installation/docker-compose-erpnext.yml \ + -f installation/erpnext-publish.yml \ + ps -q erpnext-python) + docker logs $CONTAINER_ID + exit 1 + fi +done +echo -e "\n" + +echo -e "\e[1m\e[4mPing new pgsql site\e[0m" +echo $RESTORE_STATUS +echo -e "\n" + +echo -e "\e[1m\e[4mCheck New PGSQL Index Page\e[0m" +curl -s http://pgsql.localhost | w3m -T text/html -dump +echo -e "\n" + echo -e "\e[1m\e[4mBackup site\e[0m" docker run -it \ -e "WITH_FILES=1" \ @@ -198,7 +246,7 @@ docker run \ frappe/erpnext-worker:edge restore-backup echo -e "\n" -echo -e "\e[1m\e[4mCheck Restored Site\e[0m" +echo -e "\e[1m\e[4mCheck Restored Site (test)\e[0m" sleep 3 RESTORE_STATUS=$(curl -sS http://test.localhost/api/method/version || echo "") INCREMENT=0 @@ -219,14 +267,43 @@ while [[ -z "$RESTORE_STATUS" && $INCREMENT -lt 60 ]]; do fi done -echo -e "\e[1m\e[4mPing restored site\e[0m" +echo -e "\e[1m\e[4mPing restored site (test)\e[0m" echo $RESTORE_STATUS echo -e "\n" -echo -e "\e[1m\e[4mCheck Restored Site Index Page\e[0m" +echo -e "\e[1m\e[4mCheck Restored Site Index Page (test)\e[0m" curl -s http://test.localhost | w3m -T text/html -dump echo -e "\n" +echo -e "\e[1m\e[4mCheck Restored Site (pgsql)\e[0m" +sleep 3 +RESTORE_STATUS=$(curl -sS http://pgsql.localhost/api/method/version || echo "") +INCREMENT=0 +while [[ -z "$RESTORE_STATUS" && $INCREMENT -lt 60 ]]; do + sleep 1 + echo "Wait for restoration to complete ..." + RESTORE_STATUS=$(curl -sS http://pgsql.localhost/api/method/version || echo "") + ((INCREMENT=INCREMENT+1)) + if [[ -z "$RESTORE_STATUS" && $INCREMENT -eq 60 ]]; then + CONTAINER_ID=$(docker-compose \ + --project-name frappebench00 \ + -f installation/docker-compose-common.yml \ + -f installation/docker-compose-erpnext.yml \ + -f installation/erpnext-publish.yml \ + ps -q erpnext-python) + docker logs $CONTAINER_ID + exit 1 + fi +done + +echo -e "\e[1m\e[4mPing restored site (pgsql)\e[0m" +echo $RESTORE_STATUS +echo -e "\n" + +echo -e "\e[1m\e[4mCheck Restored Site Index Page (pgsql)\e[0m" +curl -s http://pgsql.localhost | w3m -T text/html -dump +echo -e "\n" + echo -e "\e[1m\e[4mCreate new site (edge)\e[0m" docker run -it \ -e "SITE_NAME=edge.localhost" \ @@ -325,3 +402,9 @@ docker run \ -v frappebench00_sites-vol:/home/frappe/frappe-bench/sites \ --network frappebench00_default \ frappe/erpnext-worker:edge console test.localhost + +echo -e "\e[1m\e[4mCheck console command for site pgsql.localhost\e[0m" +docker run \ + -v frappebench00_sites-vol:/home/frappe/frappe-bench/sites \ + --network frappebench00_default \ + frappe/erpnext-worker:edge console pgsql.localhost