From 479ce83f5c2714349c9db830f8f7abd8b973ce51 Mon Sep 17 00:00:00 2001 From: Revant Nandgaonkar Date: Sun, 26 Apr 2020 22:48:35 +0530 Subject: [PATCH 01/13] fix: required region s3 compatible backup/restore --- README.md | 4 ++++ build/common/commands/push_backup.py | 6 ++++++ build/common/commands/restore_backup.py | 1 + 3 files changed, 11 insertions(+) diff --git a/README.md b/README.md index eeb18692..9b87e994 100644 --- a/README.md +++ b/README.md @@ -259,6 +259,7 @@ The backup will be available in the `sites-vol` volume. Environment Variables - `BUCKET_NAME`, Required to set bucket created on S3 compatible storage. +- `REGION`, Required to set region for S3 compatible storage. - `ACCESS_KEY_ID`, Required to set access key. - `SECRET_ACCESS_KEY`, Required to set secret access key. - `ENDPOINT_URL`, Required to set URL of S3 compatible storage. @@ -268,6 +269,7 @@ Environment Variables ```sh docker run \ -e "BUCKET_NAME=backups" \ + -e "REGION=region" \ -e "ACCESS_KEY_ID=access_id_from_provider" \ -e "SECRET_ACCESS_KEY=secret_access_from_provider" \ -e "ENDPOINT_URL=https://region.storage-provider.com" \ @@ -320,12 +322,14 @@ Environment Variables - `ACCESS_KEY_ID`, Required to set access key. - `SECRET_ACCESS_KEY`, Required to set secret access key. - `ENDPOINT_URL`, Required to set URL of S3 compatible storage. +- `REGION`, Required to set region for s3 compatible storage. - `BUCKET_DIR`, Required to set directory in bucket where sites from this deployment will be backed up. ```sh docker run \ -e "MYSQL_ROOT_PASSWORD=admin" \ -e "BUCKET_NAME=backups" \ + -e "REGION=region" \ -e "ACCESS_KEY_ID=access_id_from_provider" \ -e "SECRET_ACCESS_KEY=secret_access_from_provider" \ -e "ENDPOINT_URL=https://region.storage-provider.com" \ diff --git a/build/common/commands/push_backup.py b/build/common/commands/push_backup.py index e795b3ef..26b5c464 100644 --- a/build/common/commands/push_backup.py +++ b/build/common/commands/push_backup.py @@ -47,6 +47,7 @@ def get_s3_config(): conn = boto3.client( 's3', + region_name=os.environ.get('REGION'), aws_access_key_id=os.environ.get('ACCESS_KEY_ID'), aws_secret_access_key=os.environ.get('SECRET_ACCESS_KEY'), endpoint_url=os.environ.get('ENDPOINT_URL') @@ -75,6 +76,10 @@ def check_environment_variables(): print('Variable BUCKET_DIR not set') exit(1) + if not 'REGION' in os.environ: + print('Variable REGION not set') + exit(1) + def upload_file_to_s3(filename, folder, conn, bucket): destpath = os.path.join(folder, os.path.basename(filename)) @@ -96,6 +101,7 @@ def delete_old_backups(limit, bucket, site_name): s3 = boto3.resource( 's3', + region_name=os.environ.get('REGION'), aws_access_key_id=os.environ.get('ACCESS_KEY_ID'), aws_secret_access_key=os.environ.get('SECRET_ACCESS_KEY'), endpoint_url=os.environ.get('ENDPOINT_URL') diff --git a/build/common/commands/restore_backup.py b/build/common/commands/restore_backup.py index f248bbf7..38c1494b 100644 --- a/build/common/commands/restore_backup.py +++ b/build/common/commands/restore_backup.py @@ -121,6 +121,7 @@ def pull_backup_from_s3(): # https://stackoverflow.com/a/54672690 s3 = boto3.resource( 's3', + region_name=os.environ.get('REGION'), aws_access_key_id=os.environ.get('ACCESS_KEY_ID'), aws_secret_access_key=os.environ.get('SECRET_ACCESS_KEY'), endpoint_url=os.environ.get('ENDPOINT_URL') From 0361b281f98cadbb18e1d41fe293393cca2233ef Mon Sep 17 00:00:00 2001 From: Revant Nandgaonkar Date: Mon, 27 Apr 2020 13:36:33 +0530 Subject: [PATCH 02/13] fix: README single command to deploy locally env file placed where docker-compose will be executed --- README.md | 133 ++++++------------------ installation/env-example => env-example | 0 installation/erpnext-publish.yml | 6 ++ installation/frappe-publish.yml | 6 ++ 4 files changed, 44 insertions(+), 101 deletions(-) rename installation/env-example => env-example (100%) create mode 100644 installation/erpnext-publish.yml create mode 100644 installation/frappe-publish.yml diff --git a/README.md b/README.md index 9b87e994..5e24bf91 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ For docker based development refer to this [README](development/README.md) ### Setting up Pre-requisites -This repository requires Docker and Git to be setup on the instance to be used. +This repository requires Docker, docker-compose and Git to be setup on the instance to be used. ### Cloning the repository and preliminary steps @@ -33,7 +33,7 @@ cd frappe_docker Copy the example docker environment file to `.env`: ```sh -cp installation/env-example installation/.env +cp env-example .env ``` ### Setup Environment Variables @@ -63,37 +63,23 @@ The first command will start the containers; the second command will publish the For Erpnext: ```sh -# Start services docker-compose \ --project-name \ -f installation/docker-compose-common.yml \ -f installation/docker-compose-erpnext.yml \ - --project-directory installation up -d - -# Publish port -docker-compose \ - --project-name \ - -f installation/docker-compose-common.yml \ - -f installation/docker-compose-erpnext.yml \ - --project-directory installation run --publish 80:80 -d erpnext-nginx + -f installation/erpnext-publish.yml \ + up -d ``` For Frappe: ```sh -# Start services docker-compose \ --project-name \ -f installation/docker-compose-common.yml \ -f installation/docker-compose-frappe.yml \ - --project-directory installation up -d - -# Publish port -docker-compose \ - --project-name \ - -f installation/docker-compose-common.yml \ - -f installation/docker-compose-frappe.yml \ - --project-directory installation run --publish 80:80 -d frappe-nginx + -f installation/frappe-publish.yml \ + up -d ``` Make sure to replace `` with the desired name you wish to set for the project. @@ -124,8 +110,16 @@ cp .env.sample .env ./start.sh ``` +It will create the required network and configure containers for Letencrypt ACME. + For more details, see the [Letsencrypt Nginx Proxy Companion github repo](https://github.com/evertramos/docker-compose-letsencrypt-nginx-proxy-companion). Letsencrypt Nginx Proxy Companion github repo works by automatically proxying to containers with the `VIRTUAL_HOST` environmental variable. +Notes: + +- `SITES` variables from `env-example` is set as `VIRTUAL_HOST` +- `LETSENCRYPT_EMAIL` variables from `env-example` is used as it is. +- This is simple nginx + letsencrypt solution. Any other solution can be setup. Above two variables can be re-used or removed in case any other reverse-proxy is used. + #### Start Frappe/ERPNext Services To start the Frappe/ERPNext services for production, run the following command: @@ -136,7 +130,7 @@ docker-compose \ -f installation/docker-compose-common.yml \ -f installation/docker-compose-erpnext.yml \ -f installation/docker-compose-networks.yml \ - --project-directory installation up -d + up -d ``` Make sure to replace `` with any desired name you wish to set for the project. @@ -159,7 +153,7 @@ docker run \ Instead of `alpine` you can use any image you like. -For full instructions, refer to the [wiki](https://github.com/frappe/frappe/wiki/Using-Frappe-with-Amazon-RDS-(or-any-other-DBaaS). Common question can be found in Issues and on forum. +For full instructions, refer to the [wiki](https://github.com/frappe/frappe/wiki/Using-Frappe-with-Amazon-RDS-(or-any-other-DBaaS)). Common question can be found in Issues and on forum. ### Docker containers @@ -201,6 +195,9 @@ This repository contains the following docker-compose files, each one containing * docker-compose-networks.yml: this yaml define the network to communicate with *Letsencrypt Nginx Proxy Companion*. +* erpnext-publish.yml: this yml extends erpnext-nginx service to publish port 80, can only be used with docker-compose-erpnext.yml + +* frappe-publish.yml: this yml extends frappe-nginx service to publish port 80, can only be used with docker-compose-frappe.yml ### Site operations @@ -215,13 +212,15 @@ Note: ```sh # Create ERPNext site -docker exec -it \ +docker run \ -e "SITE_NAME=$SITE_NAME" \ -e "DB_ROOT_USER=$DB_ROOT_USER" \ -e "MYSQL_ROOT_PASSWORD=$MYSQL_ROOT_PASSWORD" \ -e "ADMIN_PASSWORD=$ADMIN_PASSWORD" \ -e "INSTALL_APPS=erpnext" \ - _erpnext-python_1 docker-entrypoint.sh new + -v _sites-vol:/home/frappe/frappe-bench/sites \ + --network _default \ + frappe/erpnext-worker:edge new ``` Environment Variables needed: @@ -246,10 +245,12 @@ Environment Variables - All the files generated by backup are placed at volume location `sites-vol:/{site-name}/private/backups/*` ```sh -docker exec -it \ +docker run \ -e "SITES=site1.domain.com:site2.domain.com" \ -e "WITH_FILES=1" \ - _erpnext-python_1 docker-entrypoint.sh backup + -v _sites-vol:/home/frappe/frappe-bench/sites \ + --network _default \ + frappe/erpnext-worker:edge backup ``` The backup will be available in the `sites-vol` volume. @@ -306,11 +307,13 @@ docker-compose \ -f installation/docker-compose-common.yml \ -f installation/docker-compose-erpnext.yml \ -f installation/docker-compose-networks.yml \ - --project-directory installation up -d + up -d -docker exec -it \ +docker run \ -e "MAINTENANCE_MODE=1" \ - _erpnext-python_1 docker-entrypoint.sh migrate + -v _sites-vol:/home/frappe/frappe-bench/sites \ + --network _default \ + frappe/erpnext-worker:edge migrate ``` #### Restore backups @@ -410,78 +413,6 @@ sed -i "s#\[app\]#[custom]#" ./installation/docker-compose-custom.yml Install like usual, except that when you set the `INSTALL_APPS` variable to `erpnext,[custom]`. -## Troubleshoot - -### Failed migration after image upgrade - -Issue: After upgrade of the containers, the automatic migration fails. -Solution: Remove containers and volumes, and clear redis cache: - -```shell -# change to repo root -cd $HOME/frappe_docker - -# Stop all bench containers -docker-compose \ - --project-name \ - -f installation/docker-compose-common.yml \ - -f installation/docker-compose-erpnext.yml \ - -f installation/docker-compose-networks.yml \ - --project-directory installation stop - -# Remove redis containers -docker-compose \ - --project-name \ - -f installation/docker-compose-common.yml \ - -f installation/docker-compose-erpnext.yml \ - -f installation/docker-compose-networks.yml \ - --project-directory installation rm redis-cache redis-queue redis-socketio - -# Clean redis volumes -docker volume rm \ - _redis-cache-vol \ - _redis-queue-vol \ - _redis-socketio-vol - -# Restart project -docker-compose \ - --project-name \ - -f installation/docker-compose-common.yml \ - -f installation/docker-compose-erpnext.yml \ - -f installation/docker-compose-networks.yml \ - --project-directory installation up -d -``` - -### ValueError: There exists an active worker named XXX already - -Issue: You have the following error during container restart - - -``` -frappe-worker-short_1 | Traceback (most recent call last): -frappe-worker-short_1 | File "/home/frappe/frappe-bench/commands/worker.py", line 5, in -frappe-worker-short_1 | start_worker(queue, False) -frappe-worker-short_1 | File "/home/frappe/frappe-bench/apps/frappe/frappe/utils/background_jobs.py", line 147, in start_worker -frappe-worker-short_1 | Worker(queues, name=get_worker_name(queue)).work(logging_level = logging_level) -frappe-worker-short_1 | File "/home/frappe/frappe-bench/env/lib/python3.7/site-packages/rq/worker.py", line 474, in work -frappe-worker-short_1 | self.register_birth() -frappe-worker-short_1 | File "/home/frappe/frappe-bench/env/lib/python3.7/site-packages/rq/worker.py", line 261, in register_birth -frappe-worker-short_1 | raise ValueError(msg.format(self.name)) -frappe-worker-short_1 | ValueError: There exists an active worker named '8dfe5c234085.10.short' already -``` - -Solution: Clear redis cache using `docker exec` command (take care of replacing `` accordingly): - -```sh -# Clear the cache which is causing problem. - -docker exec -it _redis-cache_1 redis-cli FLUSHALL -docker exec -it _redis-queue_1 redis-cli FLUSHALL -docker exec -it _redis-socketio_1 redis-cli FLUSHALL -``` - -Note: Environment variables from `.env` file located at the current working directory will be used. - ## Development This repository includes a complete setup to develop with Frappe/ERPNext and Bench, Including the following features: diff --git a/installation/env-example b/env-example similarity index 100% rename from installation/env-example rename to env-example diff --git a/installation/erpnext-publish.yml b/installation/erpnext-publish.yml new file mode 100644 index 00000000..6975442a --- /dev/null +++ b/installation/erpnext-publish.yml @@ -0,0 +1,6 @@ +version: "3" + +services: + erpnext-nginx: + ports: + - "80:80" diff --git a/installation/frappe-publish.yml b/installation/frappe-publish.yml new file mode 100644 index 00000000..a3f4caa9 --- /dev/null +++ b/installation/frappe-publish.yml @@ -0,0 +1,6 @@ +version: "3" + +services: + frappe-nginx: + ports: + - "80:80" From 081ca6f085eb8c888542a4402372fcb1bfa3eecc Mon Sep 17 00:00:00 2001 From: Revant Nandgaonkar Date: Mon, 27 Apr 2020 14:38:33 +0530 Subject: [PATCH 03/13] fix: README source additional .env for site operations --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 5e24bf91..8806909f 100644 --- a/README.md +++ b/README.md @@ -201,6 +201,8 @@ This repository contains the following docker-compose files, each one containing ### Site operations +Use `source .env` file or specify environment variables instead of passing secrets as command arguments. Refer notes section for environment variables required + #### Setup New Sites Note: @@ -208,7 +210,6 @@ Note: - Wait for the MariaDB service to start before trying to create a new site. - If new site creation fails, retry after the MariaDB container is up and running. - If you're using a managed database instance, make sure that the database is running before setting up a new site. -- Use `.env` file or environment variables instead of passing secrets as command arguments. ```sh # Create ERPNext site From 2bfc7d69586ff9dc3d2a55e9c61a513140214fc5 Mon Sep 17 00:00:00 2001 From: Revant Nandgaonkar Date: Mon, 27 Apr 2020 17:32:28 +0530 Subject: [PATCH 04/13] fix: README site operations image version tag --- README.md | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 8806909f..badb0721 100644 --- a/README.md +++ b/README.md @@ -201,7 +201,13 @@ This repository contains the following docker-compose files, each one containing ### Site operations -Use `source .env` file or specify environment variables instead of passing secrets as command arguments. Refer notes section for environment variables required +Use env file, + +```sh +source .env +``` + +Or specify environment variables instead of passing secrets as command arguments. Refer notes section for environment variables required #### Setup New Sites @@ -221,7 +227,7 @@ docker run \ -e "INSTALL_APPS=erpnext" \ -v _sites-vol:/home/frappe/frappe-bench/sites \ --network _default \ - frappe/erpnext-worker:edge new + frappe/erpnext-worker:$VERSION new ``` Environment Variables needed: @@ -251,7 +257,7 @@ docker run \ -e "WITH_FILES=1" \ -v _sites-vol:/home/frappe/frappe-bench/sites \ --network _default \ - frappe/erpnext-worker:edge backup + frappe/erpnext-worker:$VERSION backup ``` The backup will be available in the `sites-vol` volume. @@ -275,10 +281,10 @@ Environment Variables -e "ACCESS_KEY_ID=access_id_from_provider" \ -e "SECRET_ACCESS_KEY=secret_access_from_provider" \ -e "ENDPOINT_URL=https://region.storage-provider.com" \ - -e "BUCKET_DIR=frappe-bench-v12" \ + -e "BUCKET_DIR=frappe-bench" \ -v _sites-vol:/home/frappe/frappe-bench/sites \ --network _default \ - frappe/frappe-worker:v12 push-backup + frappe/frappe-worker:$VERSION push-backup ``` Note: @@ -314,7 +320,7 @@ docker run \ -e "MAINTENANCE_MODE=1" \ -v _sites-vol:/home/frappe/frappe-bench/sites \ --network _default \ - frappe/erpnext-worker:edge migrate + frappe/erpnext-worker:$VERSION migrate ``` #### Restore backups @@ -337,11 +343,11 @@ docker run \ -e "ACCESS_KEY_ID=access_id_from_provider" \ -e "SECRET_ACCESS_KEY=secret_access_from_provider" \ -e "ENDPOINT_URL=https://region.storage-provider.com" \ - -e "BUCKET_DIR=frappe-bench-v12" \ + -e "BUCKET_DIR=frappe-bench" \ -v _sites-vol:/home/frappe/frappe-bench/sites \ -v ./backups:/home/frappe/backups \ --network _default \ - frappe/frappe-worker:v12 restore-backup + frappe/frappe-worker:$VERSION restore-backup ``` Note: From 884a82d81454d2d97b4a03631085b7caf13970fd Mon Sep 17 00:00:00 2001 From: "Chinmay D. Pai" Date: Wed, 29 Apr 2020 01:45:59 +0530 Subject: [PATCH 05/13] fix: make semantic changes to commands * add missing __main__ call to commands.py * remove unnecessary imports * fix backup WITH_FILES logic * follow python semantics (?) Signed-off-by: Chinmay D. Pai --- build/common/commands/auto_migrate.py | 22 +++++++++------ build/common/commands/background.py | 3 ++- build/common/commands/backup.py | 11 +++++--- build/common/commands/check_connection.py | 26 +++++++++++++----- build/common/commands/console.py | 5 ++++ build/common/commands/doctor.py | 6 ++--- build/common/commands/migrate.py | 17 ++++++++---- build/common/commands/new.py | 11 +++++--- build/common/commands/push_backup.py | 20 +++++++++----- build/common/commands/restore_backup.py | 33 ++++++++++++----------- build/common/commands/worker.py | 4 ++- 11 files changed, 103 insertions(+), 55 deletions(-) diff --git a/build/common/commands/auto_migrate.py b/build/common/commands/auto_migrate.py index 1c096ef7..3c53db8d 100644 --- a/build/common/commands/auto_migrate.py +++ b/build/common/commands/auto_migrate.py @@ -9,10 +9,12 @@ from check_connection import get_config APP_VERSIONS_JSON_FILE = 'app_versions.json' APPS_TXT_FILE = 'apps.txt' + def save_version_file(versions): with open(APP_VERSIONS_JSON_FILE, 'w') as f: return json.dump(versions, f, indent=1, sort_keys=True) + def get_apps(): apps = [] try: @@ -24,40 +26,43 @@ def get_apps(): except FileNotFoundError as exception: print(exception) exit(1) - except: + except Exception: print(APPS_TXT_FILE+" is not valid") exit(1) return apps + def get_container_versions(apps): versions = {} for app in apps: try: version = __import__(app).__version__ - versions.update({app:version}) - except: + versions.update({app: version}) + except Exception: pass try: - path = os.path.join('..','apps', app) + path = os.path.join('..', 'apps', app) repo = git.Repo(path) commit_hash = repo.head.object.hexsha - versions.update({app+'_git_hash':commit_hash}) - except: + versions.update({app+'_git_hash': commit_hash}) + except Exception: pass return versions + def get_version_file(): versions = None try: with open(APP_VERSIONS_JSON_FILE) as versions_file: versions = json.load(versions_file) - except: + except Exception: pass return versions + def main(): is_ready = False apps = get_apps() @@ -76,7 +81,7 @@ def main(): version_file_hash = None container_hash = None - repo = git.Repo(os.path.join('..','apps',app)) + repo = git.Repo(os.path.join('..', 'apps', app)) branch = repo.active_branch.name if branch == 'develop': @@ -105,5 +110,6 @@ def main(): version_file = container_versions save_version_file(version_file) + if __name__ == "__main__": main() diff --git a/build/common/commands/background.py b/build/common/commands/background.py index fb008ba9..a26b7934 100644 --- a/build/common/commands/background.py +++ b/build/common/commands/background.py @@ -1,10 +1,11 @@ -import frappe from frappe.utils.scheduler import start_scheduler + def main(): print("Starting background scheduler . . .") start_scheduler() exit(0) + if __name__ == "__main__": main() diff --git a/build/common/commands/backup.py b/build/common/commands/backup.py index 9fa01cfd..c26468dd 100644 --- a/build/common/commands/backup.py +++ b/build/common/commands/backup.py @@ -1,7 +1,8 @@ -import os, frappe, compileall, re +import os +import frappe from frappe.utils.backups import scheduled_backup -from frappe.utils import now -from frappe.utils import get_sites +from frappe.utils import cint, get_sites, now + def backup(sites, with_files=False): for site in sites: @@ -20,13 +21,15 @@ def backup(sites, with_files=False): print("private files backup taken -", odb.backup_path_private_files, "- on", now()) frappe.destroy() + def main(): installed_sites = ":".join(get_sites()) sites = os.environ.get("SITES", installed_sites).split(":") - with_files=True if os.environ.get("WITH_FILES") else False + with_files = cint(os.environ.get("WITH_FILES")) backup(sites, with_files) exit(0) + if __name__ == "__main__": main() diff --git a/build/common/commands/check_connection.py b/build/common/commands/check_connection.py index b44ea12b..35fe93ff 100644 --- a/build/common/commands/check_connection.py +++ b/build/common/commands/check_connection.py @@ -1,4 +1,6 @@ -import socket, os, json, time +import socket +import json +import time from six.moves.urllib.parse import urlparse COMMON_SITE_CONFIG_FILE = 'common_site_config.json' @@ -8,6 +10,7 @@ REDIS_SOCKETIO_KEY = 'redis_socketio' DB_HOST_KEY = 'db_host' DB_PORT = 3306 + def is_open(ip, port, timeout=30): s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) s.settimeout(timeout) @@ -15,16 +18,17 @@ def is_open(ip, port, timeout=30): s.connect((ip, int(port))) s.shutdown(socket.SHUT_RDWR) return True - except: + except Exception: return False finally: s.close() + def check_host(ip, port, retry=10, delay=3, print_attempt=True): ipup = False for i in range(retry): if print_attempt: - print("Attempt {i} to connect to {ip}:{port}".format(ip=ip,port=port,i=i+1)) + print("Attempt {i} to connect to {ip}:{port}".format(ip=ip, port=port, i=i+1)) if is_open(ip, port): ipup = True break @@ -32,6 +36,7 @@ def check_host(ip, port, retry=10, delay=3, print_attempt=True): time.sleep(delay) return ipup + # Check connection to servers def get_config(): config = None @@ -41,11 +46,12 @@ def get_config(): except FileNotFoundError as exception: print(exception) exit(1) - except: + except Exception: print(COMMON_SITE_CONFIG_FILE+" is not valid") exit(1) return config + # Check mariadb def check_mariadb(retry=10, delay=3, print_attempt=True): config = get_config() @@ -60,11 +66,12 @@ def check_mariadb(retry=10, delay=3, print_attempt=True): print("Connection to MariaDB timed out") exit(1) + # Check redis queue def check_redis_queue(retry=10, delay=3, print_attempt=True): check_redis_queue = False config = get_config() - redis_queue_url = urlparse(config.get(REDIS_QUEUE_KEY,"redis://redis-queue:6379")).netloc + redis_queue_url = urlparse(config.get(REDIS_QUEUE_KEY, "redis://redis-queue:6379")).netloc redis_queue, redis_queue_port = redis_queue_url.split(":") check_redis_queue = check_host( redis_queue, @@ -76,11 +83,12 @@ def check_redis_queue(retry=10, delay=3, print_attempt=True): print("Connection to redis queue timed out") exit(1) + # Check redis cache def check_redis_cache(retry=10, delay=3, print_attempt=True): check_redis_cache = False config = get_config() - redis_cache_url = urlparse(config.get(REDIS_CACHE_KEY,"redis://redis-cache:6379")).netloc + redis_cache_url = urlparse(config.get(REDIS_CACHE_KEY, "redis://redis-cache:6379")).netloc redis_cache, redis_cache_port = redis_cache_url.split(":") check_redis_cache = check_host( redis_cache, @@ -92,11 +100,12 @@ def check_redis_cache(retry=10, delay=3, print_attempt=True): print("Connection to redis cache timed out") exit(1) + # Check redis socketio def check_redis_socketio(retry=10, delay=3, print_attempt=True): check_redis_socketio = False config = get_config() - redis_socketio_url = urlparse(config.get(REDIS_SOCKETIO_KEY,"redis://redis-socketio:6379")).netloc + redis_socketio_url = urlparse(config.get(REDIS_SOCKETIO_KEY, "redis://redis-socketio:6379")).netloc redis_socketio, redis_socketio_port = redis_socketio_url.split(":") check_redis_socketio = check_host( redis_socketio, @@ -108,6 +117,7 @@ def check_redis_socketio(retry=10, delay=3, print_attempt=True): print("Connection to redis socketio timed out") exit(1) + # Get site_config.json def get_site_config(site_name): site_config = None @@ -115,6 +125,7 @@ def get_site_config(site_name): site_config = json.load(site_config_file) return site_config + def main(): check_mariadb() check_redis_queue() @@ -122,5 +133,6 @@ def main(): check_redis_socketio() print('Connections OK') + if __name__ == "__main__": main() diff --git a/build/common/commands/console.py b/build/common/commands/console.py index a9ade863..369f049a 100644 --- a/build/common/commands/console.py +++ b/build/common/commands/console.py @@ -20,6 +20,11 @@ def console(site): print("Apps in this namespace:\n{}".format(", ".join(all_apps))) IPython.embed(display_banner="", header="") + def main(): site = sys.argv[-1] console(site) + + +if __name__ == "__main__": + main() diff --git a/build/common/commands/doctor.py b/build/common/commands/doctor.py index 98f2509f..c317f291 100644 --- a/build/common/commands/doctor.py +++ b/build/common/commands/doctor.py @@ -1,7 +1,3 @@ -import frappe -import json -import redis -from rq import Worker from check_connection import ( check_mariadb, check_redis_cache, @@ -9,6 +5,7 @@ from check_connection import ( check_redis_socketio, ) + def main(): check_mariadb(retry=1, delay=0, print_attempt=False) print("MariaDB Connected") @@ -22,5 +19,6 @@ def main(): print("Health check successful") exit(0) + if __name__ == "__main__": main() diff --git a/build/common/commands/migrate.py b/build/common/commands/migrate.py index 7f8a1602..70471981 100644 --- a/build/common/commands/migrate.py +++ b/build/common/commands/migrate.py @@ -1,29 +1,34 @@ -import os, frappe, compileall, re, json +import os +import frappe +import json from frappe.migrate import migrate -from frappe.utils import get_sites +from frappe.utils import cint, get_sites from check_connection import get_config + def save_config(config): with open('common_site_config.json', 'w') as f: return json.dump(config, f, indent=1, sort_keys=True) + def set_maintenance_mode(enable=True): conf = get_config() if enable: - conf.update({ "maintenance_mode": 1, "pause_scheduler": 1 }) + conf.update({"maintenance_mode": 1, "pause_scheduler": 1}) save_config(conf) if not enable: - conf.update({ "maintenance_mode": 0, "pause_scheduler": 0 }) + conf.update({"maintenance_mode": 0, "pause_scheduler": 0}) save_config(conf) + def migrate_sites(maintenance_mode=False): installed_sites = ":".join(get_sites()) sites = os.environ.get("SITES", installed_sites).split(":") if not maintenance_mode: - maintenance_mode = True if os.environ.get("MAINTENANCE_MODE") else False + maintenance_mode = cint(os.environ.get("MAINTENANCE_MODE")) if maintenance_mode: set_maintenance_mode(True) @@ -40,9 +45,11 @@ def migrate_sites(maintenance_mode=False): if maintenance_mode: set_maintenance_mode(False) + def main(): migrate_sites() exit(0) + if __name__ == "__main__": main() diff --git a/build/common/commands/new.py b/build/common/commands/new.py index 48ea44fb..0db5616c 100644 --- a/build/common/commands/new.py +++ b/build/common/commands/new.py @@ -1,17 +1,19 @@ -import os, frappe, json +import os +import frappe from frappe.commands.site import _new_site from check_connection import get_config, get_site_config + def get_password(env_var, default=None): return os.environ.get(env_var) or _get_password_from_secret(f"{env_var}_FILE") or default def _get_password_from_secret(env_var): - """Fetches the secret value from the docker secret file + """Fetches the secret value from the docker secret file usually located inside /run/secrets/ Arguments: - env_var {str} -- Name of the environment variable + env_var {str} -- Name of the environment variable containing the path to the secret file. Returns: [str] -- Secret value @@ -71,11 +73,12 @@ def main(): os.system(command) # Grant permission to database - command = mysql_command + "\"GRANT ALL PRIVILEGES ON \`{db_name}\`.* TO '{db_name}'@'%'; FLUSH PRIVILEGES;\"".format( + command = mysql_command + "\"GRANT ALL PRIVILEGES ON `{db_name}`.* TO '{db_name}'@'%'; FLUSH PRIVILEGES;\"".format( db_name=site_config.get('db_name') ) os.system(command) exit(0) + if __name__ == "__main__": main() diff --git a/build/common/commands/push_backup.py b/build/common/commands/push_backup.py index 26b5c464..8cf2b484 100644 --- a/build/common/commands/push_backup.py +++ b/build/common/commands/push_backup.py @@ -8,6 +8,7 @@ from frappe.utils import get_sites DATE_FORMAT = "%Y%m%d_%H%M%S" + def get_file_ext(): return { "database": "-database.sql.gz", @@ -15,6 +16,7 @@ def get_file_ext(): "public_files": "-files.tar" } + def get_backup_details(sitename): backup_details = dict() file_ext = get_file_ext() @@ -41,6 +43,7 @@ def get_backup_details(sitename): return backup_details + def get_s3_config(): check_environment_variables() bucket = os.environ.get('BUCKET_NAME') @@ -55,31 +58,33 @@ def get_s3_config(): return conn, bucket + def check_environment_variables(): - if not 'BUCKET_NAME' in os.environ: + if 'BUCKET_NAME' not in os.environ: print('Variable BUCKET_NAME not set') exit(1) - if not 'ACCESS_KEY_ID' in os.environ: + if 'ACCESS_KEY_ID' not in os.environ: print('Variable ACCESS_KEY_ID not set') exit(1) - if not 'SECRET_ACCESS_KEY' in os.environ: + if 'SECRET_ACCESS_KEY' not in os.environ: print('Variable SECRET_ACCESS_KEY not set') exit(1) - if not 'ENDPOINT_URL' in os.environ: + if 'ENDPOINT_URL' not in os.environ: print('Variable ENDPOINT_URL not set') exit(1) - if not 'BUCKET_DIR' in os.environ: + if 'BUCKET_DIR' not in os.environ: print('Variable BUCKET_DIR not set') exit(1) - if not 'REGION' in os.environ: + if 'REGION' not in os.environ: print('Variable REGION not set') exit(1) + def upload_file_to_s3(filename, folder, conn, bucket): destpath = os.path.join(folder, os.path.basename(filename)) @@ -91,6 +96,7 @@ def upload_file_to_s3(filename, folder, conn, bucket): print("Error uploading: %s" % (e)) exit(1) + def delete_old_backups(limit, bucket, site_name): all_backups = list() all_backup_dates = list() @@ -156,6 +162,7 @@ def delete_old_backups(limit, bucket, site_name): print('Deleteing ' + obj.key) s3.Object(bucket.name, obj.key).delete() + def main(): details = dict() sites = get_sites() @@ -184,5 +191,6 @@ def main(): print('push-backup complete') exit(0) + if __name__ == "__main__": main() diff --git a/build/common/commands/restore_backup.py b/build/common/commands/restore_backup.py index 38c1494b..341849cd 100644 --- a/build/common/commands/restore_backup.py +++ b/build/common/commands/restore_backup.py @@ -8,10 +8,10 @@ import boto3 from new import get_password from push_backup import DATE_FORMAT, check_environment_variables from frappe.utils import get_sites, random_string -from frappe.commands.site import _new_site from frappe.installer import make_conf, get_conf_params, make_site_dirs from check_connection import get_site_config, get_config + def list_directories(path): directories = [] for name in os.listdir(path): @@ -19,25 +19,25 @@ def list_directories(path): directories.append(name) return directories + def get_backup_dir(): return os.path.join( os.path.expanduser('~'), 'backups' ) + def decompress_db(files_base, site): database_file = files_base + '-database.sql.gz' - config = get_config() - site_config = get_site_config(site) - db_root_user = os.environ.get('DB_ROOT_USER', 'root') command = 'gunzip -c {database_file} > {database_extract}'.format( database_file=database_file, - database_extract=database_file.replace('.gz','') + database_extract=database_file.replace('.gz', '') ) print('Extract Database GZip for site {}'.format(site)) os.system(command) + def restore_database(files_base, site): db_root_password = get_password('MYSQL_ROOT_PASSWORD') if not db_root_password: @@ -60,13 +60,13 @@ def restore_database(files_base, site): ) # drop db if exists for clean restore - drop_database = mysql_command + "\"DROP DATABASE IF EXISTS \`{db_name}\`;\"".format( + drop_database = mysql_command + "\"DROP DATABASE IF EXISTS `{db_name}`;\"".format( db_name=site_config.get('db_name') ) os.system(drop_database) # create db - create_database = mysql_command + "\"CREATE DATABASE IF NOT EXISTS \`{db_name}\`;\"".format( + create_database = mysql_command + "\"CREATE DATABASE IF NOT EXISTS `{db_name}`;\"".format( db_name=site_config.get('db_name') ) os.system(create_database) @@ -86,7 +86,7 @@ def restore_database(files_base, site): os.system(set_user_password) # grant db privileges to user - grant_privileges = mysql_command + "\"GRANT ALL PRIVILEGES ON \`{db_name}\`.* TO '{db_name}'@'%'; FLUSH PRIVILEGES;\"".format( + grant_privileges = mysql_command + "\"GRANT ALL PRIVILEGES ON `{db_name}`.* TO '{db_name}'@'%'; FLUSH PRIVILEGES;\"".format( db_name=site_config.get('db_name') ) os.system(grant_privileges) @@ -96,12 +96,13 @@ def restore_database(files_base, site): db_host=config.get('db_host'), db_password=db_root_password, db_name=site_config.get('db_name'), - database_file=database_file.replace('.gz',''), + database_file=database_file.replace('.gz', ''), ) print('Restoring database for site: {}'.format(site)) os.system(command) + def restore_files(files_base): public_files = files_base + '-files.tar' # extract tar @@ -109,12 +110,14 @@ def restore_files(files_base): print('Extracting {}'.format(public_files)) public_tar.extractall() + def restore_private_files(files_base): private_files = files_base + '-private-files.tar' private_tar = tarfile.open(private_files) print('Extracting {}'.format(private_files)) private_tar.extractall() + def pull_backup_from_s3(): check_environment_variables() @@ -134,8 +137,8 @@ def pull_backup_from_s3(): # Change directory to /home/frappe/backups os.chdir(get_backup_dir()) - for obj in bucket.objects.filter(Prefix = bucket_dir): - backup_file = obj.key.replace(os.path.join(bucket_dir,''),'') + for obj in bucket.objects.filter(Prefix=bucket_dir): + backup_file = obj.key.replace(os.path.join(bucket_dir, ''), '') if not os.path.exists(os.path.dirname(backup_file)): os.makedirs(os.path.dirname(backup_file)) print('Downloading {}'.format(backup_file)) @@ -143,6 +146,7 @@ def pull_backup_from_s3(): os.chdir(os.path.join(os.path.expanduser('~'), 'frappe-bench', 'sites')) + def main(): backup_dir = get_backup_dir() @@ -150,8 +154,8 @@ def main(): pull_backup_from_s3() for site in list_directories(backup_dir): - site_slug = site.replace('.','_') - backups = [datetime.datetime.strptime(backup, DATE_FORMAT) for backup in list_directories(os.path.join(backup_dir,site))] + site_slug = site.replace('.', '_') + backups = [datetime.datetime.strptime(backup, DATE_FORMAT) for backup in list_directories(os.path.join(backup_dir, site))] latest_backup = max(backups).strftime(DATE_FORMAT) files_base = os.path.join(backup_dir, site, latest_backup, '') files_base += latest_backup + '-' + site_slug @@ -164,8 +168,6 @@ def main(): if not mariadb_root_password: print('Variable MYSQL_ROOT_PASSWORD not set') exit(1) - mariadb_root_username = os.environ.get('DB_ROOT_USER', 'root') - database_file = files_base + '-database.sql.gz' site_config = get_conf_params( db_name='_' + hashlib.sha1(site.encode()).hexdigest()[:16], @@ -186,5 +188,6 @@ def main(): exit(0) + if __name__ == "__main__": main() diff --git a/build/common/commands/worker.py b/build/common/commands/worker.py index c810adf9..ba107220 100644 --- a/build/common/commands/worker.py +++ b/build/common/commands/worker.py @@ -1,10 +1,12 @@ -import os, frappe +import os from frappe.utils.background_jobs import start_worker + def main(): queue = os.environ.get("WORKER_TYPE", "default") start_worker(queue, False) exit(0) + if __name__ == "__main__": main() From 8f20e5e00d6071ce138e0dacb1437b55e3175965 Mon Sep 17 00:00:00 2001 From: "Chinmay D. Pai" Date: Wed, 29 Apr 2020 11:44:58 +0530 Subject: [PATCH 06/13] chore: escape backticks on db commands Signed-off-by: Chinmay D. Pai --- build/common/commands/new.py | 2 +- build/common/commands/restore_backup.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/build/common/commands/new.py b/build/common/commands/new.py index 0db5616c..82c0ab7b 100644 --- a/build/common/commands/new.py +++ b/build/common/commands/new.py @@ -73,7 +73,7 @@ def main(): os.system(command) # Grant permission to database - command = mysql_command + "\"GRANT ALL PRIVILEGES ON `{db_name}`.* TO '{db_name}'@'%'; FLUSH PRIVILEGES;\"".format( + command = mysql_command + "\"GRANT ALL PRIVILEGES ON \`{db_name}\`.* TO '{db_name}'@'%'; FLUSH PRIVILEGES;\"".format( db_name=site_config.get('db_name') ) os.system(command) diff --git a/build/common/commands/restore_backup.py b/build/common/commands/restore_backup.py index 341849cd..1c098157 100644 --- a/build/common/commands/restore_backup.py +++ b/build/common/commands/restore_backup.py @@ -60,13 +60,13 @@ def restore_database(files_base, site): ) # drop db if exists for clean restore - drop_database = mysql_command + "\"DROP DATABASE IF EXISTS `{db_name}`;\"".format( + drop_database = mysql_command + "\"DROP DATABASE IF EXISTS \`{db_name}\`;\"".format( db_name=site_config.get('db_name') ) os.system(drop_database) # create db - create_database = mysql_command + "\"CREATE DATABASE IF NOT EXISTS `{db_name}`;\"".format( + create_database = mysql_command + "\"CREATE DATABASE IF NOT EXISTS \`{db_name}\`;\"".format( db_name=site_config.get('db_name') ) os.system(create_database) @@ -86,7 +86,7 @@ def restore_database(files_base, site): os.system(set_user_password) # grant db privileges to user - grant_privileges = mysql_command + "\"GRANT ALL PRIVILEGES ON `{db_name}`.* TO '{db_name}'@'%'; FLUSH PRIVILEGES;\"".format( + grant_privileges = mysql_command + "\"GRANT ALL PRIVILEGES ON \`{db_name}\`.* TO '{db_name}'@'%'; FLUSH PRIVILEGES;\"".format( db_name=site_config.get('db_name') ) os.system(grant_privileges) From e8b9710052a0aab28edf98593783323ccbee0a25 Mon Sep 17 00:00:00 2001 From: Revant Nandgaonkar Date: Wed, 29 Apr 2020 14:03:11 +0530 Subject: [PATCH 07/13] fix: mysql error during set db password --- build/common/commands/new.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/common/commands/new.py b/build/common/commands/new.py index 82c0ab7b..4771b21d 100644 --- a/build/common/commands/new.py +++ b/build/common/commands/new.py @@ -66,7 +66,7 @@ def main(): os.system(command) # Set db password - command = mysql_command + "\"UPDATE mysql.user SET authentication_string = PASSWORD('{db_password}') WHERE User = \'{db_name}\' AND Host = \'%\';\"".format( + command = mysql_command + "\"ALTER USER '{db_name}'@'%' IDENTIFIED BY '{db_password}'; FLUSH PRIVILEGES;\"".format( db_name=site_config.get('db_name'), db_password=site_config.get('db_password') ) From 8ac55c86f9ab72fbfded3d3cdc5cc9015a3ebe82 Mon Sep 17 00:00:00 2001 From: Revant Nandgaonkar Date: Wed, 29 Apr 2020 14:03:28 +0530 Subject: [PATCH 08/13] fix: always disable maintenance mode after migration --- build/common/commands/migrate.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build/common/commands/migrate.py b/build/common/commands/migrate.py index 70471981..06280a0f 100644 --- a/build/common/commands/migrate.py +++ b/build/common/commands/migrate.py @@ -42,8 +42,8 @@ def migrate_sites(maintenance_mode=False): finally: frappe.destroy() - if maintenance_mode: - set_maintenance_mode(False) + # Disable maintenance mode after migration + set_maintenance_mode(False) def main(): From 3181bc66b35ee095eaa6597966bdd09a76fa156f Mon Sep 17 00:00:00 2001 From: Revant Nandgaonkar Date: Wed, 29 Apr 2020 19:32:51 +0530 Subject: [PATCH 09/13] fix: restore backup create user password command --- build/common/commands/restore_backup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/common/commands/restore_backup.py b/build/common/commands/restore_backup.py index 1c098157..be85fab8 100644 --- a/build/common/commands/restore_backup.py +++ b/build/common/commands/restore_backup.py @@ -79,7 +79,7 @@ def restore_database(files_base, site): os.system(create_user) # create user password - set_user_password = mysql_command + "\"UPDATE mysql.user SET authentication_string = PASSWORD('{db_password}') WHERE User = \'{db_name}\' AND Host = \'%\';\"".format( + set_user_password = mysql_command + "\"ALTER USER '{db_name}'@'%' IDENTIFIED BY '{db_password}'; FLUSH PRIVILEGES;\"".format( db_name=site_config.get('db_name'), db_password=site_config.get('db_password') ) From e1f0e62211b46954ca6d481381284b317f7bef88 Mon Sep 17 00:00:00 2001 From: Revant Nandgaonkar Date: Wed, 29 Apr 2020 22:52:15 +0530 Subject: [PATCH 10/13] fix: restore backup grant privileges command --- build/common/commands/restore_backup.py | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/build/common/commands/restore_backup.py b/build/common/commands/restore_backup.py index be85fab8..29b0143f 100644 --- a/build/common/commands/restore_backup.py +++ b/build/common/commands/restore_backup.py @@ -78,17 +78,11 @@ def restore_database(files_base, site): ) os.system(create_user) - # create user password - set_user_password = mysql_command + "\"ALTER USER '{db_name}'@'%' IDENTIFIED BY '{db_password}'; FLUSH PRIVILEGES;\"".format( + # grant db privileges to user + grant_privileges = mysql_command + "\"GRANT ALL PRIVILEGES ON \`{db_name}\`.* TO '{db_name}'@'%' IDENTIFIED BY '{db_password}'; FLUSH PRIVILEGES;\"".format( db_name=site_config.get('db_name'), db_password=site_config.get('db_password') ) - os.system(set_user_password) - - # grant db privileges to user - grant_privileges = mysql_command + "\"GRANT ALL PRIVILEGES ON \`{db_name}\`.* TO '{db_name}'@'%'; FLUSH PRIVILEGES;\"".format( - db_name=site_config.get('db_name') - ) os.system(grant_privileges) command = "mysql -u{db_root_user} -h{db_host} -p{db_password} '{db_name}' < {database_file}".format( From 4f12e61a7a04bd202d97bc7113c073e604c713f0 Mon Sep 17 00:00:00 2001 From: Revant Nandgaonkar Date: Fri, 1 May 2020 16:54:52 +0530 Subject: [PATCH 11/13] test: all worker commands --- .travis.yml | 11 +++ tests/docker-test.sh | 216 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 227 insertions(+) create mode 100755 tests/docker-test.sh diff --git a/.travis.yml b/.travis.yml index 73f9a74b..150acdc3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -110,3 +110,14 @@ jobs: - ./travis.py erpnext --nginx --git-version 11 - ./travis.py erpnext --nginx --tag v11 --tag-only - ./travis.py erpnext --nginx --tag version-11 --tag-only + - stage: "Build and test develop branch" + if: type = pull_request OR branch = develop + before_install: + - sudo apt-get update && sudo apt-get -y install docker-compose + script: + - docker build -t frappe/frappe-socketio:edge -f build/frappe-socketio/Dockerfile . + - docker build -t frappe/frappe-worker:develop -f build/frappe-worker/Dockerfile . + - docker build -t frappe/erpnext-worker:edge -f build/erpnext-worker/Dockerfile . + - docker build -t frappe/frappe-nginx:develop -f build/frappe-nginx/Dockerfile . + - docker build -t frappe/erpnext-nginx:edge -f build/erpnext-nginx/Dockerfile . + - ./tests/docker-test.sh diff --git a/tests/docker-test.sh b/tests/docker-test.sh new file mode 100755 index 00000000..dded6bb7 --- /dev/null +++ b/tests/docker-test.sh @@ -0,0 +1,216 @@ +#!/bin/bash + +function checkMigrationComplete() { + echo "Check Auto Migration" + CONTAINER_ID=$(docker-compose \ + --project-name frappebench00 \ + -f installation/docker-compose-common.yml \ + -f installation/docker-compose-erpnext.yml \ + -f installation/erpnext-publish.yml \ + ps -q erpnext-python) + + DOCKER_LOG=$(docker logs $CONTAINER_ID 2>&1 | grep "Starting gunicorn") + INCREMENT=0 + while [[ $DOCKER_LOG != *"Starting gunicorn"* && $INCREMENT -lt 60 ]]; do + sleep 3 + echo "Wait for migration to complete ..." + ((INCREMENT=INCREMENT+1)) + DOCKER_LOG=$(docker logs $CONTAINER_ID 2>&1 | grep "Starting gunicorn") + if [[ $DOCKER_LOG != *"Starting gunicorn"* && $INCREMENT -eq 60 ]]; then + docker logs $CONTAINER_ID + exit 1 + fi + done +} + +function loopHealthCheck() { + echo "Create Container to Check MariaDB" + docker run --name frappe_doctor \ + -v frappebench00_sites-vol:/home/frappe/frappe-bench/sites \ + --network frappebench00_default \ + frappe/erpnext-worker:edge doctor || true + + echo "Loop Health Check" + FRAPPE_LOG=$(docker logs frappe_doctor | grep "Health check successful" || echo "") + while [[ -z "$FRAPPE_LOG" ]]; do + sleep 1 + CONTAINER=$(docker start frappe_doctor) + echo "Restarting $CONTAINER ..." + FRAPPE_LOG=$(docker logs frappe_doctor | grep "Health check successful" || echo "") + done + echo "Health check successful" +} + +echo "Copy env-example file" +cp env-example .env + +echo "Set version to v12" +sed -i -e "s/edge/v12/g" .env + +echo "Start Services" +docker-compose \ + --project-name frappebench00 \ + -f installation/docker-compose-common.yml \ + -f installation/docker-compose-erpnext.yml \ + -f installation/erpnext-publish.yml \ + up -d + +loopHealthCheck + +echo "Create new site (v12)" +docker run -it \ + -e "SITE_NAME=test.localhost" \ + -e "INSTALL_APPS=erpnext" \ + -v frappebench00_sites-vol:/home/frappe/frappe-bench/sites \ + --network frappebench00_default \ + frappe/erpnext-worker:v12 new + +echo "Ping created site" +curl -S http://test.localhost/api/method/version +echo "" + +echo "Set version to edge" +sed -i -e "s/v12/edge/g" .env + +echo "Restart containers with edge image" +docker-compose \ + --project-name frappebench00 \ + -f installation/docker-compose-common.yml \ + -f installation/docker-compose-erpnext.yml \ + -f installation/erpnext-publish.yml \ + stop +docker-compose \ + --project-name frappebench00 \ + -f installation/docker-compose-common.yml \ + -f installation/docker-compose-erpnext.yml \ + -f installation/erpnext-publish.yml \ + up -d + +checkMigrationComplete + +echo "Ping migrated site" +sleep 3 +curl -S http://test.localhost/api/method/version +echo "" + +echo "Backup site" +docker run -it \ + -e "WITH_FILES=1" \ + -v frappebench00_sites-vol:/home/frappe/frappe-bench/sites \ + --network frappebench00_default \ + frappe/erpnext-worker:edge backup + +export MINIO_ACCESS_KEY="AKIAIOSFODNN7EXAMPLE" +export MINIO_SECRET_KEY="wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" + +echo "Start MinIO container for s3 compatible storage" +docker run -d --name minio \ + -e "MINIO_ACCESS_KEY=$MINIO_ACCESS_KEY" \ + -e "MINIO_SECRET_KEY=$MINIO_SECRET_KEY" \ + --network frappebench00_default \ + minio/minio server /data + +echo "Create bucket named erpnext after 3 sec." +sleep 3 +docker run \ + --network frappebench00_default \ + vltgroup/s3cmd:latest s3cmd --access_key=$MINIO_ACCESS_KEY \ + --secret_key=$MINIO_SECRET_KEY \ + --region=us-east-1 \ + --no-ssl \ + --host=minio:9000 \ + --host-bucket=minio:9000 \ + mb s3://erpnext + +echo "Push backup to MinIO s3" +docker run \ + -e BUCKET_NAME=erpnext \ + -e REGION=us-east-1 \ + -e BUCKET_DIR=local \ + -e ACCESS_KEY_ID=$MINIO_ACCESS_KEY \ + -e SECRET_ACCESS_KEY=$MINIO_SECRET_KEY \ + -e ENDPOINT_URL=http://minio:9000 \ + -v frappebench00_sites-vol:/home/frappe/frappe-bench/sites \ + --network frappebench00_default \ + frappe/erpnext-worker:edge push-backup + +echo "Stop Services" +docker-compose \ + --project-name frappebench00 \ + -f installation/docker-compose-common.yml \ + -f installation/docker-compose-erpnext.yml \ + -f installation/erpnext-publish.yml \ + stop + +echo "Prune Containers" +docker container prune -f && docker volume prune -f + +echo "Start Services" +docker-compose \ + --project-name frappebench00 \ + -f installation/docker-compose-common.yml \ + -f installation/docker-compose-erpnext.yml \ + -f installation/erpnext-publish.yml \ + up -d + +loopHealthCheck + +echo "Restore backup from MinIO / S3" +docker run \ + -e MYSQL_ROOT_PASSWORD=admin \ + -e BUCKET_NAME=erpnext \ + -e BUCKET_DIR=local \ + -e ACCESS_KEY_ID=$MINIO_ACCESS_KEY \ + -e SECRET_ACCESS_KEY=$MINIO_SECRET_KEY \ + -e ENDPOINT_URL=http://minio:9000 \ + -e REGION=us-east-1 \ + -v frappebench00_sites-vol:/home/frappe/frappe-bench/sites \ + --network frappebench00_default \ + frappe/erpnext-worker:edge restore-backup + +echo "Check Restored Site" +sleep 3 +RESTORE_STATUS=$(curl -S http://test.localhost/api/method/version || echo "") +INCREMENT=0 +while [[ -z "$RESTORE_STATUS" && $INCREMENT -lt 60 ]]; do + sleep 1 + echo "Wait for restoration to complete ..." + RESTORE_STATUS=$(curl -S http://test.localhost/api/method/version || echo "") + ((INCREMENT=INCREMENT+1)) + if [[ -z "$RESTORE_STATUS" && $INCREMENT -eq 60 ]]; then + CONTAINER_ID=$(docker-compose \ + --project-name frappebench00 \ + -f installation/docker-compose-common.yml \ + -f installation/docker-compose-erpnext.yml \ + -f installation/erpnext-publish.yml \ + ps -q erpnext-python) + docker logs $CONTAINER_ID + exit 1 + fi +done + +echo "Ping restored site" +echo $RESTORE_STATUS + +echo "Migrate command in edge container" +docker run -it \ + -e "MAINTENANCE_MODE=1" \ + -v frappebench00_sites-vol:/home/frappe/frappe-bench/sites \ + --network frappebench00_default \ + frappe/erpnext-worker:edge migrate + +checkMigrationComplete + +echo "Create new site (edge)" +docker run -it \ + -e "SITE_NAME=edge.localhost" \ + -e "INSTALL_APPS=erpnext" \ + -v frappebench00_sites-vol:/home/frappe/frappe-bench/sites \ + --network frappebench00_default \ + frappe/erpnext-worker:edge new + +echo "Check console command for site test.localhost" +docker run \ + -v frappebench00_sites-vol:/home/frappe/frappe-bench/sites \ + --network frappebench00_default \ + frappe/erpnext-worker:edge console test.localhost From da4f65438dba426e5c59133413c690c1028afbab Mon Sep 17 00:00:00 2001 From: Revant Nandgaonkar Date: Fri, 1 May 2020 19:42:19 +0530 Subject: [PATCH 12/13] test: separate builds for develop branch and PR Develop branch jobs pull edge images and test PR jobs build images and test --- .travis.yml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 150acdc3..355180a7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -110,8 +110,8 @@ jobs: - ./travis.py erpnext --nginx --git-version 11 - ./travis.py erpnext --nginx --tag v11 --tag-only - ./travis.py erpnext --nginx --tag version-11 --tag-only - - stage: "Build and test develop branch" - if: type = pull_request OR branch = develop + - stage: "Build and test edge images" + if: type = pull_request before_install: - sudo apt-get update && sudo apt-get -y install docker-compose script: @@ -121,3 +121,9 @@ jobs: - docker build -t frappe/frappe-nginx:develop -f build/frappe-nginx/Dockerfile . - docker build -t frappe/erpnext-nginx:edge -f build/erpnext-nginx/Dockerfile . - ./tests/docker-test.sh + - stage: "Pull and test edge images" + if: branch = develop AND type != pull_request + before_install: + - sudo apt-get update && sudo apt-get -y install docker-compose + script: + - ./tests/docker-test.sh From 05cb6a45968e380985871451b19a93d60ead0b54 Mon Sep 17 00:00:00 2001 From: Revant Nandgaonkar Date: Sat, 2 May 2020 00:21:56 +0530 Subject: [PATCH 13/13] build v13-beta images --- .travis.yml | 30 +++++++++++++ build/erpnext-nginx/v13-beta.Dockerfile | 16 +++++++ build/erpnext-worker/v13-beta.Dockerfile | 3 ++ build/frappe-nginx/v13-beta.Dockerfile | 52 +++++++++++++++++++++++ build/frappe-socketio/v13-beta.Dockerfile | 36 ++++++++++++++++ build/frappe-worker/v13-beta.Dockerfile | 48 +++++++++++++++++++++ travis.py | 6 ++- 7 files changed, 190 insertions(+), 1 deletion(-) create mode 100644 build/erpnext-nginx/v13-beta.Dockerfile create mode 100644 build/erpnext-worker/v13-beta.Dockerfile create mode 100644 build/frappe-nginx/v13-beta.Dockerfile create mode 100644 build/frappe-socketio/v13-beta.Dockerfile create mode 100644 build/frappe-worker/v13-beta.Dockerfile diff --git a/.travis.yml b/.travis.yml index 355180a7..9bb9b583 100644 --- a/.travis.yml +++ b/.travis.yml @@ -50,6 +50,36 @@ jobs: script: - ./travis.py erpnext --nginx --tag edge - ./travis.py erpnext --nginx --tag develop --tag-only + - stage: "Frappe (v13-beta)" + if: branch = master AND type != pull_request + script: + - ./travis.py frappe --worker --git-version 13-beta + - ./travis.py frappe --worker --tag v13-beta --tag-only + - ./travis.py frappe --worker --tag version-13-beta --tag-only + - stage: "Frappe (v13-beta)" + if: branch = master AND type != pull_request + script: + - ./travis.py frappe --nginx --git-version 13-beta + - ./travis.py frappe --nginx --tag v13-beta --tag-only + - ./travis.py frappe --nginx --tag version-13-beta --tag-only + - stage: "Frappe (v13-beta)" + if: branch = master AND type != pull_request + script: + - ./travis.py frappe --socketio --git-version 13-beta + - ./travis.py frappe --socketio --tag v13-beta --tag-only + - ./travis.py frappe --socketio --tag version-13-beta --tag-only + - stage: "ERPNext (v13-beta)" + if: branch = master AND type != pull_request + script: + - ./travis.py erpnext --worker --git-version 13-beta + - ./travis.py erpnext --worker --tag v13-beta --tag-only + - ./travis.py erpnext --worker --tag version-13-beta --tag-only + - stage: "ERPNext (v13-beta)" + if: branch = master AND type != pull_request + script: + - ./travis.py erpnext --nginx --git-version 13-beta + - ./travis.py erpnext --nginx --tag v13-beta --tag-only + - ./travis.py erpnext --nginx --tag version-13-beta --tag-only - stage: "Frappe (v12)" if: branch = master AND type != pull_request script: diff --git a/build/erpnext-nginx/v13-beta.Dockerfile b/build/erpnext-nginx/v13-beta.Dockerfile new file mode 100644 index 00000000..952859aa --- /dev/null +++ b/build/erpnext-nginx/v13-beta.Dockerfile @@ -0,0 +1,16 @@ +FROM bitnami/node:12-prod + +COPY build/erpnext-nginx/install_app.sh /install_app + +RUN /install_app erpnext https://github.com/frappe/erpnext version-13-beta + +FROM frappe/frappe-nginx:v13-beta + +COPY --from=0 /home/frappe/frappe-bench/sites/ /var/www/html/ +COPY --from=0 /rsync /rsync +RUN echo -n "\nerpnext" >> /var/www/html/apps.txt + +VOLUME [ "/assets" ] + +ENTRYPOINT ["/docker-entrypoint.sh"] +CMD ["nginx", "-g", "daemon off;"] diff --git a/build/erpnext-worker/v13-beta.Dockerfile b/build/erpnext-worker/v13-beta.Dockerfile new file mode 100644 index 00000000..cfb4ca08 --- /dev/null +++ b/build/erpnext-worker/v13-beta.Dockerfile @@ -0,0 +1,3 @@ +FROM frappe/frappe-worker:v13-beta + +RUN install_app erpnext https://github.com/frappe/erpnext version-13-beta \ No newline at end of file diff --git a/build/frappe-nginx/v13-beta.Dockerfile b/build/frappe-nginx/v13-beta.Dockerfile new file mode 100644 index 00000000..1a6577bc --- /dev/null +++ b/build/frappe-nginx/v13-beta.Dockerfile @@ -0,0 +1,52 @@ +FROM bitnami/node:12-prod + +WORKDIR /home/frappe/frappe-bench +RUN mkdir -p /home/frappe/frappe-bench/sites \ + && echo "frappe" > /home/frappe/frappe-bench/sites/apps.txt + +RUN install_packages git + +RUN mkdir -p apps sites/assets/css \ + && cd apps \ + && git clone --depth 1 https://github.com/frappe/frappe --branch version-13-beta + +COPY build/frappe-nginx/generate_standard_style_css.js \ + /home/frappe/frappe-bench/apps/frappe/generate_standard_style_css.js + +RUN cd /home/frappe/frappe-bench/apps/frappe \ + && yarn \ + && yarn run production \ + && yarn add nunjucks -D \ + && node generate_standard_style_css.js \ + frappe/website/doctype/website_theme/website_theme_template.scss > \ + /home/frappe/standard_templates_string \ + && node generate_bootstrap_theme.js \ + /home/frappe/frappe-bench/sites/assets/css/standard_style.css \ + "$(cat /home/frappe/standard_templates_string)" \ + && rm -fr node_modules \ + && yarn install --production=true \ + && node --version \ + && npm --version \ + && yarn --version + +RUN git clone --depth 1 https://github.com/frappe/bench /tmp/bench \ + && mkdir -p /var/www/error_pages \ + && cp -r /tmp/bench/bench/config/templates /var/www/error_pages + +RUN cp -R /home/frappe/frappe-bench/apps/frappe/frappe/public/* /home/frappe/frappe-bench/sites/assets/frappe \ + && cp -R /home/frappe/frappe-bench/apps/frappe/node_modules /home/frappe/frappe-bench/sites/assets/frappe/ + +FROM nginx:latest +COPY --from=0 /home/frappe/frappe-bench/sites /var/www/html/ +COPY --from=0 /var/www/error_pages /var/www/ +COPY build/common/nginx-default.conf.template /etc/nginx/conf.d/default.conf.template +COPY build/frappe-nginx/docker-entrypoint.sh / + +RUN apt-get update && apt-get install -y rsync && apt-get clean \ + && echo "#!/bin/bash" > /rsync \ + && chmod +x /rsync + +VOLUME [ "/assets" ] + +ENTRYPOINT ["/docker-entrypoint.sh"] +CMD ["nginx", "-g", "daemon off;"] diff --git a/build/frappe-socketio/v13-beta.Dockerfile b/build/frappe-socketio/v13-beta.Dockerfile new file mode 100644 index 00000000..4ae4f7ea --- /dev/null +++ b/build/frappe-socketio/v13-beta.Dockerfile @@ -0,0 +1,36 @@ +FROM node:slim + +# Install needed packages +RUN apt-get update && apt-get install -y curl && apt-get clean + +RUN useradd -ms /bin/bash frappe + +# Make bench directories +RUN mkdir -p /home/frappe/frappe-bench/sites /home/frappe/frappe-bench/apps/frappe + +COPY build/frappe-socketio/package.json /home/frappe/frappe-bench/apps/frappe + + +# get socketio +RUN cd /home/frappe/frappe-bench/apps/frappe \ + && curl https://raw.githubusercontent.com/frappe/frappe/version-13-beta/socketio.js \ + --output /home/frappe/frappe-bench/apps/frappe/socketio.js \ + && curl https://raw.githubusercontent.com/frappe/frappe/version-13-beta/node_utils.js \ + --output /home/frappe/frappe-bench/apps/frappe/node_utils.js + +RUN cd /home/frappe/frappe-bench/apps/frappe \ + && npm install --only=production \ + && node --version \ + && npm --version + +COPY build/frappe-socketio/health.js /home/frappe/frappe-bench/apps/frappe/health.js +RUN chown -R frappe:frappe /home/frappe + +# Setup docker-entrypoint +COPY build/frappe-socketio/docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh +RUN ln -s /usr/local/bin/docker-entrypoint.sh / # backwards compat + +WORKDIR /home/frappe/frappe-bench/sites + +ENTRYPOINT ["docker-entrypoint.sh"] +CMD ["start"] diff --git a/build/frappe-worker/v13-beta.Dockerfile b/build/frappe-worker/v13-beta.Dockerfile new file mode 100644 index 00000000..1f2ed290 --- /dev/null +++ b/build/frappe-worker/v13-beta.Dockerfile @@ -0,0 +1,48 @@ +FROM bitnami/python:latest-prod + +RUN useradd -ms /bin/bash frappe +WORKDIR /home/frappe/frappe-bench +RUN install_packages \ + git \ + wkhtmltopdf \ + mariadb-client \ + gettext-base \ + wget \ + # for PDF + libssl-dev \ + fonts-cantarell \ + xfonts-75dpi \ + xfonts-base \ + # For psycopg2 + libpq-dev \ + build-essential + +# Install wkhtmltox correctly +RUN wget https://github.com/wkhtmltopdf/wkhtmltopdf/releases/download/0.12.5/wkhtmltox_0.12.5-1.stretch_amd64.deb +RUN dpkg -i wkhtmltox_0.12.5-1.stretch_amd64.deb && rm wkhtmltox_0.12.5-1.stretch_amd64.deb + +RUN mkdir -p apps logs commands /home/frappe/backups + +RUN virtualenv env \ + && . env/bin/activate \ + && cd apps \ + && git clone --depth 1 -o upstream https://github.com/frappe/frappe --branch version-13-beta \ + && pip3 install --no-cache-dir -e /home/frappe/frappe-bench/apps/frappe + +COPY build/common/commands/* /home/frappe/frappe-bench/commands/ +COPY build/common/common_site_config.json.template /opt/frappe/common_site_config.json.template + +# Setup docker-entrypoint +COPY build/common/worker/docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh +RUN ln -s /usr/local/bin/docker-entrypoint.sh / # backwards compat + +COPY build/common/worker/install_app.sh /usr/local/bin/install_app + +WORKDIR /home/frappe/frappe-bench/sites + +RUN chown -R frappe:frappe /home/frappe/frappe-bench/sites /home/frappe/backups + +VOLUME [ "/home/frappe/frappe-bench/sites", "/home/frappe/backups" ] + +ENTRYPOINT ["docker-entrypoint.sh"] +CMD ["start"] diff --git a/travis.py b/travis.py index 02101369..a9cb936e 100755 --- a/travis.py +++ b/travis.py @@ -15,7 +15,7 @@ image_type.add_argument('-s', '--socketio', action='store_const', dest='image_ty image_type.add_argument('-w', '--worker', action='store_const', dest='image_type', const='worker', help='Build the python environment image') tag_type = parser.add_mutually_exclusive_group(required=True) -tag_type.add_argument('-g', '--git-version', action='store', type=int, dest='version', help='The version number of --service (i.e. "11", "12", etc.)') +tag_type.add_argument('-g', '--git-version', action='store', type=str, dest='version', help='The version number of --service (i.e. "11", "12", etc.)') tag_type.add_argument('-t', '--tag', action='store', type=str, dest='tag', help='The image tag (i.e. erpnext-worker:$TAG )') args = parser.parse_args() @@ -26,6 +26,10 @@ def git_version(service, version): cd = os.getcwd() os.chdir(os.getcwd() + f'/{service}') subprocess.run('git fetch --tags', shell=True) + + # XX-beta becomes XX for tags search + version = version.split('-')[0] + version_tag = subprocess.check_output(f'git tag --list --sort=-version:refname "v{version}*" | sed -n 1p | sed -e \'s#.*@\(\)#\\1#\'', shell=True).strip().decode('ascii') os.chdir(cd) return version_tag