From aeadd23214cf4e859cda7d8d00a1860e2bc20438 Mon Sep 17 00:00:00 2001 From: Revant Nandgaonkar Date: Mon, 10 Feb 2020 13:30:54 +0530 Subject: [PATCH] feat: docker production images --- .travis.yml | 61 ++++--- build/erpnext-assets/.dockerignore | 2 + build/erpnext-assets/Dockerfile | 39 +++++ build/erpnext-assets/docker-entrypoint.sh | 35 ++++ .../nginx-default.conf.template | 90 ++++++++++ build/erpnext-assets/sites/apps.txt | 2 + build/erpnext-assets/v11.Dockerfile | 39 +++++ build/erpnext-assets/v12.Dockerfile | 39 +++++ build/erpnext-python/Dockerfile | 44 +++++ build/erpnext-python/commands/background.py | 7 + build/erpnext-python/commands/backup.py | 29 ++++ .../commands/check_connection.py | 67 ++++++++ build/erpnext-python/commands/doctor.py | 4 + build/erpnext-python/commands/migrate.py | 47 ++++++ build/erpnext-python/commands/new.py | 68 ++++++++ build/erpnext-python/commands/update.py | 55 ++++++ build/erpnext-python/commands/worker.py | 7 + .../common_site_config.json.template | 7 + build/erpnext-python/docker-entrypoint.sh | 158 ++++++++++++++++++ build/erpnext-python/v11.Dockerfile | 41 +++++ build/erpnext-python/v12.Dockerfile | 44 +++++ build/frappe-socketio/Dockerfile | 34 ++++ build/frappe-socketio/docker-entrypoint.sh | 29 ++++ build/frappe-socketio/health.js | 19 +++ build/frappe-socketio/package.json | 17 ++ build/frappe-socketio/v11.Dockerfile | 34 ++++ build/frappe-socketio/v12.Dockerfile | 34 ++++ 27 files changed, 1031 insertions(+), 21 deletions(-) create mode 100644 build/erpnext-assets/.dockerignore create mode 100644 build/erpnext-assets/Dockerfile create mode 100755 build/erpnext-assets/docker-entrypoint.sh create mode 100644 build/erpnext-assets/nginx-default.conf.template create mode 100644 build/erpnext-assets/sites/apps.txt create mode 100644 build/erpnext-assets/v11.Dockerfile create mode 100644 build/erpnext-assets/v12.Dockerfile create mode 100644 build/erpnext-python/Dockerfile create mode 100644 build/erpnext-python/commands/background.py create mode 100644 build/erpnext-python/commands/backup.py create mode 100644 build/erpnext-python/commands/check_connection.py create mode 100644 build/erpnext-python/commands/doctor.py create mode 100644 build/erpnext-python/commands/migrate.py create mode 100644 build/erpnext-python/commands/new.py create mode 100644 build/erpnext-python/commands/update.py create mode 100644 build/erpnext-python/commands/worker.py create mode 100755 build/erpnext-python/common_site_config.json.template create mode 100755 build/erpnext-python/docker-entrypoint.sh create mode 100644 build/erpnext-python/v11.Dockerfile create mode 100644 build/erpnext-python/v12.Dockerfile create mode 100644 build/frappe-socketio/Dockerfile create mode 100755 build/frappe-socketio/docker-entrypoint.sh create mode 100644 build/frappe-socketio/health.js create mode 100644 build/frappe-socketio/package.json create mode 100644 build/frappe-socketio/v11.Dockerfile create mode 100644 build/frappe-socketio/v12.Dockerfile diff --git a/.travis.yml b/.travis.yml index de8802ae..bb56485f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,30 +1,49 @@ +sudo: required + dist: xenial -env: - - DOCKER_COMPOSE_VERSION: 1.23.1 - - services: - docker before_install: - - curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - - - sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" - - sudo apt-get update - - sudo apt-get -y -o Dpkg::Options::="--force-confnew" install docker-ce - - sudo rm /usr/local/bin/docker-compose - - curl -L https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-`uname -s`-`uname -m` > docker-compose - - chmod +x docker-compose - - sudo mv docker-compose /usr/local/bin + - echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin + - if [ $BUILD == "development" ];then + sudo apt-get update; + sudo apt-get -y install docker-compose; + fi install: - - chmod ugo+x ./dbench - - chmod ugo+x ./test.sh - - ./dbench setup docker - - ./dbench init frappe-bench - - ./dbench new-site site1.local - - ./dbench setup hosts + - if [ $BUILD == "development" ];then + chmod ugo+x ./dbench; + chmod ugo+x ./test.sh; + ./dbench setup docker; + ./dbench init frappe-bench; + ./dbench new-site site1.local; + ./dbench setup hosts; + fi -script: - - ./test.sh - - ./dbench setup docker stop +after_success: + - docker --version + +matrix: + include: + - name: "Test frappe / erpnext development" + env: BUILD=development + script: + - ./test.sh + - ./dbench setup docker stop + - name: "Build frappe / erpnext python environment" + script: + - docker build -t erpnext-python build/erpnext-python + - docker tag erpnext-python frappe/erpnext-python:edge + - docker push frappe/erpnext-python:edge + - name: "Build frappe / erpnext nginx + static assets" + script: + - docker build -t erpnext-assets build/erpnext-assets + - docker tag erpnext-assets frappe/erpnext-assets:edge + - docker push frappe/erpnext-assets:edge + - name: "Build frappe socketio service" + script: + - docker build -t frappe-socketio build/frappe-socketio + - docker tag frappe-socketio frappe/frappe-socketio:edge + - docker push frappe/frappe-socketio:edge diff --git a/build/erpnext-assets/.dockerignore b/build/erpnext-assets/.dockerignore new file mode 100644 index 00000000..c58728b5 --- /dev/null +++ b/build/erpnext-assets/.dockerignore @@ -0,0 +1,2 @@ +Dockerfile +/docker/ diff --git a/build/erpnext-assets/Dockerfile b/build/erpnext-assets/Dockerfile new file mode 100644 index 00000000..90bd0a4d --- /dev/null +++ b/build/erpnext-assets/Dockerfile @@ -0,0 +1,39 @@ +FROM bitnami/node:12-prod + +WORKDIR /home/frappe/frappe-bench +COPY sites/apps.txt /home/frappe/frappe-bench/sites/apps.txt + +RUN install_packages git + +RUN mkdir -p apps sites/assets \ + && cd apps \ + && git clone --depth 1 https://github.com/frappe/frappe \ + && git clone --depth 1 https://github.com/frappe/erpnext + +RUN cd /home/frappe/frappe-bench/apps/frappe \ + && yarn \ + && yarn run production \ + && rm -fr node_modules \ + && yarn install --production=true + +RUN git clone --depth 1 https://github.com/frappe/bench /tmp/bench \ + && mkdir -p /var/www/error_pages \ + && mkdir -p /home/frappe/frappe-bench/sites/assets/erpnext \ + && cp -r /tmp/bench/bench/config/templates /var/www/error_pages + +RUN cp -R /home/frappe/frappe-bench/apps/frappe/frappe/public/* /home/frappe/frappe-bench/sites/assets/frappe \ + && cp -R /home/frappe/frappe-bench/apps/frappe/node_modules /home/frappe/frappe-bench/sites/assets/frappe/ \ + && cp -R /home/frappe/frappe-bench/apps/erpnext/erpnext/public/* /home/frappe/frappe-bench/sites/assets/erpnext + +FROM nginx:latest +COPY --from=0 /home/frappe/frappe-bench/sites /var/www/html/ +COPY --from=0 /var/www/error_pages /var/www/ +COPY nginx-default.conf.template /etc/nginx/conf.d/default.conf.template +COPY docker-entrypoint.sh / + +RUN apt-get update && apt-get install -y rsync && apt-get clean + +VOLUME [ "/assets" ] + +ENTRYPOINT ["/docker-entrypoint.sh"] +CMD ["nginx", "-g", "daemon off;"] diff --git a/build/erpnext-assets/docker-entrypoint.sh b/build/erpnext-assets/docker-entrypoint.sh new file mode 100755 index 00000000..80fbf003 --- /dev/null +++ b/build/erpnext-assets/docker-entrypoint.sh @@ -0,0 +1,35 @@ +#!/bin/bash + +## Thanks +# https://serverfault.com/a/919212 +## + +set -e + +rsync -a --delete /var/www/html/assets/js /assets +rsync -a --delete /var/www/html/assets/css /assets +rsync -a --delete /var/www/html/assets/frappe /assets +rsync -a --delete /var/www/html/assets/erpnext /assets + +chmod -R 755 /assets + +if [[ -z "$ERPNEXT_PY" ]]; then + export ERPNEXT_PY=0.0.0.0 +fi + +if [[ -z "$ERPNEXT_PY_PORT" ]]; then + export ERPNEXT_PY_PORT=8000 +fi + +if [[ -z "$FRAPPE_SOCKETIO" ]]; then + export FRAPPE_SOCKETIO=0.0.0.0 +fi + +if [[ -z "$FRAPPE_SOCKETIO_PORT" ]]; then + export FRAPPE_SOCKETIO_PORT=9000 +fi + +envsubst '${API_HOST} ${API_PORT} ${ERPNEXT_PY} ${ERPNEXT_PY_PORT} ${FRAPPE_SOCKETIO} ${FRAPPE_SOCKETIO_PORT}' \ + < /etc/nginx/conf.d/default.conf.template > /etc/nginx/conf.d/default.conf + +exec "$@" diff --git a/build/erpnext-assets/nginx-default.conf.template b/build/erpnext-assets/nginx-default.conf.template new file mode 100644 index 00000000..17578b6a --- /dev/null +++ b/build/erpnext-assets/nginx-default.conf.template @@ -0,0 +1,90 @@ +upstream erpnext-server { + server ${ERPNEXT_PY}:${ERPNEXT_PY_PORT} fail_timeout=0; +} + +upstream socketio-server { + server ${FRAPPE_SOCKETIO}:${FRAPPE_SOCKETIO_PORT} fail_timeout=0; +} + +server { + listen 80; + server_name $http_host; + root /var/www/html; + + location /assets { + try_files $uri =404; + } + + location ~ ^/protected/(.*) { + internal; + try_files /sites/$http_host/$1 =404; + } + + location /socket.io { + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + proxy_set_header X-Frappe-Site-Name $http_host; + proxy_set_header Origin $scheme://$http_host; + proxy_set_header Host $host; + + proxy_pass http://socketio-server; + } + + location / { + try_files /sites/$http_host/public/$uri @webserver; + } + + location @webserver { + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Frappe-Site-Name $http_host; + proxy_set_header Host $host; + proxy_set_header X-Use-X-Accel-Redirect True; + proxy_read_timeout 120; + proxy_redirect off; + + proxy_pass http://erpnext-server; + } + + # error pages + error_page 502 /502.html; + location /502.html { + root /var/www/templates; + internal; + } + + # optimizations + sendfile on; + keepalive_timeout 15; + client_max_body_size 50m; + client_body_buffer_size 16K; + client_header_buffer_size 1k; + + # enable gzip compresion + # based on https://mattstauffer.co/blog/enabling-gzip-on-nginx-servers-including-laravel-forge + gzip on; + gzip_http_version 1.1; + gzip_comp_level 5; + gzip_min_length 256; + gzip_proxied any; + gzip_vary on; + gzip_types + application/atom+xml + application/javascript + application/json + application/rss+xml + application/vnd.ms-fontobject + application/x-font-ttf + application/font-woff + application/x-web-app-manifest+json + application/xhtml+xml + application/xml + font/opentype + image/svg+xml + image/x-icon + text/css + text/plain + text/x-component; + # text/html is always compressed by HttpGzipModule +} diff --git a/build/erpnext-assets/sites/apps.txt b/build/erpnext-assets/sites/apps.txt new file mode 100644 index 00000000..d700de5f --- /dev/null +++ b/build/erpnext-assets/sites/apps.txt @@ -0,0 +1,2 @@ +frappe +erpnext diff --git a/build/erpnext-assets/v11.Dockerfile b/build/erpnext-assets/v11.Dockerfile new file mode 100644 index 00000000..d2024670 --- /dev/null +++ b/build/erpnext-assets/v11.Dockerfile @@ -0,0 +1,39 @@ +FROM bitnami/node:10-prod + +WORKDIR /home/frappe/frappe-bench +COPY sites/apps.txt /home/frappe/frappe-bench/sites/apps.txt + +RUN install_packages git + +RUN mkdir -p apps sites/assets \ + && cd apps \ + && git clone --depth 1 https://github.com/frappe/frappe --branch version-11 \ + && git clone --depth 1 https://github.com/frappe/erpnext --branch version-11 + +RUN cd /home/frappe/frappe-bench/apps/frappe \ + && yarn \ + && yarn run production \ + && rm -fr node_modules \ + && yarn install --production=true + +RUN git clone --depth 1 https://github.com/frappe/bench /tmp/bench \ + && mkdir -p /var/www/error_pages \ + && mkdir -p /home/frappe/frappe-bench/sites/assets/erpnext \ + && cp -r /tmp/bench/bench/config/templates /var/www/error_pages + +RUN cp -R /home/frappe/frappe-bench/apps/frappe/frappe/public/* /home/frappe/frappe-bench/sites/assets/frappe \ + && cp -R /home/frappe/frappe-bench/apps/frappe/node_modules /home/frappe/frappe-bench/sites/assets/frappe/ \ + && cp -R /home/frappe/frappe-bench/apps/erpnext/erpnext/public/* /home/frappe/frappe-bench/sites/assets/erpnext + +FROM nginx:latest +COPY --from=0 /home/frappe/frappe-bench/sites /var/www/html/ +COPY --from=0 /var/www/error_pages /var/www/ +COPY nginx-default.conf.template /etc/nginx/conf.d/default.conf.template +COPY docker-entrypoint.sh / + +RUN apt-get update && apt-get install -y rsync && apt-get clean + +VOLUME [ "/assets" ] + +ENTRYPOINT ["/docker-entrypoint.sh"] +CMD ["nginx", "-g", "daemon off;"] diff --git a/build/erpnext-assets/v12.Dockerfile b/build/erpnext-assets/v12.Dockerfile new file mode 100644 index 00000000..1dd2e5d6 --- /dev/null +++ b/build/erpnext-assets/v12.Dockerfile @@ -0,0 +1,39 @@ +FROM bitnami/node:12-prod + +WORKDIR /home/frappe/frappe-bench +COPY sites/apps.txt /home/frappe/frappe-bench/sites/apps.txt + +RUN install_packages git + +RUN mkdir -p apps sites/assets \ + && cd apps \ + && git clone --depth 1 https://github.com/frappe/frappe --branch version-12 \ + && git clone --depth 1 https://github.com/frappe/erpnext --branch version-12 + +RUN cd /home/frappe/frappe-bench/apps/frappe \ + && yarn \ + && yarn run production \ + && rm -fr node_modules \ + && yarn install --production=true + +RUN git clone --depth 1 https://github.com/frappe/bench /tmp/bench \ + && mkdir -p /var/www/error_pages \ + && mkdir -p /home/frappe/frappe-bench/sites/assets/erpnext \ + && cp -r /tmp/bench/bench/config/templates /var/www/error_pages + +RUN cp -R /home/frappe/frappe-bench/apps/frappe/frappe/public/* /home/frappe/frappe-bench/sites/assets/frappe \ + && cp -R /home/frappe/frappe-bench/apps/frappe/node_modules /home/frappe/frappe-bench/sites/assets/frappe/ \ + && cp -R /home/frappe/frappe-bench/apps/erpnext/erpnext/public/* /home/frappe/frappe-bench/sites/assets/erpnext + +FROM nginx:latest +COPY --from=0 /home/frappe/frappe-bench/sites /var/www/html/ +COPY --from=0 /var/www/error_pages /var/www/ +COPY nginx-default.conf.template /etc/nginx/conf.d/default.conf.template +COPY docker-entrypoint.sh / + +RUN apt-get update && apt-get install -y rsync && apt-get clean + +VOLUME [ "/assets" ] + +ENTRYPOINT ["/docker-entrypoint.sh"] +CMD ["nginx", "-g", "daemon off;"] diff --git a/build/erpnext-python/Dockerfile b/build/erpnext-python/Dockerfile new file mode 100644 index 00000000..59175254 --- /dev/null +++ b/build/erpnext-python/Dockerfile @@ -0,0 +1,44 @@ +FROM bitnami/python:latest-prod + +RUN useradd -ms /bin/bash frappe +WORKDIR /home/frappe/frappe-bench +RUN install_packages \ + git \ + wkhtmltopdf \ + mariadb-client \ + gettext-base \ + wget \ + # for PDF + libssl-dev \ + fonts-cantarell \ + xfonts-75dpi \ + xfonts-base \ + # For psycopg2 + libpq-dev \ + build-essential + +# Install wkhtmltox correctly +RUN wget https://github.com/wkhtmltopdf/wkhtmltopdf/releases/download/0.12.5/wkhtmltox_0.12.5-1.stretch_amd64.deb +RUN dpkg -i wkhtmltox_0.12.5-1.stretch_amd64.deb && rm wkhtmltox_0.12.5-1.stretch_amd64.deb + +RUN mkdir -p apps logs commands + +RUN virtualenv env \ + && . env/bin/activate \ + && cd apps \ + && git clone --depth 1 -o upstream https://github.com/frappe/frappe \ + && git clone --depth 1 -o upstream https://github.com/frappe/erpnext \ + && pip3 install --no-cache-dir -e /home/frappe/frappe-bench/apps/frappe \ + && pip3 install --no-cache-dir -e /home/frappe/frappe-bench/apps/erpnext + +COPY ./commands/* /home/frappe/frappe-bench/commands/ +COPY ./common_site_config.json.template /opt/frappe/common_site_config.json.template + +# Setup docker-entrypoint +COPY docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh +RUN ln -s /usr/local/bin/docker-entrypoint.sh / # backwards compat + +WORKDIR /home/frappe/frappe-bench/sites + +ENTRYPOINT ["docker-entrypoint.sh"] +CMD ["start"] diff --git a/build/erpnext-python/commands/background.py b/build/erpnext-python/commands/background.py new file mode 100644 index 00000000..7065efd9 --- /dev/null +++ b/build/erpnext-python/commands/background.py @@ -0,0 +1,7 @@ +import frappe +from frappe.utils.scheduler import start_scheduler + +print("Starting background scheduler . . .") +start_scheduler() + +exit(0) diff --git a/build/erpnext-python/commands/backup.py b/build/erpnext-python/commands/backup.py new file mode 100644 index 00000000..e6fbcbb6 --- /dev/null +++ b/build/erpnext-python/commands/backup.py @@ -0,0 +1,29 @@ +import os, frappe, compileall, re +from frappe.utils.backups import scheduled_backup +from frappe.utils import now +from frappe.utils import get_sites + +def backup(sites, with_files=False): + for site in sites: + frappe.init(site) + frappe.connect() + odb = scheduled_backup( + ignore_files=not with_files, + backup_path_db=None, + backup_path_files=None, + backup_path_private_files=None, + force=True + ) + print("database backup taken -", odb.backup_path_db, "- on", now()) + if with_files: + print("files backup taken -", odb.backup_path_files, "- on", now()) + print("private files backup taken -", odb.backup_path_private_files, "- on", now()) + frappe.destroy() + +installed_sites = ":".join(get_sites()) +sites = os.environ.get("SITES", installed_sites).split(":") +with_files=True if os.environ.get("WITH_FILES") else False + +backup(sites, with_files) + +exit(0) diff --git a/build/erpnext-python/commands/check_connection.py b/build/erpnext-python/commands/check_connection.py new file mode 100644 index 00000000..abe17199 --- /dev/null +++ b/build/erpnext-python/commands/check_connection.py @@ -0,0 +1,67 @@ +import socket, os, json, time +from six.moves.urllib.parse import urlparse + +def is_open(ip, port, timeout=30): + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + s.settimeout(timeout) + try: + s.connect((ip, int(port))) + s.shutdown(socket.SHUT_RDWR) + return True + except: + return False + finally: + s.close() + +def check_host(ip, port, retry=10, delay=3): + ipup = False + for i in range(retry): + print("Attempt {i} to connect to {ip}:{port}".format(ip=ip,port=port,i=i+1)) + if is_open(ip, port): + ipup = True + break + else: + time.sleep(delay) + return ipup + +# Check connection to servers +config = None +try: + with open('common_site_config.json') as config_file: + config = json.load(config_file) +except FileNotFoundError: + raise FileNotFoundError("common_site_config.json missing") +except: + raise ValueError("common_site_config.json is not valid") + +# Check mariadb +check_mariadb = False +check_mariadb = check_host(config.get('db_host', 'mariadb'), 3306) +if not check_mariadb: + raise ConnectionError("Connection to mariadb timed out") + +# Check redis queue +check_redis_queue = False +redis_queue_url = urlparse(config.get("redis_queue","redis://redis:6379")).netloc +redis_queue, redis_queue_port = redis_queue_url.split(":") +check_redis_queue = check_host(redis_queue, redis_queue_port) +if not check_redis_queue: + raise ConnectionError("Connection to redis queue timed out") + +# Check redis cache +check_redis_cache = False +redis_cache_url = urlparse(config.get("redis_cache","redis://redis:6379")).netloc +redis_cache, redis_cache_port = redis_cache_url.split(":") +check_redis_cache = check_host(redis_cache, redis_cache_port) +if not check_redis_cache: + raise ConnectionError("Connection to redis cache timed out") + +# Check redis socketio +check_redis_socketio = False +redis_socketio_url = urlparse(config.get("redis_socketio","redis://redis:6379")).netloc +redis_socketio, redis_socketio_port = redis_socketio_url.split(":") +check_redis_socketio = check_host(redis_socketio, redis_socketio_port) +if not check_redis_socketio: + raise ConnectionError("Connection to redis socketio timed out") + +print('Connections OK') diff --git a/build/erpnext-python/commands/doctor.py b/build/erpnext-python/commands/doctor.py new file mode 100644 index 00000000..54f508fb --- /dev/null +++ b/build/erpnext-python/commands/doctor.py @@ -0,0 +1,4 @@ +import frappe +from frappe.utils.doctor import doctor + +doctor() diff --git a/build/erpnext-python/commands/migrate.py b/build/erpnext-python/commands/migrate.py new file mode 100644 index 00000000..be382bea --- /dev/null +++ b/build/erpnext-python/commands/migrate.py @@ -0,0 +1,47 @@ +import os, frappe, compileall, re, json + +from frappe.migrate import migrate +from frappe.utils import get_sites + +def get_config(): + config = None + with open('common_site_config.json') as config_file: + config = json.load(config_file) + return config + +def save_config(config): + with open('common_site_config.json', 'w') as f: + return json.dump(config, f, indent=1, sort_keys=True) + +def set_maintenance_mode(enable=True): + conf = get_config() + + if enable: + conf.update({ "maintenance_mode": 1, "pause_scheduler": 1 }) + save_config(conf) + + if not enable: + conf.update({ "maintenance_mode": 0, "pause_scheduler": 0 }) + save_config(conf) + + +installed_sites = ":".join(get_sites()) +sites = os.environ.get("SITES", installed_sites).split(":") +maintenance_mode = True if os.environ.get("MAINTENANCE_MODE") else False + +if maintenance_mode: + set_maintenance_mode(True) + +for site in sites: + print('Migrating', site) + frappe.init(site=site) + frappe.connect() + try: + migrate() + finally: + frappe.destroy() + +if maintenance_mode: + set_maintenance_mode(False) + +exit(0) diff --git a/build/erpnext-python/commands/new.py b/build/erpnext-python/commands/new.py new file mode 100644 index 00000000..b6589c79 --- /dev/null +++ b/build/erpnext-python/commands/new.py @@ -0,0 +1,68 @@ +import os, frappe, json + +from frappe.commands.site import _new_site + +site_name = os.environ.get("SITE_NAME", 'site1.localhost') +mariadb_root_username = os.environ.get("DB_ROOT_USER", 'root') +mariadb_root_password = os.environ.get("DB_ROOT_PASSWORD", 'admin') +install_erpnext = os.environ.get("INSTALL_ERPNEXT", None) +force = True if os.environ.get("FORCE", None) else False + +frappe.init(site_name, new_site=True) + +_new_site( + None, + site_name, + mariadb_root_username=mariadb_root_username, + mariadb_root_password=mariadb_root_password, + admin_password=os.environ.get("ADMIN_PASSWORD", 'admin'), + verbose=True, + install_apps=['erpnext'] if install_erpnext else [], + source_sql=None, + force=force, + reinstall=False, +) + +config = None +with open('common_site_config.json') as config_file: + config = json.load(config_file) + +site_config = None +with open('{site_name}/site_config.json'.format(site_name=site_name)) as site_config_file: + site_config = json.load(site_config_file) + +# update User's host to '%' required to connect from any container +command = 'mysql -h{db_host} -u{mariadb_root_username} -p{mariadb_root_password} -e '.format( + db_host=config.get('db_host'), + mariadb_root_username=mariadb_root_username, + mariadb_root_password=mariadb_root_password +) +command += "\"UPDATE mysql.user SET Host = '%' where User = '{db_name}'; FLUSH PRIVILEGES;\"".format( + db_name=site_config.get('db_name') +) +os.system(command) + +# Set db password +command = 'mysql -h{db_host} -u{mariadb_root_username} -p{mariadb_root_password} -e '.format( + db_host=config.get('db_host'), + mariadb_root_username=mariadb_root_username, + mariadb_root_password=mariadb_root_password +) +command += "\"SET PASSWORD FOR '{db_name}'@'%' = PASSWORD('{db_password}'); FLUSH PRIVILEGES;\"".format( + db_name=site_config.get('db_name'), + db_password=site_config.get('db_password') +) +os.system(command) + +# Grant permission to database +command = 'mysql -h{db_host} -u{mariadb_root_username} -p{mariadb_root_password} -e '.format( + db_host=config.get('db_host'), + mariadb_root_username=mariadb_root_username, + mariadb_root_password=mariadb_root_password +) +command += "\"GRANT ALL PRIVILEGES ON \`{db_name}\`.* TO '{db_name}'@'%'; FLUSH PRIVILEGES;\"".format( + db_name=site_config.get('db_name') +) +os.system(command) + +exit(0) diff --git a/build/erpnext-python/commands/update.py b/build/erpnext-python/commands/update.py new file mode 100644 index 00000000..fa47fc44 --- /dev/null +++ b/build/erpnext-python/commands/update.py @@ -0,0 +1,55 @@ +def update(pull=False, patch=False, build=False, update_bench=False, auto=False, restart_supervisor=False, + restart_systemd=False, requirements=False, no_backup=False, bench_path='.', force=False, reset=False): + conf = get_config(bench_path=bench_path) + version_upgrade = is_version_upgrade(bench_path=bench_path) + + if version_upgrade[0] or (not version_upgrade[0] and force): + validate_upgrade(version_upgrade[1], version_upgrade[2], bench_path=bench_path) + + before_update(bench_path=bench_path, requirements=requirements) + + conf.update({ "maintenance_mode": 1, "pause_scheduler": 1 }) + update_config(conf, bench_path=bench_path) + + if not no_backup: + print('Backing up sites...') + backup_all_sites(bench_path=bench_path) + + if pull: + pull_all_apps(bench_path=bench_path, reset=reset) + + if requirements: + update_requirements(bench_path=bench_path) + update_node_packages(bench_path=bench_path) + + if version_upgrade[0] or (not version_upgrade[0] and force): + pre_upgrade(version_upgrade[1], version_upgrade[2], bench_path=bench_path) + import bench.utils, bench.app + print('Reloading bench...') + if sys.version_info >= (3, 4): + import importlib + importlib.reload(bench.utils) + importlib.reload(bench.app) + else: + reload(bench.utils) + reload(bench.app) + + if patch: + print('Patching sites...') + patch_sites(bench_path=bench_path) + if build: + build_assets(bench_path=bench_path) + if version_upgrade[0] or (not version_upgrade[0] and force): + post_upgrade(version_upgrade[1], version_upgrade[2], bench_path=bench_path) + if restart_supervisor or conf.get('restart_supervisor_on_update'): + restart_supervisor_processes(bench_path=bench_path) + if restart_systemd or conf.get('restart_systemd_on_update'): + restart_systemd_processes(bench_path=bench_path) + + conf.update({ "maintenance_mode": 0, "pause_scheduler": 0 }) + update_config(conf, bench_path=bench_path) + + print("_"*80) + print("Bench: Deployment tool for Frappe and ERPNext (https://erpnext.org).") + print("Open source depends on your contributions, so please contribute bug reports, patches, fixes or cash and be a part of the community") + print() \ No newline at end of file diff --git a/build/erpnext-python/commands/worker.py b/build/erpnext-python/commands/worker.py new file mode 100644 index 00000000..6ec0bcbf --- /dev/null +++ b/build/erpnext-python/commands/worker.py @@ -0,0 +1,7 @@ +import os, frappe +from frappe.utils.background_jobs import start_worker + +queue = os.environ.get("WORKER_TYPE", "default") +start_worker(queue, False) + +exit(0) diff --git a/build/erpnext-python/common_site_config.json.template b/build/erpnext-python/common_site_config.json.template new file mode 100755 index 00000000..27593289 --- /dev/null +++ b/build/erpnext-python/common_site_config.json.template @@ -0,0 +1,7 @@ +{ + "db_host": "${MARIADB_HOST}", + "redis_cache": "redis://${REDIS_CACHE}", + "redis_queue": "redis://${REDIS_QUEUE}", + "redis_socketio": "redis://${REDIS_SOCKETIO}", + "socketio_port": ${SOCKETIO_PORT} +} diff --git a/build/erpnext-python/docker-entrypoint.sh b/build/erpnext-python/docker-entrypoint.sh new file mode 100755 index 00000000..fef4eda9 --- /dev/null +++ b/build/erpnext-python/docker-entrypoint.sh @@ -0,0 +1,158 @@ +#!/bin/bash + +function configureEnv() { + if [ ! -f /home/frappe/frappe-bench/sites/common_site_config.json ]; then + + if [[ -z "$MARIADB_HOST" ]]; then + echo "MARIADB_HOST is not set" + exit 1 + fi + + if [[ -z "$REDIS_CACHE" ]]; then + echo "REDIS_CACHE is not set" + exit 1 + fi + + if [[ -z "$REDIS_QUEUE" ]]; then + echo "REDIS_QUEUE is not set" + exit 1 + fi + + if [[ -z "$REDIS_SOCKETIO" ]]; then + echo "REDIS_SOCKETIO is not set" + exit 1 + fi + + if [[ -z "$SOCKETIO_PORT" ]]; then + echo "SOCKETIO_PORT is not set" + exit 1 + fi + + envsubst '${MARIADB_HOST} + ${REDIS_CACHE} + ${REDIS_QUEUE} + ${REDIS_SOCKETIO} + ${SOCKETIO_PORT}' < /opt/frappe/common_site_config.json.template > /home/frappe/frappe-bench/sites/common_site_config.json + fi +} + +function checkConnection() { + su frappe -c ". /home/frappe/frappe-bench/env/bin/activate \ + && python /home/frappe/frappe-bench/commands/check_connection.py" +} + +function checkConfigExists() { + COUNTER=0 + while [[ ! -e /home/frappe/frappe-bench/sites/common_site_config.json ]] && [[ $COUNTER -le 30 ]] ; do + sleep 1 + let COUNTER=COUNTER+1 + echo "config file not created, retry $COUNTER" + done + + if [[ ! -e /home/frappe/frappe-bench/sites/common_site_config.json ]]; then + echo "timeout: config file not created" + exit 1 + fi +} + +if [[ ! -e /home/frappe/frappe-bench/sites/apps.txt ]]; then + ls -1 /home/frappe/frappe-bench/apps | sort -r > /home/frappe/frappe-bench/sites/apps.txt +fi + +if [ "$1" = 'start' ]; then + configureEnv + checkConnection + + chown frappe:frappe /home/frappe/frappe-bench/sites/common_site_config.json + + if [[ -z "$WORKERS" ]]; then + export WORKERS=2 + fi + + if [[ -z "$FRAPPE_PORT" ]]; then + export FRAPPE_PORT=8000 + fi + + + if [[ -z "$RUN_AS_ROOT" ]]; then + su frappe -c ". /home/frappe/frappe-bench/env/bin/activate \ + && gunicorn -b 0.0.0.0:$FRAPPE_PORT \ + --worker-tmp-dir /dev/shm \ + --threads=4 \ + --workers $WORKERS \ + --worker-class=gthread \ + --log-file=- \ + -t 120 frappe.app:application --preload" + else + . /home/frappe/frappe-bench/env/bin/activate + gunicorn -b 0.0.0.0:$FRAPPE_PORT \ + --worker-tmp-dir /dev/shm \ + --threads=4 \ + --workers $WORKERS \ + --worker-class=gthread \ + --log-file=- \ + -t 120 frappe.app:application --preload + fi + +elif [ "$1" = 'worker' ]; then + checkConfigExists + checkConnection + # default WORKER_TYPE=default + if [[ -z "$RUN_AS_ROOT" ]]; then + su frappe -c ". /home/frappe/frappe-bench/env/bin/activate \ + && python /home/frappe/frappe-bench/commands/worker.py" + else + . /home/frappe/frappe-bench/env/bin/activate + python /home/frappe/frappe-bench/commands/worker.py + fi + +elif [ "$1" = 'schedule' ]; then + checkConfigExists + checkConnection + if [[ -z "$RUN_AS_ROOT" ]]; then + su frappe -c ". /home/frappe/frappe-bench/env/bin/activate \ + && python /home/frappe/frappe-bench/commands/background.py" + else + . /home/frappe/frappe-bench/env/bin/activate + python /home/frappe/frappe-bench/commands/background.py + fi + +elif [ "$1" = 'new' ]; then + + if [[ -z "$RUN_AS_ROOT" ]]; then + su frappe -c ". /home/frappe/frappe-bench/env/bin/activate \ + && python /home/frappe/frappe-bench/commands/new.py" + exit + else + . /home/frappe/frappe-bench/env/bin/activate + python /home/frappe/frappe-bench/commands/new.py + fi + +elif [ "$1" = 'migrate' ]; then + + su frappe -c ". /home/frappe/frappe-bench/env/bin/activate \ + && python /home/frappe/frappe-bench/commands/migrate.py" + exit + +elif [ "$1" = 'doctor' ]; then + + su frappe -c ". /home/frappe/frappe-bench/env/bin/activate \ + && python /home/frappe/frappe-bench/commands/doctor.py" + exit + +elif [ "$1" = 'backup' ]; then + + if [[ -z "$RUN_AS_ROOT" ]]; then + su frappe -c ". /home/frappe/frappe-bench/env/bin/activate \ + && python /home/frappe/frappe-bench/commands/backup.py" + exit + else + . /home/frappe/frappe-bench/env/bin/activate + python /home/frappe/frappe-bench/commands/backup.py + fi + +else + + exec su frappe -c "$@" + +fi diff --git a/build/erpnext-python/v11.Dockerfile b/build/erpnext-python/v11.Dockerfile new file mode 100644 index 00000000..c56a5e96 --- /dev/null +++ b/build/erpnext-python/v11.Dockerfile @@ -0,0 +1,41 @@ +FROM bitnami/python:latest-prod + +RUN useradd -ms /bin/bash frappe +WORKDIR /home/frappe/frappe-bench +RUN install_packages \ + git \ + wkhtmltopdf \ + mariadb-client \ + gettext-base \ + wget \ + # for PDF + libssl-dev \ + fonts-cantarell \ + xfonts-75dpi \ + xfonts-base + +# Install wkhtmltox correctly +RUN wget https://github.com/wkhtmltopdf/wkhtmltopdf/releases/download/0.12.5/wkhtmltox_0.12.5-1.stretch_amd64.deb +RUN dpkg -i wkhtmltox_0.12.5-1.stretch_amd64.deb && rm wkhtmltox_0.12.5-1.stretch_amd64.deb + +RUN mkdir -p apps logs commands + +RUN virtualenv env \ + && . env/bin/activate \ + && cd apps \ + && git clone --depth 1 -o upstream https://github.com/frappe/frappe --branch version-11 \ + && git clone --depth 1 -o upstream https://github.com/frappe/erpnext --branch version-11 \ + && pip3 install --no-cache-dir -e /home/frappe/frappe-bench/apps/frappe \ + && pip3 install --no-cache-dir -e /home/frappe/frappe-bench/apps/erpnext + +COPY ./commands/* /home/frappe/frappe-bench/commands/ +COPY ./common_site_config.json.template /opt/frappe/common_site_config.json.template + +# Setup docker-entrypoint +COPY docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh +RUN ln -s /usr/local/bin/docker-entrypoint.sh / # backwards compat + +WORKDIR /home/frappe/frappe-bench/sites + +ENTRYPOINT ["docker-entrypoint.sh"] +CMD ["start"] diff --git a/build/erpnext-python/v12.Dockerfile b/build/erpnext-python/v12.Dockerfile new file mode 100644 index 00000000..d892c81d --- /dev/null +++ b/build/erpnext-python/v12.Dockerfile @@ -0,0 +1,44 @@ +FROM bitnami/python:latest-prod + +RUN useradd -ms /bin/bash frappe +WORKDIR /home/frappe/frappe-bench +RUN install_packages \ + git \ + wkhtmltopdf \ + mariadb-client \ + gettext-base \ + wget \ + # for PDF + libssl-dev \ + fonts-cantarell \ + xfonts-75dpi \ + xfonts-base \ + # For psycopg2 + libpq-dev \ + build-essential + +# Install wkhtmltox correctly +RUN wget https://github.com/wkhtmltopdf/wkhtmltopdf/releases/download/0.12.5/wkhtmltox_0.12.5-1.stretch_amd64.deb +RUN dpkg -i wkhtmltox_0.12.5-1.stretch_amd64.deb && rm wkhtmltox_0.12.5-1.stretch_amd64.deb + +RUN mkdir -p apps logs commands + +RUN virtualenv env \ + && . env/bin/activate \ + && cd apps \ + && git clone --depth 1 -o upstream https://github.com/frappe/frappe --branch version-12 \ + && git clone --depth 1 -o upstream https://github.com/frappe/erpnext --branch version-12 \ + && pip3 install --no-cache-dir -e /home/frappe/frappe-bench/apps/frappe \ + && pip3 install --no-cache-dir -e /home/frappe/frappe-bench/apps/erpnext + +COPY ./commands/* /home/frappe/frappe-bench/commands/ +COPY ./common_site_config.json.template /opt/frappe/common_site_config.json.template + +# Setup docker-entrypoint +COPY docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh +RUN ln -s /usr/local/bin/docker-entrypoint.sh / # backwards compat + +WORKDIR /home/frappe/frappe-bench/sites + +ENTRYPOINT ["docker-entrypoint.sh"] +CMD ["start"] diff --git a/build/frappe-socketio/Dockerfile b/build/frappe-socketio/Dockerfile new file mode 100644 index 00000000..4987af4e --- /dev/null +++ b/build/frappe-socketio/Dockerfile @@ -0,0 +1,34 @@ +FROM node:slim + +# Install needed packages +RUN apt-get update && apt-get install -y curl && apt-get clean + +RUN useradd -ms /bin/bash frappe + +# Make bench directories +RUN mkdir -p /home/frappe/frappe-bench/sites /home/frappe/frappe-bench/apps/frappe + +COPY package.json /home/frappe/frappe-bench/apps/frappe + + +# get socketio +RUN cd /home/frappe/frappe-bench/apps/frappe \ + && curl https://raw.githubusercontent.com/frappe/frappe/develop/socketio.js \ + --output /home/frappe/frappe-bench/apps/frappe/socketio.js \ + && curl https://raw.githubusercontent.com/frappe/frappe/develop/node_utils.js \ + --output /home/frappe/frappe-bench/apps/frappe/node_utils.js + +RUN cd /home/frappe/frappe-bench/apps/frappe \ + && npm install --only=production + +COPY health.js /home/frappe/frappe-bench/apps/frappe/health.js +RUN chown -R frappe:frappe /home/frappe + +# Setup docker-entrypoint +COPY docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh +RUN ln -s /usr/local/bin/docker-entrypoint.sh / # backwards compat + +WORKDIR /home/frappe/frappe-bench/sites + +ENTRYPOINT ["docker-entrypoint.sh"] +CMD ["start"] diff --git a/build/frappe-socketio/docker-entrypoint.sh b/build/frappe-socketio/docker-entrypoint.sh new file mode 100755 index 00000000..600a744c --- /dev/null +++ b/build/frappe-socketio/docker-entrypoint.sh @@ -0,0 +1,29 @@ +#!/bin/bash + +function checkConfigExists() { + COUNTER=0 + while [[ ! -e /home/frappe/frappe-bench/sites/common_site_config.json ]] && [[ $COUNTER -le 30 ]] ; do + sleep 1 + let COUNTER=COUNTER+1 + echo "config file not created, retry $COUNTER" + done + + if [[ ! -e /home/frappe/frappe-bench/sites/common_site_config.json ]]; then + echo "timeout: config file not created" + exit 1 + fi +} + +if [ "$1" = 'start' ]; then + checkConfigExists + su frappe -c "node /home/frappe/frappe-bench/apps/frappe/socketio.js" + +elif [ "$1" = 'doctor' ]; then + + su frappe -c "node /home/frappe/frappe-bench/apps/frappe/health.js" + +else + + exec su frappe -c "$@" + +fi diff --git a/build/frappe-socketio/health.js b/build/frappe-socketio/health.js new file mode 100644 index 00000000..b85981bd --- /dev/null +++ b/build/frappe-socketio/health.js @@ -0,0 +1,19 @@ +var net = require('net'); +var { get_conf } = require('./node_utils'); + +var conf = get_conf(); +var server = ['localhost', conf.socketio_port || 9000]; + +var sock = new net.Socket(); +sock.setTimeout(2500); +sock.on('connect', function() { + console.info(server[0]+':'+server[1]+' is up.'); + sock.destroy(); + process.exit(); +}).on('error', function(e) { + console.error(server[0]+':'+server[1]+' is down: ' + e.message); + process.exit(1); +}).on('timeout', function(e) { + console.error(server[0]+':'+server[1]+' is down: timeout'); + process.exit(1); +}).connect(server[1], server[0]); diff --git a/build/frappe-socketio/package.json b/build/frappe-socketio/package.json new file mode 100644 index 00000000..4f8572d5 --- /dev/null +++ b/build/frappe-socketio/package.json @@ -0,0 +1,17 @@ +{ + "name": "frappe-socketio", + "version": "1.0.0", + "description": "Frappe SocketIO Server", + "main": "socketio.js", + "scripts": { + "start": "node socketio.js" + }, + "author": "Revant Nandgaonkar", + "license": "MIT", + "dependencies": { + "express": "^4.17.1", + "redis": "^2.8.0", + "socket.io": "^2.3.0", + "superagent": "^5.1.0" + } +} diff --git a/build/frappe-socketio/v11.Dockerfile b/build/frappe-socketio/v11.Dockerfile new file mode 100644 index 00000000..78986e0c --- /dev/null +++ b/build/frappe-socketio/v11.Dockerfile @@ -0,0 +1,34 @@ +FROM node:slim + +# Install needed packages +RUN apt-get update && apt-get install -y curl && apt-get clean + +RUN useradd -ms /bin/bash frappe + +# Make bench directories +RUN mkdir -p /home/frappe/frappe-bench/sites /home/frappe/frappe-bench/apps/frappe + +COPY package.json /home/frappe/frappe-bench/apps/frappe + + +# get socketio +RUN cd /home/frappe/frappe-bench/apps/frappe \ + && curl https://raw.githubusercontent.com/frappe/frappe/version-11/socketio.js \ + --output /home/frappe/frappe-bench/apps/frappe/socketio.js \ + && curl https://raw.githubusercontent.com/frappe/frappe/version-11/node_utils.js \ + --output /home/frappe/frappe-bench/apps/frappe/node_utils.js + +RUN cd /home/frappe/frappe-bench/apps/frappe \ + && npm install --only=production + +COPY health.js /home/frappe/frappe-bench/apps/frappe/health.js +RUN chown -R frappe:frappe /home/frappe + +# Setup docker-entrypoint +COPY docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh +RUN ln -s /usr/local/bin/docker-entrypoint.sh / # backwards compat + +WORKDIR /home/frappe/frappe-bench/sites + +ENTRYPOINT ["docker-entrypoint.sh"] +CMD ["start"] diff --git a/build/frappe-socketio/v12.Dockerfile b/build/frappe-socketio/v12.Dockerfile new file mode 100644 index 00000000..022000d7 --- /dev/null +++ b/build/frappe-socketio/v12.Dockerfile @@ -0,0 +1,34 @@ +FROM node:slim + +# Install needed packages +RUN apt-get update && apt-get install -y curl && apt-get clean + +RUN useradd -ms /bin/bash frappe + +# Make bench directories +RUN mkdir -p /home/frappe/frappe-bench/sites /home/frappe/frappe-bench/apps/frappe + +COPY package.json /home/frappe/frappe-bench/apps/frappe + + +# get socketio +RUN cd /home/frappe/frappe-bench/apps/frappe \ + && curl https://raw.githubusercontent.com/frappe/frappe/version-12/socketio.js \ + --output /home/frappe/frappe-bench/apps/frappe/socketio.js \ + && curl https://raw.githubusercontent.com/frappe/frappe/version-12/node_utils.js \ + --output /home/frappe/frappe-bench/apps/frappe/node_utils.js + +RUN cd /home/frappe/frappe-bench/apps/frappe \ + && npm install --only=production + +COPY health.js /home/frappe/frappe-bench/apps/frappe/health.js +RUN chown -R frappe:frappe /home/frappe + +# Setup docker-entrypoint +COPY docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh +RUN ln -s /usr/local/bin/docker-entrypoint.sh / # backwards compat + +WORKDIR /home/frappe/frappe-bench/sites + +ENTRYPOINT ["docker-entrypoint.sh"] +CMD ["start"]