From e1e1c6b6f96f27135128f0cafb45bac805f1af4b Mon Sep 17 00:00:00 2001 From: Revant Nandgaonkar Date: Mon, 17 Feb 2020 20:35:50 +0530 Subject: [PATCH] refactor: rename images erpnext-assets - ERPNext nginx config + static assets erpnext-worker - ERPNext python environment frappe-assets - Frappe nginx config + static assets frappe-worker - Frappe python environment frappe-socketio - Frappe socketio process common for apps --- .travis.yml | 24 ++- .../Dockerfile | 0 .../commands/background.py | 0 .../commands/backup.py | 0 .../commands/check_connection.py | 0 .../commands/doctor.py | 0 .../commands/migrate.py | 0 .../commands/new.py | 0 .../commands/update.py | 0 .../commands/worker.py | 0 .../common_site_config.json.template | 0 .../docker-entrypoint.sh | 0 .../v11.Dockerfile | 0 .../v12.Dockerfile | 0 build/frappe-assets/.dockerignore | 2 + build/frappe-assets/Dockerfile | 36 ++++ build/frappe-assets/docker-entrypoint.sh | 35 ++++ .../frappe-assets/nginx-default.conf.template | 90 ++++++++++ build/frappe-assets/sites/apps.txt | 1 + build/frappe-assets/v11.Dockerfile | 36 ++++ build/frappe-assets/v12.Dockerfile | 36 ++++ build/frappe-worker/Dockerfile | 42 +++++ build/frappe-worker/commands/background.py | 7 + build/frappe-worker/commands/backup.py | 29 ++++ .../commands/check_connection.py | 67 ++++++++ build/frappe-worker/commands/doctor.py | 4 + build/frappe-worker/commands/migrate.py | 47 ++++++ build/frappe-worker/commands/new.py | 67 ++++++++ build/frappe-worker/commands/update.py | 55 ++++++ build/frappe-worker/commands/worker.py | 7 + .../common_site_config.json.template | 7 + build/frappe-worker/docker-entrypoint.sh | 158 ++++++++++++++++++ build/frappe-worker/v11.Dockerfile | 39 +++++ build/frappe-worker/v12.Dockerfile | 42 +++++ 34 files changed, 824 insertions(+), 7 deletions(-) rename build/{erpnext-python => erpnext-worker}/Dockerfile (100%) rename build/{erpnext-python => erpnext-worker}/commands/background.py (100%) rename build/{erpnext-python => erpnext-worker}/commands/backup.py (100%) rename build/{erpnext-python => erpnext-worker}/commands/check_connection.py (100%) rename build/{erpnext-python => erpnext-worker}/commands/doctor.py (100%) rename build/{erpnext-python => erpnext-worker}/commands/migrate.py (100%) rename build/{erpnext-python => erpnext-worker}/commands/new.py (100%) rename build/{erpnext-python => erpnext-worker}/commands/update.py (100%) rename build/{erpnext-python => erpnext-worker}/commands/worker.py (100%) rename build/{erpnext-python => erpnext-worker}/common_site_config.json.template (100%) rename build/{erpnext-python => erpnext-worker}/docker-entrypoint.sh (100%) rename build/{erpnext-python => erpnext-worker}/v11.Dockerfile (100%) rename build/{erpnext-python => erpnext-worker}/v12.Dockerfile (100%) create mode 100644 build/frappe-assets/.dockerignore create mode 100644 build/frappe-assets/Dockerfile create mode 100755 build/frappe-assets/docker-entrypoint.sh create mode 100644 build/frappe-assets/nginx-default.conf.template create mode 100644 build/frappe-assets/sites/apps.txt create mode 100644 build/frappe-assets/v11.Dockerfile create mode 100644 build/frappe-assets/v12.Dockerfile create mode 100644 build/frappe-worker/Dockerfile create mode 100644 build/frappe-worker/commands/background.py create mode 100644 build/frappe-worker/commands/backup.py create mode 100644 build/frappe-worker/commands/check_connection.py create mode 100644 build/frappe-worker/commands/doctor.py create mode 100644 build/frappe-worker/commands/migrate.py create mode 100644 build/frappe-worker/commands/new.py create mode 100644 build/frappe-worker/commands/update.py create mode 100644 build/frappe-worker/commands/worker.py create mode 100755 build/frappe-worker/common_site_config.json.template create mode 100755 build/frappe-worker/docker-entrypoint.sh create mode 100644 build/frappe-worker/v11.Dockerfile create mode 100644 build/frappe-worker/v12.Dockerfile diff --git a/.travis.yml b/.travis.yml index bb56485f..ae4bb89f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -27,22 +27,32 @@ after_success: matrix: include: - - name: "Test frappe / erpnext development" + - name: "Test Frappe / ERPNext development" env: BUILD=development script: - ./test.sh - ./dbench setup docker stop - - name: "Build frappe / erpnext python environment" + - name: "Build Frappe python environment" script: - - docker build -t erpnext-python build/erpnext-python - - docker tag erpnext-python frappe/erpnext-python:edge - - docker push frappe/erpnext-python:edge - - name: "Build frappe / erpnext nginx + static assets" + - docker build -t frappe-worker build/frappe-worker + - docker tag frappe-worker frappe/frappe-worker:edge + - docker push frappe/frappe-worker:edge + - name: "Build Frappe nginx + static assets" + script: + - docker build -t frappe-assets build/frappe-assets + - docker tag frappe-assets frappe/frappe-assets:edge + - docker push frappe/frappe-assets:edge + - name: "Build ERPNext python environment" + script: + - docker build -t erpnext-worker build/erpnext-worker + - docker tag erpnext-worker frappe/erpnext-worker:edge + - docker push frappe/erpnext-worker:edge + - name: "Build ERPNext nginx + static assets" script: - docker build -t erpnext-assets build/erpnext-assets - docker tag erpnext-assets frappe/erpnext-assets:edge - docker push frappe/erpnext-assets:edge - - name: "Build frappe socketio service" + - name: "Build Frappe socketio service" script: - docker build -t frappe-socketio build/frappe-socketio - docker tag frappe-socketio frappe/frappe-socketio:edge diff --git a/build/erpnext-python/Dockerfile b/build/erpnext-worker/Dockerfile similarity index 100% rename from build/erpnext-python/Dockerfile rename to build/erpnext-worker/Dockerfile diff --git a/build/erpnext-python/commands/background.py b/build/erpnext-worker/commands/background.py similarity index 100% rename from build/erpnext-python/commands/background.py rename to build/erpnext-worker/commands/background.py diff --git a/build/erpnext-python/commands/backup.py b/build/erpnext-worker/commands/backup.py similarity index 100% rename from build/erpnext-python/commands/backup.py rename to build/erpnext-worker/commands/backup.py diff --git a/build/erpnext-python/commands/check_connection.py b/build/erpnext-worker/commands/check_connection.py similarity index 100% rename from build/erpnext-python/commands/check_connection.py rename to build/erpnext-worker/commands/check_connection.py diff --git a/build/erpnext-python/commands/doctor.py b/build/erpnext-worker/commands/doctor.py similarity index 100% rename from build/erpnext-python/commands/doctor.py rename to build/erpnext-worker/commands/doctor.py diff --git a/build/erpnext-python/commands/migrate.py b/build/erpnext-worker/commands/migrate.py similarity index 100% rename from build/erpnext-python/commands/migrate.py rename to build/erpnext-worker/commands/migrate.py diff --git a/build/erpnext-python/commands/new.py b/build/erpnext-worker/commands/new.py similarity index 100% rename from build/erpnext-python/commands/new.py rename to build/erpnext-worker/commands/new.py diff --git a/build/erpnext-python/commands/update.py b/build/erpnext-worker/commands/update.py similarity index 100% rename from build/erpnext-python/commands/update.py rename to build/erpnext-worker/commands/update.py diff --git a/build/erpnext-python/commands/worker.py b/build/erpnext-worker/commands/worker.py similarity index 100% rename from build/erpnext-python/commands/worker.py rename to build/erpnext-worker/commands/worker.py diff --git a/build/erpnext-python/common_site_config.json.template b/build/erpnext-worker/common_site_config.json.template similarity index 100% rename from build/erpnext-python/common_site_config.json.template rename to build/erpnext-worker/common_site_config.json.template diff --git a/build/erpnext-python/docker-entrypoint.sh b/build/erpnext-worker/docker-entrypoint.sh similarity index 100% rename from build/erpnext-python/docker-entrypoint.sh rename to build/erpnext-worker/docker-entrypoint.sh diff --git a/build/erpnext-python/v11.Dockerfile b/build/erpnext-worker/v11.Dockerfile similarity index 100% rename from build/erpnext-python/v11.Dockerfile rename to build/erpnext-worker/v11.Dockerfile diff --git a/build/erpnext-python/v12.Dockerfile b/build/erpnext-worker/v12.Dockerfile similarity index 100% rename from build/erpnext-python/v12.Dockerfile rename to build/erpnext-worker/v12.Dockerfile diff --git a/build/frappe-assets/.dockerignore b/build/frappe-assets/.dockerignore new file mode 100644 index 00000000..c58728b5 --- /dev/null +++ b/build/frappe-assets/.dockerignore @@ -0,0 +1,2 @@ +Dockerfile +/docker/ diff --git a/build/frappe-assets/Dockerfile b/build/frappe-assets/Dockerfile new file mode 100644 index 00000000..ef0e7dcc --- /dev/null +++ b/build/frappe-assets/Dockerfile @@ -0,0 +1,36 @@ +FROM bitnami/node:12-prod + +WORKDIR /home/frappe/frappe-bench +COPY sites/apps.txt /home/frappe/frappe-bench/sites/apps.txt + +RUN install_packages git + +RUN mkdir -p apps sites/assets \ + && cd apps \ + && git clone --depth 1 https://github.com/frappe/frappe + +RUN cd /home/frappe/frappe-bench/apps/frappe \ + && yarn \ + && yarn run production \ + && rm -fr node_modules \ + && yarn install --production=true + +RUN git clone --depth 1 https://github.com/frappe/bench /tmp/bench \ + && mkdir -p /var/www/error_pages \ + && cp -r /tmp/bench/bench/config/templates /var/www/error_pages + +RUN cp -R /home/frappe/frappe-bench/apps/frappe/frappe/public/* /home/frappe/frappe-bench/sites/assets/frappe \ + && cp -R /home/frappe/frappe-bench/apps/frappe/node_modules /home/frappe/frappe-bench/sites/assets/frappe/ + +FROM nginx:latest +COPY --from=0 /home/frappe/frappe-bench/sites /var/www/html/ +COPY --from=0 /var/www/error_pages /var/www/ +COPY nginx-default.conf.template /etc/nginx/conf.d/default.conf.template +COPY docker-entrypoint.sh / + +RUN apt-get update && apt-get install -y rsync && apt-get clean + +VOLUME [ "/assets" ] + +ENTRYPOINT ["/docker-entrypoint.sh"] +CMD ["nginx", "-g", "daemon off;"] diff --git a/build/frappe-assets/docker-entrypoint.sh b/build/frappe-assets/docker-entrypoint.sh new file mode 100755 index 00000000..3cae6744 --- /dev/null +++ b/build/frappe-assets/docker-entrypoint.sh @@ -0,0 +1,35 @@ +#!/bin/bash + +## Thanks +# https://serverfault.com/a/919212 +## + +set -e + +rsync -a --delete /var/www/html/assets/js /assets +rsync -a --delete /var/www/html/assets/css /assets +rsync -a --delete /var/www/html/assets/frappe /assets +rsync -a --delete /var/www/html/assets/erpnext /assets + +chmod -R 755 /assets + +if [[ -z "$FRAPPE_PY" ]]; then + export FRAPPE_PY=0.0.0.0 +fi + +if [[ -z "$FRAPPE_PY_PORT" ]]; then + export FRAPPE_PY_PORT=8000 +fi + +if [[ -z "$FRAPPE_SOCKETIO" ]]; then + export FRAPPE_SOCKETIO=0.0.0.0 +fi + +if [[ -z "$FRAPPE_SOCKETIO_PORT" ]]; then + export FRAPPE_SOCKETIO_PORT=9000 +fi + +envsubst '${API_HOST} ${API_PORT} ${FRAPPE_PY} ${FRAPPE_PY_PORT} ${FRAPPE_SOCKETIO} ${FRAPPE_SOCKETIO_PORT}' \ + < /etc/nginx/conf.d/default.conf.template > /etc/nginx/conf.d/default.conf + +exec "$@" diff --git a/build/frappe-assets/nginx-default.conf.template b/build/frappe-assets/nginx-default.conf.template new file mode 100644 index 00000000..c2e0e0ba --- /dev/null +++ b/build/frappe-assets/nginx-default.conf.template @@ -0,0 +1,90 @@ +upstream frappe-server { + server ${FRAPPE_PY}:${FRAPPE_PY_PORT} fail_timeout=0; +} + +upstream socketio-server { + server ${FRAPPE_SOCKETIO}:${FRAPPE_SOCKETIO_PORT} fail_timeout=0; +} + +server { + listen 80; + server_name $http_host; + root /var/www/html; + + location /assets { + try_files $uri =404; + } + + location ~ ^/protected/(.*) { + internal; + try_files /sites/$http_host/$1 =404; + } + + location /socket.io { + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + proxy_set_header X-Frappe-Site-Name $http_host; + proxy_set_header Origin $scheme://$http_host; + proxy_set_header Host $host; + + proxy_pass http://socketio-server; + } + + location / { + try_files /sites/$http_host/public/$uri @webserver; + } + + location @webserver { + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Frappe-Site-Name $http_host; + proxy_set_header Host $host; + proxy_set_header X-Use-X-Accel-Redirect True; + proxy_read_timeout 120; + proxy_redirect off; + + proxy_pass http://frappe-server; + } + + # error pages + error_page 502 /502.html; + location /502.html { + root /var/www/templates; + internal; + } + + # optimizations + sendfile on; + keepalive_timeout 15; + client_max_body_size 50m; + client_body_buffer_size 16K; + client_header_buffer_size 1k; + + # enable gzip compresion + # based on https://mattstauffer.co/blog/enabling-gzip-on-nginx-servers-including-laravel-forge + gzip on; + gzip_http_version 1.1; + gzip_comp_level 5; + gzip_min_length 256; + gzip_proxied any; + gzip_vary on; + gzip_types + application/atom+xml + application/javascript + application/json + application/rss+xml + application/vnd.ms-fontobject + application/x-font-ttf + application/font-woff + application/x-web-app-manifest+json + application/xhtml+xml + application/xml + font/opentype + image/svg+xml + image/x-icon + text/css + text/plain + text/x-component; + # text/html is always compressed by HttpGzipModule +} diff --git a/build/frappe-assets/sites/apps.txt b/build/frappe-assets/sites/apps.txt new file mode 100644 index 00000000..f47ffaf5 --- /dev/null +++ b/build/frappe-assets/sites/apps.txt @@ -0,0 +1 @@ +frappe diff --git a/build/frappe-assets/v11.Dockerfile b/build/frappe-assets/v11.Dockerfile new file mode 100644 index 00000000..ef98aded --- /dev/null +++ b/build/frappe-assets/v11.Dockerfile @@ -0,0 +1,36 @@ +FROM bitnami/node:10-prod + +WORKDIR /home/frappe/frappe-bench +COPY sites/apps.txt /home/frappe/frappe-bench/sites/apps.txt + +RUN install_packages git + +RUN mkdir -p apps sites/assets \ + && cd apps \ + && git clone --depth 1 https://github.com/frappe/frappe --branch version-11 + +RUN cd /home/frappe/frappe-bench/apps/frappe \ + && yarn \ + && yarn run production \ + && rm -fr node_modules \ + && yarn install --production=true + +RUN git clone --depth 1 https://github.com/frappe/bench /tmp/bench \ + && mkdir -p /var/www/error_pages \ + && cp -r /tmp/bench/bench/config/templates /var/www/error_pages + +RUN cp -R /home/frappe/frappe-bench/apps/frappe/frappe/public/* /home/frappe/frappe-bench/sites/assets/frappe \ + && cp -R /home/frappe/frappe-bench/apps/frappe/node_modules /home/frappe/frappe-bench/sites/assets/frappe/ + +FROM nginx:latest +COPY --from=0 /home/frappe/frappe-bench/sites /var/www/html/ +COPY --from=0 /var/www/error_pages /var/www/ +COPY nginx-default.conf.template /etc/nginx/conf.d/default.conf.template +COPY docker-entrypoint.sh / + +RUN apt-get update && apt-get install -y rsync && apt-get clean + +VOLUME [ "/assets" ] + +ENTRYPOINT ["/docker-entrypoint.sh"] +CMD ["nginx", "-g", "daemon off;"] diff --git a/build/frappe-assets/v12.Dockerfile b/build/frappe-assets/v12.Dockerfile new file mode 100644 index 00000000..2e16296b --- /dev/null +++ b/build/frappe-assets/v12.Dockerfile @@ -0,0 +1,36 @@ +FROM bitnami/node:12-prod + +WORKDIR /home/frappe/frappe-bench +COPY sites/apps.txt /home/frappe/frappe-bench/sites/apps.txt + +RUN install_packages git + +RUN mkdir -p apps sites/assets \ + && cd apps \ + && git clone --depth 1 https://github.com/frappe/frappe --branch version-12 + +RUN cd /home/frappe/frappe-bench/apps/frappe \ + && yarn \ + && yarn run production \ + && rm -fr node_modules \ + && yarn install --production=true + +RUN git clone --depth 1 https://github.com/frappe/bench /tmp/bench \ + && mkdir -p /var/www/error_pages \ + && cp -r /tmp/bench/bench/config/templates /var/www/error_pages + +RUN cp -R /home/frappe/frappe-bench/apps/frappe/frappe/public/* /home/frappe/frappe-bench/sites/assets/frappe \ + && cp -R /home/frappe/frappe-bench/apps/frappe/node_modules /home/frappe/frappe-bench/sites/assets/frappe/ + +FROM nginx:latest +COPY --from=0 /home/frappe/frappe-bench/sites /var/www/html/ +COPY --from=0 /var/www/error_pages /var/www/ +COPY nginx-default.conf.template /etc/nginx/conf.d/default.conf.template +COPY docker-entrypoint.sh / + +RUN apt-get update && apt-get install -y rsync && apt-get clean + +VOLUME [ "/assets" ] + +ENTRYPOINT ["/docker-entrypoint.sh"] +CMD ["nginx", "-g", "daemon off;"] diff --git a/build/frappe-worker/Dockerfile b/build/frappe-worker/Dockerfile new file mode 100644 index 00000000..39013537 --- /dev/null +++ b/build/frappe-worker/Dockerfile @@ -0,0 +1,42 @@ +FROM bitnami/python:latest-prod + +RUN useradd -ms /bin/bash frappe +WORKDIR /home/frappe/frappe-bench +RUN install_packages \ + git \ + wkhtmltopdf \ + mariadb-client \ + gettext-base \ + wget \ + # for PDF + libssl-dev \ + fonts-cantarell \ + xfonts-75dpi \ + xfonts-base \ + # For psycopg2 + libpq-dev \ + build-essential + +# Install wkhtmltox correctly +RUN wget https://github.com/wkhtmltopdf/wkhtmltopdf/releases/download/0.12.5/wkhtmltox_0.12.5-1.stretch_amd64.deb +RUN dpkg -i wkhtmltox_0.12.5-1.stretch_amd64.deb && rm wkhtmltox_0.12.5-1.stretch_amd64.deb + +RUN mkdir -p apps logs commands + +RUN virtualenv env \ + && . env/bin/activate \ + && cd apps \ + && git clone --depth 1 -o upstream https://github.com/frappe/frappe \ + && pip3 install --no-cache-dir -e /home/frappe/frappe-bench/apps/frappe + +COPY ./commands/* /home/frappe/frappe-bench/commands/ +COPY ./common_site_config.json.template /opt/frappe/common_site_config.json.template + +# Setup docker-entrypoint +COPY docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh +RUN ln -s /usr/local/bin/docker-entrypoint.sh / # backwards compat + +WORKDIR /home/frappe/frappe-bench/sites + +ENTRYPOINT ["docker-entrypoint.sh"] +CMD ["start"] diff --git a/build/frappe-worker/commands/background.py b/build/frappe-worker/commands/background.py new file mode 100644 index 00000000..7065efd9 --- /dev/null +++ b/build/frappe-worker/commands/background.py @@ -0,0 +1,7 @@ +import frappe +from frappe.utils.scheduler import start_scheduler + +print("Starting background scheduler . . .") +start_scheduler() + +exit(0) diff --git a/build/frappe-worker/commands/backup.py b/build/frappe-worker/commands/backup.py new file mode 100644 index 00000000..e6fbcbb6 --- /dev/null +++ b/build/frappe-worker/commands/backup.py @@ -0,0 +1,29 @@ +import os, frappe, compileall, re +from frappe.utils.backups import scheduled_backup +from frappe.utils import now +from frappe.utils import get_sites + +def backup(sites, with_files=False): + for site in sites: + frappe.init(site) + frappe.connect() + odb = scheduled_backup( + ignore_files=not with_files, + backup_path_db=None, + backup_path_files=None, + backup_path_private_files=None, + force=True + ) + print("database backup taken -", odb.backup_path_db, "- on", now()) + if with_files: + print("files backup taken -", odb.backup_path_files, "- on", now()) + print("private files backup taken -", odb.backup_path_private_files, "- on", now()) + frappe.destroy() + +installed_sites = ":".join(get_sites()) +sites = os.environ.get("SITES", installed_sites).split(":") +with_files=True if os.environ.get("WITH_FILES") else False + +backup(sites, with_files) + +exit(0) diff --git a/build/frappe-worker/commands/check_connection.py b/build/frappe-worker/commands/check_connection.py new file mode 100644 index 00000000..abe17199 --- /dev/null +++ b/build/frappe-worker/commands/check_connection.py @@ -0,0 +1,67 @@ +import socket, os, json, time +from six.moves.urllib.parse import urlparse + +def is_open(ip, port, timeout=30): + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + s.settimeout(timeout) + try: + s.connect((ip, int(port))) + s.shutdown(socket.SHUT_RDWR) + return True + except: + return False + finally: + s.close() + +def check_host(ip, port, retry=10, delay=3): + ipup = False + for i in range(retry): + print("Attempt {i} to connect to {ip}:{port}".format(ip=ip,port=port,i=i+1)) + if is_open(ip, port): + ipup = True + break + else: + time.sleep(delay) + return ipup + +# Check connection to servers +config = None +try: + with open('common_site_config.json') as config_file: + config = json.load(config_file) +except FileNotFoundError: + raise FileNotFoundError("common_site_config.json missing") +except: + raise ValueError("common_site_config.json is not valid") + +# Check mariadb +check_mariadb = False +check_mariadb = check_host(config.get('db_host', 'mariadb'), 3306) +if not check_mariadb: + raise ConnectionError("Connection to mariadb timed out") + +# Check redis queue +check_redis_queue = False +redis_queue_url = urlparse(config.get("redis_queue","redis://redis:6379")).netloc +redis_queue, redis_queue_port = redis_queue_url.split(":") +check_redis_queue = check_host(redis_queue, redis_queue_port) +if not check_redis_queue: + raise ConnectionError("Connection to redis queue timed out") + +# Check redis cache +check_redis_cache = False +redis_cache_url = urlparse(config.get("redis_cache","redis://redis:6379")).netloc +redis_cache, redis_cache_port = redis_cache_url.split(":") +check_redis_cache = check_host(redis_cache, redis_cache_port) +if not check_redis_cache: + raise ConnectionError("Connection to redis cache timed out") + +# Check redis socketio +check_redis_socketio = False +redis_socketio_url = urlparse(config.get("redis_socketio","redis://redis:6379")).netloc +redis_socketio, redis_socketio_port = redis_socketio_url.split(":") +check_redis_socketio = check_host(redis_socketio, redis_socketio_port) +if not check_redis_socketio: + raise ConnectionError("Connection to redis socketio timed out") + +print('Connections OK') diff --git a/build/frappe-worker/commands/doctor.py b/build/frappe-worker/commands/doctor.py new file mode 100644 index 00000000..54f508fb --- /dev/null +++ b/build/frappe-worker/commands/doctor.py @@ -0,0 +1,4 @@ +import frappe +from frappe.utils.doctor import doctor + +doctor() diff --git a/build/frappe-worker/commands/migrate.py b/build/frappe-worker/commands/migrate.py new file mode 100644 index 00000000..be382bea --- /dev/null +++ b/build/frappe-worker/commands/migrate.py @@ -0,0 +1,47 @@ +import os, frappe, compileall, re, json + +from frappe.migrate import migrate +from frappe.utils import get_sites + +def get_config(): + config = None + with open('common_site_config.json') as config_file: + config = json.load(config_file) + return config + +def save_config(config): + with open('common_site_config.json', 'w') as f: + return json.dump(config, f, indent=1, sort_keys=True) + +def set_maintenance_mode(enable=True): + conf = get_config() + + if enable: + conf.update({ "maintenance_mode": 1, "pause_scheduler": 1 }) + save_config(conf) + + if not enable: + conf.update({ "maintenance_mode": 0, "pause_scheduler": 0 }) + save_config(conf) + + +installed_sites = ":".join(get_sites()) +sites = os.environ.get("SITES", installed_sites).split(":") +maintenance_mode = True if os.environ.get("MAINTENANCE_MODE") else False + +if maintenance_mode: + set_maintenance_mode(True) + +for site in sites: + print('Migrating', site) + frappe.init(site=site) + frappe.connect() + try: + migrate() + finally: + frappe.destroy() + +if maintenance_mode: + set_maintenance_mode(False) + +exit(0) diff --git a/build/frappe-worker/commands/new.py b/build/frappe-worker/commands/new.py new file mode 100644 index 00000000..10d6a8e4 --- /dev/null +++ b/build/frappe-worker/commands/new.py @@ -0,0 +1,67 @@ +import os, frappe, json + +from frappe.commands.site import _new_site + +site_name = os.environ.get("SITE_NAME", 'site1.localhost') +mariadb_root_username = os.environ.get("DB_ROOT_USER", 'root') +mariadb_root_password = os.environ.get("DB_ROOT_PASSWORD", 'admin') +force = True if os.environ.get("FORCE", None) else False + +frappe.init(site_name, new_site=True) + +_new_site( + None, + site_name, + mariadb_root_username=mariadb_root_username, + mariadb_root_password=mariadb_root_password, + admin_password=os.environ.get("ADMIN_PASSWORD", 'admin'), + verbose=True, + install_apps=[], + source_sql=None, + force=force, + reinstall=False, +) + +config = None +with open('common_site_config.json') as config_file: + config = json.load(config_file) + +site_config = None +with open('{site_name}/site_config.json'.format(site_name=site_name)) as site_config_file: + site_config = json.load(site_config_file) + +# update User's host to '%' required to connect from any container +command = 'mysql -h{db_host} -u{mariadb_root_username} -p{mariadb_root_password} -e '.format( + db_host=config.get('db_host'), + mariadb_root_username=mariadb_root_username, + mariadb_root_password=mariadb_root_password +) +command += "\"UPDATE mysql.user SET Host = '%' where User = '{db_name}'; FLUSH PRIVILEGES;\"".format( + db_name=site_config.get('db_name') +) +os.system(command) + +# Set db password +command = 'mysql -h{db_host} -u{mariadb_root_username} -p{mariadb_root_password} -e '.format( + db_host=config.get('db_host'), + mariadb_root_username=mariadb_root_username, + mariadb_root_password=mariadb_root_password +) +command += "\"SET PASSWORD FOR '{db_name}'@'%' = PASSWORD('{db_password}'); FLUSH PRIVILEGES;\"".format( + db_name=site_config.get('db_name'), + db_password=site_config.get('db_password') +) +os.system(command) + +# Grant permission to database +command = 'mysql -h{db_host} -u{mariadb_root_username} -p{mariadb_root_password} -e '.format( + db_host=config.get('db_host'), + mariadb_root_username=mariadb_root_username, + mariadb_root_password=mariadb_root_password +) +command += "\"GRANT ALL PRIVILEGES ON \`{db_name}\`.* TO '{db_name}'@'%'; FLUSH PRIVILEGES;\"".format( + db_name=site_config.get('db_name') +) +os.system(command) + +exit(0) diff --git a/build/frappe-worker/commands/update.py b/build/frappe-worker/commands/update.py new file mode 100644 index 00000000..fa47fc44 --- /dev/null +++ b/build/frappe-worker/commands/update.py @@ -0,0 +1,55 @@ +def update(pull=False, patch=False, build=False, update_bench=False, auto=False, restart_supervisor=False, + restart_systemd=False, requirements=False, no_backup=False, bench_path='.', force=False, reset=False): + conf = get_config(bench_path=bench_path) + version_upgrade = is_version_upgrade(bench_path=bench_path) + + if version_upgrade[0] or (not version_upgrade[0] and force): + validate_upgrade(version_upgrade[1], version_upgrade[2], bench_path=bench_path) + + before_update(bench_path=bench_path, requirements=requirements) + + conf.update({ "maintenance_mode": 1, "pause_scheduler": 1 }) + update_config(conf, bench_path=bench_path) + + if not no_backup: + print('Backing up sites...') + backup_all_sites(bench_path=bench_path) + + if pull: + pull_all_apps(bench_path=bench_path, reset=reset) + + if requirements: + update_requirements(bench_path=bench_path) + update_node_packages(bench_path=bench_path) + + if version_upgrade[0] or (not version_upgrade[0] and force): + pre_upgrade(version_upgrade[1], version_upgrade[2], bench_path=bench_path) + import bench.utils, bench.app + print('Reloading bench...') + if sys.version_info >= (3, 4): + import importlib + importlib.reload(bench.utils) + importlib.reload(bench.app) + else: + reload(bench.utils) + reload(bench.app) + + if patch: + print('Patching sites...') + patch_sites(bench_path=bench_path) + if build: + build_assets(bench_path=bench_path) + if version_upgrade[0] or (not version_upgrade[0] and force): + post_upgrade(version_upgrade[1], version_upgrade[2], bench_path=bench_path) + if restart_supervisor or conf.get('restart_supervisor_on_update'): + restart_supervisor_processes(bench_path=bench_path) + if restart_systemd or conf.get('restart_systemd_on_update'): + restart_systemd_processes(bench_path=bench_path) + + conf.update({ "maintenance_mode": 0, "pause_scheduler": 0 }) + update_config(conf, bench_path=bench_path) + + print("_"*80) + print("Bench: Deployment tool for Frappe and ERPNext (https://erpnext.org).") + print("Open source depends on your contributions, so please contribute bug reports, patches, fixes or cash and be a part of the community") + print() \ No newline at end of file diff --git a/build/frappe-worker/commands/worker.py b/build/frappe-worker/commands/worker.py new file mode 100644 index 00000000..6ec0bcbf --- /dev/null +++ b/build/frappe-worker/commands/worker.py @@ -0,0 +1,7 @@ +import os, frappe +from frappe.utils.background_jobs import start_worker + +queue = os.environ.get("WORKER_TYPE", "default") +start_worker(queue, False) + +exit(0) diff --git a/build/frappe-worker/common_site_config.json.template b/build/frappe-worker/common_site_config.json.template new file mode 100755 index 00000000..27593289 --- /dev/null +++ b/build/frappe-worker/common_site_config.json.template @@ -0,0 +1,7 @@ +{ + "db_host": "${MARIADB_HOST}", + "redis_cache": "redis://${REDIS_CACHE}", + "redis_queue": "redis://${REDIS_QUEUE}", + "redis_socketio": "redis://${REDIS_SOCKETIO}", + "socketio_port": ${SOCKETIO_PORT} +} diff --git a/build/frappe-worker/docker-entrypoint.sh b/build/frappe-worker/docker-entrypoint.sh new file mode 100755 index 00000000..fef4eda9 --- /dev/null +++ b/build/frappe-worker/docker-entrypoint.sh @@ -0,0 +1,158 @@ +#!/bin/bash + +function configureEnv() { + if [ ! -f /home/frappe/frappe-bench/sites/common_site_config.json ]; then + + if [[ -z "$MARIADB_HOST" ]]; then + echo "MARIADB_HOST is not set" + exit 1 + fi + + if [[ -z "$REDIS_CACHE" ]]; then + echo "REDIS_CACHE is not set" + exit 1 + fi + + if [[ -z "$REDIS_QUEUE" ]]; then + echo "REDIS_QUEUE is not set" + exit 1 + fi + + if [[ -z "$REDIS_SOCKETIO" ]]; then + echo "REDIS_SOCKETIO is not set" + exit 1 + fi + + if [[ -z "$SOCKETIO_PORT" ]]; then + echo "SOCKETIO_PORT is not set" + exit 1 + fi + + envsubst '${MARIADB_HOST} + ${REDIS_CACHE} + ${REDIS_QUEUE} + ${REDIS_SOCKETIO} + ${SOCKETIO_PORT}' < /opt/frappe/common_site_config.json.template > /home/frappe/frappe-bench/sites/common_site_config.json + fi +} + +function checkConnection() { + su frappe -c ". /home/frappe/frappe-bench/env/bin/activate \ + && python /home/frappe/frappe-bench/commands/check_connection.py" +} + +function checkConfigExists() { + COUNTER=0 + while [[ ! -e /home/frappe/frappe-bench/sites/common_site_config.json ]] && [[ $COUNTER -le 30 ]] ; do + sleep 1 + let COUNTER=COUNTER+1 + echo "config file not created, retry $COUNTER" + done + + if [[ ! -e /home/frappe/frappe-bench/sites/common_site_config.json ]]; then + echo "timeout: config file not created" + exit 1 + fi +} + +if [[ ! -e /home/frappe/frappe-bench/sites/apps.txt ]]; then + ls -1 /home/frappe/frappe-bench/apps | sort -r > /home/frappe/frappe-bench/sites/apps.txt +fi + +if [ "$1" = 'start' ]; then + configureEnv + checkConnection + + chown frappe:frappe /home/frappe/frappe-bench/sites/common_site_config.json + + if [[ -z "$WORKERS" ]]; then + export WORKERS=2 + fi + + if [[ -z "$FRAPPE_PORT" ]]; then + export FRAPPE_PORT=8000 + fi + + + if [[ -z "$RUN_AS_ROOT" ]]; then + su frappe -c ". /home/frappe/frappe-bench/env/bin/activate \ + && gunicorn -b 0.0.0.0:$FRAPPE_PORT \ + --worker-tmp-dir /dev/shm \ + --threads=4 \ + --workers $WORKERS \ + --worker-class=gthread \ + --log-file=- \ + -t 120 frappe.app:application --preload" + else + . /home/frappe/frappe-bench/env/bin/activate + gunicorn -b 0.0.0.0:$FRAPPE_PORT \ + --worker-tmp-dir /dev/shm \ + --threads=4 \ + --workers $WORKERS \ + --worker-class=gthread \ + --log-file=- \ + -t 120 frappe.app:application --preload + fi + +elif [ "$1" = 'worker' ]; then + checkConfigExists + checkConnection + # default WORKER_TYPE=default + if [[ -z "$RUN_AS_ROOT" ]]; then + su frappe -c ". /home/frappe/frappe-bench/env/bin/activate \ + && python /home/frappe/frappe-bench/commands/worker.py" + else + . /home/frappe/frappe-bench/env/bin/activate + python /home/frappe/frappe-bench/commands/worker.py + fi + +elif [ "$1" = 'schedule' ]; then + checkConfigExists + checkConnection + if [[ -z "$RUN_AS_ROOT" ]]; then + su frappe -c ". /home/frappe/frappe-bench/env/bin/activate \ + && python /home/frappe/frappe-bench/commands/background.py" + else + . /home/frappe/frappe-bench/env/bin/activate + python /home/frappe/frappe-bench/commands/background.py + fi + +elif [ "$1" = 'new' ]; then + + if [[ -z "$RUN_AS_ROOT" ]]; then + su frappe -c ". /home/frappe/frappe-bench/env/bin/activate \ + && python /home/frappe/frappe-bench/commands/new.py" + exit + else + . /home/frappe/frappe-bench/env/bin/activate + python /home/frappe/frappe-bench/commands/new.py + fi + +elif [ "$1" = 'migrate' ]; then + + su frappe -c ". /home/frappe/frappe-bench/env/bin/activate \ + && python /home/frappe/frappe-bench/commands/migrate.py" + exit + +elif [ "$1" = 'doctor' ]; then + + su frappe -c ". /home/frappe/frappe-bench/env/bin/activate \ + && python /home/frappe/frappe-bench/commands/doctor.py" + exit + +elif [ "$1" = 'backup' ]; then + + if [[ -z "$RUN_AS_ROOT" ]]; then + su frappe -c ". /home/frappe/frappe-bench/env/bin/activate \ + && python /home/frappe/frappe-bench/commands/backup.py" + exit + else + . /home/frappe/frappe-bench/env/bin/activate + python /home/frappe/frappe-bench/commands/backup.py + fi + +else + + exec su frappe -c "$@" + +fi diff --git a/build/frappe-worker/v11.Dockerfile b/build/frappe-worker/v11.Dockerfile new file mode 100644 index 00000000..934a316b --- /dev/null +++ b/build/frappe-worker/v11.Dockerfile @@ -0,0 +1,39 @@ +FROM bitnami/python:latest-prod + +RUN useradd -ms /bin/bash frappe +WORKDIR /home/frappe/frappe-bench +RUN install_packages \ + git \ + wkhtmltopdf \ + mariadb-client \ + gettext-base \ + wget \ + # for PDF + libssl-dev \ + fonts-cantarell \ + xfonts-75dpi \ + xfonts-base + +# Install wkhtmltox correctly +RUN wget https://github.com/wkhtmltopdf/wkhtmltopdf/releases/download/0.12.5/wkhtmltox_0.12.5-1.stretch_amd64.deb +RUN dpkg -i wkhtmltox_0.12.5-1.stretch_amd64.deb && rm wkhtmltox_0.12.5-1.stretch_amd64.deb + +RUN mkdir -p apps logs commands + +RUN virtualenv env \ + && . env/bin/activate \ + && cd apps \ + && git clone --depth 1 -o upstream https://github.com/frappe/frappe --branch version-11 \ + && pip3 install --no-cache-dir -e /home/frappe/frappe-bench/apps/frappe + +COPY ./commands/* /home/frappe/frappe-bench/commands/ +COPY ./common_site_config.json.template /opt/frappe/common_site_config.json.template + +# Setup docker-entrypoint +COPY docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh +RUN ln -s /usr/local/bin/docker-entrypoint.sh / # backwards compat + +WORKDIR /home/frappe/frappe-bench/sites + +ENTRYPOINT ["docker-entrypoint.sh"] +CMD ["start"] diff --git a/build/frappe-worker/v12.Dockerfile b/build/frappe-worker/v12.Dockerfile new file mode 100644 index 00000000..86e63041 --- /dev/null +++ b/build/frappe-worker/v12.Dockerfile @@ -0,0 +1,42 @@ +FROM bitnami/python:latest-prod + +RUN useradd -ms /bin/bash frappe +WORKDIR /home/frappe/frappe-bench +RUN install_packages \ + git \ + wkhtmltopdf \ + mariadb-client \ + gettext-base \ + wget \ + # for PDF + libssl-dev \ + fonts-cantarell \ + xfonts-75dpi \ + xfonts-base \ + # For psycopg2 + libpq-dev \ + build-essential + +# Install wkhtmltox correctly +RUN wget https://github.com/wkhtmltopdf/wkhtmltopdf/releases/download/0.12.5/wkhtmltox_0.12.5-1.stretch_amd64.deb +RUN dpkg -i wkhtmltox_0.12.5-1.stretch_amd64.deb && rm wkhtmltox_0.12.5-1.stretch_amd64.deb + +RUN mkdir -p apps logs commands + +RUN virtualenv env \ + && . env/bin/activate \ + && cd apps \ + && git clone --depth 1 -o upstream https://github.com/frappe/frappe --branch version-12 \ + && pip3 install --no-cache-dir -e /home/frappe/frappe-bench/apps/frappe + +COPY ./commands/* /home/frappe/frappe-bench/commands/ +COPY ./common_site_config.json.template /opt/frappe/common_site_config.json.template + +# Setup docker-entrypoint +COPY docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh +RUN ln -s /usr/local/bin/docker-entrypoint.sh / # backwards compat + +WORKDIR /home/frappe/frappe-bench/sites + +ENTRYPOINT ["docker-entrypoint.sh"] +CMD ["start"]