Update frappe-worker

build/frappe-worker/Dockerfile

@@ -1,108 +1,86 @@
-ARG PYTHON_VERSION=3.9
+ARG PYTHON_VERSION
-FROM python:${PYTHON_VERSION}-slim-bullseye
+FROM python:${PYTHON_VERSION}-slim-bullseye as base
 
-# Add non root user without password
-RUN useradd -ms /bin/bash frappe
-
-ARG GIT_REPO=https://github.com/frappe/frappe
-ARG GIT_BRANCH=develop
-
-ARG ARCH=amd64
-ENV PYTHONUNBUFFERED 1
-
-ENV NODE_VERSION=14.18.1
-ENV NVM_DIR /home/frappe/.nvm
-ENV PATH ${NVM_DIR}/versions/node/v${NODE_VERSION}/bin/:${PATH}
-ENV WKHTMLTOPDF_VERSION 0.12.6-1
-
-# Install apt dependencies
 RUN apt-get update \
     && apt-get install --no-install-recommends -y \
-    # For frappe framework
+    # MariaDB
-    git \
     mariadb-client \
+    # Postgres
     postgresql-client \
-    gettext-base \
-    wget \
-    wait-for-it \
-    # For PDF
-    libjpeg62-turbo \
-    libx11-6 \
-    libxcb1 \
-    libxext6 \
-    libxrender1 \
-    libssl-dev \
-    fonts-cantarell \
-    xfonts-75dpi \
-    xfonts-base \
-    libxml2 \
-    libffi-dev \
-    libjpeg-dev \
-    zlib1g-dev \
-    # For psycopg2
     libpq-dev \
-    # For arm64 python wheel builds
+    # wkhtmltopdf
-    && if [ "$(uname -m)" = "aarch64" ]; then \
+    xvfb \
-    apt-get install --no-install-recommends -y \
+    libfontconfig \
-    gcc \
+    wkhtmltopdf \
-    g++; \
+    # For healthcheck.sh in helm chart
-    fi \
+    wait-for-it \
-    # Install additional requirements for develop branch
-    && if [ "${GIT_BRANCH}" = 'develop' ]; then \
-    apt-get install --no-install-recommends -y \
-    libcairo2 \
-    python3-cffi \
-    python3-brotli \
-    libpango-1.0-0 \
-    libpangoft2-1.0-0 \
-    libpangocairo-1.0-0; \
-    fi \
     && rm -rf /var/lib/apt/lists/*
 
-# Detect arch, download and install wkhtmltopdf
+RUN useradd -ms /bin/bash frappe
-RUN if [ "$(uname -m)" = "aarch64" ]; then export ARCH=arm64; fi \
+USER frappe
-    && if [ "$(uname -m)" = "x86_64" ]; then export ARCH=amd64; fi \
+RUN mkdir -p /home/frappe/frappe-bench/apps /home/frappe/frappe-bench/logs /home/frappe/frappe-bench/sites
-    && downloaded_file=wkhtmltox_$WKHTMLTOPDF_VERSION.buster_${ARCH}.deb \
-    && wget -q https://github.com/wkhtmltopdf/packaging/releases/download/$WKHTMLTOPDF_VERSION/$downloaded_file \
-    && dpkg -i $downloaded_file \
-    && rm $downloaded_file
-
-# Setup docker-entrypoint
-COPY build/frappe-worker/docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh
-RUN ln -s /usr/local/bin/docker-entrypoint.sh / # backwards compat
-
 WORKDIR /home/frappe/frappe-bench
-RUN chown -R frappe:frappe /home/frappe
+
+RUN pip install --no-cache-dir -U pip wheel \
+    && python -m venv env \
+    && env/bin/pip install --no-cache-dir -U pip wheel
+
+
+FROM base as frappe_builder
+
+USER root
+RUN apt-get update \
+    && apt-get install --no-install-recommends -y \
+    # Install git here because it is not required in production
+    git \
+    # gcc and g++ are required for building different packages across different versions
+    # of Frappe and ERPNext and also on different platforms (for example, linux/arm64).
+    # It is safe to install build deps even if they are not required
+    # because they won't be included in final images.
+    gcc \
+    g++ \
+    && rm -rf /var/lib/apt/lists/*
 USER frappe
 
-# Create frappe-bench directories
+ARG FRAPPE_VERSION
-RUN mkdir -p apps logs commands sites /home/frappe/backups
+RUN git clone --depth 1 -b ${FRAPPE_VERSION} https://github.com/frappe/frappe apps/frappe \
+    && env/bin/pip install --no-cache-dir -e apps/frappe \
+    && rm -r apps/frappe/.git
 
-# Setup python environment
-RUN python -m venv env
-RUN env/bin/pip install --no-cache-dir wheel gevent
 
-# Install nvm with node
+FROM frappe_builder as erpnext_builder
-RUN wget -qO- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.0/install.sh | bash \
-    && . ${NVM_DIR}/nvm.sh \
-    && nvm install ${NODE_VERSION} \
-    && rm -rf ${NVM_DIR}/.cache
 
-# Install Frappe
+ARG ERPNEXT_VERSION
-RUN git clone --depth 1 -o upstream -b ${GIT_BRANCH} ${GIT_REPO} apps/frappe \
+RUN git clone --depth 1 -b ${ERPNEXT_VERSION} https://github.com/frappe/erpnext apps/erpnext \
-    && env/bin/pip install --no-cache-dir -e apps/frappe
+    && env/bin/pip install --no-cache-dir -e apps/erpnext \
+    && rm -r apps/erpnext/.git
 
-# Copy scripts and templates
-COPY build/frappe-worker/commands/* /home/frappe/frappe-bench/commands/
-COPY build/frappe-worker/common_site_config.json.template /opt/frappe/common_site_config.json.template
-COPY build/frappe-worker/install_app.sh /usr/local/bin/install_app
-COPY build/frappe-worker/bench /usr/local/bin/bench
-COPY build/frappe-worker/healthcheck.sh /usr/local/bin/healthcheck.sh
 
-# Use sites volume as working directory
+FROM base as configured_base
+
+COPY pretend-bench.sh /usr/local/bin/bench
+# healthcheck.sh used in helm chart
+COPY entrypoint.sh patched_bench_helper.py healthcheck.sh /usr/local/bin/
+
 WORKDIR /home/frappe/frappe-bench/sites
 
-VOLUME [ "/home/frappe/frappe-bench/sites", "/home/frappe/backups", "/home/frappe/frappe-bench/logs" ]
+VOLUME [ "/home/frappe/frappe-bench/sites", "/home/frappe/frappe-bench/logs" ]
 
-ENTRYPOINT ["docker-entrypoint.sh"]
+ENTRYPOINT [ "entrypoint.sh" ]
-CMD ["start"]
+
+CMD [ "/home/frappe/frappe-bench/env/bin/gunicorn", "-b", "0.0.0.0:8000", "frappe.app:application", "--access-logfile", "-" ]
+
+
+FROM configured_base as frappe
+
+RUN echo "frappe" >/home/frappe/frappe-bench/sites/apps.txt
+COPY --from=frappe_builder /home/frappe/frappe-bench/apps/frappe /home/frappe/frappe-bench/apps/frappe
+COPY --from=frappe_builder /home/frappe/frappe-bench/env /home/frappe/frappe-bench/env
+
+
+FROM configured_base as erpnext
+
+RUN echo "frappe\nerpnext" >/home/frappe/frappe-bench/sites/apps.txt
+COPY --from=frappe_builder /home/frappe/frappe-bench/apps/frappe /home/frappe/frappe-bench/apps/frappe
+COPY --from=erpnext_builder /home/frappe/frappe-bench/apps/erpnext /home/frappe/frappe-bench/apps/erpnext
+COPY --from=erpnext_builder /home/frappe/frappe-bench/env /home/frappe/frappe-bench/env

build/frappe-worker/bench

@@ -1,22 +0,0 @@
-#!/home/frappe/frappe-bench/env/bin/python
-
-import os
-import subprocess
-import sys
-
-if __name__ == "__main__":
-    bench_dir = os.path.join(os.sep, "home", "frappe", "frappe-bench")
-    sites_dir = os.path.join(bench_dir, "sites")
-    bench_helper = os.path.join(
-        bench_dir,
-        "apps",
-        "frappe",
-        "frappe",
-        "utils",
-        "bench_helper.py",
-    )
-    cwd = os.getcwd()
-    os.chdir(sites_dir)
-    subprocess.check_call(
-        [sys.executable, bench_helper, "frappe"] + sys.argv[1:],
-    )

build/frappe-worker/commands/auto_migrate.py

@@ -1,64 +0,0 @@
-import os
-
-import git
-import semantic_version
-from migrate import migrate_sites
-from utils import (
-    get_apps,
-    get_config,
-    get_container_versions,
-    get_version_file,
-    save_version_file,
-)
-
-
-def main():
-    is_ready = False
-    apps = get_apps()
-
-    container_versions = get_container_versions(apps)
-
-    version_file = get_version_file()
-
-    if not version_file:
-        version_file = container_versions
-        save_version_file(version_file)
-
-    for app in apps:
-        container_version = None
-        file_version = None
-        version_file_hash = None
-        container_hash = None
-
-        repo = git.Repo(os.path.join("..", "apps", app))
-        branch = repo.active_branch.name
-
-        if branch == "develop":
-            version_file_hash = version_file.get(app + "_git_hash")
-            container_hash = container_versions.get(app + "_git_hash")
-            if container_hash and version_file_hash:
-                if container_hash != version_file_hash:
-                    is_ready = True
-                    break
-
-        if version_file.get(app):
-            file_version = semantic_version.Version(version_file.get(app))
-
-        if container_versions.get(app):
-            container_version = semantic_version.Version(container_versions.get(app))
-
-        if file_version and container_version:
-            if container_version > file_version:
-                is_ready = True
-                break
-
-    config = get_config()
-
-    if is_ready and config.get("maintenance_mode") != 1:
-        migrate_sites(maintenance_mode=True)
-        version_file = container_versions
-        save_version_file(version_file)
-
-
-if __name__ == "__main__":
-    main()

build/frappe-worker/commands/backup.py

@@ -1,45 +0,0 @@
-import os
-
-import frappe
-from frappe.utils import cint, get_sites, now
-from frappe.utils.backups import scheduled_backup
-
-
-def backup(sites, with_files=False):
-    for site in sites:
-        frappe.init(site)
-        frappe.connect()
-        odb = scheduled_backup(
-            ignore_files=not with_files,
-            backup_path_db=None,
-            backup_path_files=None,
-            backup_path_private_files=None,
-            force=True,
-        )
-        print("database backup taken -", odb.backup_path_db, "- on", now())
-        if with_files:
-            print("files backup taken -", odb.backup_path_files, "- on", now())
-            print(
-                "private files backup taken -",
-                odb.backup_path_private_files,
-                "- on",
-                now(),
-            )
-        frappe.destroy()
-
-
-def main():
-    installed_sites = ":".join(get_sites())
-    sites = os.environ.get("SITES", installed_sites).split(":")
-    with_files = cint(os.environ.get("WITH_FILES"))
-
-    backup(sites, with_files)
-
-    if frappe.redis_server:
-        frappe.redis_server.connection_pool.disconnect()
-
-    exit(0)
-
-
-if __name__ == "__main__":
-    main()

build/frappe-worker/commands/check_connection.py

@@ -1,124 +0,0 @@
-import socket
-import time
-
-from constants import (
-    DB_HOST_KEY,
-    DB_PORT,
-    DB_PORT_KEY,
-    REDIS_CACHE_KEY,
-    REDIS_QUEUE_KEY,
-    REDIS_SOCKETIO_KEY,
-)
-from six.moves.urllib.parse import urlparse
-from utils import get_config
-
-
-def is_open(ip, port, timeout=30):
-    s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
-    s.settimeout(timeout)
-    try:
-        s.connect((ip, int(port)))
-        s.shutdown(socket.SHUT_RDWR)
-        return True
-    except Exception:
-        return False
-    finally:
-        s.close()
-
-
-def check_host(ip, port, retry=10, delay=3, print_attempt=True):
-    ipup = False
-    for i in range(retry):
-        if print_attempt:
-            print(f"Attempt {i+1} to connect to {ip}:{port}")
-        if is_open(ip, port):
-            ipup = True
-            break
-        else:
-            time.sleep(delay)
-    return ipup
-
-
-# Check service
-def check_service(
-    retry=10, delay=3, print_attempt=True, service_name=None, service_port=None
-):
-
-    config = get_config()
-    if not service_name:
-        service_name = config.get(DB_HOST_KEY, "mariadb")
-    if not service_port:
-        service_port = config.get(DB_PORT_KEY, DB_PORT)
-
-    is_db_connected = False
-    is_db_connected = check_host(
-        service_name, service_port, retry, delay, print_attempt
-    )
-    if not is_db_connected:
-        print(
-            "Connection to {service_name}:{service_port} timed out".format(
-                service_name=service_name,
-                service_port=service_port,
-            )
-        )
-        exit(1)
-
-
-# Check redis queue
-def check_redis_queue(retry=10, delay=3, print_attempt=True):
-    check_redis_queue = False
-    config = get_config()
-    redis_queue_url = urlparse(
-        config.get(REDIS_QUEUE_KEY, "redis://redis-queue:6379")
-    ).netloc
-    redis_queue, redis_queue_port = redis_queue_url.split(":")
-    check_redis_queue = check_host(
-        redis_queue, redis_queue_port, retry, delay, print_attempt
-    )
-    if not check_redis_queue:
-        print("Connection to redis queue timed out")
-        exit(1)
-
-
-# Check redis cache
-def check_redis_cache(retry=10, delay=3, print_attempt=True):
-    check_redis_cache = False
-    config = get_config()
-    redis_cache_url = urlparse(
-        config.get(REDIS_CACHE_KEY, "redis://redis-cache:6379")
-    ).netloc
-    redis_cache, redis_cache_port = redis_cache_url.split(":")
-    check_redis_cache = check_host(
-        redis_cache, redis_cache_port, retry, delay, print_attempt
-    )
-    if not check_redis_cache:
-        print("Connection to redis cache timed out")
-        exit(1)
-
-
-# Check redis socketio
-def check_redis_socketio(retry=10, delay=3, print_attempt=True):
-    check_redis_socketio = False
-    config = get_config()
-    redis_socketio_url = urlparse(
-        config.get(REDIS_SOCKETIO_KEY, "redis://redis-socketio:6379")
-    ).netloc
-    redis_socketio, redis_socketio_port = redis_socketio_url.split(":")
-    check_redis_socketio = check_host(
-        redis_socketio, redis_socketio_port, retry, delay, print_attempt
-    )
-    if not check_redis_socketio:
-        print("Connection to redis socketio timed out")
-        exit(1)
-
-
-def main():
-    check_service()
-    check_redis_queue()
-    check_redis_cache()
-    check_redis_socketio()
-    print("Connections OK")
-
-
-if __name__ == "__main__":
-    main()

build/frappe-worker/commands/constants.py

@@ -1,13 +0,0 @@
-REDIS_QUEUE_KEY = "redis_queue"
-REDIS_CACHE_KEY = "redis_cache"
-REDIS_SOCKETIO_KEY = "redis_socketio"
-DB_HOST_KEY = "db_host"
-DB_PORT_KEY = "db_port"
-DB_PORT = 3306
-APP_VERSIONS_JSON_FILE = "app_versions.json"
-APPS_TXT_FILE = "apps.txt"
-COMMON_SITE_CONFIG_FILE = "common_site_config.json"
-DATE_FORMAT = "%Y%m%d_%H%M%S"
-RDS_DB = "rds_db"
-RDS_PRIVILEGES = "SELECT, INSERT, UPDATE, DELETE, CREATE, DROP, INDEX, ALTER, CREATE TEMPORARY TABLES, CREATE VIEW, EVENT, TRIGGER, SHOW VIEW, CREATE ROUTINE, ALTER ROUTINE, EXECUTE, LOCK TABLES"
-ARCHIVE_SITES_PATH = "/home/frappe/frappe-bench/sites/archive_sites"

build/frappe-worker/commands/doctor.py

@@ -1,61 +0,0 @@
-import argparse
-
-from check_connection import (
-    check_redis_cache,
-    check_redis_queue,
-    check_redis_socketio,
-    check_service,
-)
-
-
-def parse_args():
-    parser = argparse.ArgumentParser()
-    parser.add_argument(
-        "-p",
-        "--ping-service",
-        dest="ping_services",
-        action="append",
-        type=str,
-        help='list of services to ping, e.g. doctor -p "postgres:5432" --ping-service "mariadb:3306"',
-    )
-    args = parser.parse_args()
-    return args
-
-
-def main():
-    args = parse_args()
-    check_service(retry=1, delay=0, print_attempt=False)
-    print("Bench database Connected")
-    check_redis_cache(retry=1, delay=0, print_attempt=False)
-    print("Redis Cache Connected")
-    check_redis_queue(retry=1, delay=0, print_attempt=False)
-    print("Redis Queue Connected")
-    check_redis_socketio(retry=1, delay=0, print_attempt=False)
-    print("Redis SocketIO Connected")
-
-    if args.ping_services:
-        for service in args.ping_services:
-            service_name = None
-            service_port = None
-
-            try:
-                service_name, service_port = service.split(":")
-            except ValueError:
-                print("Service should be in format host:port, e.g postgres:5432")
-                exit(1)
-
-            check_service(
-                retry=1,
-                delay=0,
-                print_attempt=False,
-                service_name=service_name,
-                service_port=service_port,
-            )
-            print(f"{service_name}:{service_port} Connected")
-
-    print("Health check successful")
-    exit(0)
-
-
-if __name__ == "__main__":
-    main()

build/frappe-worker/commands/gevent_patch.py

@@ -1,3 +0,0 @@
-import gevent.monkey
-
-gevent.monkey.patch_all()

build/frappe-worker/commands/migrate.py

@@ -1,52 +0,0 @@
-import os
-
-import frappe
-from frappe.utils import cint, get_sites
-from utils import get_config, save_config
-
-
-def set_maintenance_mode(enable=True):
-    conf = get_config()
-
-    if enable:
-        conf.update({"maintenance_mode": 1, "pause_scheduler": 1})
-        save_config(conf)
-
-    if not enable:
-        conf.update({"maintenance_mode": 0, "pause_scheduler": 0})
-        save_config(conf)
-
-
-def migrate_sites(maintenance_mode=False):
-    installed_sites = ":".join(get_sites())
-    sites = os.environ.get("SITES", installed_sites).split(":")
-    if not maintenance_mode:
-        maintenance_mode = cint(os.environ.get("MAINTENANCE_MODE"))
-
-    if maintenance_mode:
-        set_maintenance_mode(True)
-
-    for site in sites:
-        print("Migrating", site)
-        frappe.init(site=site)
-        frappe.connect()
-        try:
-            from frappe.migrate import migrate
-
-            migrate()
-        finally:
-            frappe.destroy()
-
-    # Disable maintenance mode after migration
-    set_maintenance_mode(False)
-
-
-def main():
-    migrate_sites()
-    if frappe.redis_server:
-        frappe.redis_server.connection_pool.disconnect()
-    exit(0)
-
-
-if __name__ == "__main__":
-    main()

build/frappe-worker/commands/new.py

@@ -1,132 +0,0 @@
-import os
-
-import frappe
-import semantic_version
-from constants import COMMON_SITE_CONFIG_FILE, RDS_DB, RDS_PRIVILEGES
-from frappe.installer import update_site_config
-from utils import get_config, get_password, get_site_config, run_command
-
-# try to import _new_site from frappe, which could possibly
-# exist in either commands.py or installer.py, and so we need
-# to maintain compatibility across all frappe versions.
-try:
-    # <= version-{11,12}
-    from frappe.commands.site import _new_site
-except ImportError:
-    # >= version-13 and develop
-    from frappe.installer import _new_site
-
-
-def main():
-    config = get_config()
-    db_type = "mariadb"
-    db_port = config.get("db_port", 3306)
-    db_host = config.get("db_host")
-    site_name = os.environ.get("SITE_NAME", "site1.localhost")
-    db_root_username = os.environ.get("DB_ROOT_USER", "root")
-    mariadb_root_password = get_password("MYSQL_ROOT_PASSWORD", "admin")
-    postgres_root_password = get_password("POSTGRES_PASSWORD")
-    db_root_password = mariadb_root_password
-
-    if postgres_root_password:
-        db_type = "postgres"
-        db_host = os.environ.get("POSTGRES_HOST")
-        db_port = 5432
-        db_root_password = postgres_root_password
-        if not db_host:
-            db_host = config.get("db_host")
-            print("Environment variable POSTGRES_HOST not found.")
-            print("Using db_host from common_site_config.json")
-
-        sites_path = os.getcwd()
-        common_site_config_path = os.path.join(sites_path, COMMON_SITE_CONFIG_FILE)
-        update_site_config(
-            "root_login",
-            db_root_username,
-            validate=False,
-            site_config_path=common_site_config_path,
-        )
-        update_site_config(
-            "root_password",
-            db_root_password,
-            validate=False,
-            site_config_path=common_site_config_path,
-        )
-
-    force = True if os.environ.get("FORCE", None) else False
-    install_apps = os.environ.get("INSTALL_APPS", None)
-    install_apps = install_apps.split(",") if install_apps else []
-    frappe.init(site_name, new_site=True)
-
-    if semantic_version.Version(frappe.__version__).major > 11:
-        _new_site(
-            None,
-            site_name,
-            mariadb_root_username=db_root_username,
-            mariadb_root_password=db_root_password,
-            admin_password=get_password("ADMIN_PASSWORD", "admin"),
-            verbose=True,
-            install_apps=install_apps,
-            source_sql=None,
-            force=force,
-            db_type=db_type,
-            reinstall=False,
-            db_host=db_host,
-            db_port=db_port,
-        )
-    else:
-        _new_site(
-            None,
-            site_name,
-            mariadb_root_username=db_root_username,
-            mariadb_root_password=db_root_password,
-            admin_password=get_password("ADMIN_PASSWORD", "admin"),
-            verbose=True,
-            install_apps=install_apps,
-            source_sql=None,
-            force=force,
-            reinstall=False,
-        )
-
-    if db_type == "mariadb":
-        site_config = get_site_config(site_name)
-        db_name = site_config.get("db_name")
-        db_password = site_config.get("db_password")
-
-        mysql_command = [
-            "mysql",
-            f"-h{db_host}",
-            f"-u{db_root_username}",
-            f"-p{mariadb_root_password}",
-            "-e",
-        ]
-
-        # Drop User if exists
-        command = mysql_command + [
-            f"DROP USER IF EXISTS '{db_name}'; FLUSH PRIVILEGES;"
-        ]
-        run_command(command)
-
-        # Grant permission to database and set password
-        grant_privileges = "ALL PRIVILEGES"
-
-        # for Amazon RDS
-        if config.get(RDS_DB) or site_config.get(RDS_DB):
-            grant_privileges = RDS_PRIVILEGES
-
-        command = mysql_command + [
-            f"\
-            CREATE USER IF NOT EXISTS '{db_name}'@'%' IDENTIFIED BY '{db_password}'; \
-            GRANT {grant_privileges} ON `{db_name}`.* TO '{db_name}'@'%'; \
-            FLUSH PRIVILEGES;"
-        ]
-        run_command(command)
-
-    if frappe.redis_server:
-        frappe.redis_server.connection_pool.disconnect()
-
-    exit(0)
-
-
-if __name__ == "__main__":
-    main()

build/frappe-worker/commands/push_backup.py

@@ -1,183 +0,0 @@
-import datetime
-import os
-import time
-from glob import glob
-
-import boto3
-from constants import DATE_FORMAT
-from frappe.utils import get_sites
-from utils import check_s3_environment_variables, get_s3_config, upload_file_to_s3
-
-
-def get_file_ext():
-    return {
-        "database": "-database.sql.gz",
-        "private_files": "-private-files.tar",
-        "public_files": "-files.tar",
-        "site_config": "-site_config_backup.json",
-    }
-
-
-def get_backup_details(sitename):
-    backup_details = dict()
-    file_ext = get_file_ext()
-
-    # add trailing slash https://stackoverflow.com/a/15010678
-    site_backup_path = os.path.join(os.getcwd(), sitename, "private", "backups", "")
-
-    if os.path.exists(site_backup_path):
-        for filetype, ext in file_ext.items():
-            site_slug = sitename.replace(".", "_")
-            pattern = site_backup_path + "*-" + site_slug + ext
-            backup_files = list(filter(os.path.isfile, glob(pattern)))
-
-            if len(backup_files) > 0:
-                backup_files.sort(
-                    key=lambda file: os.stat(
-                        os.path.join(site_backup_path, file)
-                    ).st_ctime
-                )
-                backup_date = datetime.datetime.strptime(
-                    time.ctime(os.path.getmtime(backup_files[0])),
-                    "%a %b %d %H:%M:%S %Y",
-                )
-                backup_details[filetype] = {
-                    "sitename": sitename,
-                    "file_size_in_bytes": os.stat(backup_files[-1]).st_size,
-                    "file_path": os.path.abspath(backup_files[-1]),
-                    "filename": os.path.basename(backup_files[-1]),
-                    "backup_date": backup_date.date().strftime("%Y-%m-%d %H:%M:%S"),
-                }
-
-    return backup_details
-
-
-def delete_old_backups(limit, bucket, site_name):
-    all_backups = list()
-    all_backup_dates = list()
-    backup_limit = int(limit)
-    check_s3_environment_variables()
-    bucket_dir = os.environ.get("BUCKET_DIR")
-    oldest_backup_date = None
-
-    s3 = boto3.resource(
-        "s3",
-        region_name=os.environ.get("REGION"),
-        aws_access_key_id=os.environ.get("ACCESS_KEY_ID"),
-        aws_secret_access_key=os.environ.get("SECRET_ACCESS_KEY"),
-        endpoint_url=os.environ.get("ENDPOINT_URL"),
-    )
-
-    bucket = s3.Bucket(bucket)
-    objects = bucket.meta.client.list_objects_v2(Bucket=bucket.name, Delimiter="/")
-
-    if objects:
-        for obj in objects.get("CommonPrefixes"):
-            if obj.get("Prefix") == bucket_dir + "/":
-                for backup_obj in bucket.objects.filter(Prefix=obj.get("Prefix")):
-                    if backup_obj.get()["ContentType"] == "application/x-directory":
-                        continue
-                    try:
-                        # backup_obj.key is bucket_dir/site/date_time/backupfile.extension
-                        (
-                            bucket_dir,
-                            site_slug,
-                            date_time,
-                            backupfile,
-                        ) = backup_obj.key.split("/")
-                        date_time_object = datetime.datetime.strptime(
-                            date_time, DATE_FORMAT
-                        )
-
-                        if site_name in backup_obj.key:
-                            all_backup_dates.append(date_time_object)
-                            all_backups.append(backup_obj.key)
-                    except IndexError as error:
-                        print(error)
-                        exit(1)
-
-    if len(all_backup_dates) > 0:
-        oldest_backup_date = min(all_backup_dates)
-
-    if len(all_backups) / 3 > backup_limit:
-        oldest_backup = None
-        for backup in all_backups:
-            try:
-                # backup is bucket_dir/site/date_time/backupfile.extension
-                backup_dir, site_slug, backup_dt_string, filename = backup.split("/")
-                backup_datetime = datetime.datetime.strptime(
-                    backup_dt_string, DATE_FORMAT
-                )
-                if backup_datetime == oldest_backup_date:
-                    oldest_backup = backup
-
-            except IndexError as error:
-                print(error)
-                exit(1)
-
-            if oldest_backup:
-                for obj in bucket.objects.filter(Prefix=oldest_backup):
-                    # delete all keys that are inside the oldest_backup
-                    if bucket_dir in obj.key:
-                        print("Deleting " + obj.key)
-                        s3.Object(bucket.name, obj.key).delete()
-
-
-def main():
-    details = dict()
-    sites = get_sites()
-    conn, bucket = get_s3_config()
-
-    for site in sites:
-        details = get_backup_details(site)
-        db_file = details.get("database", {}).get("file_path")
-        folder = os.environ.get("BUCKET_DIR") + "/" + site + "/"
-        if db_file:
-            folder = (
-                os.environ.get("BUCKET_DIR")
-                + "/"
-                + site
-                + "/"
-                + os.path.basename(db_file)[:15]
-                + "/"
-            )
-            upload_file_to_s3(db_file, folder, conn, bucket)
-
-            # Archive site_config.json
-            site_config_file = details.get("site_config", {}).get("file_path")
-            if not site_config_file:
-                site_config_file = os.path.join(os.getcwd(), site, "site_config.json")
-            upload_file_to_s3(site_config_file, folder, conn, bucket)
-
-        public_files = details.get("public_files", {}).get("file_path")
-        if public_files:
-            folder = (
-                os.environ.get("BUCKET_DIR")
-                + "/"
-                + site
-                + "/"
-                + os.path.basename(public_files)[:15]
-                + "/"
-            )
-            upload_file_to_s3(public_files, folder, conn, bucket)
-
-        private_files = details.get("private_files", {}).get("file_path")
-        if private_files:
-            folder = (
-                os.environ.get("BUCKET_DIR")
-                + "/"
-                + site
-                + "/"
-                + os.path.basename(private_files)[:15]
-                + "/"
-            )
-            upload_file_to_s3(private_files, folder, conn, bucket)
-
-        delete_old_backups(os.environ.get("BACKUP_LIMIT", "3"), bucket, site)
-
-    print("push-backup complete")
-    exit(0)
-
-
-if __name__ == "__main__":
-    main()

build/frappe-worker/commands/restore_backup.py

@@ -1,313 +0,0 @@
-import datetime
-import hashlib
-import os
-import tarfile
-
-import boto3
-import frappe
-from constants import COMMON_SITE_CONFIG_FILE, DATE_FORMAT, RDS_DB, RDS_PRIVILEGES
-from frappe.installer import (
-    get_conf_params,
-    make_conf,
-    make_site_dirs,
-    update_site_config,
-)
-from frappe.utils import get_sites, random_string
-from utils import (
-    check_s3_environment_variables,
-    get_config,
-    get_password,
-    get_site_config,
-    list_directories,
-    run_command,
-    set_key_in_site_config,
-)
-
-
-def get_backup_dir():
-    return os.path.join(os.path.expanduser("~"), "backups")
-
-
-def decompress_db(database_file, site):
-    command = ["gunzip", "-c", database_file]
-    with open(database_file.replace(".gz", ""), "w") as db_file:
-        print(f"Extract Database GZip for site {site}")
-        run_command(command, stdout=db_file)
-
-
-def restore_database(files_base, site_config_path, site):
-    # restore database
-    database_file = files_base + "-database.sql.gz"
-    decompress_db(database_file, site)
-    config = get_config()
-
-    # Set db_type if it exists in backup site_config.json
-    set_key_in_site_config("db_type", site, site_config_path)
-    # Set db_host if it exists in backup site_config.json
-    set_key_in_site_config("db_host", site, site_config_path)
-    # Set db_port if it exists in backup site_config.json
-    set_key_in_site_config("db_port", site, site_config_path)
-
-    # get updated site_config
-    site_config = get_site_config(site)
-
-    # if no db_type exists, default to mariadb
-    db_type = site_config.get("db_type", "mariadb")
-    is_database_restored = False
-
-    if db_type == "mariadb":
-        restore_mariadb(
-            config=config, site_config=site_config, database_file=database_file
-        )
-        is_database_restored = True
-    elif db_type == "postgres":
-        restore_postgres(
-            config=config, site_config=site_config, database_file=database_file
-        )
-        is_database_restored = True
-
-    if is_database_restored:
-        # Set encryption_key if it exists in backup site_config.json
-        set_key_in_site_config("encryption_key", site, site_config_path)
-
-
-def restore_files(files_base):
-    public_files = files_base + "-files.tar"
-    # extract tar
-    public_tar = tarfile.open(public_files)
-    print(f"Extracting {public_files}")
-    public_tar.extractall()
-
-
-def restore_private_files(files_base):
-    private_files = files_base + "-private-files.tar"
-    private_tar = tarfile.open(private_files)
-    print(f"Extracting {private_files}")
-    private_tar.extractall()
-
-
-def pull_backup_from_s3():
-    check_s3_environment_variables()
-
-    # https://stackoverflow.com/a/54672690
-    s3 = boto3.resource(
-        "s3",
-        region_name=os.environ.get("REGION"),
-        aws_access_key_id=os.environ.get("ACCESS_KEY_ID"),
-        aws_secret_access_key=os.environ.get("SECRET_ACCESS_KEY"),
-        endpoint_url=os.environ.get("ENDPOINT_URL"),
-    )
-
-    bucket_dir = os.environ.get("BUCKET_DIR")
-    bucket_name = os.environ.get("BUCKET_NAME")
-    bucket = s3.Bucket(bucket_name)
-
-    # Change directory to /home/frappe/backups
-    os.chdir(get_backup_dir())
-
-    backup_files = []
-    sites = set()
-    site_timestamps = set()
-    download_backups = []
-
-    for obj in bucket.objects.filter(Prefix=bucket_dir):
-        if obj.get()["ContentType"] == "application/x-directory":
-            continue
-        backup_file = obj.key.replace(os.path.join(bucket_dir, ""), "")
-        backup_files.append(backup_file)
-        site_name, timestamp, backup_type = backup_file.split("/")
-        site_timestamp = site_name + "/" + timestamp
-        sites.add(site_name)
-        site_timestamps.add(site_timestamp)
-
-    # sort sites for latest backups
-    for site in sites:
-        backup_timestamps = []
-        for site_timestamp in site_timestamps:
-            site_name, timestamp = site_timestamp.split("/")
-            if site == site_name:
-                timestamp_datetime = datetime.datetime.strptime(timestamp, DATE_FORMAT)
-                backup_timestamps.append(timestamp)
-        download_backups.append(site + "/" + max(backup_timestamps))
-
-    # Only download latest backups
-    for backup_file in backup_files:
-        for backup in download_backups:
-            if backup in backup_file:
-                if not os.path.exists(os.path.dirname(backup_file)):
-                    os.makedirs(os.path.dirname(backup_file))
-                print(f"Downloading {backup_file}")
-                bucket.download_file(bucket_dir + "/" + backup_file, backup_file)
-
-    os.chdir(os.path.join(os.path.expanduser("~"), "frappe-bench", "sites"))
-
-
-def restore_postgres(config, site_config, database_file):
-    # common config
-    common_site_config_path = os.path.join(os.getcwd(), COMMON_SITE_CONFIG_FILE)
-
-    db_root_user = config.get("root_login")
-    if not db_root_user:
-        postgres_user = os.environ.get("DB_ROOT_USER")
-        if not postgres_user:
-            print("Variable DB_ROOT_USER not set")
-            exit(1)
-
-        db_root_user = postgres_user
-        update_site_config(
-            "root_login",
-            db_root_user,
-            validate=False,
-            site_config_path=common_site_config_path,
-        )
-
-    db_root_password = config.get("root_password")
-    if not db_root_password:
-        root_password = get_password("POSTGRES_PASSWORD")
-        if not root_password:
-            print("Variable POSTGRES_PASSWORD not set")
-            exit(1)
-
-        db_root_password = root_password
-        update_site_config(
-            "root_password",
-            db_root_password,
-            validate=False,
-            site_config_path=common_site_config_path,
-        )
-
-    # site config
-    db_host = site_config.get("db_host")
-    db_port = site_config.get("db_port", 5432)
-    db_name = site_config.get("db_name")
-    db_password = site_config.get("db_password")
-
-    psql_command = ["psql"]
-    psql_uri = f"postgres://{db_root_user}:{db_root_password}@{db_host}:{db_port}"
-
-    print("Restoring PostgreSQL")
-    run_command(psql_command + [psql_uri, "-c", f'DROP DATABASE IF EXISTS "{db_name}"'])
-    run_command(psql_command + [psql_uri, "-c", f"DROP USER IF EXISTS {db_name}"])
-    run_command(psql_command + [psql_uri, "-c", f'CREATE DATABASE "{db_name}"'])
-    run_command(
-        psql_command
-        + [psql_uri, "-c", f"CREATE user {db_name} password '{db_password}'"]
-    )
-    run_command(
-        psql_command
-        + [psql_uri, "-c", f'GRANT ALL PRIVILEGES ON DATABASE "{db_name}" TO {db_name}']
-    )
-    with open(database_file.replace(".gz", "")) as db_file:
-        run_command(psql_command + [f"{psql_uri}/{db_name}", "<"], stdin=db_file)
-
-
-def restore_mariadb(config, site_config, database_file):
-    db_root_password = get_password("MYSQL_ROOT_PASSWORD")
-    if not db_root_password:
-        print("Variable MYSQL_ROOT_PASSWORD not set")
-        exit(1)
-
-    db_root_user = os.environ.get("DB_ROOT_USER", "root")
-
-    db_host = site_config.get("db_host", config.get("db_host"))
-    db_port = site_config.get("db_port", config.get("db_port", 3306))
-    db_name = site_config.get("db_name")
-    db_password = site_config.get("db_password")
-
-    # mysql command prefix
-    mysql_command = [
-        "mysql",
-        f"-u{db_root_user}",
-        f"-h{db_host}",
-        f"-p{db_root_password}",
-        f"-P{db_port}",
-    ]
-
-    # drop db if exists for clean restore
-    drop_database = mysql_command + ["-e", f"DROP DATABASE IF EXISTS `{db_name}`;"]
-    run_command(drop_database)
-
-    # create db
-    create_database = mysql_command + [
-        "-e",
-        f"CREATE DATABASE IF NOT EXISTS `{db_name}`;",
-    ]
-    run_command(create_database)
-
-    # create user
-    create_user = mysql_command + [
-        "-e",
-        f"CREATE USER IF NOT EXISTS '{db_name}'@'%' IDENTIFIED BY '{db_password}'; FLUSH PRIVILEGES;",
-    ]
-    run_command(create_user)
-
-    # grant db privileges to user
-
-    grant_privileges = "ALL PRIVILEGES"
-
-    # for Amazon RDS
-    if config.get(RDS_DB) or site_config.get(RDS_DB):
-        grant_privileges = RDS_PRIVILEGES
-
-    grant_privileges_command = mysql_command + [
-        "-e",
-        f"GRANT {grant_privileges} ON `{db_name}`.* TO '{db_name}'@'%' IDENTIFIED BY '{db_password}'; FLUSH PRIVILEGES;",
-    ]
-    run_command(grant_privileges_command)
-
-    print("Restoring MariaDB")
-    with open(database_file.replace(".gz", "")) as db_file:
-        run_command(mysql_command + [f"{db_name}"], stdin=db_file)
-
-
-def main():
-    backup_dir = get_backup_dir()
-
-    if len(list_directories(backup_dir)) == 0:
-        pull_backup_from_s3()
-
-    for site in list_directories(backup_dir):
-        site_slug = site.replace(".", "_")
-        backups = [
-            datetime.datetime.strptime(backup, DATE_FORMAT)
-            for backup in list_directories(os.path.join(backup_dir, site))
-        ]
-        latest_backup = max(backups).strftime(DATE_FORMAT)
-        files_base = os.path.join(backup_dir, site, latest_backup, "")
-        files_base += latest_backup + "-" + site_slug
-        site_config_path = files_base + "-site_config_backup.json"
-        if not os.path.exists(site_config_path):
-            site_config_path = os.path.join(backup_dir, site, "site_config.json")
-        if site in get_sites():
-            print(f"Overwrite site {site}")
-            restore_database(files_base, site_config_path, site)
-            restore_private_files(files_base)
-            restore_files(files_base)
-        else:
-            site_config = get_conf_params(
-                db_name="_" + hashlib.sha1(site.encode()).hexdigest()[:16],
-                db_password=random_string(16),
-            )
-
-            frappe.local.site = site
-            frappe.local.sites_path = os.getcwd()
-            frappe.local.site_path = os.getcwd() + "/" + site
-            make_conf(
-                db_name=site_config.get("db_name"),
-                db_password=site_config.get("db_password"),
-            )
-            make_site_dirs()
-
-            print(f"Create site {site}")
-            restore_database(files_base, site_config_path, site)
-            restore_private_files(files_base)
-            restore_files(files_base)
-
-    if frappe.redis_server:
-        frappe.redis_server.connection_pool.disconnect()
-
-    exit(0)
-
-
-if __name__ == "__main__":
-    main()

build/frappe-worker/commands/utils.py

@@ -1,208 +0,0 @@
-import json
-import os
-import subprocess
-
-import boto3
-import git
-from constants import APP_VERSIONS_JSON_FILE, APPS_TXT_FILE, COMMON_SITE_CONFIG_FILE
-from frappe.installer import update_site_config
-
-
-def run_command(command, stdout=None, stdin=None, stderr=None):
-    stdout = stdout or subprocess.PIPE
-    stderr = stderr or subprocess.PIPE
-    stdin = stdin or subprocess.PIPE
-    process = subprocess.Popen(command, stdout=stdout, stdin=stdin, stderr=stderr)
-    out, error = process.communicate()
-    if process.returncode:
-        print("Something went wrong:")
-        print(f"return code: {process.returncode}")
-        print(f"stdout:\n{out}")
-        print(f"\nstderr:\n{error}")
-        exit(process.returncode)
-
-
-def save_version_file(versions):
-    with open(APP_VERSIONS_JSON_FILE, "w") as f:
-        return json.dump(versions, f, indent=1, sort_keys=True)
-
-
-def get_apps():
-    apps = []
-    try:
-        with open(APPS_TXT_FILE) as apps_file:
-            for app in apps_file.readlines():
-                if app.strip():
-                    apps.append(app.strip())
-
-    except FileNotFoundError as exception:
-        print(exception)
-        exit(1)
-    except Exception:
-        print(APPS_TXT_FILE + " is not valid")
-        exit(1)
-
-    return apps
-
-
-def get_container_versions(apps):
-    versions = {}
-    for app in apps:
-        try:
-            version = __import__(app).__version__
-            versions.update({app: version})
-        except Exception:
-            pass
-
-        try:
-            path = os.path.join("..", "apps", app)
-            repo = git.Repo(path)
-            commit_hash = repo.head.object.hexsha
-            versions.update({app + "_git_hash": commit_hash})
-        except Exception:
-            pass
-
-    return versions
-
-
-def get_version_file():
-    versions = None
-    try:
-        with open(APP_VERSIONS_JSON_FILE) as versions_file:
-            versions = json.load(versions_file)
-    except Exception:
-        pass
-    return versions
-
-
-def get_config():
-    config = None
-    try:
-        with open(COMMON_SITE_CONFIG_FILE) as config_file:
-            config = json.load(config_file)
-    except FileNotFoundError as exception:
-        print(exception)
-        exit(1)
-    except Exception:
-        print(COMMON_SITE_CONFIG_FILE + " is not valid")
-        exit(1)
-    return config
-
-
-def get_site_config(site_name):
-    site_config = None
-    with open(f"{site_name}/site_config.json") as site_config_file:
-        site_config = json.load(site_config_file)
-    return site_config
-
-
-def save_config(config):
-    with open(COMMON_SITE_CONFIG_FILE, "w") as f:
-        return json.dump(config, f, indent=1, sort_keys=True)
-
-
-def get_password(env_var, default=None):
-    return (
-        os.environ.get(env_var)
-        or get_password_from_secret(f"{env_var}_FILE")
-        or default
-    )
-
-
-def get_password_from_secret(env_var):
-    """Fetches the secret value from the docker secret file
-    usually located inside /run/secrets/
-    Arguments:
-        env_var {str} -- Name of the environment variable
-        containing the path to the secret file.
-    Returns:
-        [str] -- Secret value
-    """
-    passwd = None
-    secret_file_path = os.environ.get(env_var)
-    if secret_file_path:
-        with open(secret_file_path) as secret_file:
-            passwd = secret_file.read().strip()
-
-    return passwd
-
-
-def get_s3_config():
-    check_s3_environment_variables()
-    bucket = os.environ.get("BUCKET_NAME")
-
-    conn = boto3.client(
-        "s3",
-        region_name=os.environ.get("REGION"),
-        aws_access_key_id=os.environ.get("ACCESS_KEY_ID"),
-        aws_secret_access_key=os.environ.get("SECRET_ACCESS_KEY"),
-        endpoint_url=os.environ.get("ENDPOINT_URL"),
-    )
-
-    return conn, bucket
-
-
-def upload_file_to_s3(filename, folder, conn, bucket):
-
-    destpath = os.path.join(folder, os.path.basename(filename))
-    try:
-        print("Uploading file:", filename)
-        conn.upload_file(filename, bucket, destpath)
-
-    except Exception as e:
-        print("Error uploading: %s" % (e))
-        exit(1)
-
-
-def list_directories(path):
-    directories = []
-    for name in os.listdir(path):
-        if os.path.isdir(os.path.join(path, name)):
-            directories.append(name)
-    return directories
-
-
-def get_site_config_from_path(site_config_path):
-    site_config = dict()
-    if os.path.exists(site_config_path):
-        with open(site_config_path) as sc:
-            site_config = json.load(sc)
-    return site_config
-
-
-def set_key_in_site_config(key, site, site_config_path):
-    site_config = get_site_config_from_path(site_config_path)
-    value = site_config.get(key)
-    if value:
-        print(f"Set {key} in site config for site: {site}")
-        update_site_config(
-            key,
-            value,
-            site_config_path=os.path.join(os.getcwd(), site, "site_config.json"),
-        )
-
-
-def check_s3_environment_variables():
-    if "BUCKET_NAME" not in os.environ:
-        print("Variable BUCKET_NAME not set")
-        exit(1)
-
-    if "ACCESS_KEY_ID" not in os.environ:
-        print("Variable ACCESS_KEY_ID not set")
-        exit(1)
-
-    if "SECRET_ACCESS_KEY" not in os.environ:
-        print("Variable SECRET_ACCESS_KEY not set")
-        exit(1)
-
-    if "ENDPOINT_URL" not in os.environ:
-        print("Variable ENDPOINT_URL not set")
-        exit(1)
-
-    if "BUCKET_DIR" not in os.environ:
-        print("Variable BUCKET_DIR not set")
-        exit(1)
-
-    if "REGION" not in os.environ:
-        print("Variable REGION not set")
-        exit(1)

build/frappe-worker/common_site_config.json.template

@@ -1,8 +0,0 @@
-{
-    "db_host": "${DB_HOST}",
-    "db_port": ${DB_PORT},
-    "redis_cache": "redis://${REDIS_CACHE}",
-    "redis_queue": "redis://${REDIS_QUEUE}",
-    "redis_socketio": "redis://${REDIS_SOCKETIO}",
-    "socketio_port": ${SOCKETIO_PORT}
-}

build/frappe-worker/docker-entrypoint.sh

@@ -1,191 +0,0 @@
-#!/bin/bash
-
-function configureEnv() {
-  if [[ ! -f /home/frappe/frappe-bench/sites/common_site_config.json ]]; then
-
-    if [[ -z "${MARIADB_HOST}" && -z "${POSTGRES_HOST}" ]]; then
-      echo "MARIADB_HOST or POSTGRES_HOST is not set" >&2
-      exit 1
-    fi
-
-    if [[ -z "${REDIS_CACHE}" ]]; then
-      echo "REDIS_CACHE is not set" >&2
-      exit 1
-    fi
-
-    if [[ -z "${REDIS_QUEUE}" ]]; then
-      echo "REDIS_QUEUE is not set" >&2
-      exit 1
-    fi
-
-    if [[ -z "${REDIS_SOCKETIO}" ]]; then
-      echo "REDIS_SOCKETIO is not set" >&2
-      exit 1
-    fi
-
-    if [[ -z "${SOCKETIO_PORT}" ]]; then
-      echo "SOCKETIO_PORT is not set" >&2
-      exit 1
-    fi
-
-    if [[ -z "${DB_PORT}" ]]; then
-      export DB_PORT=3306
-    fi
-
-    export DB_HOST="${MARIADB_HOST:-$POSTGRES_HOST}"
-
-    # shellcheck disable=SC2016
-    envsubst '${DB_HOST}
-      ${DB_PORT}
-      ${REDIS_CACHE}
-      ${REDIS_QUEUE}
-      ${REDIS_SOCKETIO}
-      ${SOCKETIO_PORT}' </opt/frappe/common_site_config.json.template >/home/frappe/frappe-bench/sites/common_site_config.json
-  fi
-}
-
-function checkConnection() {
-  /home/frappe/frappe-bench/env/bin/python /home/frappe/frappe-bench/commands/check_connection.py
-}
-
-function checkConfigExists() {
-  COUNTER=0
-  while [[ ! -e /home/frappe/frappe-bench/sites/common_site_config.json && ${COUNTER} -le 30 ]]; do
-    sleep 1
-    ((COUNTER = COUNTER + 1))
-    echo "config file not created, retry ${COUNTER}" >&2
-  done
-
-  if [[ ! -e /home/frappe/frappe-bench/sites/common_site_config.json ]]; then
-    echo "timeout: config file not created" >&2
-    exit 1
-  fi
-}
-
-if [[ ! -e /home/frappe/frappe-bench/sites/apps.txt ]]; then
-  find /home/frappe/frappe-bench/apps -mindepth 1 -maxdepth 1 -type d -printf '%f\n' |
-    sort -r >/home/frappe/frappe-bench/sites/apps.txt
-fi
-
-# symlink node_modules
-ln -sfn /home/frappe/frappe-bench/sites/assets/frappe/node_modules \
-  /home/frappe/frappe-bench/apps/frappe/node_modules
-
-case "$1" in
-
-start)
-  configureEnv
-  checkConnection
-
-  [[ -z "${WORKERS}" ]] && WORKERS='2'
-
-  [[ -z "${FRAPPE_PORT}" ]] && FRAPPE_PORT='8000'
-
-  [[ -z "${WORKER_CLASS}" ]] && WORKER_CLASS='gthread'
-
-  LOAD_CONFIG_FILE=""
-  [[ "${WORKER_CLASS}" == "gevent" ]] &&
-    LOAD_CONFIG_FILE="-c /home/frappe/frappe-bench/commands/gevent_patch.py"
-
-  if [[ -n "${AUTO_MIGRATE}" ]]; then
-    /home/frappe/frappe-bench/env/bin/python /home/frappe/frappe-bench/commands/auto_migrate.py
-  fi
-
-  # shellcheck disable=SC2086
-  /home/frappe/frappe-bench/env/bin/gunicorn ${LOAD_CONFIG_FILE} -b 0.0.0.0:${FRAPPE_PORT} \
-    --worker-tmp-dir /dev/shm \
-    --threads=4 \
-    --workers ${WORKERS} \
-    --worker-class=${WORKER_CLASS} \
-    --log-file=- \
-    -t 120 frappe.app:application --preload
-  ;;
-
-worker)
-  checkConfigExists
-  checkConnection
-
-  : "${WORKER_TYPE:=default}"
-  bench worker --queue $WORKER_TYPE
-  ;;
-
-schedule)
-  checkConfigExists
-  checkConnection
-
-  bench schedule
-  ;;
-
-new)
-  checkConfigExists
-  checkConnection
-
-  /home/frappe/frappe-bench/env/bin/python /home/frappe/frappe-bench/commands/new.py
-  exit
-  ;;
-
-drop)
-  checkConfigExists
-  checkConnection
-
-  : "${SITE_NAME:=site1.localhost}"
-  : "${DB_ROOT_USER:=root}"
-  : "${DB_ROOT_PASSWORD:=$POSTGRES_PASSWORD}"
-  : "${DB_ROOT_PASSWORD:=$MYSQL_ROOT_PASSWORD}"
-  : "${DB_ROOT_PASSWORD:=admin}"
-
-  FLAGS=
-  if [[ ${NO_BACKUP} == 1 ]]; then
-    FLAGS="${FLAGS} --no-backup"
-  fi
-  if [[ ${FORCE} == 1 ]]; then
-    FLAGS="${FLAGS} --force"
-  fi
-
-  # shellcheck disable=SC2086
-  bench drop-site \
-    ${SITE_NAME} \
-    --root-login ${DB_ROOT_USER} \
-    --root-password ${DB_ROOT_PASSWORD} \
-    --archived-sites-path /home/frappe/frappe-bench/sites/archive_sites \
-    ${FLAGS}
-  ;;
-
-migrate)
-  /home/frappe/frappe-bench/env/bin/python /home/frappe/frappe-bench/commands/migrate.py
-  exit
-  ;;
-
-doctor)
-  /home/frappe/frappe-bench/env/bin/python /home/frappe/frappe-bench/commands/doctor.py "${@:2}"
-  exit
-  ;;
-
-backup)
-  /home/frappe/frappe-bench/env/bin/python /home/frappe/frappe-bench/commands/backup.py
-  exit
-  ;;
-
-console)
-  if [[ -z "$2" ]]; then
-    echo "Need to specify a sitename with the command:" >&2
-    echo "console <sitename>" >&2
-    exit 1
-  fi
-
-  bench --site "$2" console
-  ;;
-
-push-backup)
-  /home/frappe/frappe-bench/env/bin/python /home/frappe/frappe-bench/commands/push_backup.py
-  exit
-  ;;
-
-restore-backup)
-  /home/frappe/frappe-bench/env/bin/python /home/frappe/frappe-bench/commands/restore_backup.py
-  exit
-  ;;
-*)
-  exec "$@"
-  ;;
-esac

build/frappe-worker/entrypoint.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+
+# TODO: Set config dynamically, so changes in compose file have affect
+
+set -e
+
+function create_common_site_config() {
+  if [[ ! -f common_site_config.json ]]; then
+    config=$(
+      cat <<EOF
+{
+    "db_host": "${DB_HOST}",
+    "db_port": ${DB_PORT},
+    "redis_cache": "redis://${REDIS_CACHE}",
+    "redis_queue": "redis://${REDIS_QUEUE}",
+    "redis_socketio": "redis://${REDIS_SOCKETIO}",
+    "socketio_port": ${SOCKETIO_PORT}
+}
+EOF
+    )
+    echo "$config" >/home/frappe/frappe-bench/sites/common_site_config.json
+  fi
+}
+
+create_common_site_config
+exec "$@"

build/frappe-worker/healthcheck.sh

@@ -1,45 +1,28 @@
 #!/bin/bash
-set -ea
+set -e
 
-function getUrl() {
+get_key() {
-  grep "$2" "$1" | awk -v word="$2" '$word { gsub(/[",]/,"",$2); print $2}' | tr -d '\n'
+  grep "$1" /home/frappe/frappe-bench/sites/common_site_config.json | awk -v word="$1" '$word { gsub(/[",]/,"",$2); print $2}' | tr -d '\n'
 }
 
-COMMON_SITE_CONFIG_JSON='/home/frappe/frappe-bench/sites/common_site_config.json'
+get_redis_url() {
-
+  URL=$(get_key "$1" | sed 's|redis://||g')
-# Set DB Host and port
+  if [[ ${URL} == *"/"* ]]; then
-DB_HOST=$(getUrl "${COMMON_SITE_CONFIG_JSON}" "db_host")
+    URL=$(echo "${URL}" | cut -f1 -d"/")
-DB_PORT=$(getUrl "${COMMON_SITE_CONFIG_JSON}" "db_port")
-if [[ -z "${DB_PORT}" ]]; then
-  DB_PORT=3306
   fi
+  echo "$URL"
+}
 
-# Set REDIS host:port
+check_connection() {
-REDIS_CACHE=$(getUrl "${COMMON_SITE_CONFIG_JSON}" "redis_cache" | sed 's|redis://||g')
+  echo "Check $1"
-if [[ "${REDIS_CACHE}" == *"/"* ]]; then
+  wait-for-it "$1" -t 1
-  REDIS_CACHE=$(echo "${REDIS_CACHE}" | cut -f1 -d"/")
+}
-fi
+
-
+check_connection "$(get_key db_host):$(get_key db_port)"
-REDIS_QUEUE=$(getUrl "${COMMON_SITE_CONFIG_JSON}" "redis_queue" | sed 's|redis://||g')
+check_connection "$(get_redis_url redis_cache)"
-if [[ "${REDIS_QUEUE}" == *"/"* ]]; then
+check_connection "$(get_redis_url redis_queue)"
-  REDIS_QUEUE=$(echo "${REDIS_QUEUE}" | cut -f1 -d"/")
+check_connection "$(get_redis_url redis_socketio)"
-fi
+
-
+if [[ "$1" = -p || "$1" = --ping-service ]]; then
-REDIS_SOCKETIO=$(getUrl "${COMMON_SITE_CONFIG_JSON}" "redis_socketio" | sed 's|redis://||g')
+  check_connection "$2"
-if [[ "${REDIS_SOCKETIO}" == *"/"* ]]; then
-  REDIS_SOCKETIO=$(echo "${REDIS_SOCKETIO}" | cut -f1 -d"/")
-fi
-
-echo "Check ${DB_HOST}:${DB_PORT}"
-wait-for-it "${DB_HOST}:${DB_PORT}" -t 1
-echo "Check ${REDIS_CACHE}"
-wait-for-it "${REDIS_CACHE}" -t 1
-echo "Check ${REDIS_QUEUE}"
-wait-for-it "${REDIS_QUEUE}" -t 1
-echo "Check ${REDIS_SOCKETIO}"
-wait-for-it "${REDIS_SOCKETIO}" -t 1
-
-if [[ "$1" = "-p" || "$1" = "--ping-service" ]]; then
-  echo "Check $2"
-  wait-for-it "$2" -t 1
 fi

build/frappe-worker/install_app.sh

@@ -1,11 +0,0 @@
-#!/bin/bash -ex
-
-APP_NAME=${1}
-APP_REPO=${2}
-APP_BRANCH=${3}
-
-[[ -n "${APP_BRANCH}" ]] && BRANCH="-b ${APP_BRANCH}"
-
-# shellcheck disable=SC2086
-git clone --depth 1 -o upstream "${APP_REPO}" ${BRANCH} "/home/frappe/frappe-bench/apps/${APP_NAME}"
-/home/frappe/frappe-bench/env/bin/pip3 install --no-cache-dir -e "/home/frappe/frappe-bench/apps/${APP_NAME}"

build/frappe-worker/patched_bench_helper.py

@@ -0,0 +1,24 @@
+import frappe.app
+import frappe.database.db_manager
+import frappe.utils.bench_helper
+
+
+def patch_database_creator():
+    """
+    We need to interrupt Frappe site database creation to monkeypatch
+    functions that resolve host for user that owns site database.
+    In frappe_docker this was implemented in "new" command:
+    https://github.com/frappe/frappe_docker/blob/c808ad1767feaf793a2d14541ac0f4d9cbab45b3/build/frappe-worker/commands/new.py#L87
+    """
+
+    frappe.database.db_manager.DbManager.get_current_host = lambda self: "%"
+
+
+def main() -> int:
+    patch_database_creator()
+    frappe.utils.bench_helper.main()
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

build/frappe-worker/pretend-bench.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+set -e
+
+# shellcheck disable=SC2068
+~/frappe-bench/env/bin/python /usr/local/bin/patched_bench_helper.py frappe $@