From b0b176c144b4092b372f8b6dda8621437ef35384 Mon Sep 17 00:00:00 2001 From: Lev Vereshchagin Date: Mon, 13 Dec 2021 19:24:15 +0300 Subject: [PATCH] Update frappe-worker --- build/frappe-worker/Dockerfile | 156 ++++----- build/frappe-worker/bench | 22 -- build/frappe-worker/commands/auto_migrate.py | 64 ---- build/frappe-worker/commands/backup.py | 45 --- .../commands/check_connection.py | 124 ------- build/frappe-worker/commands/constants.py | 13 - build/frappe-worker/commands/doctor.py | 61 ---- build/frappe-worker/commands/gevent_patch.py | 3 - build/frappe-worker/commands/migrate.py | 52 --- build/frappe-worker/commands/new.py | 132 -------- build/frappe-worker/commands/push_backup.py | 183 ---------- .../frappe-worker/commands/restore_backup.py | 313 ------------------ build/frappe-worker/commands/utils.py | 208 ------------ .../common_site_config.json.template | 8 - build/frappe-worker/docker-entrypoint.sh | 191 ----------- build/frappe-worker/entrypoint.sh | 26 ++ build/frappe-worker/healthcheck.sh | 61 ++-- build/frappe-worker/install_app.sh | 11 - build/frappe-worker/patched_bench_helper.py | 24 ++ build/frappe-worker/pretend-bench.sh | 5 + 20 files changed, 144 insertions(+), 1558 deletions(-) delete mode 100755 build/frappe-worker/bench delete mode 100644 build/frappe-worker/commands/auto_migrate.py delete mode 100644 build/frappe-worker/commands/backup.py delete mode 100644 build/frappe-worker/commands/check_connection.py delete mode 100644 build/frappe-worker/commands/constants.py delete mode 100644 build/frappe-worker/commands/doctor.py delete mode 100644 build/frappe-worker/commands/gevent_patch.py delete mode 100644 build/frappe-worker/commands/migrate.py delete mode 100644 build/frappe-worker/commands/new.py delete mode 100644 build/frappe-worker/commands/push_backup.py delete mode 100644 build/frappe-worker/commands/restore_backup.py delete mode 100644 build/frappe-worker/commands/utils.py delete mode 100755 build/frappe-worker/common_site_config.json.template delete mode 100755 build/frappe-worker/docker-entrypoint.sh create mode 100755 build/frappe-worker/entrypoint.sh delete mode 100755 build/frappe-worker/install_app.sh create mode 100644 build/frappe-worker/patched_bench_helper.py create mode 100755 build/frappe-worker/pretend-bench.sh diff --git a/build/frappe-worker/Dockerfile b/build/frappe-worker/Dockerfile index af1a71f6..a28a9648 100644 --- a/build/frappe-worker/Dockerfile +++ b/build/frappe-worker/Dockerfile @@ -1,108 +1,86 @@ -ARG PYTHON_VERSION=3.9 -FROM python:${PYTHON_VERSION}-slim-bullseye +ARG PYTHON_VERSION +FROM python:${PYTHON_VERSION}-slim-bullseye as base -# Add non root user without password -RUN useradd -ms /bin/bash frappe - -ARG GIT_REPO=https://github.com/frappe/frappe -ARG GIT_BRANCH=develop - -ARG ARCH=amd64 -ENV PYTHONUNBUFFERED 1 - -ENV NODE_VERSION=14.18.1 -ENV NVM_DIR /home/frappe/.nvm -ENV PATH ${NVM_DIR}/versions/node/v${NODE_VERSION}/bin/:${PATH} -ENV WKHTMLTOPDF_VERSION 0.12.6-1 - -# Install apt dependencies RUN apt-get update \ && apt-get install --no-install-recommends -y \ - # For frappe framework - git \ + # MariaDB mariadb-client \ + # Postgres postgresql-client \ - gettext-base \ - wget \ - wait-for-it \ - # For PDF - libjpeg62-turbo \ - libx11-6 \ - libxcb1 \ - libxext6 \ - libxrender1 \ - libssl-dev \ - fonts-cantarell \ - xfonts-75dpi \ - xfonts-base \ - libxml2 \ - libffi-dev \ - libjpeg-dev \ - zlib1g-dev \ - # For psycopg2 libpq-dev \ - # For arm64 python wheel builds - && if [ "$(uname -m)" = "aarch64" ]; then \ - apt-get install --no-install-recommends -y \ - gcc \ - g++; \ - fi \ - # Install additional requirements for develop branch - && if [ "${GIT_BRANCH}" = 'develop' ]; then \ - apt-get install --no-install-recommends -y \ - libcairo2 \ - python3-cffi \ - python3-brotli \ - libpango-1.0-0 \ - libpangoft2-1.0-0 \ - libpangocairo-1.0-0; \ - fi \ + # wkhtmltopdf + xvfb \ + libfontconfig \ + wkhtmltopdf \ + # For healthcheck.sh in helm chart + wait-for-it \ && rm -rf /var/lib/apt/lists/* -# Detect arch, download and install wkhtmltopdf -RUN if [ "$(uname -m)" = "aarch64" ]; then export ARCH=arm64; fi \ - && if [ "$(uname -m)" = "x86_64" ]; then export ARCH=amd64; fi \ - && downloaded_file=wkhtmltox_$WKHTMLTOPDF_VERSION.buster_${ARCH}.deb \ - && wget -q https://github.com/wkhtmltopdf/packaging/releases/download/$WKHTMLTOPDF_VERSION/$downloaded_file \ - && dpkg -i $downloaded_file \ - && rm $downloaded_file - -# Setup docker-entrypoint -COPY build/frappe-worker/docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh -RUN ln -s /usr/local/bin/docker-entrypoint.sh / # backwards compat - +RUN useradd -ms /bin/bash frappe +USER frappe +RUN mkdir -p /home/frappe/frappe-bench/apps /home/frappe/frappe-bench/logs /home/frappe/frappe-bench/sites WORKDIR /home/frappe/frappe-bench -RUN chown -R frappe:frappe /home/frappe + +RUN pip install --no-cache-dir -U pip wheel \ + && python -m venv env \ + && env/bin/pip install --no-cache-dir -U pip wheel + + +FROM base as frappe_builder + +USER root +RUN apt-get update \ + && apt-get install --no-install-recommends -y \ + # Install git here because it is not required in production + git \ + # gcc and g++ are required for building different packages across different versions + # of Frappe and ERPNext and also on different platforms (for example, linux/arm64). + # It is safe to install build deps even if they are not required + # because they won't be included in final images. + gcc \ + g++ \ + && rm -rf /var/lib/apt/lists/* USER frappe -# Create frappe-bench directories -RUN mkdir -p apps logs commands sites /home/frappe/backups +ARG FRAPPE_VERSION +RUN git clone --depth 1 -b ${FRAPPE_VERSION} https://github.com/frappe/frappe apps/frappe \ + && env/bin/pip install --no-cache-dir -e apps/frappe \ + && rm -r apps/frappe/.git -# Setup python environment -RUN python -m venv env -RUN env/bin/pip install --no-cache-dir wheel gevent -# Install nvm with node -RUN wget -qO- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.0/install.sh | bash \ - && . ${NVM_DIR}/nvm.sh \ - && nvm install ${NODE_VERSION} \ - && rm -rf ${NVM_DIR}/.cache +FROM frappe_builder as erpnext_builder -# Install Frappe -RUN git clone --depth 1 -o upstream -b ${GIT_BRANCH} ${GIT_REPO} apps/frappe \ - && env/bin/pip install --no-cache-dir -e apps/frappe +ARG ERPNEXT_VERSION +RUN git clone --depth 1 -b ${ERPNEXT_VERSION} https://github.com/frappe/erpnext apps/erpnext \ + && env/bin/pip install --no-cache-dir -e apps/erpnext \ + && rm -r apps/erpnext/.git -# Copy scripts and templates -COPY build/frappe-worker/commands/* /home/frappe/frappe-bench/commands/ -COPY build/frappe-worker/common_site_config.json.template /opt/frappe/common_site_config.json.template -COPY build/frappe-worker/install_app.sh /usr/local/bin/install_app -COPY build/frappe-worker/bench /usr/local/bin/bench -COPY build/frappe-worker/healthcheck.sh /usr/local/bin/healthcheck.sh -# Use sites volume as working directory +FROM base as configured_base + +COPY pretend-bench.sh /usr/local/bin/bench +# healthcheck.sh used in helm chart +COPY entrypoint.sh patched_bench_helper.py healthcheck.sh /usr/local/bin/ + WORKDIR /home/frappe/frappe-bench/sites -VOLUME [ "/home/frappe/frappe-bench/sites", "/home/frappe/backups", "/home/frappe/frappe-bench/logs" ] +VOLUME [ "/home/frappe/frappe-bench/sites", "/home/frappe/frappe-bench/logs" ] -ENTRYPOINT ["docker-entrypoint.sh"] -CMD ["start"] +ENTRYPOINT [ "entrypoint.sh" ] + +CMD [ "/home/frappe/frappe-bench/env/bin/gunicorn", "-b", "0.0.0.0:8000", "frappe.app:application", "--access-logfile", "-" ] + + +FROM configured_base as frappe + +RUN echo "frappe" >/home/frappe/frappe-bench/sites/apps.txt +COPY --from=frappe_builder /home/frappe/frappe-bench/apps/frappe /home/frappe/frappe-bench/apps/frappe +COPY --from=frappe_builder /home/frappe/frappe-bench/env /home/frappe/frappe-bench/env + + +FROM configured_base as erpnext + +RUN echo "frappe\nerpnext" >/home/frappe/frappe-bench/sites/apps.txt +COPY --from=frappe_builder /home/frappe/frappe-bench/apps/frappe /home/frappe/frappe-bench/apps/frappe +COPY --from=erpnext_builder /home/frappe/frappe-bench/apps/erpnext /home/frappe/frappe-bench/apps/erpnext +COPY --from=erpnext_builder /home/frappe/frappe-bench/env /home/frappe/frappe-bench/env diff --git a/build/frappe-worker/bench b/build/frappe-worker/bench deleted file mode 100755 index f702d0d7..00000000 --- a/build/frappe-worker/bench +++ /dev/null @@ -1,22 +0,0 @@ -#!/home/frappe/frappe-bench/env/bin/python - -import os -import subprocess -import sys - -if __name__ == "__main__": - bench_dir = os.path.join(os.sep, "home", "frappe", "frappe-bench") - sites_dir = os.path.join(bench_dir, "sites") - bench_helper = os.path.join( - bench_dir, - "apps", - "frappe", - "frappe", - "utils", - "bench_helper.py", - ) - cwd = os.getcwd() - os.chdir(sites_dir) - subprocess.check_call( - [sys.executable, bench_helper, "frappe"] + sys.argv[1:], - ) diff --git a/build/frappe-worker/commands/auto_migrate.py b/build/frappe-worker/commands/auto_migrate.py deleted file mode 100644 index a9f84743..00000000 --- a/build/frappe-worker/commands/auto_migrate.py +++ /dev/null @@ -1,64 +0,0 @@ -import os - -import git -import semantic_version -from migrate import migrate_sites -from utils import ( - get_apps, - get_config, - get_container_versions, - get_version_file, - save_version_file, -) - - -def main(): - is_ready = False - apps = get_apps() - - container_versions = get_container_versions(apps) - - version_file = get_version_file() - - if not version_file: - version_file = container_versions - save_version_file(version_file) - - for app in apps: - container_version = None - file_version = None - version_file_hash = None - container_hash = None - - repo = git.Repo(os.path.join("..", "apps", app)) - branch = repo.active_branch.name - - if branch == "develop": - version_file_hash = version_file.get(app + "_git_hash") - container_hash = container_versions.get(app + "_git_hash") - if container_hash and version_file_hash: - if container_hash != version_file_hash: - is_ready = True - break - - if version_file.get(app): - file_version = semantic_version.Version(version_file.get(app)) - - if container_versions.get(app): - container_version = semantic_version.Version(container_versions.get(app)) - - if file_version and container_version: - if container_version > file_version: - is_ready = True - break - - config = get_config() - - if is_ready and config.get("maintenance_mode") != 1: - migrate_sites(maintenance_mode=True) - version_file = container_versions - save_version_file(version_file) - - -if __name__ == "__main__": - main() diff --git a/build/frappe-worker/commands/backup.py b/build/frappe-worker/commands/backup.py deleted file mode 100644 index 9338d207..00000000 --- a/build/frappe-worker/commands/backup.py +++ /dev/null @@ -1,45 +0,0 @@ -import os - -import frappe -from frappe.utils import cint, get_sites, now -from frappe.utils.backups import scheduled_backup - - -def backup(sites, with_files=False): - for site in sites: - frappe.init(site) - frappe.connect() - odb = scheduled_backup( - ignore_files=not with_files, - backup_path_db=None, - backup_path_files=None, - backup_path_private_files=None, - force=True, - ) - print("database backup taken -", odb.backup_path_db, "- on", now()) - if with_files: - print("files backup taken -", odb.backup_path_files, "- on", now()) - print( - "private files backup taken -", - odb.backup_path_private_files, - "- on", - now(), - ) - frappe.destroy() - - -def main(): - installed_sites = ":".join(get_sites()) - sites = os.environ.get("SITES", installed_sites).split(":") - with_files = cint(os.environ.get("WITH_FILES")) - - backup(sites, with_files) - - if frappe.redis_server: - frappe.redis_server.connection_pool.disconnect() - - exit(0) - - -if __name__ == "__main__": - main() diff --git a/build/frappe-worker/commands/check_connection.py b/build/frappe-worker/commands/check_connection.py deleted file mode 100644 index d2bae21f..00000000 --- a/build/frappe-worker/commands/check_connection.py +++ /dev/null @@ -1,124 +0,0 @@ -import socket -import time - -from constants import ( - DB_HOST_KEY, - DB_PORT, - DB_PORT_KEY, - REDIS_CACHE_KEY, - REDIS_QUEUE_KEY, - REDIS_SOCKETIO_KEY, -) -from six.moves.urllib.parse import urlparse -from utils import get_config - - -def is_open(ip, port, timeout=30): - s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - s.settimeout(timeout) - try: - s.connect((ip, int(port))) - s.shutdown(socket.SHUT_RDWR) - return True - except Exception: - return False - finally: - s.close() - - -def check_host(ip, port, retry=10, delay=3, print_attempt=True): - ipup = False - for i in range(retry): - if print_attempt: - print(f"Attempt {i+1} to connect to {ip}:{port}") - if is_open(ip, port): - ipup = True - break - else: - time.sleep(delay) - return ipup - - -# Check service -def check_service( - retry=10, delay=3, print_attempt=True, service_name=None, service_port=None -): - - config = get_config() - if not service_name: - service_name = config.get(DB_HOST_KEY, "mariadb") - if not service_port: - service_port = config.get(DB_PORT_KEY, DB_PORT) - - is_db_connected = False - is_db_connected = check_host( - service_name, service_port, retry, delay, print_attempt - ) - if not is_db_connected: - print( - "Connection to {service_name}:{service_port} timed out".format( - service_name=service_name, - service_port=service_port, - ) - ) - exit(1) - - -# Check redis queue -def check_redis_queue(retry=10, delay=3, print_attempt=True): - check_redis_queue = False - config = get_config() - redis_queue_url = urlparse( - config.get(REDIS_QUEUE_KEY, "redis://redis-queue:6379") - ).netloc - redis_queue, redis_queue_port = redis_queue_url.split(":") - check_redis_queue = check_host( - redis_queue, redis_queue_port, retry, delay, print_attempt - ) - if not check_redis_queue: - print("Connection to redis queue timed out") - exit(1) - - -# Check redis cache -def check_redis_cache(retry=10, delay=3, print_attempt=True): - check_redis_cache = False - config = get_config() - redis_cache_url = urlparse( - config.get(REDIS_CACHE_KEY, "redis://redis-cache:6379") - ).netloc - redis_cache, redis_cache_port = redis_cache_url.split(":") - check_redis_cache = check_host( - redis_cache, redis_cache_port, retry, delay, print_attempt - ) - if not check_redis_cache: - print("Connection to redis cache timed out") - exit(1) - - -# Check redis socketio -def check_redis_socketio(retry=10, delay=3, print_attempt=True): - check_redis_socketio = False - config = get_config() - redis_socketio_url = urlparse( - config.get(REDIS_SOCKETIO_KEY, "redis://redis-socketio:6379") - ).netloc - redis_socketio, redis_socketio_port = redis_socketio_url.split(":") - check_redis_socketio = check_host( - redis_socketio, redis_socketio_port, retry, delay, print_attempt - ) - if not check_redis_socketio: - print("Connection to redis socketio timed out") - exit(1) - - -def main(): - check_service() - check_redis_queue() - check_redis_cache() - check_redis_socketio() - print("Connections OK") - - -if __name__ == "__main__": - main() diff --git a/build/frappe-worker/commands/constants.py b/build/frappe-worker/commands/constants.py deleted file mode 100644 index fd70bbdc..00000000 --- a/build/frappe-worker/commands/constants.py +++ /dev/null @@ -1,13 +0,0 @@ -REDIS_QUEUE_KEY = "redis_queue" -REDIS_CACHE_KEY = "redis_cache" -REDIS_SOCKETIO_KEY = "redis_socketio" -DB_HOST_KEY = "db_host" -DB_PORT_KEY = "db_port" -DB_PORT = 3306 -APP_VERSIONS_JSON_FILE = "app_versions.json" -APPS_TXT_FILE = "apps.txt" -COMMON_SITE_CONFIG_FILE = "common_site_config.json" -DATE_FORMAT = "%Y%m%d_%H%M%S" -RDS_DB = "rds_db" -RDS_PRIVILEGES = "SELECT, INSERT, UPDATE, DELETE, CREATE, DROP, INDEX, ALTER, CREATE TEMPORARY TABLES, CREATE VIEW, EVENT, TRIGGER, SHOW VIEW, CREATE ROUTINE, ALTER ROUTINE, EXECUTE, LOCK TABLES" -ARCHIVE_SITES_PATH = "/home/frappe/frappe-bench/sites/archive_sites" diff --git a/build/frappe-worker/commands/doctor.py b/build/frappe-worker/commands/doctor.py deleted file mode 100644 index c7eac75f..00000000 --- a/build/frappe-worker/commands/doctor.py +++ /dev/null @@ -1,61 +0,0 @@ -import argparse - -from check_connection import ( - check_redis_cache, - check_redis_queue, - check_redis_socketio, - check_service, -) - - -def parse_args(): - parser = argparse.ArgumentParser() - parser.add_argument( - "-p", - "--ping-service", - dest="ping_services", - action="append", - type=str, - help='list of services to ping, e.g. doctor -p "postgres:5432" --ping-service "mariadb:3306"', - ) - args = parser.parse_args() - return args - - -def main(): - args = parse_args() - check_service(retry=1, delay=0, print_attempt=False) - print("Bench database Connected") - check_redis_cache(retry=1, delay=0, print_attempt=False) - print("Redis Cache Connected") - check_redis_queue(retry=1, delay=0, print_attempt=False) - print("Redis Queue Connected") - check_redis_socketio(retry=1, delay=0, print_attempt=False) - print("Redis SocketIO Connected") - - if args.ping_services: - for service in args.ping_services: - service_name = None - service_port = None - - try: - service_name, service_port = service.split(":") - except ValueError: - print("Service should be in format host:port, e.g postgres:5432") - exit(1) - - check_service( - retry=1, - delay=0, - print_attempt=False, - service_name=service_name, - service_port=service_port, - ) - print(f"{service_name}:{service_port} Connected") - - print("Health check successful") - exit(0) - - -if __name__ == "__main__": - main() diff --git a/build/frappe-worker/commands/gevent_patch.py b/build/frappe-worker/commands/gevent_patch.py deleted file mode 100644 index 09f65ce7..00000000 --- a/build/frappe-worker/commands/gevent_patch.py +++ /dev/null @@ -1,3 +0,0 @@ -import gevent.monkey - -gevent.monkey.patch_all() diff --git a/build/frappe-worker/commands/migrate.py b/build/frappe-worker/commands/migrate.py deleted file mode 100644 index 2e99ed54..00000000 --- a/build/frappe-worker/commands/migrate.py +++ /dev/null @@ -1,52 +0,0 @@ -import os - -import frappe -from frappe.utils import cint, get_sites -from utils import get_config, save_config - - -def set_maintenance_mode(enable=True): - conf = get_config() - - if enable: - conf.update({"maintenance_mode": 1, "pause_scheduler": 1}) - save_config(conf) - - if not enable: - conf.update({"maintenance_mode": 0, "pause_scheduler": 0}) - save_config(conf) - - -def migrate_sites(maintenance_mode=False): - installed_sites = ":".join(get_sites()) - sites = os.environ.get("SITES", installed_sites).split(":") - if not maintenance_mode: - maintenance_mode = cint(os.environ.get("MAINTENANCE_MODE")) - - if maintenance_mode: - set_maintenance_mode(True) - - for site in sites: - print("Migrating", site) - frappe.init(site=site) - frappe.connect() - try: - from frappe.migrate import migrate - - migrate() - finally: - frappe.destroy() - - # Disable maintenance mode after migration - set_maintenance_mode(False) - - -def main(): - migrate_sites() - if frappe.redis_server: - frappe.redis_server.connection_pool.disconnect() - exit(0) - - -if __name__ == "__main__": - main() diff --git a/build/frappe-worker/commands/new.py b/build/frappe-worker/commands/new.py deleted file mode 100644 index 546ef9d8..00000000 --- a/build/frappe-worker/commands/new.py +++ /dev/null @@ -1,132 +0,0 @@ -import os - -import frappe -import semantic_version -from constants import COMMON_SITE_CONFIG_FILE, RDS_DB, RDS_PRIVILEGES -from frappe.installer import update_site_config -from utils import get_config, get_password, get_site_config, run_command - -# try to import _new_site from frappe, which could possibly -# exist in either commands.py or installer.py, and so we need -# to maintain compatibility across all frappe versions. -try: - # <= version-{11,12} - from frappe.commands.site import _new_site -except ImportError: - # >= version-13 and develop - from frappe.installer import _new_site - - -def main(): - config = get_config() - db_type = "mariadb" - db_port = config.get("db_port", 3306) - db_host = config.get("db_host") - site_name = os.environ.get("SITE_NAME", "site1.localhost") - db_root_username = os.environ.get("DB_ROOT_USER", "root") - mariadb_root_password = get_password("MYSQL_ROOT_PASSWORD", "admin") - postgres_root_password = get_password("POSTGRES_PASSWORD") - db_root_password = mariadb_root_password - - if postgres_root_password: - db_type = "postgres" - db_host = os.environ.get("POSTGRES_HOST") - db_port = 5432 - db_root_password = postgres_root_password - if not db_host: - db_host = config.get("db_host") - print("Environment variable POSTGRES_HOST not found.") - print("Using db_host from common_site_config.json") - - sites_path = os.getcwd() - common_site_config_path = os.path.join(sites_path, COMMON_SITE_CONFIG_FILE) - update_site_config( - "root_login", - db_root_username, - validate=False, - site_config_path=common_site_config_path, - ) - update_site_config( - "root_password", - db_root_password, - validate=False, - site_config_path=common_site_config_path, - ) - - force = True if os.environ.get("FORCE", None) else False - install_apps = os.environ.get("INSTALL_APPS", None) - install_apps = install_apps.split(",") if install_apps else [] - frappe.init(site_name, new_site=True) - - if semantic_version.Version(frappe.__version__).major > 11: - _new_site( - None, - site_name, - mariadb_root_username=db_root_username, - mariadb_root_password=db_root_password, - admin_password=get_password("ADMIN_PASSWORD", "admin"), - verbose=True, - install_apps=install_apps, - source_sql=None, - force=force, - db_type=db_type, - reinstall=False, - db_host=db_host, - db_port=db_port, - ) - else: - _new_site( - None, - site_name, - mariadb_root_username=db_root_username, - mariadb_root_password=db_root_password, - admin_password=get_password("ADMIN_PASSWORD", "admin"), - verbose=True, - install_apps=install_apps, - source_sql=None, - force=force, - reinstall=False, - ) - - if db_type == "mariadb": - site_config = get_site_config(site_name) - db_name = site_config.get("db_name") - db_password = site_config.get("db_password") - - mysql_command = [ - "mysql", - f"-h{db_host}", - f"-u{db_root_username}", - f"-p{mariadb_root_password}", - "-e", - ] - - # Drop User if exists - command = mysql_command + [ - f"DROP USER IF EXISTS '{db_name}'; FLUSH PRIVILEGES;" - ] - run_command(command) - - # Grant permission to database and set password - grant_privileges = "ALL PRIVILEGES" - - # for Amazon RDS - if config.get(RDS_DB) or site_config.get(RDS_DB): - grant_privileges = RDS_PRIVILEGES - - command = mysql_command + [ - f"\ - CREATE USER IF NOT EXISTS '{db_name}'@'%' IDENTIFIED BY '{db_password}'; \ - GRANT {grant_privileges} ON `{db_name}`.* TO '{db_name}'@'%'; \ - FLUSH PRIVILEGES;" - ] - run_command(command) - - if frappe.redis_server: - frappe.redis_server.connection_pool.disconnect() - - exit(0) - - -if __name__ == "__main__": - main() diff --git a/build/frappe-worker/commands/push_backup.py b/build/frappe-worker/commands/push_backup.py deleted file mode 100644 index 78337ade..00000000 --- a/build/frappe-worker/commands/push_backup.py +++ /dev/null @@ -1,183 +0,0 @@ -import datetime -import os -import time -from glob import glob - -import boto3 -from constants import DATE_FORMAT -from frappe.utils import get_sites -from utils import check_s3_environment_variables, get_s3_config, upload_file_to_s3 - - -def get_file_ext(): - return { - "database": "-database.sql.gz", - "private_files": "-private-files.tar", - "public_files": "-files.tar", - "site_config": "-site_config_backup.json", - } - - -def get_backup_details(sitename): - backup_details = dict() - file_ext = get_file_ext() - - # add trailing slash https://stackoverflow.com/a/15010678 - site_backup_path = os.path.join(os.getcwd(), sitename, "private", "backups", "") - - if os.path.exists(site_backup_path): - for filetype, ext in file_ext.items(): - site_slug = sitename.replace(".", "_") - pattern = site_backup_path + "*-" + site_slug + ext - backup_files = list(filter(os.path.isfile, glob(pattern))) - - if len(backup_files) > 0: - backup_files.sort( - key=lambda file: os.stat( - os.path.join(site_backup_path, file) - ).st_ctime - ) - backup_date = datetime.datetime.strptime( - time.ctime(os.path.getmtime(backup_files[0])), - "%a %b %d %H:%M:%S %Y", - ) - backup_details[filetype] = { - "sitename": sitename, - "file_size_in_bytes": os.stat(backup_files[-1]).st_size, - "file_path": os.path.abspath(backup_files[-1]), - "filename": os.path.basename(backup_files[-1]), - "backup_date": backup_date.date().strftime("%Y-%m-%d %H:%M:%S"), - } - - return backup_details - - -def delete_old_backups(limit, bucket, site_name): - all_backups = list() - all_backup_dates = list() - backup_limit = int(limit) - check_s3_environment_variables() - bucket_dir = os.environ.get("BUCKET_DIR") - oldest_backup_date = None - - s3 = boto3.resource( - "s3", - region_name=os.environ.get("REGION"), - aws_access_key_id=os.environ.get("ACCESS_KEY_ID"), - aws_secret_access_key=os.environ.get("SECRET_ACCESS_KEY"), - endpoint_url=os.environ.get("ENDPOINT_URL"), - ) - - bucket = s3.Bucket(bucket) - objects = bucket.meta.client.list_objects_v2(Bucket=bucket.name, Delimiter="/") - - if objects: - for obj in objects.get("CommonPrefixes"): - if obj.get("Prefix") == bucket_dir + "/": - for backup_obj in bucket.objects.filter(Prefix=obj.get("Prefix")): - if backup_obj.get()["ContentType"] == "application/x-directory": - continue - try: - # backup_obj.key is bucket_dir/site/date_time/backupfile.extension - ( - bucket_dir, - site_slug, - date_time, - backupfile, - ) = backup_obj.key.split("/") - date_time_object = datetime.datetime.strptime( - date_time, DATE_FORMAT - ) - - if site_name in backup_obj.key: - all_backup_dates.append(date_time_object) - all_backups.append(backup_obj.key) - except IndexError as error: - print(error) - exit(1) - - if len(all_backup_dates) > 0: - oldest_backup_date = min(all_backup_dates) - - if len(all_backups) / 3 > backup_limit: - oldest_backup = None - for backup in all_backups: - try: - # backup is bucket_dir/site/date_time/backupfile.extension - backup_dir, site_slug, backup_dt_string, filename = backup.split("/") - backup_datetime = datetime.datetime.strptime( - backup_dt_string, DATE_FORMAT - ) - if backup_datetime == oldest_backup_date: - oldest_backup = backup - - except IndexError as error: - print(error) - exit(1) - - if oldest_backup: - for obj in bucket.objects.filter(Prefix=oldest_backup): - # delete all keys that are inside the oldest_backup - if bucket_dir in obj.key: - print("Deleting " + obj.key) - s3.Object(bucket.name, obj.key).delete() - - -def main(): - details = dict() - sites = get_sites() - conn, bucket = get_s3_config() - - for site in sites: - details = get_backup_details(site) - db_file = details.get("database", {}).get("file_path") - folder = os.environ.get("BUCKET_DIR") + "/" + site + "/" - if db_file: - folder = ( - os.environ.get("BUCKET_DIR") - + "/" - + site - + "/" - + os.path.basename(db_file)[:15] - + "/" - ) - upload_file_to_s3(db_file, folder, conn, bucket) - - # Archive site_config.json - site_config_file = details.get("site_config", {}).get("file_path") - if not site_config_file: - site_config_file = os.path.join(os.getcwd(), site, "site_config.json") - upload_file_to_s3(site_config_file, folder, conn, bucket) - - public_files = details.get("public_files", {}).get("file_path") - if public_files: - folder = ( - os.environ.get("BUCKET_DIR") - + "/" - + site - + "/" - + os.path.basename(public_files)[:15] - + "/" - ) - upload_file_to_s3(public_files, folder, conn, bucket) - - private_files = details.get("private_files", {}).get("file_path") - if private_files: - folder = ( - os.environ.get("BUCKET_DIR") - + "/" - + site - + "/" - + os.path.basename(private_files)[:15] - + "/" - ) - upload_file_to_s3(private_files, folder, conn, bucket) - - delete_old_backups(os.environ.get("BACKUP_LIMIT", "3"), bucket, site) - - print("push-backup complete") - exit(0) - - -if __name__ == "__main__": - main() diff --git a/build/frappe-worker/commands/restore_backup.py b/build/frappe-worker/commands/restore_backup.py deleted file mode 100644 index 1462cd27..00000000 --- a/build/frappe-worker/commands/restore_backup.py +++ /dev/null @@ -1,313 +0,0 @@ -import datetime -import hashlib -import os -import tarfile - -import boto3 -import frappe -from constants import COMMON_SITE_CONFIG_FILE, DATE_FORMAT, RDS_DB, RDS_PRIVILEGES -from frappe.installer import ( - get_conf_params, - make_conf, - make_site_dirs, - update_site_config, -) -from frappe.utils import get_sites, random_string -from utils import ( - check_s3_environment_variables, - get_config, - get_password, - get_site_config, - list_directories, - run_command, - set_key_in_site_config, -) - - -def get_backup_dir(): - return os.path.join(os.path.expanduser("~"), "backups") - - -def decompress_db(database_file, site): - command = ["gunzip", "-c", database_file] - with open(database_file.replace(".gz", ""), "w") as db_file: - print(f"Extract Database GZip for site {site}") - run_command(command, stdout=db_file) - - -def restore_database(files_base, site_config_path, site): - # restore database - database_file = files_base + "-database.sql.gz" - decompress_db(database_file, site) - config = get_config() - - # Set db_type if it exists in backup site_config.json - set_key_in_site_config("db_type", site, site_config_path) - # Set db_host if it exists in backup site_config.json - set_key_in_site_config("db_host", site, site_config_path) - # Set db_port if it exists in backup site_config.json - set_key_in_site_config("db_port", site, site_config_path) - - # get updated site_config - site_config = get_site_config(site) - - # if no db_type exists, default to mariadb - db_type = site_config.get("db_type", "mariadb") - is_database_restored = False - - if db_type == "mariadb": - restore_mariadb( - config=config, site_config=site_config, database_file=database_file - ) - is_database_restored = True - elif db_type == "postgres": - restore_postgres( - config=config, site_config=site_config, database_file=database_file - ) - is_database_restored = True - - if is_database_restored: - # Set encryption_key if it exists in backup site_config.json - set_key_in_site_config("encryption_key", site, site_config_path) - - -def restore_files(files_base): - public_files = files_base + "-files.tar" - # extract tar - public_tar = tarfile.open(public_files) - print(f"Extracting {public_files}") - public_tar.extractall() - - -def restore_private_files(files_base): - private_files = files_base + "-private-files.tar" - private_tar = tarfile.open(private_files) - print(f"Extracting {private_files}") - private_tar.extractall() - - -def pull_backup_from_s3(): - check_s3_environment_variables() - - # https://stackoverflow.com/a/54672690 - s3 = boto3.resource( - "s3", - region_name=os.environ.get("REGION"), - aws_access_key_id=os.environ.get("ACCESS_KEY_ID"), - aws_secret_access_key=os.environ.get("SECRET_ACCESS_KEY"), - endpoint_url=os.environ.get("ENDPOINT_URL"), - ) - - bucket_dir = os.environ.get("BUCKET_DIR") - bucket_name = os.environ.get("BUCKET_NAME") - bucket = s3.Bucket(bucket_name) - - # Change directory to /home/frappe/backups - os.chdir(get_backup_dir()) - - backup_files = [] - sites = set() - site_timestamps = set() - download_backups = [] - - for obj in bucket.objects.filter(Prefix=bucket_dir): - if obj.get()["ContentType"] == "application/x-directory": - continue - backup_file = obj.key.replace(os.path.join(bucket_dir, ""), "") - backup_files.append(backup_file) - site_name, timestamp, backup_type = backup_file.split("/") - site_timestamp = site_name + "/" + timestamp - sites.add(site_name) - site_timestamps.add(site_timestamp) - - # sort sites for latest backups - for site in sites: - backup_timestamps = [] - for site_timestamp in site_timestamps: - site_name, timestamp = site_timestamp.split("/") - if site == site_name: - timestamp_datetime = datetime.datetime.strptime(timestamp, DATE_FORMAT) - backup_timestamps.append(timestamp) - download_backups.append(site + "/" + max(backup_timestamps)) - - # Only download latest backups - for backup_file in backup_files: - for backup in download_backups: - if backup in backup_file: - if not os.path.exists(os.path.dirname(backup_file)): - os.makedirs(os.path.dirname(backup_file)) - print(f"Downloading {backup_file}") - bucket.download_file(bucket_dir + "/" + backup_file, backup_file) - - os.chdir(os.path.join(os.path.expanduser("~"), "frappe-bench", "sites")) - - -def restore_postgres(config, site_config, database_file): - # common config - common_site_config_path = os.path.join(os.getcwd(), COMMON_SITE_CONFIG_FILE) - - db_root_user = config.get("root_login") - if not db_root_user: - postgres_user = os.environ.get("DB_ROOT_USER") - if not postgres_user: - print("Variable DB_ROOT_USER not set") - exit(1) - - db_root_user = postgres_user - update_site_config( - "root_login", - db_root_user, - validate=False, - site_config_path=common_site_config_path, - ) - - db_root_password = config.get("root_password") - if not db_root_password: - root_password = get_password("POSTGRES_PASSWORD") - if not root_password: - print("Variable POSTGRES_PASSWORD not set") - exit(1) - - db_root_password = root_password - update_site_config( - "root_password", - db_root_password, - validate=False, - site_config_path=common_site_config_path, - ) - - # site config - db_host = site_config.get("db_host") - db_port = site_config.get("db_port", 5432) - db_name = site_config.get("db_name") - db_password = site_config.get("db_password") - - psql_command = ["psql"] - psql_uri = f"postgres://{db_root_user}:{db_root_password}@{db_host}:{db_port}" - - print("Restoring PostgreSQL") - run_command(psql_command + [psql_uri, "-c", f'DROP DATABASE IF EXISTS "{db_name}"']) - run_command(psql_command + [psql_uri, "-c", f"DROP USER IF EXISTS {db_name}"]) - run_command(psql_command + [psql_uri, "-c", f'CREATE DATABASE "{db_name}"']) - run_command( - psql_command - + [psql_uri, "-c", f"CREATE user {db_name} password '{db_password}'"] - ) - run_command( - psql_command - + [psql_uri, "-c", f'GRANT ALL PRIVILEGES ON DATABASE "{db_name}" TO {db_name}'] - ) - with open(database_file.replace(".gz", "")) as db_file: - run_command(psql_command + [f"{psql_uri}/{db_name}", "<"], stdin=db_file) - - -def restore_mariadb(config, site_config, database_file): - db_root_password = get_password("MYSQL_ROOT_PASSWORD") - if not db_root_password: - print("Variable MYSQL_ROOT_PASSWORD not set") - exit(1) - - db_root_user = os.environ.get("DB_ROOT_USER", "root") - - db_host = site_config.get("db_host", config.get("db_host")) - db_port = site_config.get("db_port", config.get("db_port", 3306)) - db_name = site_config.get("db_name") - db_password = site_config.get("db_password") - - # mysql command prefix - mysql_command = [ - "mysql", - f"-u{db_root_user}", - f"-h{db_host}", - f"-p{db_root_password}", - f"-P{db_port}", - ] - - # drop db if exists for clean restore - drop_database = mysql_command + ["-e", f"DROP DATABASE IF EXISTS `{db_name}`;"] - run_command(drop_database) - - # create db - create_database = mysql_command + [ - "-e", - f"CREATE DATABASE IF NOT EXISTS `{db_name}`;", - ] - run_command(create_database) - - # create user - create_user = mysql_command + [ - "-e", - f"CREATE USER IF NOT EXISTS '{db_name}'@'%' IDENTIFIED BY '{db_password}'; FLUSH PRIVILEGES;", - ] - run_command(create_user) - - # grant db privileges to user - - grant_privileges = "ALL PRIVILEGES" - - # for Amazon RDS - if config.get(RDS_DB) or site_config.get(RDS_DB): - grant_privileges = RDS_PRIVILEGES - - grant_privileges_command = mysql_command + [ - "-e", - f"GRANT {grant_privileges} ON `{db_name}`.* TO '{db_name}'@'%' IDENTIFIED BY '{db_password}'; FLUSH PRIVILEGES;", - ] - run_command(grant_privileges_command) - - print("Restoring MariaDB") - with open(database_file.replace(".gz", "")) as db_file: - run_command(mysql_command + [f"{db_name}"], stdin=db_file) - - -def main(): - backup_dir = get_backup_dir() - - if len(list_directories(backup_dir)) == 0: - pull_backup_from_s3() - - for site in list_directories(backup_dir): - site_slug = site.replace(".", "_") - backups = [ - datetime.datetime.strptime(backup, DATE_FORMAT) - for backup in list_directories(os.path.join(backup_dir, site)) - ] - latest_backup = max(backups).strftime(DATE_FORMAT) - files_base = os.path.join(backup_dir, site, latest_backup, "") - files_base += latest_backup + "-" + site_slug - site_config_path = files_base + "-site_config_backup.json" - if not os.path.exists(site_config_path): - site_config_path = os.path.join(backup_dir, site, "site_config.json") - if site in get_sites(): - print(f"Overwrite site {site}") - restore_database(files_base, site_config_path, site) - restore_private_files(files_base) - restore_files(files_base) - else: - site_config = get_conf_params( - db_name="_" + hashlib.sha1(site.encode()).hexdigest()[:16], - db_password=random_string(16), - ) - - frappe.local.site = site - frappe.local.sites_path = os.getcwd() - frappe.local.site_path = os.getcwd() + "/" + site - make_conf( - db_name=site_config.get("db_name"), - db_password=site_config.get("db_password"), - ) - make_site_dirs() - - print(f"Create site {site}") - restore_database(files_base, site_config_path, site) - restore_private_files(files_base) - restore_files(files_base) - - if frappe.redis_server: - frappe.redis_server.connection_pool.disconnect() - - exit(0) - - -if __name__ == "__main__": - main() diff --git a/build/frappe-worker/commands/utils.py b/build/frappe-worker/commands/utils.py deleted file mode 100644 index 7ea48225..00000000 --- a/build/frappe-worker/commands/utils.py +++ /dev/null @@ -1,208 +0,0 @@ -import json -import os -import subprocess - -import boto3 -import git -from constants import APP_VERSIONS_JSON_FILE, APPS_TXT_FILE, COMMON_SITE_CONFIG_FILE -from frappe.installer import update_site_config - - -def run_command(command, stdout=None, stdin=None, stderr=None): - stdout = stdout or subprocess.PIPE - stderr = stderr or subprocess.PIPE - stdin = stdin or subprocess.PIPE - process = subprocess.Popen(command, stdout=stdout, stdin=stdin, stderr=stderr) - out, error = process.communicate() - if process.returncode: - print("Something went wrong:") - print(f"return code: {process.returncode}") - print(f"stdout:\n{out}") - print(f"\nstderr:\n{error}") - exit(process.returncode) - - -def save_version_file(versions): - with open(APP_VERSIONS_JSON_FILE, "w") as f: - return json.dump(versions, f, indent=1, sort_keys=True) - - -def get_apps(): - apps = [] - try: - with open(APPS_TXT_FILE) as apps_file: - for app in apps_file.readlines(): - if app.strip(): - apps.append(app.strip()) - - except FileNotFoundError as exception: - print(exception) - exit(1) - except Exception: - print(APPS_TXT_FILE + " is not valid") - exit(1) - - return apps - - -def get_container_versions(apps): - versions = {} - for app in apps: - try: - version = __import__(app).__version__ - versions.update({app: version}) - except Exception: - pass - - try: - path = os.path.join("..", "apps", app) - repo = git.Repo(path) - commit_hash = repo.head.object.hexsha - versions.update({app + "_git_hash": commit_hash}) - except Exception: - pass - - return versions - - -def get_version_file(): - versions = None - try: - with open(APP_VERSIONS_JSON_FILE) as versions_file: - versions = json.load(versions_file) - except Exception: - pass - return versions - - -def get_config(): - config = None - try: - with open(COMMON_SITE_CONFIG_FILE) as config_file: - config = json.load(config_file) - except FileNotFoundError as exception: - print(exception) - exit(1) - except Exception: - print(COMMON_SITE_CONFIG_FILE + " is not valid") - exit(1) - return config - - -def get_site_config(site_name): - site_config = None - with open(f"{site_name}/site_config.json") as site_config_file: - site_config = json.load(site_config_file) - return site_config - - -def save_config(config): - with open(COMMON_SITE_CONFIG_FILE, "w") as f: - return json.dump(config, f, indent=1, sort_keys=True) - - -def get_password(env_var, default=None): - return ( - os.environ.get(env_var) - or get_password_from_secret(f"{env_var}_FILE") - or default - ) - - -def get_password_from_secret(env_var): - """Fetches the secret value from the docker secret file - usually located inside /run/secrets/ - Arguments: - env_var {str} -- Name of the environment variable - containing the path to the secret file. - Returns: - [str] -- Secret value - """ - passwd = None - secret_file_path = os.environ.get(env_var) - if secret_file_path: - with open(secret_file_path) as secret_file: - passwd = secret_file.read().strip() - - return passwd - - -def get_s3_config(): - check_s3_environment_variables() - bucket = os.environ.get("BUCKET_NAME") - - conn = boto3.client( - "s3", - region_name=os.environ.get("REGION"), - aws_access_key_id=os.environ.get("ACCESS_KEY_ID"), - aws_secret_access_key=os.environ.get("SECRET_ACCESS_KEY"), - endpoint_url=os.environ.get("ENDPOINT_URL"), - ) - - return conn, bucket - - -def upload_file_to_s3(filename, folder, conn, bucket): - - destpath = os.path.join(folder, os.path.basename(filename)) - try: - print("Uploading file:", filename) - conn.upload_file(filename, bucket, destpath) - - except Exception as e: - print("Error uploading: %s" % (e)) - exit(1) - - -def list_directories(path): - directories = [] - for name in os.listdir(path): - if os.path.isdir(os.path.join(path, name)): - directories.append(name) - return directories - - -def get_site_config_from_path(site_config_path): - site_config = dict() - if os.path.exists(site_config_path): - with open(site_config_path) as sc: - site_config = json.load(sc) - return site_config - - -def set_key_in_site_config(key, site, site_config_path): - site_config = get_site_config_from_path(site_config_path) - value = site_config.get(key) - if value: - print(f"Set {key} in site config for site: {site}") - update_site_config( - key, - value, - site_config_path=os.path.join(os.getcwd(), site, "site_config.json"), - ) - - -def check_s3_environment_variables(): - if "BUCKET_NAME" not in os.environ: - print("Variable BUCKET_NAME not set") - exit(1) - - if "ACCESS_KEY_ID" not in os.environ: - print("Variable ACCESS_KEY_ID not set") - exit(1) - - if "SECRET_ACCESS_KEY" not in os.environ: - print("Variable SECRET_ACCESS_KEY not set") - exit(1) - - if "ENDPOINT_URL" not in os.environ: - print("Variable ENDPOINT_URL not set") - exit(1) - - if "BUCKET_DIR" not in os.environ: - print("Variable BUCKET_DIR not set") - exit(1) - - if "REGION" not in os.environ: - print("Variable REGION not set") - exit(1) diff --git a/build/frappe-worker/common_site_config.json.template b/build/frappe-worker/common_site_config.json.template deleted file mode 100755 index f7272fd6..00000000 --- a/build/frappe-worker/common_site_config.json.template +++ /dev/null @@ -1,8 +0,0 @@ -{ - "db_host": "${DB_HOST}", - "db_port": ${DB_PORT}, - "redis_cache": "redis://${REDIS_CACHE}", - "redis_queue": "redis://${REDIS_QUEUE}", - "redis_socketio": "redis://${REDIS_SOCKETIO}", - "socketio_port": ${SOCKETIO_PORT} -} diff --git a/build/frappe-worker/docker-entrypoint.sh b/build/frappe-worker/docker-entrypoint.sh deleted file mode 100755 index 74247208..00000000 --- a/build/frappe-worker/docker-entrypoint.sh +++ /dev/null @@ -1,191 +0,0 @@ -#!/bin/bash - -function configureEnv() { - if [[ ! -f /home/frappe/frappe-bench/sites/common_site_config.json ]]; then - - if [[ -z "${MARIADB_HOST}" && -z "${POSTGRES_HOST}" ]]; then - echo "MARIADB_HOST or POSTGRES_HOST is not set" >&2 - exit 1 - fi - - if [[ -z "${REDIS_CACHE}" ]]; then - echo "REDIS_CACHE is not set" >&2 - exit 1 - fi - - if [[ -z "${REDIS_QUEUE}" ]]; then - echo "REDIS_QUEUE is not set" >&2 - exit 1 - fi - - if [[ -z "${REDIS_SOCKETIO}" ]]; then - echo "REDIS_SOCKETIO is not set" >&2 - exit 1 - fi - - if [[ -z "${SOCKETIO_PORT}" ]]; then - echo "SOCKETIO_PORT is not set" >&2 - exit 1 - fi - - if [[ -z "${DB_PORT}" ]]; then - export DB_PORT=3306 - fi - - export DB_HOST="${MARIADB_HOST:-$POSTGRES_HOST}" - - # shellcheck disable=SC2016 - envsubst '${DB_HOST} - ${DB_PORT} - ${REDIS_CACHE} - ${REDIS_QUEUE} - ${REDIS_SOCKETIO} - ${SOCKETIO_PORT}' /home/frappe/frappe-bench/sites/common_site_config.json - fi -} - -function checkConnection() { - /home/frappe/frappe-bench/env/bin/python /home/frappe/frappe-bench/commands/check_connection.py -} - -function checkConfigExists() { - COUNTER=0 - while [[ ! -e /home/frappe/frappe-bench/sites/common_site_config.json && ${COUNTER} -le 30 ]]; do - sleep 1 - ((COUNTER = COUNTER + 1)) - echo "config file not created, retry ${COUNTER}" >&2 - done - - if [[ ! -e /home/frappe/frappe-bench/sites/common_site_config.json ]]; then - echo "timeout: config file not created" >&2 - exit 1 - fi -} - -if [[ ! -e /home/frappe/frappe-bench/sites/apps.txt ]]; then - find /home/frappe/frappe-bench/apps -mindepth 1 -maxdepth 1 -type d -printf '%f\n' | - sort -r >/home/frappe/frappe-bench/sites/apps.txt -fi - -# symlink node_modules -ln -sfn /home/frappe/frappe-bench/sites/assets/frappe/node_modules \ - /home/frappe/frappe-bench/apps/frappe/node_modules - -case "$1" in - -start) - configureEnv - checkConnection - - [[ -z "${WORKERS}" ]] && WORKERS='2' - - [[ -z "${FRAPPE_PORT}" ]] && FRAPPE_PORT='8000' - - [[ -z "${WORKER_CLASS}" ]] && WORKER_CLASS='gthread' - - LOAD_CONFIG_FILE="" - [[ "${WORKER_CLASS}" == "gevent" ]] && - LOAD_CONFIG_FILE="-c /home/frappe/frappe-bench/commands/gevent_patch.py" - - if [[ -n "${AUTO_MIGRATE}" ]]; then - /home/frappe/frappe-bench/env/bin/python /home/frappe/frappe-bench/commands/auto_migrate.py - fi - - # shellcheck disable=SC2086 - /home/frappe/frappe-bench/env/bin/gunicorn ${LOAD_CONFIG_FILE} -b 0.0.0.0:${FRAPPE_PORT} \ - --worker-tmp-dir /dev/shm \ - --threads=4 \ - --workers ${WORKERS} \ - --worker-class=${WORKER_CLASS} \ - --log-file=- \ - -t 120 frappe.app:application --preload - ;; - -worker) - checkConfigExists - checkConnection - - : "${WORKER_TYPE:=default}" - bench worker --queue $WORKER_TYPE - ;; - -schedule) - checkConfigExists - checkConnection - - bench schedule - ;; - -new) - checkConfigExists - checkConnection - - /home/frappe/frappe-bench/env/bin/python /home/frappe/frappe-bench/commands/new.py - exit - ;; - -drop) - checkConfigExists - checkConnection - - : "${SITE_NAME:=site1.localhost}" - : "${DB_ROOT_USER:=root}" - : "${DB_ROOT_PASSWORD:=$POSTGRES_PASSWORD}" - : "${DB_ROOT_PASSWORD:=$MYSQL_ROOT_PASSWORD}" - : "${DB_ROOT_PASSWORD:=admin}" - - FLAGS= - if [[ ${NO_BACKUP} == 1 ]]; then - FLAGS="${FLAGS} --no-backup" - fi - if [[ ${FORCE} == 1 ]]; then - FLAGS="${FLAGS} --force" - fi - - # shellcheck disable=SC2086 - bench drop-site \ - ${SITE_NAME} \ - --root-login ${DB_ROOT_USER} \ - --root-password ${DB_ROOT_PASSWORD} \ - --archived-sites-path /home/frappe/frappe-bench/sites/archive_sites \ - ${FLAGS} - ;; - -migrate) - /home/frappe/frappe-bench/env/bin/python /home/frappe/frappe-bench/commands/migrate.py - exit - ;; - -doctor) - /home/frappe/frappe-bench/env/bin/python /home/frappe/frappe-bench/commands/doctor.py "${@:2}" - exit - ;; - -backup) - /home/frappe/frappe-bench/env/bin/python /home/frappe/frappe-bench/commands/backup.py - exit - ;; - -console) - if [[ -z "$2" ]]; then - echo "Need to specify a sitename with the command:" >&2 - echo "console " >&2 - exit 1 - fi - - bench --site "$2" console - ;; - -push-backup) - /home/frappe/frappe-bench/env/bin/python /home/frappe/frappe-bench/commands/push_backup.py - exit - ;; - -restore-backup) - /home/frappe/frappe-bench/env/bin/python /home/frappe/frappe-bench/commands/restore_backup.py - exit - ;; -*) - exec "$@" - ;; -esac diff --git a/build/frappe-worker/entrypoint.sh b/build/frappe-worker/entrypoint.sh new file mode 100755 index 00000000..fa6d60a0 --- /dev/null +++ b/build/frappe-worker/entrypoint.sh @@ -0,0 +1,26 @@ +#!/bin/bash + +# TODO: Set config dynamically, so changes in compose file have affect + +set -e + +function create_common_site_config() { + if [[ ! -f common_site_config.json ]]; then + config=$( + cat </home/frappe/frappe-bench/sites/common_site_config.json + fi +} + +create_common_site_config +exec "$@" diff --git a/build/frappe-worker/healthcheck.sh b/build/frappe-worker/healthcheck.sh index 3c4266ee..642ba046 100755 --- a/build/frappe-worker/healthcheck.sh +++ b/build/frappe-worker/healthcheck.sh @@ -1,45 +1,28 @@ #!/bin/bash -set -ea +set -e -function getUrl() { - grep "$2" "$1" | awk -v word="$2" '$word { gsub(/[",]/,"",$2); print $2}' | tr -d '\n' +get_key() { + grep "$1" /home/frappe/frappe-bench/sites/common_site_config.json | awk -v word="$1" '$word { gsub(/[",]/,"",$2); print $2}' | tr -d '\n' } -COMMON_SITE_CONFIG_JSON='/home/frappe/frappe-bench/sites/common_site_config.json' +get_redis_url() { + URL=$(get_key "$1" | sed 's|redis://||g') + if [[ ${URL} == *"/"* ]]; then + URL=$(echo "${URL}" | cut -f1 -d"/") + fi + echo "$URL" +} -# Set DB Host and port -DB_HOST=$(getUrl "${COMMON_SITE_CONFIG_JSON}" "db_host") -DB_PORT=$(getUrl "${COMMON_SITE_CONFIG_JSON}" "db_port") -if [[ -z "${DB_PORT}" ]]; then - DB_PORT=3306 -fi - -# Set REDIS host:port -REDIS_CACHE=$(getUrl "${COMMON_SITE_CONFIG_JSON}" "redis_cache" | sed 's|redis://||g') -if [[ "${REDIS_CACHE}" == *"/"* ]]; then - REDIS_CACHE=$(echo "${REDIS_CACHE}" | cut -f1 -d"/") -fi - -REDIS_QUEUE=$(getUrl "${COMMON_SITE_CONFIG_JSON}" "redis_queue" | sed 's|redis://||g') -if [[ "${REDIS_QUEUE}" == *"/"* ]]; then - REDIS_QUEUE=$(echo "${REDIS_QUEUE}" | cut -f1 -d"/") -fi - -REDIS_SOCKETIO=$(getUrl "${COMMON_SITE_CONFIG_JSON}" "redis_socketio" | sed 's|redis://||g') -if [[ "${REDIS_SOCKETIO}" == *"/"* ]]; then - REDIS_SOCKETIO=$(echo "${REDIS_SOCKETIO}" | cut -f1 -d"/") -fi - -echo "Check ${DB_HOST}:${DB_PORT}" -wait-for-it "${DB_HOST}:${DB_PORT}" -t 1 -echo "Check ${REDIS_CACHE}" -wait-for-it "${REDIS_CACHE}" -t 1 -echo "Check ${REDIS_QUEUE}" -wait-for-it "${REDIS_QUEUE}" -t 1 -echo "Check ${REDIS_SOCKETIO}" -wait-for-it "${REDIS_SOCKETIO}" -t 1 - -if [[ "$1" = "-p" || "$1" = "--ping-service" ]]; then - echo "Check $2" - wait-for-it "$2" -t 1 +check_connection() { + echo "Check $1" + wait-for-it "$1" -t 1 +} + +check_connection "$(get_key db_host):$(get_key db_port)" +check_connection "$(get_redis_url redis_cache)" +check_connection "$(get_redis_url redis_queue)" +check_connection "$(get_redis_url redis_socketio)" + +if [[ "$1" = -p || "$1" = --ping-service ]]; then + check_connection "$2" fi diff --git a/build/frappe-worker/install_app.sh b/build/frappe-worker/install_app.sh deleted file mode 100755 index 6e55ec74..00000000 --- a/build/frappe-worker/install_app.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash -ex - -APP_NAME=${1} -APP_REPO=${2} -APP_BRANCH=${3} - -[[ -n "${APP_BRANCH}" ]] && BRANCH="-b ${APP_BRANCH}" - -# shellcheck disable=SC2086 -git clone --depth 1 -o upstream "${APP_REPO}" ${BRANCH} "/home/frappe/frappe-bench/apps/${APP_NAME}" -/home/frappe/frappe-bench/env/bin/pip3 install --no-cache-dir -e "/home/frappe/frappe-bench/apps/${APP_NAME}" diff --git a/build/frappe-worker/patched_bench_helper.py b/build/frappe-worker/patched_bench_helper.py new file mode 100644 index 00000000..8c69dbdd --- /dev/null +++ b/build/frappe-worker/patched_bench_helper.py @@ -0,0 +1,24 @@ +import frappe.app +import frappe.database.db_manager +import frappe.utils.bench_helper + + +def patch_database_creator(): + """ + We need to interrupt Frappe site database creation to monkeypatch + functions that resolve host for user that owns site database. + In frappe_docker this was implemented in "new" command: + https://github.com/frappe/frappe_docker/blob/c808ad1767feaf793a2d14541ac0f4d9cbab45b3/build/frappe-worker/commands/new.py#L87 + """ + + frappe.database.db_manager.DbManager.get_current_host = lambda self: "%" + + +def main() -> int: + patch_database_creator() + frappe.utils.bench_helper.main() + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/build/frappe-worker/pretend-bench.sh b/build/frappe-worker/pretend-bench.sh new file mode 100755 index 00000000..fe39f0a3 --- /dev/null +++ b/build/frappe-worker/pretend-bench.sh @@ -0,0 +1,5 @@ +#!/bin/bash +set -e + +# shellcheck disable=SC2068 +~/frappe-bench/env/bin/python /usr/local/bin/patched_bench_helper.py frappe $@