mirror of
https://github.com/frappe/frappe_docker.git
synced 2024-11-10 07:11:00 +00:00
Update frappe-worker
This commit is contained in:
Lev Vereshchagin
parent
bf95cfa0c9
commit
b0b176c144
@ -1,108 +1,86 @@
|
||||
ARG PYTHON_VERSION=3.9
|
||||
FROM python:${PYTHON_VERSION}-slim-bullseye
|
||||
ARG PYTHON_VERSION
|
||||
FROM python:${PYTHON_VERSION}-slim-bullseye as base
|
||||
|
||||
# Add non root user without password
|
||||
RUN useradd -ms /bin/bash frappe
|
||||
|
||||
ARG GIT_REPO=https://github.com/frappe/frappe
|
||||
ARG GIT_BRANCH=develop
|
||||
|
||||
ARG ARCH=amd64
|
||||
ENV PYTHONUNBUFFERED 1
|
||||
|
||||
ENV NODE_VERSION=14.18.1
|
||||
ENV NVM_DIR /home/frappe/.nvm
|
||||
ENV PATH ${NVM_DIR}/versions/node/v${NODE_VERSION}/bin/:${PATH}
|
||||
ENV WKHTMLTOPDF_VERSION 0.12.6-1
|
||||
|
||||
# Install apt dependencies
|
||||
RUN apt-get update \
|
||||
&& apt-get install --no-install-recommends -y \
|
||||
# For frappe framework
|
||||
git \
|
||||
# MariaDB
|
||||
mariadb-client \
|
||||
# Postgres
|
||||
postgresql-client \
|
||||
gettext-base \
|
||||
wget \
|
||||
wait-for-it \
|
||||
# For PDF
|
||||
libjpeg62-turbo \
|
||||
libx11-6 \
|
||||
libxcb1 \
|
||||
libxext6 \
|
||||
libxrender1 \
|
||||
libssl-dev \
|
||||
fonts-cantarell \
|
||||
xfonts-75dpi \
|
||||
xfonts-base \
|
||||
libxml2 \
|
||||
libffi-dev \
|
||||
libjpeg-dev \
|
||||
zlib1g-dev \
|
||||
# For psycopg2
|
||||
libpq-dev \
|
||||
# For arm64 python wheel builds
|
||||
&& if [ "$(uname -m)" = "aarch64" ]; then \
|
||||
apt-get install --no-install-recommends -y \
|
||||
gcc \
|
||||
g++; \
|
||||
fi \
|
||||
# Install additional requirements for develop branch
|
||||
&& if [ "${GIT_BRANCH}" = 'develop' ]; then \
|
||||
apt-get install --no-install-recommends -y \
|
||||
libcairo2 \
|
||||
python3-cffi \
|
||||
python3-brotli \
|
||||
libpango-1.0-0 \
|
||||
libpangoft2-1.0-0 \
|
||||
libpangocairo-1.0-0; \
|
||||
fi \
|
||||
# wkhtmltopdf
|
||||
xvfb \
|
||||
libfontconfig \
|
||||
wkhtmltopdf \
|
||||
# For healthcheck.sh in helm chart
|
||||
wait-for-it \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# Detect arch, download and install wkhtmltopdf
|
||||
RUN if [ "$(uname -m)" = "aarch64" ]; then export ARCH=arm64; fi \
|
||||
&& if [ "$(uname -m)" = "x86_64" ]; then export ARCH=amd64; fi \
|
||||
&& downloaded_file=wkhtmltox_$WKHTMLTOPDF_VERSION.buster_${ARCH}.deb \
|
||||
&& wget -q https://github.com/wkhtmltopdf/packaging/releases/download/$WKHTMLTOPDF_VERSION/$downloaded_file \
|
||||
&& dpkg -i $downloaded_file \
|
||||
&& rm $downloaded_file
|
||||
|
||||
# Setup docker-entrypoint
|
||||
COPY build/frappe-worker/docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh
|
||||
RUN ln -s /usr/local/bin/docker-entrypoint.sh / # backwards compat
|
||||
|
||||
RUN useradd -ms /bin/bash frappe
|
||||
USER frappe
|
||||
RUN mkdir -p /home/frappe/frappe-bench/apps /home/frappe/frappe-bench/logs /home/frappe/frappe-bench/sites
|
||||
WORKDIR /home/frappe/frappe-bench
|
||||
RUN chown -R frappe:frappe /home/frappe
|
||||
|
||||
RUN pip install --no-cache-dir -U pip wheel \
|
||||
&& python -m venv env \
|
||||
&& env/bin/pip install --no-cache-dir -U pip wheel
|
||||
|
||||
|
||||
FROM base as frappe_builder
|
||||
|
||||
USER root
|
||||
RUN apt-get update \
|
||||
&& apt-get install --no-install-recommends -y \
|
||||
# Install git here because it is not required in production
|
||||
git \
|
||||
# gcc and g++ are required for building different packages across different versions
|
||||
# of Frappe and ERPNext and also on different platforms (for example, linux/arm64).
|
||||
# It is safe to install build deps even if they are not required
|
||||
# because they won't be included in final images.
|
||||
gcc \
|
||||
g++ \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
USER frappe
|
||||
|
||||
# Create frappe-bench directories
|
||||
RUN mkdir -p apps logs commands sites /home/frappe/backups
|
||||
ARG FRAPPE_VERSION
|
||||
RUN git clone --depth 1 -b ${FRAPPE_VERSION} https://github.com/frappe/frappe apps/frappe \
|
||||
&& env/bin/pip install --no-cache-dir -e apps/frappe \
|
||||
&& rm -r apps/frappe/.git
|
||||
|
||||
# Setup python environment
|
||||
RUN python -m venv env
|
||||
RUN env/bin/pip install --no-cache-dir wheel gevent
|
||||
|
||||
# Install nvm with node
|
||||
RUN wget -qO- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.0/install.sh | bash \
|
||||
&& . ${NVM_DIR}/nvm.sh \
|
||||
&& nvm install ${NODE_VERSION} \
|
||||
&& rm -rf ${NVM_DIR}/.cache
|
||||
FROM frappe_builder as erpnext_builder
|
||||
|
||||
# Install Frappe
|
||||
RUN git clone --depth 1 -o upstream -b ${GIT_BRANCH} ${GIT_REPO} apps/frappe \
|
||||
&& env/bin/pip install --no-cache-dir -e apps/frappe
|
||||
ARG ERPNEXT_VERSION
|
||||
RUN git clone --depth 1 -b ${ERPNEXT_VERSION} https://github.com/frappe/erpnext apps/erpnext \
|
||||
&& env/bin/pip install --no-cache-dir -e apps/erpnext \
|
||||
&& rm -r apps/erpnext/.git
|
||||
|
||||
# Copy scripts and templates
|
||||
COPY build/frappe-worker/commands/* /home/frappe/frappe-bench/commands/
|
||||
COPY build/frappe-worker/common_site_config.json.template /opt/frappe/common_site_config.json.template
|
||||
COPY build/frappe-worker/install_app.sh /usr/local/bin/install_app
|
||||
COPY build/frappe-worker/bench /usr/local/bin/bench
|
||||
COPY build/frappe-worker/healthcheck.sh /usr/local/bin/healthcheck.sh
|
||||
|
||||
# Use sites volume as working directory
|
||||
FROM base as configured_base
|
||||
|
||||
COPY pretend-bench.sh /usr/local/bin/bench
|
||||
# healthcheck.sh used in helm chart
|
||||
COPY entrypoint.sh patched_bench_helper.py healthcheck.sh /usr/local/bin/
|
||||
|
||||
WORKDIR /home/frappe/frappe-bench/sites
|
||||
|
||||
VOLUME [ "/home/frappe/frappe-bench/sites", "/home/frappe/backups", "/home/frappe/frappe-bench/logs" ]
|
||||
VOLUME [ "/home/frappe/frappe-bench/sites", "/home/frappe/frappe-bench/logs" ]
|
||||
|
||||
ENTRYPOINT ["docker-entrypoint.sh"]
|
||||
CMD ["start"]
|
||||
ENTRYPOINT [ "entrypoint.sh" ]
|
||||
|
||||
CMD [ "/home/frappe/frappe-bench/env/bin/gunicorn", "-b", "0.0.0.0:8000", "frappe.app:application", "--access-logfile", "-" ]
|
||||
|
||||
|
||||
FROM configured_base as frappe
|
||||
|
||||
RUN echo "frappe" >/home/frappe/frappe-bench/sites/apps.txt
|
||||
COPY --from=frappe_builder /home/frappe/frappe-bench/apps/frappe /home/frappe/frappe-bench/apps/frappe
|
||||
COPY --from=frappe_builder /home/frappe/frappe-bench/env /home/frappe/frappe-bench/env
|
||||
|
||||
|
||||
FROM configured_base as erpnext
|
||||
|
||||
RUN echo "frappe\nerpnext" >/home/frappe/frappe-bench/sites/apps.txt
|
||||
COPY --from=frappe_builder /home/frappe/frappe-bench/apps/frappe /home/frappe/frappe-bench/apps/frappe
|
||||
COPY --from=erpnext_builder /home/frappe/frappe-bench/apps/erpnext /home/frappe/frappe-bench/apps/erpnext
|
||||
COPY --from=erpnext_builder /home/frappe/frappe-bench/env /home/frappe/frappe-bench/env
|
||||
|
||||
@ -1,22 +0,0 @@
|
||||
#!/home/frappe/frappe-bench/env/bin/python
|
||||
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
if __name__ == "__main__":
|
||||
bench_dir = os.path.join(os.sep, "home", "frappe", "frappe-bench")
|
||||
sites_dir = os.path.join(bench_dir, "sites")
|
||||
bench_helper = os.path.join(
|
||||
bench_dir,
|
||||
"apps",
|
||||
"frappe",
|
||||
"frappe",
|
||||
"utils",
|
||||
"bench_helper.py",
|
||||
)
|
||||
cwd = os.getcwd()
|
||||
os.chdir(sites_dir)
|
||||
subprocess.check_call(
|
||||
[sys.executable, bench_helper, "frappe"] + sys.argv[1:],
|
||||
)
|
||||
@ -1,64 +0,0 @@
|
||||
import os
|
||||
|
||||
import git
|
||||
import semantic_version
|
||||
from migrate import migrate_sites
|
||||
from utils import (
|
||||
get_apps,
|
||||
get_config,
|
||||
get_container_versions,
|
||||
get_version_file,
|
||||
save_version_file,
|
||||
)
|
||||
|
||||
|
||||
def main():
|
||||
is_ready = False
|
||||
apps = get_apps()
|
||||
|
||||
container_versions = get_container_versions(apps)
|
||||
|
||||
version_file = get_version_file()
|
||||
|
||||
if not version_file:
|
||||
version_file = container_versions
|
||||
save_version_file(version_file)
|
||||
|
||||
for app in apps:
|
||||
container_version = None
|
||||
file_version = None
|
||||
version_file_hash = None
|
||||
container_hash = None
|
||||
|
||||
repo = git.Repo(os.path.join("..", "apps", app))
|
||||
branch = repo.active_branch.name
|
||||
|
||||
if branch == "develop":
|
||||
version_file_hash = version_file.get(app + "_git_hash")
|
||||
container_hash = container_versions.get(app + "_git_hash")
|
||||
if container_hash and version_file_hash:
|
||||
if container_hash != version_file_hash:
|
||||
is_ready = True
|
||||
break
|
||||
|
||||
if version_file.get(app):
|
||||
file_version = semantic_version.Version(version_file.get(app))
|
||||
|
||||
if container_versions.get(app):
|
||||
container_version = semantic_version.Version(container_versions.get(app))
|
||||
|
||||
if file_version and container_version:
|
||||
if container_version > file_version:
|
||||
is_ready = True
|
||||
break
|
||||
|
||||
config = get_config()
|
||||
|
||||
if is_ready and config.get("maintenance_mode") != 1:
|
||||
migrate_sites(maintenance_mode=True)
|
||||
version_file = container_versions
|
||||
save_version_file(version_file)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@ -1,45 +0,0 @@
|
||||
import os
|
||||
|
||||
import frappe
|
||||
from frappe.utils import cint, get_sites, now
|
||||
from frappe.utils.backups import scheduled_backup
|
||||
|
||||
|
||||
def backup(sites, with_files=False):
|
||||
for site in sites:
|
||||
frappe.init(site)
|
||||
frappe.connect()
|
||||
odb = scheduled_backup(
|
||||
ignore_files=not with_files,
|
||||
backup_path_db=None,
|
||||
backup_path_files=None,
|
||||
backup_path_private_files=None,
|
||||
force=True,
|
||||
)
|
||||
print("database backup taken -", odb.backup_path_db, "- on", now())
|
||||
if with_files:
|
||||
print("files backup taken -", odb.backup_path_files, "- on", now())
|
||||
print(
|
||||
"private files backup taken -",
|
||||
odb.backup_path_private_files,
|
||||
"- on",
|
||||
now(),
|
||||
)
|
||||
frappe.destroy()
|
||||
|
||||
|
||||
def main():
|
||||
installed_sites = ":".join(get_sites())
|
||||
sites = os.environ.get("SITES", installed_sites).split(":")
|
||||
with_files = cint(os.environ.get("WITH_FILES"))
|
||||
|
||||
backup(sites, with_files)
|
||||
|
||||
if frappe.redis_server:
|
||||
frappe.redis_server.connection_pool.disconnect()
|
||||
|
||||
exit(0)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@ -1,124 +0,0 @@
|
||||
import socket
|
||||
import time
|
||||
|
||||
from constants import (
|
||||
DB_HOST_KEY,
|
||||
DB_PORT,
|
||||
DB_PORT_KEY,
|
||||
REDIS_CACHE_KEY,
|
||||
REDIS_QUEUE_KEY,
|
||||
REDIS_SOCKETIO_KEY,
|
||||
)
|
||||
from six.moves.urllib.parse import urlparse
|
||||
from utils import get_config
|
||||
|
||||
|
||||
def is_open(ip, port, timeout=30):
|
||||
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
||||
s.settimeout(timeout)
|
||||
try:
|
||||
s.connect((ip, int(port)))
|
||||
s.shutdown(socket.SHUT_RDWR)
|
||||
return True
|
||||
except Exception:
|
||||
return False
|
||||
finally:
|
||||
s.close()
|
||||
|
||||
|
||||
def check_host(ip, port, retry=10, delay=3, print_attempt=True):
|
||||
ipup = False
|
||||
for i in range(retry):
|
||||
if print_attempt:
|
||||
print(f"Attempt {i+1} to connect to {ip}:{port}")
|
||||
if is_open(ip, port):
|
||||
ipup = True
|
||||
break
|
||||
else:
|
||||
time.sleep(delay)
|
||||
return ipup
|
||||
|
||||
|
||||
# Check service
|
||||
def check_service(
|
||||
retry=10, delay=3, print_attempt=True, service_name=None, service_port=None
|
||||
):
|
||||
|
||||
config = get_config()
|
||||
if not service_name:
|
||||
service_name = config.get(DB_HOST_KEY, "mariadb")
|
||||
if not service_port:
|
||||
service_port = config.get(DB_PORT_KEY, DB_PORT)
|
||||
|
||||
is_db_connected = False
|
||||
is_db_connected = check_host(
|
||||
service_name, service_port, retry, delay, print_attempt
|
||||
)
|
||||
if not is_db_connected:
|
||||
print(
|
||||
"Connection to {service_name}:{service_port} timed out".format(
|
||||
service_name=service_name,
|
||||
service_port=service_port,
|
||||
)
|
||||
)
|
||||
exit(1)
|
||||
|
||||
|
||||
# Check redis queue
|
||||
def check_redis_queue(retry=10, delay=3, print_attempt=True):
|
||||
check_redis_queue = False
|
||||
config = get_config()
|
||||
redis_queue_url = urlparse(
|
||||
config.get(REDIS_QUEUE_KEY, "redis://redis-queue:6379")
|
||||
).netloc
|
||||
redis_queue, redis_queue_port = redis_queue_url.split(":")
|
||||
check_redis_queue = check_host(
|
||||
redis_queue, redis_queue_port, retry, delay, print_attempt
|
||||
)
|
||||
if not check_redis_queue:
|
||||
print("Connection to redis queue timed out")
|
||||
exit(1)
|
||||
|
||||
|
||||
# Check redis cache
|
||||
def check_redis_cache(retry=10, delay=3, print_attempt=True):
|
||||
check_redis_cache = False
|
||||
config = get_config()
|
||||
redis_cache_url = urlparse(
|
||||
config.get(REDIS_CACHE_KEY, "redis://redis-cache:6379")
|
||||
).netloc
|
||||
redis_cache, redis_cache_port = redis_cache_url.split(":")
|
||||
check_redis_cache = check_host(
|
||||
redis_cache, redis_cache_port, retry, delay, print_attempt
|
||||
)
|
||||
if not check_redis_cache:
|
||||
print("Connection to redis cache timed out")
|
||||
exit(1)
|
||||
|
||||
|
||||
# Check redis socketio
|
||||
def check_redis_socketio(retry=10, delay=3, print_attempt=True):
|
||||
check_redis_socketio = False
|
||||
config = get_config()
|
||||
redis_socketio_url = urlparse(
|
||||
config.get(REDIS_SOCKETIO_KEY, "redis://redis-socketio:6379")
|
||||
).netloc
|
||||
redis_socketio, redis_socketio_port = redis_socketio_url.split(":")
|
||||
check_redis_socketio = check_host(
|
||||
redis_socketio, redis_socketio_port, retry, delay, print_attempt
|
||||
)
|
||||
if not check_redis_socketio:
|
||||
print("Connection to redis socketio timed out")
|
||||
exit(1)
|
||||
|
||||
|
||||
def main():
|
||||
check_service()
|
||||
check_redis_queue()
|
||||
check_redis_cache()
|
||||
check_redis_socketio()
|
||||
print("Connections OK")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@ -1,13 +0,0 @@
|
||||
REDIS_QUEUE_KEY = "redis_queue"
|
||||
REDIS_CACHE_KEY = "redis_cache"
|
||||
REDIS_SOCKETIO_KEY = "redis_socketio"
|
||||
DB_HOST_KEY = "db_host"
|
||||
DB_PORT_KEY = "db_port"
|
||||
DB_PORT = 3306
|
||||
APP_VERSIONS_JSON_FILE = "app_versions.json"
|
||||
APPS_TXT_FILE = "apps.txt"
|
||||
COMMON_SITE_CONFIG_FILE = "common_site_config.json"
|
||||
DATE_FORMAT = "%Y%m%d_%H%M%S"
|
||||
RDS_DB = "rds_db"
|
||||
RDS_PRIVILEGES = "SELECT, INSERT, UPDATE, DELETE, CREATE, DROP, INDEX, ALTER, CREATE TEMPORARY TABLES, CREATE VIEW, EVENT, TRIGGER, SHOW VIEW, CREATE ROUTINE, ALTER ROUTINE, EXECUTE, LOCK TABLES"
|
||||
ARCHIVE_SITES_PATH = "/home/frappe/frappe-bench/sites/archive_sites"
|
||||
@ -1,61 +0,0 @@
|
||||
import argparse
|
||||
|
||||
from check_connection import (
|
||||
check_redis_cache,
|
||||
check_redis_queue,
|
||||
check_redis_socketio,
|
||||
check_service,
|
||||
)
|
||||
|
||||
|
||||
def parse_args():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument(
|
||||
"-p",
|
||||
"--ping-service",
|
||||
dest="ping_services",
|
||||
action="append",
|
||||
type=str,
|
||||
help='list of services to ping, e.g. doctor -p "postgres:5432" --ping-service "mariadb:3306"',
|
||||
)
|
||||
args = parser.parse_args()
|
||||
return args
|
||||
|
||||
|
||||
def main():
|
||||
args = parse_args()
|
||||
check_service(retry=1, delay=0, print_attempt=False)
|
||||
print("Bench database Connected")
|
||||
check_redis_cache(retry=1, delay=0, print_attempt=False)
|
||||
print("Redis Cache Connected")
|
||||
check_redis_queue(retry=1, delay=0, print_attempt=False)
|
||||
print("Redis Queue Connected")
|
||||
check_redis_socketio(retry=1, delay=0, print_attempt=False)
|
||||
print("Redis SocketIO Connected")
|
||||
|
||||
if args.ping_services:
|
||||
for service in args.ping_services:
|
||||
service_name = None
|
||||
service_port = None
|
||||
|
||||
try:
|
||||
service_name, service_port = service.split(":")
|
||||
except ValueError:
|
||||
print("Service should be in format host:port, e.g postgres:5432")
|
||||
exit(1)
|
||||
|
||||
check_service(
|
||||
retry=1,
|
||||
delay=0,
|
||||
print_attempt=False,
|
||||
service_name=service_name,
|
||||
service_port=service_port,
|
||||
)
|
||||
print(f"{service_name}:{service_port} Connected")
|
||||
|
||||
print("Health check successful")
|
||||
exit(0)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@ -1,3 +0,0 @@
|
||||
import gevent.monkey
|
||||
|
||||
gevent.monkey.patch_all()
|
||||
@ -1,52 +0,0 @@
|
||||
import os
|
||||
|
||||
import frappe
|
||||
from frappe.utils import cint, get_sites
|
||||
from utils import get_config, save_config
|
||||
|
||||
|
||||
def set_maintenance_mode(enable=True):
|
||||
conf = get_config()
|
||||
|
||||
if enable:
|
||||
conf.update({"maintenance_mode": 1, "pause_scheduler": 1})
|
||||
save_config(conf)
|
||||
|
||||
if not enable:
|
||||
conf.update({"maintenance_mode": 0, "pause_scheduler": 0})
|
||||
save_config(conf)
|
||||
|
||||
|
||||
def migrate_sites(maintenance_mode=False):
|
||||
installed_sites = ":".join(get_sites())
|
||||
sites = os.environ.get("SITES", installed_sites).split(":")
|
||||
if not maintenance_mode:
|
||||
maintenance_mode = cint(os.environ.get("MAINTENANCE_MODE"))
|
||||
|
||||
if maintenance_mode:
|
||||
set_maintenance_mode(True)
|
||||
|
||||
for site in sites:
|
||||
print("Migrating", site)
|
||||
frappe.init(site=site)
|
||||
frappe.connect()
|
||||
try:
|
||||
from frappe.migrate import migrate
|
||||
|
||||
migrate()
|
||||
finally:
|
||||
frappe.destroy()
|
||||
|
||||
# Disable maintenance mode after migration
|
||||
set_maintenance_mode(False)
|
||||
|
||||
|
||||
def main():
|
||||
migrate_sites()
|
||||
if frappe.redis_server:
|
||||
frappe.redis_server.connection_pool.disconnect()
|
||||
exit(0)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@ -1,132 +0,0 @@
|
||||
import os
|
||||
|
||||
import frappe
|
||||
import semantic_version
|
||||
from constants import COMMON_SITE_CONFIG_FILE, RDS_DB, RDS_PRIVILEGES
|
||||
from frappe.installer import update_site_config
|
||||
from utils import get_config, get_password, get_site_config, run_command
|
||||
|
||||
# try to import _new_site from frappe, which could possibly
|
||||
# exist in either commands.py or installer.py, and so we need
|
||||
# to maintain compatibility across all frappe versions.
|
||||
try:
|
||||
# <= version-{11,12}
|
||||
from frappe.commands.site import _new_site
|
||||
except ImportError:
|
||||
# >= version-13 and develop
|
||||
from frappe.installer import _new_site
|
||||
|
||||
|
||||
def main():
|
||||
config = get_config()
|
||||
db_type = "mariadb"
|
||||
db_port = config.get("db_port", 3306)
|
||||
db_host = config.get("db_host")
|
||||
site_name = os.environ.get("SITE_NAME", "site1.localhost")
|
||||
db_root_username = os.environ.get("DB_ROOT_USER", "root")
|
||||
mariadb_root_password = get_password("MYSQL_ROOT_PASSWORD", "admin")
|
||||
postgres_root_password = get_password("POSTGRES_PASSWORD")
|
||||
db_root_password = mariadb_root_password
|
||||
|
||||
if postgres_root_password:
|
||||
db_type = "postgres"
|
||||
db_host = os.environ.get("POSTGRES_HOST")
|
||||
db_port = 5432
|
||||
db_root_password = postgres_root_password
|
||||
if not db_host:
|
||||
db_host = config.get("db_host")
|
||||
print("Environment variable POSTGRES_HOST not found.")
|
||||
print("Using db_host from common_site_config.json")
|
||||
|
||||
sites_path = os.getcwd()
|
||||
common_site_config_path = os.path.join(sites_path, COMMON_SITE_CONFIG_FILE)
|
||||
update_site_config(
|
||||
"root_login",
|
||||
db_root_username,
|
||||
validate=False,
|
||||
site_config_path=common_site_config_path,
|
||||
)
|
||||
update_site_config(
|
||||
"root_password",
|
||||
db_root_password,
|
||||
validate=False,
|
||||
site_config_path=common_site_config_path,
|
||||
)
|
||||
|
||||
force = True if os.environ.get("FORCE", None) else False
|
||||
install_apps = os.environ.get("INSTALL_APPS", None)
|
||||
install_apps = install_apps.split(",") if install_apps else []
|
||||
frappe.init(site_name, new_site=True)
|
||||
|
||||
if semantic_version.Version(frappe.__version__).major > 11:
|
||||
_new_site(
|
||||
None,
|
||||
site_name,
|
||||
mariadb_root_username=db_root_username,
|
||||
mariadb_root_password=db_root_password,
|
||||
admin_password=get_password("ADMIN_PASSWORD", "admin"),
|
||||
verbose=True,
|
||||
install_apps=install_apps,
|
||||
source_sql=None,
|
||||
force=force,
|
||||
db_type=db_type,
|
||||
reinstall=False,
|
||||
db_host=db_host,
|
||||
db_port=db_port,
|
||||
)
|
||||
else:
|
||||
_new_site(
|
||||
None,
|
||||
site_name,
|
||||
mariadb_root_username=db_root_username,
|
||||
mariadb_root_password=db_root_password,
|
||||
admin_password=get_password("ADMIN_PASSWORD", "admin"),
|
||||
verbose=True,
|
||||
install_apps=install_apps,
|
||||
source_sql=None,
|
||||
force=force,
|
||||
reinstall=False,
|
||||
)
|
||||
|
||||
if db_type == "mariadb":
|
||||
site_config = get_site_config(site_name)
|
||||
db_name = site_config.get("db_name")
|
||||
db_password = site_config.get("db_password")
|
||||
|
||||
mysql_command = [
|
||||
"mysql",
|
||||
f"-h{db_host}",
|
||||
f"-u{db_root_username}",
|
||||
f"-p{mariadb_root_password}",
|
||||
"-e",
|
||||
]
|
||||
|
||||
# Drop User if exists
|
||||
command = mysql_command + [
|
||||
f"DROP USER IF EXISTS '{db_name}'; FLUSH PRIVILEGES;"
|
||||
]
|
||||
run_command(command)
|
||||
|
||||
# Grant permission to database and set password
|
||||
grant_privileges = "ALL PRIVILEGES"
|
||||
|
||||
# for Amazon RDS
|
||||
if config.get(RDS_DB) or site_config.get(RDS_DB):
|
||||
grant_privileges = RDS_PRIVILEGES
|
||||
|
||||
command = mysql_command + [
|
||||
f"\
|
||||
CREATE USER IF NOT EXISTS '{db_name}'@'%' IDENTIFIED BY '{db_password}'; \
|
||||
GRANT {grant_privileges} ON `{db_name}`.* TO '{db_name}'@'%'; \
|
||||
FLUSH PRIVILEGES;"
|
||||
]
|
||||
run_command(command)
|
||||
|
||||
if frappe.redis_server:
|
||||
frappe.redis_server.connection_pool.disconnect()
|
||||
|
||||
exit(0)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@ -1,183 +0,0 @@
|
||||
import datetime
|
||||
import os
|
||||
import time
|
||||
from glob import glob
|
||||
|
||||
import boto3
|
||||
from constants import DATE_FORMAT
|
||||
from frappe.utils import get_sites
|
||||
from utils import check_s3_environment_variables, get_s3_config, upload_file_to_s3
|
||||
|
||||
|
||||
def get_file_ext():
|
||||
return {
|
||||
"database": "-database.sql.gz",
|
||||
"private_files": "-private-files.tar",
|
||||
"public_files": "-files.tar",
|
||||
"site_config": "-site_config_backup.json",
|
||||
}
|
||||
|
||||
|
||||
def get_backup_details(sitename):
|
||||
backup_details = dict()
|
||||
file_ext = get_file_ext()
|
||||
|
||||
# add trailing slash https://stackoverflow.com/a/15010678
|
||||
site_backup_path = os.path.join(os.getcwd(), sitename, "private", "backups", "")
|
||||
|
||||
if os.path.exists(site_backup_path):
|
||||
for filetype, ext in file_ext.items():
|
||||
site_slug = sitename.replace(".", "_")
|
||||
pattern = site_backup_path + "*-" + site_slug + ext
|
||||
backup_files = list(filter(os.path.isfile, glob(pattern)))
|
||||
|
||||
if len(backup_files) > 0:
|
||||
backup_files.sort(
|
||||
key=lambda file: os.stat(
|
||||
os.path.join(site_backup_path, file)
|
||||
).st_ctime
|
||||
)
|
||||
backup_date = datetime.datetime.strptime(
|
||||
time.ctime(os.path.getmtime(backup_files[0])),
|
||||
"%a %b %d %H:%M:%S %Y",
|
||||
)
|
||||
backup_details[filetype] = {
|
||||
"sitename": sitename,
|
||||
"file_size_in_bytes": os.stat(backup_files[-1]).st_size,
|
||||
"file_path": os.path.abspath(backup_files[-1]),
|
||||
"filename": os.path.basename(backup_files[-1]),
|
||||
"backup_date": backup_date.date().strftime("%Y-%m-%d %H:%M:%S"),
|
||||
}
|
||||
|
||||
return backup_details
|
||||
|
||||
|
||||
def delete_old_backups(limit, bucket, site_name):
|
||||
all_backups = list()
|
||||
all_backup_dates = list()
|
||||
backup_limit = int(limit)
|
||||
check_s3_environment_variables()
|
||||
bucket_dir = os.environ.get("BUCKET_DIR")
|
||||
oldest_backup_date = None
|
||||
|
||||
s3 = boto3.resource(
|
||||
"s3",
|
||||
region_name=os.environ.get("REGION"),
|
||||
aws_access_key_id=os.environ.get("ACCESS_KEY_ID"),
|
||||
aws_secret_access_key=os.environ.get("SECRET_ACCESS_KEY"),
|
||||
endpoint_url=os.environ.get("ENDPOINT_URL"),
|
||||
)
|
||||
|
||||
bucket = s3.Bucket(bucket)
|
||||
objects = bucket.meta.client.list_objects_v2(Bucket=bucket.name, Delimiter="/")
|
||||
|
||||
if objects:
|
||||
for obj in objects.get("CommonPrefixes"):
|
||||
if obj.get("Prefix") == bucket_dir + "/":
|
||||
for backup_obj in bucket.objects.filter(Prefix=obj.get("Prefix")):
|
||||
if backup_obj.get()["ContentType"] == "application/x-directory":
|
||||
continue
|
||||
try:
|
||||
# backup_obj.key is bucket_dir/site/date_time/backupfile.extension
|
||||
(
|
||||
bucket_dir,
|
||||
site_slug,
|
||||
date_time,
|
||||
backupfile,
|
||||
) = backup_obj.key.split("/")
|
||||
date_time_object = datetime.datetime.strptime(
|
||||
date_time, DATE_FORMAT
|
||||
)
|
||||
|
||||
if site_name in backup_obj.key:
|
||||
all_backup_dates.append(date_time_object)
|
||||
all_backups.append(backup_obj.key)
|
||||
except IndexError as error:
|
||||
print(error)
|
||||
exit(1)
|
||||
|
||||
if len(all_backup_dates) > 0:
|
||||
oldest_backup_date = min(all_backup_dates)
|
||||
|
||||
if len(all_backups) / 3 > backup_limit:
|
||||
oldest_backup = None
|
||||
for backup in all_backups:
|
||||
try:
|
||||
# backup is bucket_dir/site/date_time/backupfile.extension
|
||||
backup_dir, site_slug, backup_dt_string, filename = backup.split("/")
|
||||
backup_datetime = datetime.datetime.strptime(
|
||||
backup_dt_string, DATE_FORMAT
|
||||
)
|
||||
if backup_datetime == oldest_backup_date:
|
||||
oldest_backup = backup
|
||||
|
||||
except IndexError as error:
|
||||
print(error)
|
||||
exit(1)
|
||||
|
||||
if oldest_backup:
|
||||
for obj in bucket.objects.filter(Prefix=oldest_backup):
|
||||
# delete all keys that are inside the oldest_backup
|
||||
if bucket_dir in obj.key:
|
||||
print("Deleting " + obj.key)
|
||||
s3.Object(bucket.name, obj.key).delete()
|
||||
|
||||
|
||||
def main():
|
||||
details = dict()
|
||||
sites = get_sites()
|
||||
conn, bucket = get_s3_config()
|
||||
|
||||
for site in sites:
|
||||
details = get_backup_details(site)
|
||||
db_file = details.get("database", {}).get("file_path")
|
||||
folder = os.environ.get("BUCKET_DIR") + "/" + site + "/"
|
||||
if db_file:
|
||||
folder = (
|
||||
os.environ.get("BUCKET_DIR")
|
||||
+ "/"
|
||||
+ site
|
||||
+ "/"
|
||||
+ os.path.basename(db_file)[:15]
|
||||
+ "/"
|
||||
)
|
||||
upload_file_to_s3(db_file, folder, conn, bucket)
|
||||
|
||||
# Archive site_config.json
|
||||
site_config_file = details.get("site_config", {}).get("file_path")
|
||||
if not site_config_file:
|
||||
site_config_file = os.path.join(os.getcwd(), site, "site_config.json")
|
||||
upload_file_to_s3(site_config_file, folder, conn, bucket)
|
||||
|
||||
public_files = details.get("public_files", {}).get("file_path")
|
||||
if public_files:
|
||||
folder = (
|
||||
os.environ.get("BUCKET_DIR")
|
||||
+ "/"
|
||||
+ site
|
||||
+ "/"
|
||||
+ os.path.basename(public_files)[:15]
|
||||
+ "/"
|
||||
)
|
||||
upload_file_to_s3(public_files, folder, conn, bucket)
|
||||
|
||||
private_files = details.get("private_files", {}).get("file_path")
|
||||
if private_files:
|
||||
folder = (
|
||||
os.environ.get("BUCKET_DIR")
|
||||
+ "/"
|
||||
+ site
|
||||
+ "/"
|
||||
+ os.path.basename(private_files)[:15]
|
||||
+ "/"
|
||||
)
|
||||
upload_file_to_s3(private_files, folder, conn, bucket)
|
||||
|
||||
delete_old_backups(os.environ.get("BACKUP_LIMIT", "3"), bucket, site)
|
||||
|
||||
print("push-backup complete")
|
||||
exit(0)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@ -1,313 +0,0 @@
|
||||
import datetime
|
||||
import hashlib
|
||||
import os
|
||||
import tarfile
|
||||
|
||||
import boto3
|
||||
import frappe
|
||||
from constants import COMMON_SITE_CONFIG_FILE, DATE_FORMAT, RDS_DB, RDS_PRIVILEGES
|
||||
from frappe.installer import (
|
||||
get_conf_params,
|
||||
make_conf,
|
||||
make_site_dirs,
|
||||
update_site_config,
|
||||
)
|
||||
from frappe.utils import get_sites, random_string
|
||||
from utils import (
|
||||
check_s3_environment_variables,
|
||||
get_config,
|
||||
get_password,
|
||||
get_site_config,
|
||||
list_directories,
|
||||
run_command,
|
||||
set_key_in_site_config,
|
||||
)
|
||||
|
||||
|
||||
def get_backup_dir():
|
||||
return os.path.join(os.path.expanduser("~"), "backups")
|
||||
|
||||
|
||||
def decompress_db(database_file, site):
|
||||
command = ["gunzip", "-c", database_file]
|
||||
with open(database_file.replace(".gz", ""), "w") as db_file:
|
||||
print(f"Extract Database GZip for site {site}")
|
||||
run_command(command, stdout=db_file)
|
||||
|
||||
|
||||
def restore_database(files_base, site_config_path, site):
|
||||
# restore database
|
||||
database_file = files_base + "-database.sql.gz"
|
||||
decompress_db(database_file, site)
|
||||
config = get_config()
|
||||
|
||||
# Set db_type if it exists in backup site_config.json
|
||||
set_key_in_site_config("db_type", site, site_config_path)
|
||||
# Set db_host if it exists in backup site_config.json
|
||||
set_key_in_site_config("db_host", site, site_config_path)
|
||||
# Set db_port if it exists in backup site_config.json
|
||||
set_key_in_site_config("db_port", site, site_config_path)
|
||||
|
||||
# get updated site_config
|
||||
site_config = get_site_config(site)
|
||||
|
||||
# if no db_type exists, default to mariadb
|
||||
db_type = site_config.get("db_type", "mariadb")
|
||||
is_database_restored = False
|
||||
|
||||
if db_type == "mariadb":
|
||||
restore_mariadb(
|
||||
config=config, site_config=site_config, database_file=database_file
|
||||
)
|
||||
is_database_restored = True
|
||||
elif db_type == "postgres":
|
||||
restore_postgres(
|
||||
config=config, site_config=site_config, database_file=database_file
|
||||
)
|
||||
is_database_restored = True
|
||||
|
||||
if is_database_restored:
|
||||
# Set encryption_key if it exists in backup site_config.json
|
||||
set_key_in_site_config("encryption_key", site, site_config_path)
|
||||
|
||||
|
||||
def restore_files(files_base):
|
||||
public_files = files_base + "-files.tar"
|
||||
# extract tar
|
||||
public_tar = tarfile.open(public_files)
|
||||
print(f"Extracting {public_files}")
|
||||
public_tar.extractall()
|
||||
|
||||
|
||||
def restore_private_files(files_base):
|
||||
private_files = files_base + "-private-files.tar"
|
||||
private_tar = tarfile.open(private_files)
|
||||
print(f"Extracting {private_files}")
|
||||
private_tar.extractall()
|
||||
|
||||
|
||||
def pull_backup_from_s3():
|
||||
check_s3_environment_variables()
|
||||
|
||||
# https://stackoverflow.com/a/54672690
|
||||
s3 = boto3.resource(
|
||||
"s3",
|
||||
region_name=os.environ.get("REGION"),
|
||||
aws_access_key_id=os.environ.get("ACCESS_KEY_ID"),
|
||||
aws_secret_access_key=os.environ.get("SECRET_ACCESS_KEY"),
|
||||
endpoint_url=os.environ.get("ENDPOINT_URL"),
|
||||
)
|
||||
|
||||
bucket_dir = os.environ.get("BUCKET_DIR")
|
||||
bucket_name = os.environ.get("BUCKET_NAME")
|
||||
bucket = s3.Bucket(bucket_name)
|
||||
|
||||
# Change directory to /home/frappe/backups
|
||||
os.chdir(get_backup_dir())
|
||||
|
||||
backup_files = []
|
||||
sites = set()
|
||||
site_timestamps = set()
|
||||
download_backups = []
|
||||
|
||||
for obj in bucket.objects.filter(Prefix=bucket_dir):
|
||||
if obj.get()["ContentType"] == "application/x-directory":
|
||||
continue
|
||||
backup_file = obj.key.replace(os.path.join(bucket_dir, ""), "")
|
||||
backup_files.append(backup_file)
|
||||
site_name, timestamp, backup_type = backup_file.split("/")
|
||||
site_timestamp = site_name + "/" + timestamp
|
||||
sites.add(site_name)
|
||||
site_timestamps.add(site_timestamp)
|
||||
|
||||
# sort sites for latest backups
|
||||
for site in sites:
|
||||
backup_timestamps = []
|
||||
for site_timestamp in site_timestamps:
|
||||
site_name, timestamp = site_timestamp.split("/")
|
||||
if site == site_name:
|
||||
timestamp_datetime = datetime.datetime.strptime(timestamp, DATE_FORMAT)
|
||||
backup_timestamps.append(timestamp)
|
||||
download_backups.append(site + "/" + max(backup_timestamps))
|
||||
|
||||
# Only download latest backups
|
||||
for backup_file in backup_files:
|
||||
for backup in download_backups:
|
||||
if backup in backup_file:
|
||||
if not os.path.exists(os.path.dirname(backup_file)):
|
||||
os.makedirs(os.path.dirname(backup_file))
|
||||
print(f"Downloading {backup_file}")
|
||||
bucket.download_file(bucket_dir + "/" + backup_file, backup_file)
|
||||
|
||||
os.chdir(os.path.join(os.path.expanduser("~"), "frappe-bench", "sites"))
|
||||
|
||||
|
||||
def restore_postgres(config, site_config, database_file):
|
||||
# common config
|
||||
common_site_config_path = os.path.join(os.getcwd(), COMMON_SITE_CONFIG_FILE)
|
||||
|
||||
db_root_user = config.get("root_login")
|
||||
if not db_root_user:
|
||||
postgres_user = os.environ.get("DB_ROOT_USER")
|
||||
if not postgres_user:
|
||||
print("Variable DB_ROOT_USER not set")
|
||||
exit(1)
|
||||
|
||||
db_root_user = postgres_user
|
||||
update_site_config(
|
||||
"root_login",
|
||||
db_root_user,
|
||||
validate=False,
|
||||
site_config_path=common_site_config_path,
|
||||
)
|
||||
|
||||
db_root_password = config.get("root_password")
|
||||
if not db_root_password:
|
||||
root_password = get_password("POSTGRES_PASSWORD")
|
||||
if not root_password:
|
||||
print("Variable POSTGRES_PASSWORD not set")
|
||||
exit(1)
|
||||
|
||||
db_root_password = root_password
|
||||
update_site_config(
|
||||
"root_password",
|
||||
db_root_password,
|
||||
validate=False,
|
||||
site_config_path=common_site_config_path,
|
||||
)
|
||||
|
||||
# site config
|
||||
db_host = site_config.get("db_host")
|
||||
db_port = site_config.get("db_port", 5432)
|
||||
db_name = site_config.get("db_name")
|
||||
db_password = site_config.get("db_password")
|
||||
|
||||
psql_command = ["psql"]
|
||||
psql_uri = f"postgres://{db_root_user}:{db_root_password}@{db_host}:{db_port}"
|
||||
|
||||
print("Restoring PostgreSQL")
|
||||
run_command(psql_command + [psql_uri, "-c", f'DROP DATABASE IF EXISTS "{db_name}"'])
|
||||
run_command(psql_command + [psql_uri, "-c", f"DROP USER IF EXISTS {db_name}"])
|
||||
run_command(psql_command + [psql_uri, "-c", f'CREATE DATABASE "{db_name}"'])
|
||||
run_command(
|
||||
psql_command
|
||||
+ [psql_uri, "-c", f"CREATE user {db_name} password '{db_password}'"]
|
||||
)
|
||||
run_command(
|
||||
psql_command
|
||||
+ [psql_uri, "-c", f'GRANT ALL PRIVILEGES ON DATABASE "{db_name}" TO {db_name}']
|
||||
)
|
||||
with open(database_file.replace(".gz", "")) as db_file:
|
||||
run_command(psql_command + [f"{psql_uri}/{db_name}", "<"], stdin=db_file)
|
||||
|
||||
|
||||
def restore_mariadb(config, site_config, database_file):
|
||||
db_root_password = get_password("MYSQL_ROOT_PASSWORD")
|
||||
if not db_root_password:
|
||||
print("Variable MYSQL_ROOT_PASSWORD not set")
|
||||
exit(1)
|
||||
|
||||
db_root_user = os.environ.get("DB_ROOT_USER", "root")
|
||||
|
||||
db_host = site_config.get("db_host", config.get("db_host"))
|
||||
db_port = site_config.get("db_port", config.get("db_port", 3306))
|
||||
db_name = site_config.get("db_name")
|
||||
db_password = site_config.get("db_password")
|
||||
|
||||
# mysql command prefix
|
||||
mysql_command = [
|
||||
"mysql",
|
||||
f"-u{db_root_user}",
|
||||
f"-h{db_host}",
|
||||
f"-p{db_root_password}",
|
||||
f"-P{db_port}",
|
||||
]
|
||||
|
||||
# drop db if exists for clean restore
|
||||
drop_database = mysql_command + ["-e", f"DROP DATABASE IF EXISTS `{db_name}`;"]
|
||||
run_command(drop_database)
|
||||
|
||||
# create db
|
||||
create_database = mysql_command + [
|
||||
"-e",
|
||||
f"CREATE DATABASE IF NOT EXISTS `{db_name}`;",
|
||||
]
|
||||
run_command(create_database)
|
||||
|
||||
# create user
|
||||
create_user = mysql_command + [
|
||||
"-e",
|
||||
f"CREATE USER IF NOT EXISTS '{db_name}'@'%' IDENTIFIED BY '{db_password}'; FLUSH PRIVILEGES;",
|
||||
]
|
||||
run_command(create_user)
|
||||
|
||||
# grant db privileges to user
|
||||
|
||||
grant_privileges = "ALL PRIVILEGES"
|
||||
|
||||
# for Amazon RDS
|
||||
if config.get(RDS_DB) or site_config.get(RDS_DB):
|
||||
grant_privileges = RDS_PRIVILEGES
|
||||
|
||||
grant_privileges_command = mysql_command + [
|
||||
"-e",
|
||||
f"GRANT {grant_privileges} ON `{db_name}`.* TO '{db_name}'@'%' IDENTIFIED BY '{db_password}'; FLUSH PRIVILEGES;",
|
||||
]
|
||||
run_command(grant_privileges_command)
|
||||
|
||||
print("Restoring MariaDB")
|
||||
with open(database_file.replace(".gz", "")) as db_file:
|
||||
run_command(mysql_command + [f"{db_name}"], stdin=db_file)
|
||||
|
||||
|
||||
def main():
|
||||
backup_dir = get_backup_dir()
|
||||
|
||||
if len(list_directories(backup_dir)) == 0:
|
||||
pull_backup_from_s3()
|
||||
|
||||
for site in list_directories(backup_dir):
|
||||
site_slug = site.replace(".", "_")
|
||||
backups = [
|
||||
datetime.datetime.strptime(backup, DATE_FORMAT)
|
||||
for backup in list_directories(os.path.join(backup_dir, site))
|
||||
]
|
||||
latest_backup = max(backups).strftime(DATE_FORMAT)
|
||||
files_base = os.path.join(backup_dir, site, latest_backup, "")
|
||||
files_base += latest_backup + "-" + site_slug
|
||||
site_config_path = files_base + "-site_config_backup.json"
|
||||
if not os.path.exists(site_config_path):
|
||||
site_config_path = os.path.join(backup_dir, site, "site_config.json")
|
||||
if site in get_sites():
|
||||
print(f"Overwrite site {site}")
|
||||
restore_database(files_base, site_config_path, site)
|
||||
restore_private_files(files_base)
|
||||
restore_files(files_base)
|
||||
else:
|
||||
site_config = get_conf_params(
|
||||
db_name="_" + hashlib.sha1(site.encode()).hexdigest()[:16],
|
||||
db_password=random_string(16),
|
||||
)
|
||||
|
||||
frappe.local.site = site
|
||||
frappe.local.sites_path = os.getcwd()
|
||||
frappe.local.site_path = os.getcwd() + "/" + site
|
||||
make_conf(
|
||||
db_name=site_config.get("db_name"),
|
||||
db_password=site_config.get("db_password"),
|
||||
)
|
||||
make_site_dirs()
|
||||
|
||||
print(f"Create site {site}")
|
||||
restore_database(files_base, site_config_path, site)
|
||||
restore_private_files(files_base)
|
||||
restore_files(files_base)
|
||||
|
||||
if frappe.redis_server:
|
||||
frappe.redis_server.connection_pool.disconnect()
|
||||
|
||||
exit(0)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@ -1,208 +0,0 @@
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
|
||||
import boto3
|
||||
import git
|
||||
from constants import APP_VERSIONS_JSON_FILE, APPS_TXT_FILE, COMMON_SITE_CONFIG_FILE
|
||||
from frappe.installer import update_site_config
|
||||
|
||||
|
||||
def run_command(command, stdout=None, stdin=None, stderr=None):
|
||||
stdout = stdout or subprocess.PIPE
|
||||
stderr = stderr or subprocess.PIPE
|
||||
stdin = stdin or subprocess.PIPE
|
||||
process = subprocess.Popen(command, stdout=stdout, stdin=stdin, stderr=stderr)
|
||||
out, error = process.communicate()
|
||||
if process.returncode:
|
||||
print("Something went wrong:")
|
||||
print(f"return code: {process.returncode}")
|
||||
print(f"stdout:\n{out}")
|
||||
print(f"\nstderr:\n{error}")
|
||||
exit(process.returncode)
|
||||
|
||||
|
||||
def save_version_file(versions):
|
||||
with open(APP_VERSIONS_JSON_FILE, "w") as f:
|
||||
return json.dump(versions, f, indent=1, sort_keys=True)
|
||||
|
||||
|
||||
def get_apps():
|
||||
apps = []
|
||||
try:
|
||||
with open(APPS_TXT_FILE) as apps_file:
|
||||
for app in apps_file.readlines():
|
||||
if app.strip():
|
||||
apps.append(app.strip())
|
||||
|
||||
except FileNotFoundError as exception:
|
||||
print(exception)
|
||||
exit(1)
|
||||
except Exception:
|
||||
print(APPS_TXT_FILE + " is not valid")
|
||||
exit(1)
|
||||
|
||||
return apps
|
||||
|
||||
|
||||
def get_container_versions(apps):
|
||||
versions = {}
|
||||
for app in apps:
|
||||
try:
|
||||
version = __import__(app).__version__
|
||||
versions.update({app: version})
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
try:
|
||||
path = os.path.join("..", "apps", app)
|
||||
repo = git.Repo(path)
|
||||
commit_hash = repo.head.object.hexsha
|
||||
versions.update({app + "_git_hash": commit_hash})
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return versions
|
||||
|
||||
|
||||
def get_version_file():
|
||||
versions = None
|
||||
try:
|
||||
with open(APP_VERSIONS_JSON_FILE) as versions_file:
|
||||
versions = json.load(versions_file)
|
||||
except Exception:
|
||||
pass
|
||||
return versions
|
||||
|
||||
|
||||
def get_config():
|
||||
config = None
|
||||
try:
|
||||
with open(COMMON_SITE_CONFIG_FILE) as config_file:
|
||||
config = json.load(config_file)
|
||||
except FileNotFoundError as exception:
|
||||
print(exception)
|
||||
exit(1)
|
||||
except Exception:
|
||||
print(COMMON_SITE_CONFIG_FILE + " is not valid")
|
||||
exit(1)
|
||||
return config
|
||||
|
||||
|
||||
def get_site_config(site_name):
|
||||
site_config = None
|
||||
with open(f"{site_name}/site_config.json") as site_config_file:
|
||||
site_config = json.load(site_config_file)
|
||||
return site_config
|
||||
|
||||
|
||||
def save_config(config):
|
||||
with open(COMMON_SITE_CONFIG_FILE, "w") as f:
|
||||
return json.dump(config, f, indent=1, sort_keys=True)
|
||||
|
||||
|
||||
def get_password(env_var, default=None):
|
||||
return (
|
||||
os.environ.get(env_var)
|
||||
or get_password_from_secret(f"{env_var}_FILE")
|
||||
or default
|
||||
)
|
||||
|
||||
|
||||
def get_password_from_secret(env_var):
|
||||
"""Fetches the secret value from the docker secret file
|
||||
usually located inside /run/secrets/
|
||||
Arguments:
|
||||
env_var {str} -- Name of the environment variable
|
||||
containing the path to the secret file.
|
||||
Returns:
|
||||
[str] -- Secret value
|
||||
"""
|
||||
passwd = None
|
||||
secret_file_path = os.environ.get(env_var)
|
||||
if secret_file_path:
|
||||
with open(secret_file_path) as secret_file:
|
||||
passwd = secret_file.read().strip()
|
||||
|
||||
return passwd
|
||||
|
||||
|
||||
def get_s3_config():
|
||||
check_s3_environment_variables()
|
||||
bucket = os.environ.get("BUCKET_NAME")
|
||||
|
||||
conn = boto3.client(
|
||||
"s3",
|
||||
region_name=os.environ.get("REGION"),
|
||||
aws_access_key_id=os.environ.get("ACCESS_KEY_ID"),
|
||||
aws_secret_access_key=os.environ.get("SECRET_ACCESS_KEY"),
|
||||
endpoint_url=os.environ.get("ENDPOINT_URL"),
|
||||
)
|
||||
|
||||
return conn, bucket
|
||||
|
||||
|
||||
def upload_file_to_s3(filename, folder, conn, bucket):
|
||||
|
||||
destpath = os.path.join(folder, os.path.basename(filename))
|
||||
try:
|
||||
print("Uploading file:", filename)
|
||||
conn.upload_file(filename, bucket, destpath)
|
||||
|
||||
except Exception as e:
|
||||
print("Error uploading: %s" % (e))
|
||||
exit(1)
|
||||
|
||||
|
||||
def list_directories(path):
|
||||
directories = []
|
||||
for name in os.listdir(path):
|
||||
if os.path.isdir(os.path.join(path, name)):
|
||||
directories.append(name)
|
||||
return directories
|
||||
|
||||
|
||||
def get_site_config_from_path(site_config_path):
|
||||
site_config = dict()
|
||||
if os.path.exists(site_config_path):
|
||||
with open(site_config_path) as sc:
|
||||
site_config = json.load(sc)
|
||||
return site_config
|
||||
|
||||
|
||||
def set_key_in_site_config(key, site, site_config_path):
|
||||
site_config = get_site_config_from_path(site_config_path)
|
||||
value = site_config.get(key)
|
||||
if value:
|
||||
print(f"Set {key} in site config for site: {site}")
|
||||
update_site_config(
|
||||
key,
|
||||
value,
|
||||
site_config_path=os.path.join(os.getcwd(), site, "site_config.json"),
|
||||
)
|
||||
|
||||
|
||||
def check_s3_environment_variables():
|
||||
if "BUCKET_NAME" not in os.environ:
|
||||
print("Variable BUCKET_NAME not set")
|
||||
exit(1)
|
||||
|
||||
if "ACCESS_KEY_ID" not in os.environ:
|
||||
print("Variable ACCESS_KEY_ID not set")
|
||||
exit(1)
|
||||
|
||||
if "SECRET_ACCESS_KEY" not in os.environ:
|
||||
print("Variable SECRET_ACCESS_KEY not set")
|
||||
exit(1)
|
||||
|
||||
if "ENDPOINT_URL" not in os.environ:
|
||||
print("Variable ENDPOINT_URL not set")
|
||||
exit(1)
|
||||
|
||||
if "BUCKET_DIR" not in os.environ:
|
||||
print("Variable BUCKET_DIR not set")
|
||||
exit(1)
|
||||
|
||||
if "REGION" not in os.environ:
|
||||
print("Variable REGION not set")
|
||||
exit(1)
|
||||
@ -1,8 +0,0 @@
|
||||
{
|
||||
"db_host": "${DB_HOST}",
|
||||
"db_port": ${DB_PORT},
|
||||
"redis_cache": "redis://${REDIS_CACHE}",
|
||||
"redis_queue": "redis://${REDIS_QUEUE}",
|
||||
"redis_socketio": "redis://${REDIS_SOCKETIO}",
|
||||
"socketio_port": ${SOCKETIO_PORT}
|
||||
}
|
||||
@ -1,191 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
function configureEnv() {
|
||||
if [[ ! -f /home/frappe/frappe-bench/sites/common_site_config.json ]]; then
|
||||
|
||||
if [[ -z "${MARIADB_HOST}" && -z "${POSTGRES_HOST}" ]]; then
|
||||
echo "MARIADB_HOST or POSTGRES_HOST is not set" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ -z "${REDIS_CACHE}" ]]; then
|
||||
echo "REDIS_CACHE is not set" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ -z "${REDIS_QUEUE}" ]]; then
|
||||
echo "REDIS_QUEUE is not set" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ -z "${REDIS_SOCKETIO}" ]]; then
|
||||
echo "REDIS_SOCKETIO is not set" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ -z "${SOCKETIO_PORT}" ]]; then
|
||||
echo "SOCKETIO_PORT is not set" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ -z "${DB_PORT}" ]]; then
|
||||
export DB_PORT=3306
|
||||
fi
|
||||
|
||||
export DB_HOST="${MARIADB_HOST:-$POSTGRES_HOST}"
|
||||
|
||||
# shellcheck disable=SC2016
|
||||
envsubst '${DB_HOST}
|
||||
${DB_PORT}
|
||||
${REDIS_CACHE}
|
||||
${REDIS_QUEUE}
|
||||
${REDIS_SOCKETIO}
|
||||
${SOCKETIO_PORT}' </opt/frappe/common_site_config.json.template >/home/frappe/frappe-bench/sites/common_site_config.json
|
||||
fi
|
||||
}
|
||||
|
||||
function checkConnection() {
|
||||
/home/frappe/frappe-bench/env/bin/python /home/frappe/frappe-bench/commands/check_connection.py
|
||||
}
|
||||
|
||||
function checkConfigExists() {
|
||||
COUNTER=0
|
||||
while [[ ! -e /home/frappe/frappe-bench/sites/common_site_config.json && ${COUNTER} -le 30 ]]; do
|
||||
sleep 1
|
||||
((COUNTER = COUNTER + 1))
|
||||
echo "config file not created, retry ${COUNTER}" >&2
|
||||
done
|
||||
|
||||
if [[ ! -e /home/frappe/frappe-bench/sites/common_site_config.json ]]; then
|
||||
echo "timeout: config file not created" >&2
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
if [[ ! -e /home/frappe/frappe-bench/sites/apps.txt ]]; then
|
||||
find /home/frappe/frappe-bench/apps -mindepth 1 -maxdepth 1 -type d -printf '%f\n' |
|
||||
sort -r >/home/frappe/frappe-bench/sites/apps.txt
|
||||
fi
|
||||
|
||||
# symlink node_modules
|
||||
ln -sfn /home/frappe/frappe-bench/sites/assets/frappe/node_modules \
|
||||
/home/frappe/frappe-bench/apps/frappe/node_modules
|
||||
|
||||
case "$1" in
|
||||
|
||||
start)
|
||||
configureEnv
|
||||
checkConnection
|
||||
|
||||
[[ -z "${WORKERS}" ]] && WORKERS='2'
|
||||
|
||||
[[ -z "${FRAPPE_PORT}" ]] && FRAPPE_PORT='8000'
|
||||
|
||||
[[ -z "${WORKER_CLASS}" ]] && WORKER_CLASS='gthread'
|
||||
|
||||
LOAD_CONFIG_FILE=""
|
||||
[[ "${WORKER_CLASS}" == "gevent" ]] &&
|
||||
LOAD_CONFIG_FILE="-c /home/frappe/frappe-bench/commands/gevent_patch.py"
|
||||
|
||||
if [[ -n "${AUTO_MIGRATE}" ]]; then
|
||||
/home/frappe/frappe-bench/env/bin/python /home/frappe/frappe-bench/commands/auto_migrate.py
|
||||
fi
|
||||
|
||||
# shellcheck disable=SC2086
|
||||
/home/frappe/frappe-bench/env/bin/gunicorn ${LOAD_CONFIG_FILE} -b 0.0.0.0:${FRAPPE_PORT} \
|
||||
--worker-tmp-dir /dev/shm \
|
||||
--threads=4 \
|
||||
--workers ${WORKERS} \
|
||||
--worker-class=${WORKER_CLASS} \
|
||||
--log-file=- \
|
||||
-t 120 frappe.app:application --preload
|
||||
;;
|
||||
|
||||
worker)
|
||||
checkConfigExists
|
||||
checkConnection
|
||||
|
||||
: "${WORKER_TYPE:=default}"
|
||||
bench worker --queue $WORKER_TYPE
|
||||
;;
|
||||
|
||||
schedule)
|
||||
checkConfigExists
|
||||
checkConnection
|
||||
|
||||
bench schedule
|
||||
;;
|
||||
|
||||
new)
|
||||
checkConfigExists
|
||||
checkConnection
|
||||
|
||||
/home/frappe/frappe-bench/env/bin/python /home/frappe/frappe-bench/commands/new.py
|
||||
exit
|
||||
;;
|
||||
|
||||
drop)
|
||||
checkConfigExists
|
||||
checkConnection
|
||||
|
||||
: "${SITE_NAME:=site1.localhost}"
|
||||
: "${DB_ROOT_USER:=root}"
|
||||
: "${DB_ROOT_PASSWORD:=$POSTGRES_PASSWORD}"
|
||||
: "${DB_ROOT_PASSWORD:=$MYSQL_ROOT_PASSWORD}"
|
||||
: "${DB_ROOT_PASSWORD:=admin}"
|
||||
|
||||
FLAGS=
|
||||
if [[ ${NO_BACKUP} == 1 ]]; then
|
||||
FLAGS="${FLAGS} --no-backup"
|
||||
fi
|
||||
if [[ ${FORCE} == 1 ]]; then
|
||||
FLAGS="${FLAGS} --force"
|
||||
fi
|
||||
|
||||
# shellcheck disable=SC2086
|
||||
bench drop-site \
|
||||
${SITE_NAME} \
|
||||
--root-login ${DB_ROOT_USER} \
|
||||
--root-password ${DB_ROOT_PASSWORD} \
|
||||
--archived-sites-path /home/frappe/frappe-bench/sites/archive_sites \
|
||||
${FLAGS}
|
||||
;;
|
||||
|
||||
migrate)
|
||||
/home/frappe/frappe-bench/env/bin/python /home/frappe/frappe-bench/commands/migrate.py
|
||||
exit
|
||||
;;
|
||||
|
||||
doctor)
|
||||
/home/frappe/frappe-bench/env/bin/python /home/frappe/frappe-bench/commands/doctor.py "${@:2}"
|
||||
exit
|
||||
;;
|
||||
|
||||
backup)
|
||||
/home/frappe/frappe-bench/env/bin/python /home/frappe/frappe-bench/commands/backup.py
|
||||
exit
|
||||
;;
|
||||
|
||||
console)
|
||||
if [[ -z "$2" ]]; then
|
||||
echo "Need to specify a sitename with the command:" >&2
|
||||
echo "console <sitename>" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
bench --site "$2" console
|
||||
;;
|
||||
|
||||
push-backup)
|
||||
/home/frappe/frappe-bench/env/bin/python /home/frappe/frappe-bench/commands/push_backup.py
|
||||
exit
|
||||
;;
|
||||
|
||||
restore-backup)
|
||||
/home/frappe/frappe-bench/env/bin/python /home/frappe/frappe-bench/commands/restore_backup.py
|
||||
exit
|
||||
;;
|
||||
*)
|
||||
exec "$@"
|
||||
;;
|
||||
esac
|
||||
26
build/frappe-worker/entrypoint.sh
Executable file
26
build/frappe-worker/entrypoint.sh
Executable file
@ -0,0 +1,26 @@
|
||||
#!/bin/bash
|
||||
|
||||
# TODO: Set config dynamically, so changes in compose file have affect
|
||||
|
||||
set -e
|
||||
|
||||
function create_common_site_config() {
|
||||
if [[ ! -f common_site_config.json ]]; then
|
||||
config=$(
|
||||
cat <<EOF
|
||||
{
|
||||
"db_host": "${DB_HOST}",
|
||||
"db_port": ${DB_PORT},
|
||||
"redis_cache": "redis://${REDIS_CACHE}",
|
||||
"redis_queue": "redis://${REDIS_QUEUE}",
|
||||
"redis_socketio": "redis://${REDIS_SOCKETIO}",
|
||||
"socketio_port": ${SOCKETIO_PORT}
|
||||
}
|
||||
EOF
|
||||
)
|
||||
echo "$config" >/home/frappe/frappe-bench/sites/common_site_config.json
|
||||
fi
|
||||
}
|
||||
|
||||
create_common_site_config
|
||||
exec "$@"
|
||||
@ -1,45 +1,28 @@
|
||||
#!/bin/bash
|
||||
set -ea
|
||||
set -e
|
||||
|
||||
function getUrl() {
|
||||
grep "$2" "$1" | awk -v word="$2" '$word { gsub(/[",]/,"",$2); print $2}' | tr -d '\n'
|
||||
get_key() {
|
||||
grep "$1" /home/frappe/frappe-bench/sites/common_site_config.json | awk -v word="$1" '$word { gsub(/[",]/,"",$2); print $2}' | tr -d '\n'
|
||||
}
|
||||
|
||||
COMMON_SITE_CONFIG_JSON='/home/frappe/frappe-bench/sites/common_site_config.json'
|
||||
|
||||
# Set DB Host and port
|
||||
DB_HOST=$(getUrl "${COMMON_SITE_CONFIG_JSON}" "db_host")
|
||||
DB_PORT=$(getUrl "${COMMON_SITE_CONFIG_JSON}" "db_port")
|
||||
if [[ -z "${DB_PORT}" ]]; then
|
||||
DB_PORT=3306
|
||||
get_redis_url() {
|
||||
URL=$(get_key "$1" | sed 's|redis://||g')
|
||||
if [[ ${URL} == *"/"* ]]; then
|
||||
URL=$(echo "${URL}" | cut -f1 -d"/")
|
||||
fi
|
||||
echo "$URL"
|
||||
}
|
||||
|
||||
# Set REDIS host:port
|
||||
REDIS_CACHE=$(getUrl "${COMMON_SITE_CONFIG_JSON}" "redis_cache" | sed 's|redis://||g')
|
||||
if [[ "${REDIS_CACHE}" == *"/"* ]]; then
|
||||
REDIS_CACHE=$(echo "${REDIS_CACHE}" | cut -f1 -d"/")
|
||||
fi
|
||||
|
||||
REDIS_QUEUE=$(getUrl "${COMMON_SITE_CONFIG_JSON}" "redis_queue" | sed 's|redis://||g')
|
||||
if [[ "${REDIS_QUEUE}" == *"/"* ]]; then
|
||||
REDIS_QUEUE=$(echo "${REDIS_QUEUE}" | cut -f1 -d"/")
|
||||
fi
|
||||
|
||||
REDIS_SOCKETIO=$(getUrl "${COMMON_SITE_CONFIG_JSON}" "redis_socketio" | sed 's|redis://||g')
|
||||
if [[ "${REDIS_SOCKETIO}" == *"/"* ]]; then
|
||||
REDIS_SOCKETIO=$(echo "${REDIS_SOCKETIO}" | cut -f1 -d"/")
|
||||
fi
|
||||
|
||||
echo "Check ${DB_HOST}:${DB_PORT}"
|
||||
wait-for-it "${DB_HOST}:${DB_PORT}" -t 1
|
||||
echo "Check ${REDIS_CACHE}"
|
||||
wait-for-it "${REDIS_CACHE}" -t 1
|
||||
echo "Check ${REDIS_QUEUE}"
|
||||
wait-for-it "${REDIS_QUEUE}" -t 1
|
||||
echo "Check ${REDIS_SOCKETIO}"
|
||||
wait-for-it "${REDIS_SOCKETIO}" -t 1
|
||||
|
||||
if [[ "$1" = "-p" || "$1" = "--ping-service" ]]; then
|
||||
echo "Check $2"
|
||||
wait-for-it "$2" -t 1
|
||||
check_connection() {
|
||||
echo "Check $1"
|
||||
wait-for-it "$1" -t 1
|
||||
}
|
||||
|
||||
check_connection "$(get_key db_host):$(get_key db_port)"
|
||||
check_connection "$(get_redis_url redis_cache)"
|
||||
check_connection "$(get_redis_url redis_queue)"
|
||||
check_connection "$(get_redis_url redis_socketio)"
|
||||
|
||||
if [[ "$1" = -p || "$1" = --ping-service ]]; then
|
||||
check_connection "$2"
|
||||
fi
|
||||
|
||||
@ -1,11 +0,0 @@
|
||||
#!/bin/bash -ex
|
||||
|
||||
APP_NAME=${1}
|
||||
APP_REPO=${2}
|
||||
APP_BRANCH=${3}
|
||||
|
||||
[[ -n "${APP_BRANCH}" ]] && BRANCH="-b ${APP_BRANCH}"
|
||||
|
||||
# shellcheck disable=SC2086
|
||||
git clone --depth 1 -o upstream "${APP_REPO}" ${BRANCH} "/home/frappe/frappe-bench/apps/${APP_NAME}"
|
||||
/home/frappe/frappe-bench/env/bin/pip3 install --no-cache-dir -e "/home/frappe/frappe-bench/apps/${APP_NAME}"
|
||||
24
build/frappe-worker/patched_bench_helper.py
Normal file
24
build/frappe-worker/patched_bench_helper.py
Normal file
@ -0,0 +1,24 @@
|
||||
import frappe.app
|
||||
import frappe.database.db_manager
|
||||
import frappe.utils.bench_helper
|
||||
|
||||
|
||||
def patch_database_creator():
|
||||
"""
|
||||
We need to interrupt Frappe site database creation to monkeypatch
|
||||
functions that resolve host for user that owns site database.
|
||||
In frappe_docker this was implemented in "new" command:
|
||||
https://github.com/frappe/frappe_docker/blob/c808ad1767feaf793a2d14541ac0f4d9cbab45b3/build/frappe-worker/commands/new.py#L87
|
||||
"""
|
||||
|
||||
frappe.database.db_manager.DbManager.get_current_host = lambda self: "%"
|
||||
|
||||
|
||||
def main() -> int:
|
||||
patch_database_creator()
|
||||
frappe.utils.bench_helper.main()
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
5
build/frappe-worker/pretend-bench.sh
Executable file
5
build/frappe-worker/pretend-bench.sh
Executable file
@ -0,0 +1,5 @@
|
||||
#!/bin/bash
|
||||
set -e
|
||||
|
||||
# shellcheck disable=SC2068
|
||||
~/frappe-bench/env/bin/python /usr/local/bin/patched_bench_helper.py frappe $@
|
||||
Loading…
Reference in New Issue
Block a user