accounting/frappe_docker
2
0
Fork 0
mirror of https://github.com/frappe/frappe_docker.git synced 2024-11-09 23:00:56 +00:00
Code Issues Releases Activity

feat: add postgres support to frappe-worker

Browse Source
This commit is contained in:
Revant Nandgaonkar 2020-06-29 23:28:10 +05:30
parent aa3ef11b69
commit 87558b7f0d
13 changed files with 461 additions and 104 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
build/common/commands/backup.py
View File

@ -28,6 +28,10 @@ def main():
with_files = cint(os.environ.get("WITH_FILES"))
backup(sites, with_files)
if frappe.redis_server:
frappe.redis_server.connection_pool.disconnect()
exit(0)

9
build/common/commands/check_connection.py
View File

@ -8,6 +8,7 @@ REDIS_QUEUE_KEY = 'redis_queue'
REDIS_CACHE_KEY = 'redis_cache'
REDIS_SOCKETIO_KEY = 'redis_socketio'
DB_HOST_KEY = 'db_host'
DB_PORT_KEY = 'db_port'
DB_PORT = 3306
@ -55,14 +56,14 @@ def get_config():
# Check mariadb
def check_mariadb(retry=10, delay=3, print_attempt=True):
config = get_config()
check_mariadb = False
check_mariadb = check_host(
is_db_connected = False
is_db_connected = check_host(
config.get(DB_HOST_KEY, 'mariadb'),
DB_PORT,
config.get(DB_PORT_KEY, DB_PORT),
retry,
delay,
print_attempt)
if not check_mariadb:
if not is_db_connected:
print("Connection to MariaDB timed out")
exit(1)

2
build/common/commands/console.py
View File

@ -24,6 +24,8 @@ def console(site):
def main():
site = sys.argv[-1]
console(site)
if frappe.redis_server:
frappe.redis_server.connection_pool.disconnect()
if __name__ == "__main__":

6
build/common/commands/migrate.py
View File

@ -3,11 +3,11 @@ import frappe
import json
from frappe.utils import cint, get_sites
from check_connection import get_config
from check_connection import get_config, COMMON_SITE_CONFIG_FILE
def save_config(config):
with open('common_site_config.json', 'w') as f:
with open(COMMON_SITE_CONFIG_FILE, 'w') as f:
return json.dump(config, f, indent=1, sort_keys=True)
@ -48,6 +48,8 @@ def migrate_sites(maintenance_mode=False):
def main():
migrate_sites()
if frappe.redis_server:
frappe.redis_server.connection_pool.disconnect()
exit(0)

46
build/common/commands/new.py
View File

@ -1,8 +1,10 @@
import os
import frappe
import semantic_version
from frappe.commands.site import _new_site
from check_connection import get_config, get_site_config
from frappe.installer import update_site_config
from check_connection import get_config, get_site_config, COMMON_SITE_CONFIG_FILE
def get_password(env_var, default=None):
@ -28,14 +30,50 @@ def _get_password_from_secret(env_var):
def main():
db_type = 'mariadb'
db_host = None
db_port = None
config = get_config()
site_name = os.environ.get("SITE_NAME", 'site1.localhost')
mariadb_root_username = os.environ.get("DB_ROOT_USER", 'root')
mariadb_root_password = get_password("MYSQL_ROOT_PASSWORD", 'admin')
postgres_root_password = get_password("POSTGRES_PASSWORD")
if postgres_root_password:
db_type = 'postgres'
db_host = os.environ.get("POSTGRES_HOST")
db_port = '5432'
if not db_host:
print('Environment variable POSTGRES_HOST not found.')
print('Using db_host from common_site_config.json')
sites_path = os.getcwd()
common_site_config_path = os.path.join(sites_path, COMMON_SITE_CONFIG_FILE)
update_site_config("root_login", mariadb_root_username, validate = False, site_config_path = common_site_config_path)
update_site_config("root_password", postgres_root_password, validate = False, site_config_path = common_site_config_path)
force = True if os.environ.get("FORCE", None) else False
install_apps = os.environ.get("INSTALL_APPS", None)
install_apps = install_apps.split(',') if install_apps else []
frappe.init(site_name, new_site=True)
if semantic_version.Version(frappe.__version__).major > 11:
_new_site(
None,
site_name,
mariadb_root_username=mariadb_root_username,
mariadb_root_password=mariadb_root_password,
admin_password=get_password("ADMIN_PASSWORD", 'admin'),
verbose=True,
install_apps=install_apps,
source_sql=None,
force=force,
db_type=db_type,
reinstall=False,
db_host=db_host,
db_port=db_port,
)
else:
_new_site(
None,
site_name,
@ -49,8 +87,8 @@ def main():
reinstall=False,
)
config = get_config()
if db_type == "mariadb":
site_config = get_site_config(site_name)
mysql_command = 'mysql -h{db_host} -u{mariadb_root_username} -p{mariadb_root_password} -e '.format(
@ -77,6 +115,10 @@ def main():
db_name=site_config.get('db_name')
)
os.system(command)
if frappe.redis_server:
frappe.redis_server.connection_pool.disconnect()
exit(0)

215
build/common/commands/restore_backup.py
View File

@ -10,7 +10,7 @@ from new import get_password
from push_backup import DATE_FORMAT, check_environment_variables
from frappe.utils import get_sites, random_string
from frappe.installer import make_conf, get_conf_params, make_site_dirs, update_site_config
from check_connection import get_site_config, get_config
from check_connection import get_site_config, get_config, COMMON_SITE_CONFIG_FILE
def list_directories(path):
@ -40,71 +40,58 @@ def decompress_db(files_base, site):
def restore_database(files_base, site_config_path, site):
db_root_password = get_password('MYSQL_ROOT_PASSWORD')
if not db_root_password:
print('Variable MYSQL_ROOT_PASSWORD not set')
exit(1)
db_root_user = os.environ.get("DB_ROOT_USER", 'root')
# restore database
database_file = files_base + '-database.sql.gz'
decompress_db(files_base, site)
config = get_config()
# Set db_type if it exists in backup site_config.json
set_key_in_site_config('db_type', site, site_config_path)
# Set db_host if it exists in backup site_config.json
set_key_in_site_config('db_host', site, site_config_path)
# Set db_port if it exists in backup site_config.json
set_key_in_site_config('db_port', site, site_config_path)
# get updated site_config
site_config = get_site_config(site)
# mysql command prefix
mysql_command = 'mysql -u{db_root_user} -h{db_host} -p{db_password} -e '.format(
db_root_user=db_root_user,
db_host=config.get('db_host'),
db_password=db_root_password
)
# if no db_type exists, default to mariadb
db_type = site_config.get('db_type', 'mariadb')
is_database_restored = False
# drop db if exists for clean restore
drop_database = mysql_command + "\"DROP DATABASE IF EXISTS \`{db_name}\`;\"".format(
db_name=site_config.get('db_name')
)
os.system(drop_database)
if db_type == 'mariadb':
restore_mariadb(
config=config,
site_config=site_config,
database_file=database_file)
is_database_restored = True
elif db_type == 'postgres':
restore_postgres(
config=config,
site_config=site_config,
database_file=database_file)
is_database_restored = True
# create db
create_database = mysql_command + "\"CREATE DATABASE IF NOT EXISTS \`{db_name}\`;\"".format(
db_name=site_config.get('db_name')
)
os.system(create_database)
if is_database_restored:
# Set encryption_key if it exists in backup site_config.json
set_key_in_site_config('encryption_key', site, site_config_path)
# create user
create_user = mysql_command + "\"CREATE USER IF NOT EXISTS \'{db_name}\'@\'%\' IDENTIFIED BY \'{db_password}\'; FLUSH PRIVILEGES;\"".format(
db_name=site_config.get('db_name'),
db_password=site_config.get('db_password')
)
os.system(create_user)
# grant db privileges to user
grant_privileges = mysql_command + "\"GRANT ALL PRIVILEGES ON \`{db_name}\`.* TO '{db_name}'@'%' IDENTIFIED BY '{db_password}'; FLUSH PRIVILEGES;\"".format(
db_name=site_config.get('db_name'),
db_password=site_config.get('db_password')
)
os.system(grant_privileges)
def set_key_in_site_config(key, site, site_config_path):
site_config = get_site_config_from_path(site_config_path)
value = site_config.get(key)
if value:
print('Set {key} in site config for site: {site}'.format(key=key, site=site))
update_site_config(key, value,
site_config_path=os.path.join(os.getcwd(), site, "site_config.json"))
command = "mysql -u{db_root_user} -h{db_host} -p{db_password} '{db_name}' < {database_file}".format(
db_root_user=db_root_user,
db_host=config.get('db_host'),
db_password=db_root_password,
db_name=site_config.get('db_name'),
database_file=database_file.replace('.gz', ''),
)
print('Restoring database for site: {}'.format(site))
os.system(command)
def get_site_config_from_path(site_config_path):
site_config = dict()
if os.path.exists(site_config_path):
with open(site_config_path, 'r') as sc:
site_config = json.load(sc)
encryption_key = site_config.get("encryption_key")
if encryption_key:
print('Restoring site config for site: {}'.format(site))
update_site_config('encryption_key', encryption_key,
site_config_path=os.path.join(os.getcwd(), site, "site_config.json"))
return site_config
def restore_files(files_base):
@ -178,6 +165,121 @@ def pull_backup_from_s3():
os.chdir(os.path.join(os.path.expanduser('~'), 'frappe-bench', 'sites'))
def restore_postgres(config, site_config, database_file):
# common config
common_site_config_path = os.path.join(os.getcwd(), COMMON_SITE_CONFIG_FILE)
db_root_user = config.get('root_login')
if not db_root_user:
postgres_user = os.environ.get('DB_ROOT_USER')
if not postgres_user:
print('Variable DB_ROOT_USER not set')
exit(1)
db_root_user = postgres_user
update_site_config(
"root_login",
db_root_user,
validate=False,
site_config_path=common_site_config_path)
db_root_password = config.get('root_password')
if not db_root_password:
root_password = get_password('POSTGRES_PASSWORD')
if not root_password:
print('Variable POSTGRES_PASSWORD not set')
exit(1)
db_root_password = root_password
update_site_config(
"root_password",
db_root_password,
validate=False,
site_config_path=common_site_config_path)
# site config
db_host = site_config.get('db_host')
db_port = site_config.get('db_port', '5432')
db_name = site_config.get('db_name')
db_password = site_config.get('db_password')
psql_command = "psql postgres://{root_login}:{root_password}@{db_host}:{db_port}".format(
root_login=db_root_user,
root_password=db_root_password,
db_host=db_host,
db_port=db_port
)
print('Restoring PostgreSQL')
os.system(psql_command + ' -c "DROP DATABASE IF EXISTS \"{db_name}\""'.format(db_name=db_name))
os.system(psql_command + ' -c "DROP USER IF EXISTS {db_name}"'.format(db_name=db_name))
os.system(psql_command + ' -c "CREATE DATABASE \"{db_name}\""'.format(db_name=db_name))
os.system(psql_command + ' -c "CREATE user {db_name} password \'{db_password}\'"'.format(
db_name=db_name,
db_password=db_password))
os.system(psql_command + ' -c "GRANT ALL PRIVILEGES ON DATABASE \"{db_name}\" TO {db_name}"'.format(
db_name=db_name))
os.system("{psql_command}/{db_name} < {database_file}".format(
psql_command=psql_command,
database_file=database_file.replace('.gz', ''),
db_name=db_name,
))
def restore_mariadb(config, site_config, database_file):
db_root_password = get_password('MYSQL_ROOT_PASSWORD')
if not db_root_password:
print('Variable MYSQL_ROOT_PASSWORD not set')
exit(1)
db_root_user = os.environ.get("DB_ROOT_USER", 'root')
# mysql command prefix
mysql_command = 'mysql -u{db_root_user} -h{db_host} -p{db_password} -e '.format(
db_root_user=db_root_user,
db_host=config.get('db_host'),
db_password=db_root_password
)
# drop db if exists for clean restore
drop_database = mysql_command + "\"DROP DATABASE IF EXISTS \`{db_name}\`;\"".format(
db_name=site_config.get('db_name')
)
os.system(drop_database)
# create db
create_database = mysql_command + "\"CREATE DATABASE IF NOT EXISTS \`{db_name}\`;\"".format(
db_name=site_config.get('db_name')
)
os.system(create_database)
# create user
create_user = mysql_command + "\"CREATE USER IF NOT EXISTS \'{db_name}\'@\'%\' IDENTIFIED BY \'{db_password}\'; FLUSH PRIVILEGES;\"".format(
db_name=site_config.get('db_name'),
db_password=site_config.get('db_password')
)
os.system(create_user)
# grant db privileges to user
grant_privileges = mysql_command + "\"GRANT ALL PRIVILEGES ON \`{db_name}\`.* TO '{db_name}'@'%' IDENTIFIED BY '{db_password}'; FLUSH PRIVILEGES;\"".format(
db_name=site_config.get('db_name'),
db_password=site_config.get('db_password')
)
os.system(grant_privileges)
command = "mysql -u{db_root_user} -h{db_host} -p{db_password} '{db_name}' < {database_file}".format(
db_root_user=db_root_user,
db_host=config.get('db_host'),
db_password=db_root_password,
db_name=site_config.get('db_name'),
database_file=database_file.replace('.gz', ''),
)
print('Restoring MariaDB')
os.system(command)
def main():
backup_dir = get_backup_dir()
@ -194,15 +296,11 @@ def main():
if not os.path.exists(site_config_path):
site_config_path = os.path.join(backup_dir, site, 'site_config.json')
if site in get_sites():
print('Overwrite site {}'.format(site))
restore_database(files_base, site_config_path, site)
restore_private_files(files_base)
restore_files(files_base)
else:
mariadb_root_password = get_password('MYSQL_ROOT_PASSWORD')
if not mariadb_root_password:
print('Variable MYSQL_ROOT_PASSWORD not set')
exit(1)
site_config = get_conf_params(
db_name='_' + hashlib.sha1(site.encode()).hexdigest()[:16],
db_password=random_string(16)
@ -216,10 +314,15 @@ def main():
db_password=site_config.get('db_password'),
)
make_site_dirs()
print('Create site {}'.format(site))
restore_database(files_base, site_config_path, site)
restore_private_files(files_base)
restore_files(files_base)
if frappe.redis_server:
frappe.redis_server.connection_pool.disconnect()
exit(0)

3
build/common/common_site_config.json.template
View File

@ -1,5 +1,6 @@
{
"db_host": "${MARIADB_HOST}",
"db_host": "${DB_HOST}",
"db_port": "${DB_PORT}",
"redis_cache": "redis://${REDIS_CACHE}",
"redis_queue": "redis://${REDIS_QUEUE}",
"redis_socketio": "redis://${REDIS_SOCKETIO}",

13
build/common/worker/docker-entrypoint.sh
View File

@ -4,9 +4,11 @@ function configureEnv() {
if [ ! -f /home/frappe/frappe-bench/sites/common_site_config.json ]; then
if [[ -z "$MARIADB_HOST" ]]; then
echo "MARIADB_HOST is not set"
if [[ -z "$POSTGRES_HOST" ]]; then
echo "MARIADB_HOST or POSTGRES_HOST is not set"
exit 1
fi
fi
if [[ -z "$REDIS_CACHE" ]]; then
echo "REDIS_CACHE is not set"
@ -28,7 +30,14 @@ function configureEnv() {
exit 1
fi
envsubst '${MARIADB_HOST}
if [[ -z "$DB_PORT" ]]; then
export DB_PORT=3306
fi
export DB_HOST="${MARIADB_HOST:-$POSTGRES_HOST}"
envsubst '${DB_HOST}
${DB_PORT}
${REDIS_CACHE}
${REDIS_QUEUE}
${REDIS_SOCKETIO}

2
build/erpnext-nginx/Dockerfile
View File

@ -11,7 +11,7 @@ FROM frappe/frappe-nginx:${GIT_BRANCH}
COPY --from=0 /home/frappe/frappe-bench/sites/ /var/www/html/
COPY --from=0 /rsync /rsync
RUN echo -n "\nerpnext" >> /var/www/html/apps.txt
RUN echo "erpnext" >> /var/www/html/apps.txt
VOLUME [ "/assets" ]

2
build/erpnext-nginx/install_app.sh
View File

@ -19,7 +19,7 @@ git clone --depth 1 ${APP_REPO} ${BRANCH}
cd /home/frappe/frappe-bench/apps/frappe
yarn
yarn production
yarn production --app ${APP_NAME}
rm -fr node_modules
yarn install --production=true

165
installation/frappe-postgresql/docker-compose.yml Normal file
View File

@ -0,0 +1,165 @@
version: "3"
services:
traefik:
image: "traefik:v2.2"
command:
- "--log.level=DEBUG"
- "--providers.docker=true"
- "--providers.docker.exposedbydefault=false"
- "--entrypoints.web.address=:80"
- "--entrypoints.websecure.address=:443"
- "--certificatesresolvers.myresolver.acme.httpchallenge=true"
- "--certificatesresolvers.myresolver.acme.httpchallenge.entrypoint=web"
- "--certificatesresolvers.myresolver.acme.email=${LETSENCRYPT_EMAIL}"
- "--certificatesresolvers.myresolver.acme.storage=/letsencrypt/acme.json"
labels:
# enable traefik
- "traefik.enable=true"
# global redirect to https for production only
- "${HTTPS_REDIRECT_RULE_LABEL}"
- "${HTTPS_REDIRECT_ENTRYPOINT_LABEL}"
- "${HTTPS_REDIRECT_MIDDLEWARE_LABEL}"
# middleware redirect for production only
- "${HTTPS_USE_REDIRECT_MIDDLEWARE_LABEL}"
ports:
- "80:80"
- "443:443"
volumes:
- cert-vol:/letsencrypt
- /var/run/docker.sock:/var/run/docker.sock:ro
frappe-nginx:
image: frappe/frappe-nginx:${FRAPPE_VERSION}
restart: on-failure
environment:
- FRAPPE_PY=frappe-python
- FRAPPE_PY_PORT=8000
- FRAPPE_SOCKETIO=frappe-socketio
- SOCKETIO_PORT=9000
labels:
- "traefik.enable=true"
- "traefik.http.routers.frappe-nginx.rule=Host(${SITES})"
- "${ENTRYPOINT_LABEL}"
- "${CERT_RESOLVER_LABEL}"
- "traefik.http.services.frappe-nginx.loadbalancer.server.port=80"
volumes:
- sites-vol:/var/www/html/sites:rw
- assets-vol:/assets:rw
frappe-python:
image: frappe/frappe-worker:${FRAPPE_VERSION}
restart: on-failure
environment:
- POSTGRES_HOST=${POSTGRES_HOST}
- REDIS_CACHE=redis-cache:6379
- REDIS_QUEUE=redis-queue:6379
- REDIS_SOCKETIO=redis-socketio:6379
- SOCKETIO_PORT=9000
- AUTO_MIGRATE=1
volumes:
- sites-vol:/home/frappe/frappe-bench/sites:rw
- assets-vol:/home/frappe/frappe-bench/sites/assets:rw
frappe-socketio:
image: frappe/frappe-socketio:${FRAPPE_VERSION}
restart: on-failure
depends_on:
- redis-socketio
volumes:
- sites-vol:/home/frappe/frappe-bench/sites:rw
frappe-worker-default:
image: frappe/frappe-worker:${FRAPPE_VERSION}
restart: on-failure
command: worker
depends_on:
- redis-queue
- redis-cache
volumes:
- sites-vol:/home/frappe/frappe-bench/sites:rw
frappe-worker-short:
image: frappe/frappe-worker:${FRAPPE_VERSION}
restart: on-failure
command: worker
environment:
- WORKER_TYPE=short
depends_on:
- redis-queue
- redis-cache
volumes:
- sites-vol:/home/frappe/frappe-bench/sites:rw
frappe-worker-long:
image: frappe/frappe-worker:${FRAPPE_VERSION}
restart: on-failure
command: worker
environment:
- WORKER_TYPE=long
depends_on:
- redis-queue
- redis-cache
volumes:
- sites-vol:/home/frappe/frappe-bench/sites:rw
frappe-schedule:
image: frappe/frappe-worker:${FRAPPE_VERSION}
restart: on-failure
command: schedule
depends_on:
- redis-queue
- redis-cache
volumes:
- sites-vol:/home/frappe/frappe-bench/sites:rw
redis-cache:
image: redis:latest
restart: on-failure
volumes:
- redis-cache-vol:/data
redis-queue:
image: redis:latest
restart: on-failure
volumes:
- redis-queue-vol:/data
redis-socketio:
image: redis:latest
restart: on-failure
volumes:
- redis-socketio-vol:/data
postgresql:
image: postgres:11.8
restart: on-failure
environment:
- POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
volumes:
- postgresql-vol:/var/lib/postgresql/data
site-creator:
image: frappe/frappe-worker:${FRAPPE_VERSION}
restart: "no"
command: new
depends_on:
- frappe-python
environment:
- POSTGRES_HOST=${POSTGRES_HOST}
- SITE_NAME=${SITE_NAME}
- DB_ROOT_USER=${DB_ROOT_USER}
- POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
- ADMIN_PASSWORD=${ADMIN_PASSWORD}
- INSTALL_APPS=${INSTALL_APPS}
volumes:
- sites-vol:/home/frappe/frappe-bench/sites:rw
volumes:
postgresql-vol:
redis-cache-vol:
redis-queue-vol:
redis-socketio-vol:
assets-vol:
sites-vol:
cert-vol:

14
installation/frappe-postgresql/env-local Normal file
View File

@ -0,0 +1,14 @@
LETSENCRYPT_EMAIL=email@example.com
FRAPPE_VERSION=edge
POSTGRES_HOST=postgresql
POSTGRES_PASSWORD=admin
SITE_NAME=mysite.localhost
SITES=`mysite.localhost`
DB_ROOT_USER=postgres
ADMIN_PASSWORD=admin
ENTRYPOINT_LABEL=traefik.http.routers.erpnext-nginx.entrypoints=web
CERT_RESOLVER_LABEL=erpnext.local.no-cert-resolver
HTTPS_REDIRECT_RULE_LABEL=erpnext.local.no-redirect-rule
HTTPS_REDIRECT_ENTRYPOINT_LABEL=erpnext.local.no-entrypoint
HTTPS_REDIRECT_MIDDLEWARE_LABEL=erpnext.local.no-middleware
HTTPS_USE_REDIRECT_MIDDLEWARE_LABEL=erpnext.local-no-redirect-middleware

14
installation/frappe-postgresql/env-production Normal file
View File

@ -0,0 +1,14 @@
LETSENCRYPT_EMAIL=email@example.com
FRAPPE_VERSION=edge
POSTGRES_HOST=postgresql
POSTGRES_PASSWORD=admin
SITE_NAME=erp.example.com
SITES=`erp.example.com`
DB_ROOT_USER=postgres
ADMIN_PASSWORD=admin
ENTRYPOINT_LABEL=traefik.http.routers.erpnext-nginx.entrypoints=websecure
CERT_RESOLVER_LABEL=traefik.http.routers.erpnext-nginx.tls.certresolver=myresolver
HTTPS_REDIRECT_RULE_LABEL=traefik.http.routers.http-catchall.rule=hostregexp(`{host:.+}`)
HTTPS_REDIRECT_ENTRYPOINT_LABEL=traefik.http.routers.http-catchall.entrypoints=web
HTTPS_REDIRECT_MIDDLEWARE_LABEL=traefik.http.routers.http-catchall.middlewares=redirect-to-https
HTTPS_USE_REDIRECT_MIDDLEWARE_LABEL=traefik.http.middlewares.redirect-to-https.redirectscheme.scheme=https