2
0
mirror of https://github.com/frappe/frappe_docker.git synced 2024-12-23 18:48:58 +00:00

Merge pull request #225 from frappe/develop

Frappe & ERPNext Version 13.0.0 Beta.1 Released
This commit is contained in:
Revant Nandgaonkar 2020-05-02 00:28:53 +05:30 committed by GitHub
commit 2b8b5fff80
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
23 changed files with 593 additions and 168 deletions

View File

@ -50,6 +50,36 @@ jobs:
script:
- ./travis.py erpnext --nginx --tag edge
- ./travis.py erpnext --nginx --tag develop --tag-only
- stage: "Frappe (v13-beta)"
if: branch = master AND type != pull_request
script:
- ./travis.py frappe --worker --git-version 13-beta
- ./travis.py frappe --worker --tag v13-beta --tag-only
- ./travis.py frappe --worker --tag version-13-beta --tag-only
- stage: "Frappe (v13-beta)"
if: branch = master AND type != pull_request
script:
- ./travis.py frappe --nginx --git-version 13-beta
- ./travis.py frappe --nginx --tag v13-beta --tag-only
- ./travis.py frappe --nginx --tag version-13-beta --tag-only
- stage: "Frappe (v13-beta)"
if: branch = master AND type != pull_request
script:
- ./travis.py frappe --socketio --git-version 13-beta
- ./travis.py frappe --socketio --tag v13-beta --tag-only
- ./travis.py frappe --socketio --tag version-13-beta --tag-only
- stage: "ERPNext (v13-beta)"
if: branch = master AND type != pull_request
script:
- ./travis.py erpnext --worker --git-version 13-beta
- ./travis.py erpnext --worker --tag v13-beta --tag-only
- ./travis.py erpnext --worker --tag version-13-beta --tag-only
- stage: "ERPNext (v13-beta)"
if: branch = master AND type != pull_request
script:
- ./travis.py erpnext --nginx --git-version 13-beta
- ./travis.py erpnext --nginx --tag v13-beta --tag-only
- ./travis.py erpnext --nginx --tag version-13-beta --tag-only
- stage: "Frappe (v12)"
if: branch = master AND type != pull_request
script:
@ -110,3 +140,20 @@ jobs:
- ./travis.py erpnext --nginx --git-version 11
- ./travis.py erpnext --nginx --tag v11 --tag-only
- ./travis.py erpnext --nginx --tag version-11 --tag-only
- stage: "Build and test edge images"
if: type = pull_request
before_install:
- sudo apt-get update && sudo apt-get -y install docker-compose
script:
- docker build -t frappe/frappe-socketio:edge -f build/frappe-socketio/Dockerfile .
- docker build -t frappe/frappe-worker:develop -f build/frappe-worker/Dockerfile .
- docker build -t frappe/erpnext-worker:edge -f build/erpnext-worker/Dockerfile .
- docker build -t frappe/frappe-nginx:develop -f build/frappe-nginx/Dockerfile .
- docker build -t frappe/erpnext-nginx:edge -f build/erpnext-nginx/Dockerfile .
- ./tests/docker-test.sh
- stage: "Pull and test edge images"
if: branch = develop AND type != pull_request
before_install:
- sudo apt-get update && sudo apt-get -y install docker-compose
script:
- ./tests/docker-test.sh

154
README.md
View File

@ -19,7 +19,7 @@ For docker based development refer to this [README](development/README.md)
### Setting up Pre-requisites
This repository requires Docker and Git to be setup on the instance to be used.
This repository requires Docker, docker-compose and Git to be setup on the instance to be used.
### Cloning the repository and preliminary steps
@ -33,7 +33,7 @@ cd frappe_docker
Copy the example docker environment file to `.env`:
```sh
cp installation/env-example installation/.env
cp env-example .env
```
### Setup Environment Variables
@ -63,37 +63,23 @@ The first command will start the containers; the second command will publish the
For Erpnext:
```sh
# Start services
docker-compose \
--project-name <project-name> \
-f installation/docker-compose-common.yml \
-f installation/docker-compose-erpnext.yml \
--project-directory installation up -d
# Publish port
docker-compose \
--project-name <project-name> \
-f installation/docker-compose-common.yml \
-f installation/docker-compose-erpnext.yml \
--project-directory installation run --publish 80:80 -d erpnext-nginx
-f installation/erpnext-publish.yml \
up -d
```
For Frappe:
```sh
# Start services
docker-compose \
--project-name <project-name> \
-f installation/docker-compose-common.yml \
-f installation/docker-compose-frappe.yml \
--project-directory installation up -d
# Publish port
docker-compose \
--project-name <project-name> \
-f installation/docker-compose-common.yml \
-f installation/docker-compose-frappe.yml \
--project-directory installation run --publish 80:80 -d frappe-nginx
-f installation/frappe-publish.yml \
up -d
```
Make sure to replace `<project-name>` with the desired name you wish to set for the project.
@ -124,8 +110,16 @@ cp .env.sample .env
./start.sh
```
It will create the required network and configure containers for Letencrypt ACME.
For more details, see the [Letsencrypt Nginx Proxy Companion github repo](https://github.com/evertramos/docker-compose-letsencrypt-nginx-proxy-companion). Letsencrypt Nginx Proxy Companion github repo works by automatically proxying to containers with the `VIRTUAL_HOST` environmental variable.
Notes:
- `SITES` variables from `env-example` is set as `VIRTUAL_HOST`
- `LETSENCRYPT_EMAIL` variables from `env-example` is used as it is.
- This is simple nginx + letsencrypt solution. Any other solution can be setup. Above two variables can be re-used or removed in case any other reverse-proxy is used.
#### Start Frappe/ERPNext Services
To start the Frappe/ERPNext services for production, run the following command:
@ -136,7 +130,7 @@ docker-compose \
-f installation/docker-compose-common.yml \
-f installation/docker-compose-erpnext.yml \
-f installation/docker-compose-networks.yml \
--project-directory installation up -d
up -d
```
Make sure to replace `<project-name>` with any desired name you wish to set for the project.
@ -159,7 +153,7 @@ docker run \
Instead of `alpine` you can use any image you like.
For full instructions, refer to the [wiki](https://github.com/frappe/frappe/wiki/Using-Frappe-with-Amazon-RDS-(or-any-other-DBaaS). Common question can be found in Issues and on forum.
For full instructions, refer to the [wiki](https://github.com/frappe/frappe/wiki/Using-Frappe-with-Amazon-RDS-(or-any-other-DBaaS)). Common question can be found in Issues and on forum.
### Docker containers
@ -201,9 +195,20 @@ This repository contains the following docker-compose files, each one containing
* docker-compose-networks.yml: this yaml define the network to communicate with *Letsencrypt Nginx Proxy Companion*.
* erpnext-publish.yml: this yml extends erpnext-nginx service to publish port 80, can only be used with docker-compose-erpnext.yml
* frappe-publish.yml: this yml extends frappe-nginx service to publish port 80, can only be used with docker-compose-frappe.yml
### Site operations
Use env file,
```sh
source .env
```
Or specify environment variables instead of passing secrets as command arguments. Refer notes section for environment variables required
#### Setup New Sites
Note:
@ -211,17 +216,18 @@ Note:
- Wait for the MariaDB service to start before trying to create a new site.
- If new site creation fails, retry after the MariaDB container is up and running.
- If you're using a managed database instance, make sure that the database is running before setting up a new site.
- Use `.env` file or environment variables instead of passing secrets as command arguments.
```sh
# Create ERPNext site
docker exec -it \
docker run \
-e "SITE_NAME=$SITE_NAME" \
-e "DB_ROOT_USER=$DB_ROOT_USER" \
-e "MYSQL_ROOT_PASSWORD=$MYSQL_ROOT_PASSWORD" \
-e "ADMIN_PASSWORD=$ADMIN_PASSWORD" \
-e "INSTALL_APPS=erpnext" \
<project-name>_erpnext-python_1 docker-entrypoint.sh new
-v <project-name>_sites-vol:/home/frappe/frappe-bench/sites \
--network <project-name>_default \
frappe/erpnext-worker:$VERSION new
```
Environment Variables needed:
@ -246,10 +252,12 @@ Environment Variables
- All the files generated by backup are placed at volume location `sites-vol:/{site-name}/private/backups/*`
```sh
docker exec -it \
docker run \
-e "SITES=site1.domain.com:site2.domain.com" \
-e "WITH_FILES=1" \
<project-name>_erpnext-python_1 docker-entrypoint.sh backup
-v <project-name>_sites-vol:/home/frappe/frappe-bench/sites \
--network <project-name>_default \
frappe/erpnext-worker:$VERSION backup
```
The backup will be available in the `sites-vol` volume.
@ -259,6 +267,7 @@ The backup will be available in the `sites-vol` volume.
Environment Variables
- `BUCKET_NAME`, Required to set bucket created on S3 compatible storage.
- `REGION`, Required to set region for S3 compatible storage.
- `ACCESS_KEY_ID`, Required to set access key.
- `SECRET_ACCESS_KEY`, Required to set secret access key.
- `ENDPOINT_URL`, Required to set URL of S3 compatible storage.
@ -268,13 +277,14 @@ Environment Variables
```sh
docker run \
-e "BUCKET_NAME=backups" \
-e "REGION=region" \
-e "ACCESS_KEY_ID=access_id_from_provider" \
-e "SECRET_ACCESS_KEY=secret_access_from_provider" \
-e "ENDPOINT_URL=https://region.storage-provider.com" \
-e "BUCKET_DIR=frappe-bench-v12" \
-e "BUCKET_DIR=frappe-bench" \
-v <project-name>_sites-vol:/home/frappe/frappe-bench/sites \
--network <project-name>_default \
frappe/frappe-worker:v12 push-backup
frappe/frappe-worker:$VERSION push-backup
```
Note:
@ -304,11 +314,13 @@ docker-compose \
-f installation/docker-compose-common.yml \
-f installation/docker-compose-erpnext.yml \
-f installation/docker-compose-networks.yml \
--project-directory installation up -d
up -d
docker exec -it \
docker run \
-e "MAINTENANCE_MODE=1" \
<project-name>_erpnext-python_1 docker-entrypoint.sh migrate
-v <project-name>_sites-vol:/home/frappe/frappe-bench/sites \
--network <project-name>_default \
frappe/erpnext-worker:$VERSION migrate
```
#### Restore backups
@ -320,20 +332,22 @@ Environment Variables
- `ACCESS_KEY_ID`, Required to set access key.
- `SECRET_ACCESS_KEY`, Required to set secret access key.
- `ENDPOINT_URL`, Required to set URL of S3 compatible storage.
- `REGION`, Required to set region for s3 compatible storage.
- `BUCKET_DIR`, Required to set directory in bucket where sites from this deployment will be backed up.
```sh
docker run \
-e "MYSQL_ROOT_PASSWORD=admin" \
-e "BUCKET_NAME=backups" \
-e "REGION=region" \
-e "ACCESS_KEY_ID=access_id_from_provider" \
-e "SECRET_ACCESS_KEY=secret_access_from_provider" \
-e "ENDPOINT_URL=https://region.storage-provider.com" \
-e "BUCKET_DIR=frappe-bench-v12" \
-e "BUCKET_DIR=frappe-bench" \
-v <project-name>_sites-vol:/home/frappe/frappe-bench/sites \
-v ./backups:/home/frappe/backups \
--network <project-name>_default \
frappe/frappe-worker:v12 restore-backup
frappe/frappe-worker:$VERSION restore-backup
```
Note:
@ -406,78 +420,6 @@ sed -i "s#\[app\]#[custom]#" ./installation/docker-compose-custom.yml
Install like usual, except that when you set the `INSTALL_APPS` variable to `erpnext,[custom]`.
## Troubleshoot
### Failed migration after image upgrade
Issue: After upgrade of the containers, the automatic migration fails.
Solution: Remove containers and volumes, and clear redis cache:
```shell
# change to repo root
cd $HOME/frappe_docker
# Stop all bench containers
docker-compose \
--project-name <project-name> \
-f installation/docker-compose-common.yml \
-f installation/docker-compose-erpnext.yml \
-f installation/docker-compose-networks.yml \
--project-directory installation stop
# Remove redis containers
docker-compose \
--project-name <project-name> \
-f installation/docker-compose-common.yml \
-f installation/docker-compose-erpnext.yml \
-f installation/docker-compose-networks.yml \
--project-directory installation rm redis-cache redis-queue redis-socketio
# Clean redis volumes
docker volume rm \
<project-name>_redis-cache-vol \
<project-name>_redis-queue-vol \
<project-name>_redis-socketio-vol
# Restart project
docker-compose \
--project-name <project-name> \
-f installation/docker-compose-common.yml \
-f installation/docker-compose-erpnext.yml \
-f installation/docker-compose-networks.yml \
--project-directory installation up -d
```
### ValueError: There exists an active worker named XXX already
Issue: You have the following error during container restart
```
frappe-worker-short_1 | Traceback (most recent call last):
frappe-worker-short_1 | File "/home/frappe/frappe-bench/commands/worker.py", line 5, in <module>
frappe-worker-short_1 | start_worker(queue, False)
frappe-worker-short_1 | File "/home/frappe/frappe-bench/apps/frappe/frappe/utils/background_jobs.py", line 147, in start_worker
frappe-worker-short_1 | Worker(queues, name=get_worker_name(queue)).work(logging_level = logging_level)
frappe-worker-short_1 | File "/home/frappe/frappe-bench/env/lib/python3.7/site-packages/rq/worker.py", line 474, in work
frappe-worker-short_1 | self.register_birth()
frappe-worker-short_1 | File "/home/frappe/frappe-bench/env/lib/python3.7/site-packages/rq/worker.py", line 261, in register_birth
frappe-worker-short_1 | raise ValueError(msg.format(self.name))
frappe-worker-short_1 | ValueError: There exists an active worker named '8dfe5c234085.10.short' already
```
Solution: Clear redis cache using `docker exec` command (take care of replacing `<project-name>` accordingly):
```sh
# Clear the cache which is causing problem.
docker exec -it <project-name>_redis-cache_1 redis-cli FLUSHALL
docker exec -it <project-name>_redis-queue_1 redis-cli FLUSHALL
docker exec -it <project-name>_redis-socketio_1 redis-cli FLUSHALL
```
Note: Environment variables from `.env` file located at the current working directory will be used.
## Development
This repository includes a complete setup to develop with Frappe/ERPNext and Bench, Including the following features:

View File

@ -9,10 +9,12 @@ from check_connection import get_config
APP_VERSIONS_JSON_FILE = 'app_versions.json'
APPS_TXT_FILE = 'apps.txt'
def save_version_file(versions):
with open(APP_VERSIONS_JSON_FILE, 'w') as f:
return json.dump(versions, f, indent=1, sort_keys=True)
def get_apps():
apps = []
try:
@ -24,40 +26,43 @@ def get_apps():
except FileNotFoundError as exception:
print(exception)
exit(1)
except:
except Exception:
print(APPS_TXT_FILE+" is not valid")
exit(1)
return apps
def get_container_versions(apps):
versions = {}
for app in apps:
try:
version = __import__(app).__version__
versions.update({app:version})
except:
versions.update({app: version})
except Exception:
pass
try:
path = os.path.join('..','apps', app)
path = os.path.join('..', 'apps', app)
repo = git.Repo(path)
commit_hash = repo.head.object.hexsha
versions.update({app+'_git_hash':commit_hash})
except:
versions.update({app+'_git_hash': commit_hash})
except Exception:
pass
return versions
def get_version_file():
versions = None
try:
with open(APP_VERSIONS_JSON_FILE) as versions_file:
versions = json.load(versions_file)
except:
except Exception:
pass
return versions
def main():
is_ready = False
apps = get_apps()
@ -76,7 +81,7 @@ def main():
version_file_hash = None
container_hash = None
repo = git.Repo(os.path.join('..','apps',app))
repo = git.Repo(os.path.join('..', 'apps', app))
branch = repo.active_branch.name
if branch == 'develop':
@ -105,5 +110,6 @@ def main():
version_file = container_versions
save_version_file(version_file)
if __name__ == "__main__":
main()

View File

@ -1,10 +1,11 @@
import frappe
from frappe.utils.scheduler import start_scheduler
def main():
print("Starting background scheduler . . .")
start_scheduler()
exit(0)
if __name__ == "__main__":
main()

View File

@ -1,7 +1,8 @@
import os, frappe, compileall, re
import os
import frappe
from frappe.utils.backups import scheduled_backup
from frappe.utils import now
from frappe.utils import get_sites
from frappe.utils import cint, get_sites, now
def backup(sites, with_files=False):
for site in sites:
@ -20,13 +21,15 @@ def backup(sites, with_files=False):
print("private files backup taken -", odb.backup_path_private_files, "- on", now())
frappe.destroy()
def main():
installed_sites = ":".join(get_sites())
sites = os.environ.get("SITES", installed_sites).split(":")
with_files=True if os.environ.get("WITH_FILES") else False
with_files = cint(os.environ.get("WITH_FILES"))
backup(sites, with_files)
exit(0)
if __name__ == "__main__":
main()

View File

@ -1,4 +1,6 @@
import socket, os, json, time
import socket
import json
import time
from six.moves.urllib.parse import urlparse
COMMON_SITE_CONFIG_FILE = 'common_site_config.json'
@ -8,6 +10,7 @@ REDIS_SOCKETIO_KEY = 'redis_socketio'
DB_HOST_KEY = 'db_host'
DB_PORT = 3306
def is_open(ip, port, timeout=30):
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.settimeout(timeout)
@ -15,16 +18,17 @@ def is_open(ip, port, timeout=30):
s.connect((ip, int(port)))
s.shutdown(socket.SHUT_RDWR)
return True
except:
except Exception:
return False
finally:
s.close()
def check_host(ip, port, retry=10, delay=3, print_attempt=True):
ipup = False
for i in range(retry):
if print_attempt:
print("Attempt {i} to connect to {ip}:{port}".format(ip=ip,port=port,i=i+1))
print("Attempt {i} to connect to {ip}:{port}".format(ip=ip, port=port, i=i+1))
if is_open(ip, port):
ipup = True
break
@ -32,6 +36,7 @@ def check_host(ip, port, retry=10, delay=3, print_attempt=True):
time.sleep(delay)
return ipup
# Check connection to servers
def get_config():
config = None
@ -41,11 +46,12 @@ def get_config():
except FileNotFoundError as exception:
print(exception)
exit(1)
except:
except Exception:
print(COMMON_SITE_CONFIG_FILE+" is not valid")
exit(1)
return config
# Check mariadb
def check_mariadb(retry=10, delay=3, print_attempt=True):
config = get_config()
@ -60,11 +66,12 @@ def check_mariadb(retry=10, delay=3, print_attempt=True):
print("Connection to MariaDB timed out")
exit(1)
# Check redis queue
def check_redis_queue(retry=10, delay=3, print_attempt=True):
check_redis_queue = False
config = get_config()
redis_queue_url = urlparse(config.get(REDIS_QUEUE_KEY,"redis://redis-queue:6379")).netloc
redis_queue_url = urlparse(config.get(REDIS_QUEUE_KEY, "redis://redis-queue:6379")).netloc
redis_queue, redis_queue_port = redis_queue_url.split(":")
check_redis_queue = check_host(
redis_queue,
@ -76,11 +83,12 @@ def check_redis_queue(retry=10, delay=3, print_attempt=True):
print("Connection to redis queue timed out")
exit(1)
# Check redis cache
def check_redis_cache(retry=10, delay=3, print_attempt=True):
check_redis_cache = False
config = get_config()
redis_cache_url = urlparse(config.get(REDIS_CACHE_KEY,"redis://redis-cache:6379")).netloc
redis_cache_url = urlparse(config.get(REDIS_CACHE_KEY, "redis://redis-cache:6379")).netloc
redis_cache, redis_cache_port = redis_cache_url.split(":")
check_redis_cache = check_host(
redis_cache,
@ -92,11 +100,12 @@ def check_redis_cache(retry=10, delay=3, print_attempt=True):
print("Connection to redis cache timed out")
exit(1)
# Check redis socketio
def check_redis_socketio(retry=10, delay=3, print_attempt=True):
check_redis_socketio = False
config = get_config()
redis_socketio_url = urlparse(config.get(REDIS_SOCKETIO_KEY,"redis://redis-socketio:6379")).netloc
redis_socketio_url = urlparse(config.get(REDIS_SOCKETIO_KEY, "redis://redis-socketio:6379")).netloc
redis_socketio, redis_socketio_port = redis_socketio_url.split(":")
check_redis_socketio = check_host(
redis_socketio,
@ -108,6 +117,7 @@ def check_redis_socketio(retry=10, delay=3, print_attempt=True):
print("Connection to redis socketio timed out")
exit(1)
# Get site_config.json
def get_site_config(site_name):
site_config = None
@ -115,6 +125,7 @@ def get_site_config(site_name):
site_config = json.load(site_config_file)
return site_config
def main():
check_mariadb()
check_redis_queue()
@ -122,5 +133,6 @@ def main():
check_redis_socketio()
print('Connections OK')
if __name__ == "__main__":
main()

View File

@ -20,6 +20,11 @@ def console(site):
print("Apps in this namespace:\n{}".format(", ".join(all_apps)))
IPython.embed(display_banner="", header="")
def main():
site = sys.argv[-1]
console(site)
if __name__ == "__main__":
main()

View File

@ -1,7 +1,3 @@
import frappe
import json
import redis
from rq import Worker
from check_connection import (
check_mariadb,
check_redis_cache,
@ -9,6 +5,7 @@ from check_connection import (
check_redis_socketio,
)
def main():
check_mariadb(retry=1, delay=0, print_attempt=False)
print("MariaDB Connected")
@ -22,5 +19,6 @@ def main():
print("Health check successful")
exit(0)
if __name__ == "__main__":
main()

View File

@ -1,29 +1,34 @@
import os, frappe, compileall, re, json
import os
import frappe
import json
from frappe.migrate import migrate
from frappe.utils import get_sites
from frappe.utils import cint, get_sites
from check_connection import get_config
def save_config(config):
with open('common_site_config.json', 'w') as f:
return json.dump(config, f, indent=1, sort_keys=True)
def set_maintenance_mode(enable=True):
conf = get_config()
if enable:
conf.update({ "maintenance_mode": 1, "pause_scheduler": 1 })
conf.update({"maintenance_mode": 1, "pause_scheduler": 1})
save_config(conf)
if not enable:
conf.update({ "maintenance_mode": 0, "pause_scheduler": 0 })
conf.update({"maintenance_mode": 0, "pause_scheduler": 0})
save_config(conf)
def migrate_sites(maintenance_mode=False):
installed_sites = ":".join(get_sites())
sites = os.environ.get("SITES", installed_sites).split(":")
if not maintenance_mode:
maintenance_mode = True if os.environ.get("MAINTENANCE_MODE") else False
maintenance_mode = cint(os.environ.get("MAINTENANCE_MODE"))
if maintenance_mode:
set_maintenance_mode(True)
@ -37,12 +42,14 @@ def migrate_sites(maintenance_mode=False):
finally:
frappe.destroy()
if maintenance_mode:
set_maintenance_mode(False)
# Disable maintenance mode after migration
set_maintenance_mode(False)
def main():
migrate_sites()
exit(0)
if __name__ == "__main__":
main()

View File

@ -1,17 +1,19 @@
import os, frappe, json
import os
import frappe
from frappe.commands.site import _new_site
from check_connection import get_config, get_site_config
def get_password(env_var, default=None):
return os.environ.get(env_var) or _get_password_from_secret(f"{env_var}_FILE") or default
def _get_password_from_secret(env_var):
"""Fetches the secret value from the docker secret file
"""Fetches the secret value from the docker secret file
usually located inside /run/secrets/
Arguments:
env_var {str} -- Name of the environment variable
env_var {str} -- Name of the environment variable
containing the path to the secret file.
Returns:
[str] -- Secret value
@ -64,7 +66,7 @@ def main():
os.system(command)
# Set db password
command = mysql_command + "\"UPDATE mysql.user SET authentication_string = PASSWORD('{db_password}') WHERE User = \'{db_name}\' AND Host = \'%\';\"".format(
command = mysql_command + "\"ALTER USER '{db_name}'@'%' IDENTIFIED BY '{db_password}'; FLUSH PRIVILEGES;\"".format(
db_name=site_config.get('db_name'),
db_password=site_config.get('db_password')
)
@ -77,5 +79,6 @@ def main():
os.system(command)
exit(0)
if __name__ == "__main__":
main()

View File

@ -8,6 +8,7 @@ from frappe.utils import get_sites
DATE_FORMAT = "%Y%m%d_%H%M%S"
def get_file_ext():
return {
"database": "-database.sql.gz",
@ -15,6 +16,7 @@ def get_file_ext():
"public_files": "-files.tar"
}
def get_backup_details(sitename):
backup_details = dict()
file_ext = get_file_ext()
@ -41,12 +43,14 @@ def get_backup_details(sitename):
return backup_details
def get_s3_config():
check_environment_variables()
bucket = os.environ.get('BUCKET_NAME')
conn = boto3.client(
's3',
region_name=os.environ.get('REGION'),
aws_access_key_id=os.environ.get('ACCESS_KEY_ID'),
aws_secret_access_key=os.environ.get('SECRET_ACCESS_KEY'),
endpoint_url=os.environ.get('ENDPOINT_URL')
@ -54,27 +58,33 @@ def get_s3_config():
return conn, bucket
def check_environment_variables():
if not 'BUCKET_NAME' in os.environ:
if 'BUCKET_NAME' not in os.environ:
print('Variable BUCKET_NAME not set')
exit(1)
if not 'ACCESS_KEY_ID' in os.environ:
if 'ACCESS_KEY_ID' not in os.environ:
print('Variable ACCESS_KEY_ID not set')
exit(1)
if not 'SECRET_ACCESS_KEY' in os.environ:
if 'SECRET_ACCESS_KEY' not in os.environ:
print('Variable SECRET_ACCESS_KEY not set')
exit(1)
if not 'ENDPOINT_URL' in os.environ:
if 'ENDPOINT_URL' not in os.environ:
print('Variable ENDPOINT_URL not set')
exit(1)
if not 'BUCKET_DIR' in os.environ:
if 'BUCKET_DIR' not in os.environ:
print('Variable BUCKET_DIR not set')
exit(1)
if 'REGION' not in os.environ:
print('Variable REGION not set')
exit(1)
def upload_file_to_s3(filename, folder, conn, bucket):
destpath = os.path.join(folder, os.path.basename(filename))
@ -86,6 +96,7 @@ def upload_file_to_s3(filename, folder, conn, bucket):
print("Error uploading: %s" % (e))
exit(1)
def delete_old_backups(limit, bucket, site_name):
all_backups = list()
all_backup_dates = list()
@ -96,6 +107,7 @@ def delete_old_backups(limit, bucket, site_name):
s3 = boto3.resource(
's3',
region_name=os.environ.get('REGION'),
aws_access_key_id=os.environ.get('ACCESS_KEY_ID'),
aws_secret_access_key=os.environ.get('SECRET_ACCESS_KEY'),
endpoint_url=os.environ.get('ENDPOINT_URL')
@ -150,6 +162,7 @@ def delete_old_backups(limit, bucket, site_name):
print('Deleteing ' + obj.key)
s3.Object(bucket.name, obj.key).delete()
def main():
details = dict()
sites = get_sites()
@ -178,5 +191,6 @@ def main():
print('push-backup complete')
exit(0)
if __name__ == "__main__":
main()

View File

@ -8,10 +8,10 @@ import boto3
from new import get_password
from push_backup import DATE_FORMAT, check_environment_variables
from frappe.utils import get_sites, random_string
from frappe.commands.site import _new_site
from frappe.installer import make_conf, get_conf_params, make_site_dirs
from check_connection import get_site_config, get_config
def list_directories(path):
directories = []
for name in os.listdir(path):
@ -19,25 +19,25 @@ def list_directories(path):
directories.append(name)
return directories
def get_backup_dir():
return os.path.join(
os.path.expanduser('~'),
'backups'
)
def decompress_db(files_base, site):
database_file = files_base + '-database.sql.gz'
config = get_config()
site_config = get_site_config(site)
db_root_user = os.environ.get('DB_ROOT_USER', 'root')
command = 'gunzip -c {database_file} > {database_extract}'.format(
database_file=database_file,
database_extract=database_file.replace('.gz','')
database_extract=database_file.replace('.gz', '')
)
print('Extract Database GZip for site {}'.format(site))
os.system(command)
def restore_database(files_base, site):
db_root_password = get_password('MYSQL_ROOT_PASSWORD')
if not db_root_password:
@ -78,17 +78,11 @@ def restore_database(files_base, site):
)
os.system(create_user)
# create user password
set_user_password = mysql_command + "\"UPDATE mysql.user SET authentication_string = PASSWORD('{db_password}') WHERE User = \'{db_name}\' AND Host = \'%\';\"".format(
# grant db privileges to user
grant_privileges = mysql_command + "\"GRANT ALL PRIVILEGES ON \`{db_name}\`.* TO '{db_name}'@'%' IDENTIFIED BY '{db_password}'; FLUSH PRIVILEGES;\"".format(
db_name=site_config.get('db_name'),
db_password=site_config.get('db_password')
)
os.system(set_user_password)
# grant db privileges to user
grant_privileges = mysql_command + "\"GRANT ALL PRIVILEGES ON \`{db_name}\`.* TO '{db_name}'@'%'; FLUSH PRIVILEGES;\"".format(
db_name=site_config.get('db_name')
)
os.system(grant_privileges)
command = "mysql -u{db_root_user} -h{db_host} -p{db_password} '{db_name}' < {database_file}".format(
@ -96,12 +90,13 @@ def restore_database(files_base, site):
db_host=config.get('db_host'),
db_password=db_root_password,
db_name=site_config.get('db_name'),
database_file=database_file.replace('.gz',''),
database_file=database_file.replace('.gz', ''),
)
print('Restoring database for site: {}'.format(site))
os.system(command)
def restore_files(files_base):
public_files = files_base + '-files.tar'
# extract tar
@ -109,18 +104,21 @@ def restore_files(files_base):
print('Extracting {}'.format(public_files))
public_tar.extractall()
def restore_private_files(files_base):
private_files = files_base + '-private-files.tar'
private_tar = tarfile.open(private_files)
print('Extracting {}'.format(private_files))
private_tar.extractall()
def pull_backup_from_s3():
check_environment_variables()
# https://stackoverflow.com/a/54672690
s3 = boto3.resource(
's3',
region_name=os.environ.get('REGION'),
aws_access_key_id=os.environ.get('ACCESS_KEY_ID'),
aws_secret_access_key=os.environ.get('SECRET_ACCESS_KEY'),
endpoint_url=os.environ.get('ENDPOINT_URL')
@ -133,8 +131,8 @@ def pull_backup_from_s3():
# Change directory to /home/frappe/backups
os.chdir(get_backup_dir())
for obj in bucket.objects.filter(Prefix = bucket_dir):
backup_file = obj.key.replace(os.path.join(bucket_dir,''),'')
for obj in bucket.objects.filter(Prefix=bucket_dir):
backup_file = obj.key.replace(os.path.join(bucket_dir, ''), '')
if not os.path.exists(os.path.dirname(backup_file)):
os.makedirs(os.path.dirname(backup_file))
print('Downloading {}'.format(backup_file))
@ -142,6 +140,7 @@ def pull_backup_from_s3():
os.chdir(os.path.join(os.path.expanduser('~'), 'frappe-bench', 'sites'))
def main():
backup_dir = get_backup_dir()
@ -149,8 +148,8 @@ def main():
pull_backup_from_s3()
for site in list_directories(backup_dir):
site_slug = site.replace('.','_')
backups = [datetime.datetime.strptime(backup, DATE_FORMAT) for backup in list_directories(os.path.join(backup_dir,site))]
site_slug = site.replace('.', '_')
backups = [datetime.datetime.strptime(backup, DATE_FORMAT) for backup in list_directories(os.path.join(backup_dir, site))]
latest_backup = max(backups).strftime(DATE_FORMAT)
files_base = os.path.join(backup_dir, site, latest_backup, '')
files_base += latest_backup + '-' + site_slug
@ -163,8 +162,6 @@ def main():
if not mariadb_root_password:
print('Variable MYSQL_ROOT_PASSWORD not set')
exit(1)
mariadb_root_username = os.environ.get('DB_ROOT_USER', 'root')
database_file = files_base + '-database.sql.gz'
site_config = get_conf_params(
db_name='_' + hashlib.sha1(site.encode()).hexdigest()[:16],
@ -185,5 +182,6 @@ def main():
exit(0)
if __name__ == "__main__":
main()

View File

@ -1,10 +1,12 @@
import os, frappe
import os
from frappe.utils.background_jobs import start_worker
def main():
queue = os.environ.get("WORKER_TYPE", "default")
start_worker(queue, False)
exit(0)
if __name__ == "__main__":
main()

View File

@ -0,0 +1,16 @@
FROM bitnami/node:12-prod
COPY build/erpnext-nginx/install_app.sh /install_app
RUN /install_app erpnext https://github.com/frappe/erpnext version-13-beta
FROM frappe/frappe-nginx:v13-beta
COPY --from=0 /home/frappe/frappe-bench/sites/ /var/www/html/
COPY --from=0 /rsync /rsync
RUN echo -n "\nerpnext" >> /var/www/html/apps.txt
VOLUME [ "/assets" ]
ENTRYPOINT ["/docker-entrypoint.sh"]
CMD ["nginx", "-g", "daemon off;"]

View File

@ -0,0 +1,3 @@
FROM frappe/frappe-worker:v13-beta
RUN install_app erpnext https://github.com/frappe/erpnext version-13-beta

View File

@ -0,0 +1,52 @@
FROM bitnami/node:12-prod
WORKDIR /home/frappe/frappe-bench
RUN mkdir -p /home/frappe/frappe-bench/sites \
&& echo "frappe" > /home/frappe/frappe-bench/sites/apps.txt
RUN install_packages git
RUN mkdir -p apps sites/assets/css \
&& cd apps \
&& git clone --depth 1 https://github.com/frappe/frappe --branch version-13-beta
COPY build/frappe-nginx/generate_standard_style_css.js \
/home/frappe/frappe-bench/apps/frappe/generate_standard_style_css.js
RUN cd /home/frappe/frappe-bench/apps/frappe \
&& yarn \
&& yarn run production \
&& yarn add nunjucks -D \
&& node generate_standard_style_css.js \
frappe/website/doctype/website_theme/website_theme_template.scss > \
/home/frappe/standard_templates_string \
&& node generate_bootstrap_theme.js \
/home/frappe/frappe-bench/sites/assets/css/standard_style.css \
"$(cat /home/frappe/standard_templates_string)" \
&& rm -fr node_modules \
&& yarn install --production=true \
&& node --version \
&& npm --version \
&& yarn --version
RUN git clone --depth 1 https://github.com/frappe/bench /tmp/bench \
&& mkdir -p /var/www/error_pages \
&& cp -r /tmp/bench/bench/config/templates /var/www/error_pages
RUN cp -R /home/frappe/frappe-bench/apps/frappe/frappe/public/* /home/frappe/frappe-bench/sites/assets/frappe \
&& cp -R /home/frappe/frappe-bench/apps/frappe/node_modules /home/frappe/frappe-bench/sites/assets/frappe/
FROM nginx:latest
COPY --from=0 /home/frappe/frappe-bench/sites /var/www/html/
COPY --from=0 /var/www/error_pages /var/www/
COPY build/common/nginx-default.conf.template /etc/nginx/conf.d/default.conf.template
COPY build/frappe-nginx/docker-entrypoint.sh /
RUN apt-get update && apt-get install -y rsync && apt-get clean \
&& echo "#!/bin/bash" > /rsync \
&& chmod +x /rsync
VOLUME [ "/assets" ]
ENTRYPOINT ["/docker-entrypoint.sh"]
CMD ["nginx", "-g", "daemon off;"]

View File

@ -0,0 +1,36 @@
FROM node:slim
# Install needed packages
RUN apt-get update && apt-get install -y curl && apt-get clean
RUN useradd -ms /bin/bash frappe
# Make bench directories
RUN mkdir -p /home/frappe/frappe-bench/sites /home/frappe/frappe-bench/apps/frappe
COPY build/frappe-socketio/package.json /home/frappe/frappe-bench/apps/frappe
# get socketio
RUN cd /home/frappe/frappe-bench/apps/frappe \
&& curl https://raw.githubusercontent.com/frappe/frappe/version-13-beta/socketio.js \
--output /home/frappe/frappe-bench/apps/frappe/socketio.js \
&& curl https://raw.githubusercontent.com/frappe/frappe/version-13-beta/node_utils.js \
--output /home/frappe/frappe-bench/apps/frappe/node_utils.js
RUN cd /home/frappe/frappe-bench/apps/frappe \
&& npm install --only=production \
&& node --version \
&& npm --version
COPY build/frappe-socketio/health.js /home/frappe/frappe-bench/apps/frappe/health.js
RUN chown -R frappe:frappe /home/frappe
# Setup docker-entrypoint
COPY build/frappe-socketio/docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh
RUN ln -s /usr/local/bin/docker-entrypoint.sh / # backwards compat
WORKDIR /home/frappe/frappe-bench/sites
ENTRYPOINT ["docker-entrypoint.sh"]
CMD ["start"]

View File

@ -0,0 +1,48 @@
FROM bitnami/python:latest-prod
RUN useradd -ms /bin/bash frappe
WORKDIR /home/frappe/frappe-bench
RUN install_packages \
git \
wkhtmltopdf \
mariadb-client \
gettext-base \
wget \
# for PDF
libssl-dev \
fonts-cantarell \
xfonts-75dpi \
xfonts-base \
# For psycopg2
libpq-dev \
build-essential
# Install wkhtmltox correctly
RUN wget https://github.com/wkhtmltopdf/wkhtmltopdf/releases/download/0.12.5/wkhtmltox_0.12.5-1.stretch_amd64.deb
RUN dpkg -i wkhtmltox_0.12.5-1.stretch_amd64.deb && rm wkhtmltox_0.12.5-1.stretch_amd64.deb
RUN mkdir -p apps logs commands /home/frappe/backups
RUN virtualenv env \
&& . env/bin/activate \
&& cd apps \
&& git clone --depth 1 -o upstream https://github.com/frappe/frappe --branch version-13-beta \
&& pip3 install --no-cache-dir -e /home/frappe/frappe-bench/apps/frappe
COPY build/common/commands/* /home/frappe/frappe-bench/commands/
COPY build/common/common_site_config.json.template /opt/frappe/common_site_config.json.template
# Setup docker-entrypoint
COPY build/common/worker/docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh
RUN ln -s /usr/local/bin/docker-entrypoint.sh / # backwards compat
COPY build/common/worker/install_app.sh /usr/local/bin/install_app
WORKDIR /home/frappe/frappe-bench/sites
RUN chown -R frappe:frappe /home/frappe/frappe-bench/sites /home/frappe/backups
VOLUME [ "/home/frappe/frappe-bench/sites", "/home/frappe/backups" ]
ENTRYPOINT ["docker-entrypoint.sh"]
CMD ["start"]

View File

@ -0,0 +1,6 @@
version: "3"
services:
erpnext-nginx:
ports:
- "80:80"

View File

@ -0,0 +1,6 @@
version: "3"
services:
frappe-nginx:
ports:
- "80:80"

216
tests/docker-test.sh Executable file
View File

@ -0,0 +1,216 @@
#!/bin/bash
function checkMigrationComplete() {
echo "Check Auto Migration"
CONTAINER_ID=$(docker-compose \
--project-name frappebench00 \
-f installation/docker-compose-common.yml \
-f installation/docker-compose-erpnext.yml \
-f installation/erpnext-publish.yml \
ps -q erpnext-python)
DOCKER_LOG=$(docker logs $CONTAINER_ID 2>&1 | grep "Starting gunicorn")
INCREMENT=0
while [[ $DOCKER_LOG != *"Starting gunicorn"* && $INCREMENT -lt 60 ]]; do
sleep 3
echo "Wait for migration to complete ..."
((INCREMENT=INCREMENT+1))
DOCKER_LOG=$(docker logs $CONTAINER_ID 2>&1 | grep "Starting gunicorn")
if [[ $DOCKER_LOG != *"Starting gunicorn"* && $INCREMENT -eq 60 ]]; then
docker logs $CONTAINER_ID
exit 1
fi
done
}
function loopHealthCheck() {
echo "Create Container to Check MariaDB"
docker run --name frappe_doctor \
-v frappebench00_sites-vol:/home/frappe/frappe-bench/sites \
--network frappebench00_default \
frappe/erpnext-worker:edge doctor || true
echo "Loop Health Check"
FRAPPE_LOG=$(docker logs frappe_doctor | grep "Health check successful" || echo "")
while [[ -z "$FRAPPE_LOG" ]]; do
sleep 1
CONTAINER=$(docker start frappe_doctor)
echo "Restarting $CONTAINER ..."
FRAPPE_LOG=$(docker logs frappe_doctor | grep "Health check successful" || echo "")
done
echo "Health check successful"
}
echo "Copy env-example file"
cp env-example .env
echo "Set version to v12"
sed -i -e "s/edge/v12/g" .env
echo "Start Services"
docker-compose \
--project-name frappebench00 \
-f installation/docker-compose-common.yml \
-f installation/docker-compose-erpnext.yml \
-f installation/erpnext-publish.yml \
up -d
loopHealthCheck
echo "Create new site (v12)"
docker run -it \
-e "SITE_NAME=test.localhost" \
-e "INSTALL_APPS=erpnext" \
-v frappebench00_sites-vol:/home/frappe/frappe-bench/sites \
--network frappebench00_default \
frappe/erpnext-worker:v12 new
echo "Ping created site"
curl -S http://test.localhost/api/method/version
echo ""
echo "Set version to edge"
sed -i -e "s/v12/edge/g" .env
echo "Restart containers with edge image"
docker-compose \
--project-name frappebench00 \
-f installation/docker-compose-common.yml \
-f installation/docker-compose-erpnext.yml \
-f installation/erpnext-publish.yml \
stop
docker-compose \
--project-name frappebench00 \
-f installation/docker-compose-common.yml \
-f installation/docker-compose-erpnext.yml \
-f installation/erpnext-publish.yml \
up -d
checkMigrationComplete
echo "Ping migrated site"
sleep 3
curl -S http://test.localhost/api/method/version
echo ""
echo "Backup site"
docker run -it \
-e "WITH_FILES=1" \
-v frappebench00_sites-vol:/home/frappe/frappe-bench/sites \
--network frappebench00_default \
frappe/erpnext-worker:edge backup
export MINIO_ACCESS_KEY="AKIAIOSFODNN7EXAMPLE"
export MINIO_SECRET_KEY="wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"
echo "Start MinIO container for s3 compatible storage"
docker run -d --name minio \
-e "MINIO_ACCESS_KEY=$MINIO_ACCESS_KEY" \
-e "MINIO_SECRET_KEY=$MINIO_SECRET_KEY" \
--network frappebench00_default \
minio/minio server /data
echo "Create bucket named erpnext after 3 sec."
sleep 3
docker run \
--network frappebench00_default \
vltgroup/s3cmd:latest s3cmd --access_key=$MINIO_ACCESS_KEY \
--secret_key=$MINIO_SECRET_KEY \
--region=us-east-1 \
--no-ssl \
--host=minio:9000 \
--host-bucket=minio:9000 \
mb s3://erpnext
echo "Push backup to MinIO s3"
docker run \
-e BUCKET_NAME=erpnext \
-e REGION=us-east-1 \
-e BUCKET_DIR=local \
-e ACCESS_KEY_ID=$MINIO_ACCESS_KEY \
-e SECRET_ACCESS_KEY=$MINIO_SECRET_KEY \
-e ENDPOINT_URL=http://minio:9000 \
-v frappebench00_sites-vol:/home/frappe/frappe-bench/sites \
--network frappebench00_default \
frappe/erpnext-worker:edge push-backup
echo "Stop Services"
docker-compose \
--project-name frappebench00 \
-f installation/docker-compose-common.yml \
-f installation/docker-compose-erpnext.yml \
-f installation/erpnext-publish.yml \
stop
echo "Prune Containers"
docker container prune -f && docker volume prune -f
echo "Start Services"
docker-compose \
--project-name frappebench00 \
-f installation/docker-compose-common.yml \
-f installation/docker-compose-erpnext.yml \
-f installation/erpnext-publish.yml \
up -d
loopHealthCheck
echo "Restore backup from MinIO / S3"
docker run \
-e MYSQL_ROOT_PASSWORD=admin \
-e BUCKET_NAME=erpnext \
-e BUCKET_DIR=local \
-e ACCESS_KEY_ID=$MINIO_ACCESS_KEY \
-e SECRET_ACCESS_KEY=$MINIO_SECRET_KEY \
-e ENDPOINT_URL=http://minio:9000 \
-e REGION=us-east-1 \
-v frappebench00_sites-vol:/home/frappe/frappe-bench/sites \
--network frappebench00_default \
frappe/erpnext-worker:edge restore-backup
echo "Check Restored Site"
sleep 3
RESTORE_STATUS=$(curl -S http://test.localhost/api/method/version || echo "")
INCREMENT=0
while [[ -z "$RESTORE_STATUS" && $INCREMENT -lt 60 ]]; do
sleep 1
echo "Wait for restoration to complete ..."
RESTORE_STATUS=$(curl -S http://test.localhost/api/method/version || echo "")
((INCREMENT=INCREMENT+1))
if [[ -z "$RESTORE_STATUS" && $INCREMENT -eq 60 ]]; then
CONTAINER_ID=$(docker-compose \
--project-name frappebench00 \
-f installation/docker-compose-common.yml \
-f installation/docker-compose-erpnext.yml \
-f installation/erpnext-publish.yml \
ps -q erpnext-python)
docker logs $CONTAINER_ID
exit 1
fi
done
echo "Ping restored site"
echo $RESTORE_STATUS
echo "Migrate command in edge container"
docker run -it \
-e "MAINTENANCE_MODE=1" \
-v frappebench00_sites-vol:/home/frappe/frappe-bench/sites \
--network frappebench00_default \
frappe/erpnext-worker:edge migrate
checkMigrationComplete
echo "Create new site (edge)"
docker run -it \
-e "SITE_NAME=edge.localhost" \
-e "INSTALL_APPS=erpnext" \
-v frappebench00_sites-vol:/home/frappe/frappe-bench/sites \
--network frappebench00_default \
frappe/erpnext-worker:edge new
echo "Check console command for site test.localhost"
docker run \
-v frappebench00_sites-vol:/home/frappe/frappe-bench/sites \
--network frappebench00_default \
frappe/erpnext-worker:edge console test.localhost

View File

@ -15,7 +15,7 @@ image_type.add_argument('-s', '--socketio', action='store_const', dest='image_ty
image_type.add_argument('-w', '--worker', action='store_const', dest='image_type', const='worker', help='Build the python environment image')
tag_type = parser.add_mutually_exclusive_group(required=True)
tag_type.add_argument('-g', '--git-version', action='store', type=int, dest='version', help='The version number of --service (i.e. "11", "12", etc.)')
tag_type.add_argument('-g', '--git-version', action='store', type=str, dest='version', help='The version number of --service (i.e. "11", "12", etc.)')
tag_type.add_argument('-t', '--tag', action='store', type=str, dest='tag', help='The image tag (i.e. erpnext-worker:$TAG )')
args = parser.parse_args()
@ -26,6 +26,10 @@ def git_version(service, version):
cd = os.getcwd()
os.chdir(os.getcwd() + f'/{service}')
subprocess.run('git fetch --tags', shell=True)
# XX-beta becomes XX for tags search
version = version.split('-')[0]
version_tag = subprocess.check_output(f'git tag --list --sort=-version:refname "v{version}*" | sed -n 1p | sed -e \'s#.*@\(\)#\\1#\'', shell=True).strip().decode('ascii')
os.chdir(cd)
return version_tag