mirror of
https://github.com/frappe/bench.git
synced 2025-02-09 22:28:26 +00:00
[test] python site-packages
This commit is contained in:
Anand Doshi
parent
073bc1c0bd
commit
26356f2b3e
@ -13,4 +13,4 @@ script:
|
||||
- cd ~
|
||||
- sudo pip install --upgrade pip
|
||||
- sudo pip install -e bench-repo
|
||||
- sudo python -m unittest bench.tests.test_setup_production
|
||||
- sudo python -m unittest bench.tests.test_setup_production.TestSetupProduction.test_setup_production
|
||||
|
||||
@ -42,7 +42,7 @@ def write_appstxt(apps, bench='.'):
|
||||
|
||||
def get_app(app, git_url, branch=None, bench='.', build_asset_files=True, verbose=False):
|
||||
logger.info('getting app {}'.format(app))
|
||||
shallow_clone = '--depth 1' if check_git_for_shallow_clone() and get_config().get('shallow_clone') else ''
|
||||
shallow_clone = '--depth 1' if check_git_for_shallow_clone() and get_config(bench).get('shallow_clone') else ''
|
||||
branch = '--branch {branch}'.format(branch=branch) if branch else ''
|
||||
exec_cmd("git clone {git_url} {branch} {shallow_clone} --origin upstream {app}".format(
|
||||
git_url=git_url,
|
||||
@ -81,7 +81,7 @@ def install_app(app, bench='.', verbose=False):
|
||||
add_to_appstxt(app, bench=bench)
|
||||
|
||||
def pull_all_apps(bench='.'):
|
||||
rebase = '--rebase' if get_config().get('rebase_on_pull') else ''
|
||||
rebase = '--rebase' if get_config(bench).get('rebase_on_pull') else ''
|
||||
|
||||
for app in get_apps(bench=bench):
|
||||
app_dir = get_repo_dir(app, bench=bench)
|
||||
|
||||
@ -13,7 +13,8 @@ from .utils import set_default_site as _set_default_site
|
||||
from .utils import (build_assets, patch_sites, exec_cmd, update_bench, get_env_cmd, get_frappe, setup_logging,
|
||||
get_config, update_config, restart_supervisor_processes, put_config, default_config, update_requirements,
|
||||
backup_all_sites, backup_site, get_sites, prime_wheel_cache, is_root, set_mariadb_host, drop_privileges,
|
||||
fix_file_perms, fix_prod_setup_perms, set_ssl_certificate, set_ssl_certificate_key, get_cmd_output, post_upgrade,
|
||||
fix_file_perms, fix_prod_setup_perms, set_ssl_certificate, set_ssl_certificate_key,
|
||||
get_cmd_output, post_upgrade, get_bench_name,
|
||||
pre_upgrade, validate_upgrade, PatchError, download_translations_p, setup_socketio, before_update)
|
||||
from .app import get_app as _get_app
|
||||
from .app import new_app as _new_app
|
||||
@ -582,7 +583,8 @@ def patch():
|
||||
def _fix_prod_perms():
|
||||
"Fix permissions if supervisor processes were run as root"
|
||||
if os.path.exists("config/supervisor.conf"):
|
||||
exec_cmd("supervisorctl stop frappe:")
|
||||
bench_name = get_bench_name(bench_path=".")
|
||||
exec_cmd("supervisorctl stop {bench_name}-processes:".format(bench_name=bench_name))
|
||||
|
||||
fix_prod_setup_perms()
|
||||
|
||||
|
||||
@ -6,7 +6,7 @@ import shutil
|
||||
import socket
|
||||
from distutils.spawn import find_executable
|
||||
from jinja2 import Environment, PackageLoader
|
||||
from .utils import get_sites, get_config, update_config, get_redis_version, update_common_site_config
|
||||
from .utils import get_sites, get_config, update_config, get_redis_version, update_common_site_config, get_bench_name
|
||||
|
||||
env = Environment(loader=PackageLoader('bench', 'templates'), trim_blocks=True)
|
||||
|
||||
@ -45,8 +45,8 @@ def generate_supervisor_config(bench='.', user=None):
|
||||
"redis_celery_broker_config": os.path.join(bench_dir, 'config', 'redis_celery_broker.conf'),
|
||||
"frappe_version": get_current_frappe_version(),
|
||||
"webserver_port": config.get('webserver_port', 8000),
|
||||
"n_workers": config.get('max_workers', 2),
|
||||
"bench_name": os.path.basename(os.path.abspath(bench))
|
||||
"gunicorn_workers": config.get('gunicorn_workers', 2),
|
||||
"bench_name": get_bench_name(bench)
|
||||
})
|
||||
write_config_file(bench, 'supervisor.conf', config)
|
||||
update_config({'restart_supervisor_on_update': True})
|
||||
@ -114,7 +114,7 @@ def generate_nginx_config(bench='.'):
|
||||
"sites": sites,
|
||||
"webserver_port": config.get('webserver_port', 8000),
|
||||
"socketio_port": config.get('socketio_port', 3000),
|
||||
"bench_name": os.path.basename(os.path.abspath(bench))
|
||||
"bench_name": get_bench_name(bench)
|
||||
})
|
||||
write_config_file(bench, 'nginx.conf', config)
|
||||
|
||||
@ -157,7 +157,7 @@ def _generate_redis_config(template_name, context, bench):
|
||||
template = env.get_template(template_name)
|
||||
|
||||
if "pid_path" not in context:
|
||||
context["pid_path"] = os.path.abspath(os.path.join(bench, "config", "files"))
|
||||
context["pid_path"] = os.path.abspath(os.path.join(bench, "config", "pids"))
|
||||
|
||||
redis_config = template.render(**context)
|
||||
write_config_file(bench, template_name, redis_config)
|
||||
|
||||
@ -1,4 +1,4 @@
|
||||
from .utils import get_program, exec_cmd, get_cmd_output, fix_prod_setup_perms, get_config
|
||||
from .utils import get_program, exec_cmd, get_cmd_output, fix_prod_setup_perms, get_config, get_bench_name
|
||||
from .config import generate_nginx_config, generate_supervisor_config
|
||||
from jinja2 import Environment, PackageLoader
|
||||
import os
|
||||
@ -55,7 +55,7 @@ def setup_production(user, bench='.'):
|
||||
fix_prod_setup_perms(bench, frappe_user=user)
|
||||
remove_default_nginx_configs()
|
||||
|
||||
bench_name = os.path.basename(os.path.abspath(bench))
|
||||
bench_name = get_bench_name(bench)
|
||||
nginx_conf = '/etc/nginx/conf.d/{bench_name}.conf'.format(bench_name=bench_name)
|
||||
|
||||
if is_centos7():
|
||||
|
||||
@ -173,7 +173,7 @@ def bump(repo, bump_type, develop='develop', master='master', remote='upstream')
|
||||
print 'Released {tag} for {repo}'.format(tag=tag_name, repo=repo)
|
||||
|
||||
def release(repo, bump_type, develop, master):
|
||||
if not get_config().get('release_bench'):
|
||||
if not get_config(".").get('release_bench'):
|
||||
print 'bench not configured to release'
|
||||
sys.exit(1)
|
||||
global github_username, github_password
|
||||
|
||||
@ -3,7 +3,7 @@
|
||||
; killasgroup=true --> send kill signal to child processes too
|
||||
|
||||
[program:{{ bench_name }}-frappe-web]
|
||||
command={{ bench_dir }}/env/bin/gunicorn -b 127.0.0.1:{{ webserver_port }} -w {{ n_workers }} -t {{ http_timeout }} frappe.app:application --preload
|
||||
command={{ bench_dir }}/env/bin/gunicorn -b 127.0.0.1:{{ webserver_port }} -w {{ gunicorn_workers }} -t {{ http_timeout }} frappe.app:application --preload
|
||||
priority=4
|
||||
autostart=true
|
||||
autorestart=true
|
||||
@ -98,7 +98,7 @@ autorestart=true
|
||||
stdout_logfile={{ bench_dir }}/logs/node-socketio.log
|
||||
stderr_logfile={{ bench_dir }}/logs/node-socketio.error.log
|
||||
user={{ user }}
|
||||
directory={{ sites_dir }}
|
||||
directory={{ bench_dir }}
|
||||
{% endif %}
|
||||
|
||||
{% endif %}
|
||||
|
||||
@ -15,7 +15,7 @@ class TestBenchInit(unittest.TestCase):
|
||||
for bench_name in self.benches:
|
||||
bench_path = os.path.join(self.benches_path, bench_name)
|
||||
if os.path.exists(bench_path):
|
||||
shutil.rmtree(bench_path)
|
||||
shutil.rmtree(bench_path, ignore_errors=True)
|
||||
|
||||
def test_init(self, bench_name="test-bench"):
|
||||
self.init_bench(bench_name)
|
||||
@ -78,11 +78,19 @@ class TestBenchInit(unittest.TestCase):
|
||||
self.assert_exists(bench_name, "apps", "frappe", "setup.py")
|
||||
|
||||
def assert_virtual_env(self, bench_name):
|
||||
self.assert_exists(bench_name, "env", "lib", "python2.7")
|
||||
self.assert_exists(bench_name, "env", "lib", "python2.7", "site-packages")
|
||||
self.assert_exists(bench_name, "env", "lib", "python2.7", "site-packages", "IPython")
|
||||
self.assert_exists(bench_name, "env", "lib", "python2.7", "site-packages", "MySQL_python-1.2.5.dist-info")
|
||||
self.assert_exists(bench_name, "env", "lib", "python2.7", "site-packages", "pip")
|
||||
bench_path = os.path.abspath(bench_name)
|
||||
python = os.path.join(bench_path, "env", "bin", "python")
|
||||
python_path = bench.utils.get_cmd_output('{python} -c "import os; print os.path.dirname(os.__file__)"'.format(python=python))
|
||||
|
||||
# part of bench's virtualenv
|
||||
self.assertTrue(python_path.startswith(bench_path))
|
||||
self.assert_exists(python_path)
|
||||
self.assert_exists(python_path, "site-packages")
|
||||
self.assert_exists(python_path, "site-packages", "IPython")
|
||||
self.assert_exists(python_path, "site-packages", "pip")
|
||||
|
||||
site_packages = os.listdir(os.path.join(python_path, "site-packages"))
|
||||
self.assertTrue(any(package.startswith("MySQL_python-1.2.5") for package in site_packages))
|
||||
|
||||
def assert_bench_config(self, bench_name):
|
||||
config_json = os.path.join(bench_name, "config.json")
|
||||
|
||||
@ -1,11 +1,12 @@
|
||||
from __future__ import unicode_literals
|
||||
from bench.tests.test_init import TestBenchInit
|
||||
from bench.production_setup import setup_production
|
||||
from bench.production_setup import setup_production, get_supervisor_confdir
|
||||
import bench.utils
|
||||
import os
|
||||
import getpass
|
||||
import re
|
||||
import unittest
|
||||
import time
|
||||
|
||||
class TestSetupProduction(TestBenchInit):
|
||||
# setUp, tearDown and other tests are defiend in TestBenchInit
|
||||
@ -20,6 +21,8 @@ class TestSetupProduction(TestBenchInit):
|
||||
self.assert_nginx_config(bench_name)
|
||||
self.assert_supervisor_config(bench_name)
|
||||
|
||||
# test after start of both benches
|
||||
for bench_name in ("test-bench-1", "test-bench-2"):
|
||||
self.assert_supervisor_process(bench_name)
|
||||
|
||||
self.assert_nginx_process()
|
||||
@ -39,14 +42,16 @@ class TestSetupProduction(TestBenchInit):
|
||||
f = f.read().decode("utf-8")
|
||||
|
||||
for key in (
|
||||
"upstream { bench_name }-frappe",
|
||||
"upstream { bench_name}-socketio-server"
|
||||
"upstream {bench_name}-frappe",
|
||||
"upstream {bench_name}-socketio-server"
|
||||
):
|
||||
self.assertTrue(key.format(bench_name) in f)
|
||||
self.assertTrue(key.format(bench_name=bench_name) in f)
|
||||
|
||||
def assert_supervisor_config(self, bench_name):
|
||||
conf_src = os.path.join(os.path.abspath(bench_name), 'config', 'supervisor.conf')
|
||||
conf_dest = "/etc/supervisor.d/{bench_name}.conf".format(bench_name=bench_name)
|
||||
|
||||
supervisor_conf_dir = get_supervisor_confdir()
|
||||
conf_dest = "{supervisor_conf_dir}/{bench_name}.conf".format(supervisor_conf_dir=supervisor_conf_dir, bench_name=bench_name)
|
||||
|
||||
self.assertTrue(os.path.exists(conf_src))
|
||||
self.assertTrue(os.path.exists(conf_dest))
|
||||
@ -59,37 +64,42 @@ class TestSetupProduction(TestBenchInit):
|
||||
f = f.read().decode("utf-8")
|
||||
|
||||
for key in (
|
||||
"program:{ bench_name }-frappe-web",
|
||||
"program:{ bench_name }-frappe-worker",
|
||||
"program:{ bench_name }-frappe-longjob-worker",
|
||||
"program:{ bench_name }-frappe-async-worker",
|
||||
"program:{ bench_name }-frappe-workerbeat",
|
||||
"program:{ bench_name }-redis-cache",
|
||||
"program:{ bench_name }-redis-celery-broker",
|
||||
"program:{ bench_name }-redis-async-broker",
|
||||
"program:{ bench_name }-node-socketio",
|
||||
"group:{ bench_name }-processes",
|
||||
"group:{ bench_name }-redis"
|
||||
"program:{bench_name}-frappe-web",
|
||||
"program:{bench_name}-frappe-worker",
|
||||
"program:{bench_name}-frappe-longjob-worker",
|
||||
"program:{bench_name}-frappe-async-worker",
|
||||
"program:{bench_name}-frappe-workerbeat",
|
||||
"program:{bench_name}-redis-cache",
|
||||
"program:{bench_name}-redis-celery-broker",
|
||||
"program:{bench_name}-redis-async-broker",
|
||||
"program:{bench_name}-node-socketio",
|
||||
"group:{bench_name}-processes",
|
||||
"group:{bench_name}-redis"
|
||||
):
|
||||
self.assertTrue(key.format(bench_name) in f)
|
||||
self.assertTrue(key.format(bench_name=bench_name) in f)
|
||||
|
||||
def assert_supervisor_process(self, bench_name):
|
||||
out = bench.utils.get_cmd_output("sudo supervisorctl status")
|
||||
|
||||
if "STARTING" in out:
|
||||
time.sleep(10)
|
||||
out = bench.utils.get_cmd_output("sudo supervisorctl status")
|
||||
|
||||
for key in (
|
||||
"{bench_name}-process:{ bench_name }-frappe-web[\s]*RUNNING",
|
||||
"{bench_name}-process:{ bench_name }-frappe-worker[\s]*RUNNING",
|
||||
"{bench_name}-process:{ bench_name }-frappe-longjob-worker[\s]*RUNNING",
|
||||
"{bench_name}-process:{ bench_name }-frappe-async-worker[\s]*RUNNING",
|
||||
"{bench_name}-process:{ bench_name }-frappe-workerbeat[\s]*RUNNING",
|
||||
"{bench_name}-process:{ bench_name }-node-socketio[\s]*RUNNING",
|
||||
"{bench_name}-redis:{ bench_name }-redis-cache[\s]*RUNNING",
|
||||
"{bench_name}-redis:{bench_name}-redis-celery-broker[\s]*RUNNING",
|
||||
"{bench_name}-redis:{bench_name}-redis-async-broker[\s]*RUNNING",
|
||||
"{bench_name}-processes:{bench_name}-frappe-web[\s]+RUNNING",
|
||||
"{bench_name}-processes:{bench_name}-frappe-worker[\s]+RUNNING",
|
||||
"{bench_name}-processes:{bench_name}-frappe-longjob-worker[\s]+RUNNING",
|
||||
"{bench_name}-processes:{bench_name}-frappe-async-worker[\s]+RUNNING",
|
||||
"{bench_name}-processes:{bench_name}-frappe-workerbeat[\s]+RUNNING",
|
||||
"{bench_name}-processes:{bench_name}-node-socketio[\s]+RUNNING",
|
||||
"{bench_name}-redis:{bench_name}-redis-cache[\s]+RUNNING",
|
||||
"{bench_name}-redis:{bench_name}-redis-celery-broker[\s]+RUNNING",
|
||||
"{bench_name}-redis:{bench_name}-redis-async-broker[\s]+RUNNING",
|
||||
):
|
||||
self.assertTrue(re.match(key.format(bench_name), out))
|
||||
self.assertTrue(re.search(key.format(bench_name=bench_name), out))
|
||||
|
||||
def assert_nginx_process(self):
|
||||
out = bench.utils.get_cmd_output("sudo nginx -t")
|
||||
out = bench.utils.get_cmd_output("sudo nginx -t 2>&1")
|
||||
self.assertTrue("nginx: configuration file /etc/nginx/nginx.conf test is successful" in out)
|
||||
|
||||
|
||||
|
||||
@ -97,16 +97,15 @@ def make_bench_config():
|
||||
bench_config = {}
|
||||
bench_config.update(default_config)
|
||||
bench_config.update(make_ports())
|
||||
bench_config.update(get_max_worker_count())
|
||||
bench_config.update(get_gunicorn_workers())
|
||||
|
||||
return bench_config
|
||||
|
||||
def get_max_worker_count():
|
||||
def get_gunicorn_workers():
|
||||
'''This function will return the maximum workers that can be started depending upon
|
||||
number of cpu's present on the machine'''
|
||||
n_cpus = multiprocessing.cpu_count()
|
||||
return {
|
||||
"max_workers": (2 * n_cpus) + 1
|
||||
"gunicorn_workers": multiprocessing.cpu_count()
|
||||
}
|
||||
|
||||
def make_ports(benches_path="."):
|
||||
@ -357,7 +356,9 @@ def get_cmd_output(cmd, cwd='.'):
|
||||
|
||||
def restart_supervisor_processes(bench='.'):
|
||||
conf = get_config(bench=bench)
|
||||
cmd = conf.get('supervisor_restart_cmd', 'sudo supervisorctl restart frappe:')
|
||||
bench_name = get_bench_name(bench)
|
||||
cmd = conf.get('supervisor_restart_cmd',
|
||||
'sudo supervisorctl restart {bench_name}-processes:'.format(bench_name=bench_name))
|
||||
exec_cmd(cmd, cwd=bench)
|
||||
|
||||
def get_site_config(site, bench='.'):
|
||||
@ -718,3 +719,6 @@ def validate_pillow_dependencies(bench, requirements):
|
||||
print "sudo apt-get install -y libtiff5-dev libjpeg8-dev zlib1g-dev libfreetype6-dev liblcms2-dev libwebp-dev tcl8.6-dev tk8.6-dev python-tk"
|
||||
|
||||
raise
|
||||
|
||||
def get_bench_name(bench_path):
|
||||
return os.path.basename(os.path.abspath(bench_path))
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user