From 26356f2b3ef98afdb9bd84ef82b4e5565ea7c589 Mon Sep 17 00:00:00 2001 From: Anand Doshi Date: Fri, 26 Feb 2016 12:28:01 +0530 Subject: [PATCH] [test] python site-packages --- .travis.yml | 2 +- bench/app.py | 4 +- bench/cli.py | 6 ++- bench/config.py | 10 ++--- bench/production_setup.py | 4 +- bench/release.py | 2 +- bench/templates/supervisor.conf | 4 +- bench/tests/test_init.py | 20 ++++++--- bench/tests/test_setup_production.py | 66 ++++++++++++++++------------ bench/utils.py | 14 +++--- 10 files changed, 78 insertions(+), 54 deletions(-) diff --git a/.travis.yml b/.travis.yml index 77b331e0..bb23ac9d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,4 +13,4 @@ script: - cd ~ - sudo pip install --upgrade pip - sudo pip install -e bench-repo - - sudo python -m unittest bench.tests.test_setup_production + - sudo python -m unittest bench.tests.test_setup_production.TestSetupProduction.test_setup_production diff --git a/bench/app.py b/bench/app.py index 0b0ac760..bf8717c9 100644 --- a/bench/app.py +++ b/bench/app.py @@ -42,7 +42,7 @@ def write_appstxt(apps, bench='.'): def get_app(app, git_url, branch=None, bench='.', build_asset_files=True, verbose=False): logger.info('getting app {}'.format(app)) - shallow_clone = '--depth 1' if check_git_for_shallow_clone() and get_config().get('shallow_clone') else '' + shallow_clone = '--depth 1' if check_git_for_shallow_clone() and get_config(bench).get('shallow_clone') else '' branch = '--branch {branch}'.format(branch=branch) if branch else '' exec_cmd("git clone {git_url} {branch} {shallow_clone} --origin upstream {app}".format( git_url=git_url, @@ -81,7 +81,7 @@ def install_app(app, bench='.', verbose=False): add_to_appstxt(app, bench=bench) def pull_all_apps(bench='.'): - rebase = '--rebase' if get_config().get('rebase_on_pull') else '' + rebase = '--rebase' if get_config(bench).get('rebase_on_pull') else '' for app in get_apps(bench=bench): app_dir = get_repo_dir(app, bench=bench) diff --git a/bench/cli.py b/bench/cli.py index f86358d7..622c0918 100644 --- a/bench/cli.py +++ b/bench/cli.py @@ -13,7 +13,8 @@ from .utils import set_default_site as _set_default_site from .utils import (build_assets, patch_sites, exec_cmd, update_bench, get_env_cmd, get_frappe, setup_logging, get_config, update_config, restart_supervisor_processes, put_config, default_config, update_requirements, backup_all_sites, backup_site, get_sites, prime_wheel_cache, is_root, set_mariadb_host, drop_privileges, - fix_file_perms, fix_prod_setup_perms, set_ssl_certificate, set_ssl_certificate_key, get_cmd_output, post_upgrade, + fix_file_perms, fix_prod_setup_perms, set_ssl_certificate, set_ssl_certificate_key, + get_cmd_output, post_upgrade, get_bench_name, pre_upgrade, validate_upgrade, PatchError, download_translations_p, setup_socketio, before_update) from .app import get_app as _get_app from .app import new_app as _new_app @@ -582,7 +583,8 @@ def patch(): def _fix_prod_perms(): "Fix permissions if supervisor processes were run as root" if os.path.exists("config/supervisor.conf"): - exec_cmd("supervisorctl stop frappe:") + bench_name = get_bench_name(bench_path=".") + exec_cmd("supervisorctl stop {bench_name}-processes:".format(bench_name=bench_name)) fix_prod_setup_perms() diff --git a/bench/config.py b/bench/config.py index fe474d84..bd104d5b 100644 --- a/bench/config.py +++ b/bench/config.py @@ -6,7 +6,7 @@ import shutil import socket from distutils.spawn import find_executable from jinja2 import Environment, PackageLoader -from .utils import get_sites, get_config, update_config, get_redis_version, update_common_site_config +from .utils import get_sites, get_config, update_config, get_redis_version, update_common_site_config, get_bench_name env = Environment(loader=PackageLoader('bench', 'templates'), trim_blocks=True) @@ -45,8 +45,8 @@ def generate_supervisor_config(bench='.', user=None): "redis_celery_broker_config": os.path.join(bench_dir, 'config', 'redis_celery_broker.conf'), "frappe_version": get_current_frappe_version(), "webserver_port": config.get('webserver_port', 8000), - "n_workers": config.get('max_workers', 2), - "bench_name": os.path.basename(os.path.abspath(bench)) + "gunicorn_workers": config.get('gunicorn_workers', 2), + "bench_name": get_bench_name(bench) }) write_config_file(bench, 'supervisor.conf', config) update_config({'restart_supervisor_on_update': True}) @@ -114,7 +114,7 @@ def generate_nginx_config(bench='.'): "sites": sites, "webserver_port": config.get('webserver_port', 8000), "socketio_port": config.get('socketio_port', 3000), - "bench_name": os.path.basename(os.path.abspath(bench)) + "bench_name": get_bench_name(bench) }) write_config_file(bench, 'nginx.conf', config) @@ -157,7 +157,7 @@ def _generate_redis_config(template_name, context, bench): template = env.get_template(template_name) if "pid_path" not in context: - context["pid_path"] = os.path.abspath(os.path.join(bench, "config", "files")) + context["pid_path"] = os.path.abspath(os.path.join(bench, "config", "pids")) redis_config = template.render(**context) write_config_file(bench, template_name, redis_config) diff --git a/bench/production_setup.py b/bench/production_setup.py index 234ee320..08e9853a 100644 --- a/bench/production_setup.py +++ b/bench/production_setup.py @@ -1,4 +1,4 @@ -from .utils import get_program, exec_cmd, get_cmd_output, fix_prod_setup_perms, get_config +from .utils import get_program, exec_cmd, get_cmd_output, fix_prod_setup_perms, get_config, get_bench_name from .config import generate_nginx_config, generate_supervisor_config from jinja2 import Environment, PackageLoader import os @@ -55,7 +55,7 @@ def setup_production(user, bench='.'): fix_prod_setup_perms(bench, frappe_user=user) remove_default_nginx_configs() - bench_name = os.path.basename(os.path.abspath(bench)) + bench_name = get_bench_name(bench) nginx_conf = '/etc/nginx/conf.d/{bench_name}.conf'.format(bench_name=bench_name) if is_centos7(): diff --git a/bench/release.py b/bench/release.py index 87d44951..5bbbe637 100755 --- a/bench/release.py +++ b/bench/release.py @@ -173,7 +173,7 @@ def bump(repo, bump_type, develop='develop', master='master', remote='upstream') print 'Released {tag} for {repo}'.format(tag=tag_name, repo=repo) def release(repo, bump_type, develop, master): - if not get_config().get('release_bench'): + if not get_config(".").get('release_bench'): print 'bench not configured to release' sys.exit(1) global github_username, github_password diff --git a/bench/templates/supervisor.conf b/bench/templates/supervisor.conf index 136e4cb1..555e4ebb 100644 --- a/bench/templates/supervisor.conf +++ b/bench/templates/supervisor.conf @@ -3,7 +3,7 @@ ; killasgroup=true --> send kill signal to child processes too [program:{{ bench_name }}-frappe-web] -command={{ bench_dir }}/env/bin/gunicorn -b 127.0.0.1:{{ webserver_port }} -w {{ n_workers }} -t {{ http_timeout }} frappe.app:application --preload +command={{ bench_dir }}/env/bin/gunicorn -b 127.0.0.1:{{ webserver_port }} -w {{ gunicorn_workers }} -t {{ http_timeout }} frappe.app:application --preload priority=4 autostart=true autorestart=true @@ -98,7 +98,7 @@ autorestart=true stdout_logfile={{ bench_dir }}/logs/node-socketio.log stderr_logfile={{ bench_dir }}/logs/node-socketio.error.log user={{ user }} -directory={{ sites_dir }} +directory={{ bench_dir }} {% endif %} {% endif %} diff --git a/bench/tests/test_init.py b/bench/tests/test_init.py index 303032d4..923ec0c9 100644 --- a/bench/tests/test_init.py +++ b/bench/tests/test_init.py @@ -15,7 +15,7 @@ class TestBenchInit(unittest.TestCase): for bench_name in self.benches: bench_path = os.path.join(self.benches_path, bench_name) if os.path.exists(bench_path): - shutil.rmtree(bench_path) + shutil.rmtree(bench_path, ignore_errors=True) def test_init(self, bench_name="test-bench"): self.init_bench(bench_name) @@ -78,11 +78,19 @@ class TestBenchInit(unittest.TestCase): self.assert_exists(bench_name, "apps", "frappe", "setup.py") def assert_virtual_env(self, bench_name): - self.assert_exists(bench_name, "env", "lib", "python2.7") - self.assert_exists(bench_name, "env", "lib", "python2.7", "site-packages") - self.assert_exists(bench_name, "env", "lib", "python2.7", "site-packages", "IPython") - self.assert_exists(bench_name, "env", "lib", "python2.7", "site-packages", "MySQL_python-1.2.5.dist-info") - self.assert_exists(bench_name, "env", "lib", "python2.7", "site-packages", "pip") + bench_path = os.path.abspath(bench_name) + python = os.path.join(bench_path, "env", "bin", "python") + python_path = bench.utils.get_cmd_output('{python} -c "import os; print os.path.dirname(os.__file__)"'.format(python=python)) + + # part of bench's virtualenv + self.assertTrue(python_path.startswith(bench_path)) + self.assert_exists(python_path) + self.assert_exists(python_path, "site-packages") + self.assert_exists(python_path, "site-packages", "IPython") + self.assert_exists(python_path, "site-packages", "pip") + + site_packages = os.listdir(os.path.join(python_path, "site-packages")) + self.assertTrue(any(package.startswith("MySQL_python-1.2.5") for package in site_packages)) def assert_bench_config(self, bench_name): config_json = os.path.join(bench_name, "config.json") diff --git a/bench/tests/test_setup_production.py b/bench/tests/test_setup_production.py index dda0e4a9..01e6bb39 100644 --- a/bench/tests/test_setup_production.py +++ b/bench/tests/test_setup_production.py @@ -1,11 +1,12 @@ from __future__ import unicode_literals from bench.tests.test_init import TestBenchInit -from bench.production_setup import setup_production +from bench.production_setup import setup_production, get_supervisor_confdir import bench.utils import os import getpass import re import unittest +import time class TestSetupProduction(TestBenchInit): # setUp, tearDown and other tests are defiend in TestBenchInit @@ -20,6 +21,8 @@ class TestSetupProduction(TestBenchInit): self.assert_nginx_config(bench_name) self.assert_supervisor_config(bench_name) + # test after start of both benches + for bench_name in ("test-bench-1", "test-bench-2"): self.assert_supervisor_process(bench_name) self.assert_nginx_process() @@ -39,14 +42,16 @@ class TestSetupProduction(TestBenchInit): f = f.read().decode("utf-8") for key in ( - "upstream { bench_name }-frappe", - "upstream { bench_name}-socketio-server" + "upstream {bench_name}-frappe", + "upstream {bench_name}-socketio-server" ): - self.assertTrue(key.format(bench_name) in f) + self.assertTrue(key.format(bench_name=bench_name) in f) def assert_supervisor_config(self, bench_name): conf_src = os.path.join(os.path.abspath(bench_name), 'config', 'supervisor.conf') - conf_dest = "/etc/supervisor.d/{bench_name}.conf".format(bench_name=bench_name) + + supervisor_conf_dir = get_supervisor_confdir() + conf_dest = "{supervisor_conf_dir}/{bench_name}.conf".format(supervisor_conf_dir=supervisor_conf_dir, bench_name=bench_name) self.assertTrue(os.path.exists(conf_src)) self.assertTrue(os.path.exists(conf_dest)) @@ -59,37 +64,42 @@ class TestSetupProduction(TestBenchInit): f = f.read().decode("utf-8") for key in ( - "program:{ bench_name }-frappe-web", - "program:{ bench_name }-frappe-worker", - "program:{ bench_name }-frappe-longjob-worker", - "program:{ bench_name }-frappe-async-worker", - "program:{ bench_name }-frappe-workerbeat", - "program:{ bench_name }-redis-cache", - "program:{ bench_name }-redis-celery-broker", - "program:{ bench_name }-redis-async-broker", - "program:{ bench_name }-node-socketio", - "group:{ bench_name }-processes", - "group:{ bench_name }-redis" + "program:{bench_name}-frappe-web", + "program:{bench_name}-frappe-worker", + "program:{bench_name}-frappe-longjob-worker", + "program:{bench_name}-frappe-async-worker", + "program:{bench_name}-frappe-workerbeat", + "program:{bench_name}-redis-cache", + "program:{bench_name}-redis-celery-broker", + "program:{bench_name}-redis-async-broker", + "program:{bench_name}-node-socketio", + "group:{bench_name}-processes", + "group:{bench_name}-redis" ): - self.assertTrue(key.format(bench_name) in f) + self.assertTrue(key.format(bench_name=bench_name) in f) def assert_supervisor_process(self, bench_name): out = bench.utils.get_cmd_output("sudo supervisorctl status") + + if "STARTING" in out: + time.sleep(10) + out = bench.utils.get_cmd_output("sudo supervisorctl status") + for key in ( - "{bench_name}-process:{ bench_name }-frappe-web[\s]*RUNNING", - "{bench_name}-process:{ bench_name }-frappe-worker[\s]*RUNNING", - "{bench_name}-process:{ bench_name }-frappe-longjob-worker[\s]*RUNNING", - "{bench_name}-process:{ bench_name }-frappe-async-worker[\s]*RUNNING", - "{bench_name}-process:{ bench_name }-frappe-workerbeat[\s]*RUNNING", - "{bench_name}-process:{ bench_name }-node-socketio[\s]*RUNNING", - "{bench_name}-redis:{ bench_name }-redis-cache[\s]*RUNNING", - "{bench_name}-redis:{bench_name}-redis-celery-broker[\s]*RUNNING", - "{bench_name}-redis:{bench_name}-redis-async-broker[\s]*RUNNING", + "{bench_name}-processes:{bench_name}-frappe-web[\s]+RUNNING", + "{bench_name}-processes:{bench_name}-frappe-worker[\s]+RUNNING", + "{bench_name}-processes:{bench_name}-frappe-longjob-worker[\s]+RUNNING", + "{bench_name}-processes:{bench_name}-frappe-async-worker[\s]+RUNNING", + "{bench_name}-processes:{bench_name}-frappe-workerbeat[\s]+RUNNING", + "{bench_name}-processes:{bench_name}-node-socketio[\s]+RUNNING", + "{bench_name}-redis:{bench_name}-redis-cache[\s]+RUNNING", + "{bench_name}-redis:{bench_name}-redis-celery-broker[\s]+RUNNING", + "{bench_name}-redis:{bench_name}-redis-async-broker[\s]+RUNNING", ): - self.assertTrue(re.match(key.format(bench_name), out)) + self.assertTrue(re.search(key.format(bench_name=bench_name), out)) def assert_nginx_process(self): - out = bench.utils.get_cmd_output("sudo nginx -t") + out = bench.utils.get_cmd_output("sudo nginx -t 2>&1") self.assertTrue("nginx: configuration file /etc/nginx/nginx.conf test is successful" in out) diff --git a/bench/utils.py b/bench/utils.py index 19c513ab..daac593a 100644 --- a/bench/utils.py +++ b/bench/utils.py @@ -97,16 +97,15 @@ def make_bench_config(): bench_config = {} bench_config.update(default_config) bench_config.update(make_ports()) - bench_config.update(get_max_worker_count()) + bench_config.update(get_gunicorn_workers()) return bench_config -def get_max_worker_count(): +def get_gunicorn_workers(): '''This function will return the maximum workers that can be started depending upon number of cpu's present on the machine''' - n_cpus = multiprocessing.cpu_count() return { - "max_workers": (2 * n_cpus) + 1 + "gunicorn_workers": multiprocessing.cpu_count() } def make_ports(benches_path="."): @@ -357,7 +356,9 @@ def get_cmd_output(cmd, cwd='.'): def restart_supervisor_processes(bench='.'): conf = get_config(bench=bench) - cmd = conf.get('supervisor_restart_cmd', 'sudo supervisorctl restart frappe:') + bench_name = get_bench_name(bench) + cmd = conf.get('supervisor_restart_cmd', + 'sudo supervisorctl restart {bench_name}-processes:'.format(bench_name=bench_name)) exec_cmd(cmd, cwd=bench) def get_site_config(site, bench='.'): @@ -718,3 +719,6 @@ def validate_pillow_dependencies(bench, requirements): print "sudo apt-get install -y libtiff5-dev libjpeg8-dev zlib1g-dev libfreetype6-dev liblcms2-dev libwebp-dev tcl8.6-dev tk8.6-dev python-tk" raise + +def get_bench_name(bench_path): + return os.path.basename(os.path.abspath(bench_path))