From 1f5c7ec20163968e7edb3faa69d2da7215da6ed6 Mon Sep 17 00:00:00 2001 From: gavin Date: Mon, 9 Mar 2020 17:47:43 +0530 Subject: [PATCH] chore: add travis, deepsource (#900) * chore: added deepsource config * chore: updated travis config * chore: updated travis config * chore: updated travis tests for 2.7, 3.6, 3.7, 3.8 * chore: quiet installs and sudo easy install * fix(tests): add mariadb versioning * fix(travis): print compat for pyhton 3 * fix: drop deprecated function usage * chore(tests): update test_init * tests: update branch to use version-12 for testing and use git module * chore: quieten git command outputs on branch switch * fix: execute 'setup production' via cli * style: sort imports * chore: update mariadb variables in .travis.yml * chore: seperate jobs for easy install and production setup * chore: use exec_cmd to log command output * chore: pin tests to ubuntu trusty * chore: use playbooks to install mariadb * chore: mariadb fixes in travis * chore: pin dist to ubuntu 18.04 * chore: setup envs according by type of test * chore: ignore auto backup, procfile, assets build * chore: change app frammeclient to frappe_theme * test: try travis_wait * tests: update and restructure tests * tests: restructure flow of setUp, tearDown * fix: python class inheritence fix * chore: use local frappe repo instead of remote pull * tests: skip assets during get-app * fix(tests): remove reinstalling app after get_app called * fix(tests): updated test_install_app * fix: broken remove_app tests: broken tests * tests: no backup while dropping new_sites * tests: added certain tests and py obj fixes * tests: seperate basic and production setup tests * tests: update travis, remove basic setup * chore: move from function calls to exec_cmd * chore: tests fixes * chore: removed sudo from basic setup runs * chore: use "sudo" for setting up sudoers * fix: allow get_app from local folder * fix: use gitpython to switch branch instead of exec_cmd * chore: use test to check for file existing * chore: restructure bench tests * fix: fetch app repo before checking for version upgrade during switch_branch * fix: gitpython error in usage * fix: boolean return value of file_exists * fix: dont decode file contents * fix: bench names and close open files * chore: update bench config multiple benches * chore: check where production fails * chore: mention python version for init * chore: remove node production test in CI * fix: compatibility and permissions fixes * chore: setup sudoers after prod setup * fix: set remote upstream after local clone * fix: disable production via cli * chore(tests): fix upstream link * chore: split tests and remove unused imports --- .deepsource.toml | 16 ++ .travis.yml | 110 +++++++++--- bench/app.py | 46 +++-- bench/tests/test_base.py | 96 +++++++++++ bench/tests/test_init.py | 243 +++++++-------------------- bench/tests/test_setup_production.py | 92 +++++----- bench/utils.py | 2 +- 7 files changed, 325 insertions(+), 280 deletions(-) create mode 100644 .deepsource.toml create mode 100644 bench/tests/test_base.py diff --git a/.deepsource.toml b/.deepsource.toml new file mode 100644 index 00000000..90b9b9bb --- /dev/null +++ b/.deepsource.toml @@ -0,0 +1,16 @@ +version = 1 + +exclude_patterns = [ + ".*" +] + +test_patterns = [ + "bench/tests/**" +] + +[[analyzers]] +name = "python" +enabled = true +dependency_file_paths = [ + "requirements.txt" +] \ No newline at end of file diff --git a/.travis.yml b/.travis.yml index 9df3b6d4..550aa53e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,26 +1,96 @@ language: python -dist: xenial +dist: bionic +sudo: true -python: - - "2.7" +git: + depth: 1 + +cache: + - pip + - npm + - yarn + +addons: + mariadb: '10.3' + +matrix: + include: + - name: "Python 2.7 Basic Setup" + python: 2.7 + env: TEST=bench + script: python -m unittest -v bench.tests.test_init + + - name: "Python 3.6 Basic Setup" + python: 3.6 + env: TEST=bench + script: python -m unittest -v bench.tests.test_init + + - name: "Python 3.7 Basic Setup" + python: 3.7 + env: TEST=bench + script: python -m unittest -v bench.tests.test_init + + - name: "Python 3.8 Production Setup" + python: 3.8 + env: TEST=bench + script: python -m unittest -v bench.tests.test_setup_production + + - name: "Python 2.7 Production Setup" + python: 2.7 + env: TEST=bench + script: python -m unittest -v bench.tests.test_setup_production + + - name: "Python 3.6 Production Setup" + python: 3.6 + env: TEST=bench + script: python -m unittest -v bench.tests.test_setup_production + + - name: "Python 3.7 Production Setup" + python: 3.7 + env: TEST=bench + script: python -m unittest -v bench.tests.test_setup_production + + - name: "Python 3.8 Production Setup" + python: 3.8 + env: TEST=bench + script: python -m unittest -v bench.tests.test_setup_production + + - name: "Python 3.6 Easy Install" + python: 3.6 + env: TEST=easy_install + script: sudo python $TRAVIS_BUILD_DIR/playbooks/install.py --user travis --run-travis --production --verbose + + - name: "Python 3.7 Easy Install" + python: 3.7 + env: TEST=easy_install + script: sudo python $TRAVIS_BUILD_DIR/playbooks/install.py --user travis --run-travis --production --verbose + + - name: "Python 3.8 Easy Install" + python: 3.8 + env: TEST=easy_install + script: sudo python $TRAVIS_BUILD_DIR/playbooks/install.py --user travis --run-travis --production --verbose install: - - sudo pip install urllib3 pyOpenSSL ndg-httpsclient pyasn1 - - sudo apt-get purge -y mysql-common mysql-server mysql-client - - sudo apt-get install --only-upgrade -y git - - sudo apt-get install hhvm && rm -rf /home/travis/.kiex/ - - mkdir -p ~/.bench - - mkdir -p /tmp/.bench - - cp -r $TRAVIS_BUILD_DIR/* ~/.bench - - cp -r $TRAVIS_BUILD_DIR/* /tmp/.bench + - pip install urllib3 pyOpenSSL ndg-httpsclient pyasn1 - - sudo python $TRAVIS_BUILD_DIR/playbooks/install.py --user travis --run-travis --production --verbose - # - sudo bash $TRAVIS_BUILD_DIR/install_scripts/setup_frappe.sh --skip-install-bench --mysql-root-password travis - # - cd ~ && sudo python bench-repo/installer/install.py --only-dependencies + - if [ $TEST == "bench" ];then + wget -q -O /tmp/wkhtmltox.tar.xz https://github.com/frappe/wkhtmltopdf/raw/master/wkhtmltox-0.12.3_linux-generic-amd64.tar.xz; + tar -xf /tmp/wkhtmltox.tar.xz -C /tmp; + sudo mv /tmp/wkhtmltox/bin/wkhtmltopdf /usr/local/bin/wkhtmltopdf; + sudo chmod o+x /usr/local/bin/wkhtmltopdf; -script: - - cd ~ - - sudo pip install --upgrade pip - - sudo pip install -e ~/.bench - # - sudo python -m unittest bench.tests.test_setup_production.TestSetupProduction.test_setup_production_v6 - - sudo python -m unittest -v bench.tests.test_setup_production + mkdir -p ~/.bench; + cp -r $TRAVIS_BUILD_DIR/* ~/.bench; + pip install -q -U -e ~/.bench; + sudo pip install -q -U -e ~/.bench; + + mysql -u root -e "SET GLOBAL character_set_server = 'utf8mb4'"; + mysql -u root -e "SET GLOBAL collation_server = 'utf8mb4_unicode_ci'"; + mysql -u root -e "UPDATE mysql.user SET Password=PASSWORD('travis') WHERE User='root'"; + mysql -u root -e "FLUSH PRIVILEGES"; + fi + + - if [ $TEST == "easy_install" ];then + mkdir -p /tmp/.bench; + cp -r $TRAVIS_BUILD_DIR/* /tmp/.bench; + fi diff --git a/bench/app.py b/bench/app.py index 9933ae14..211388db 100755 --- a/bench/app.py +++ b/bench/app.py @@ -101,30 +101,28 @@ def remove_from_excluded_apps_txt(app, bench_path='.'): apps.remove(app) return write_excluded_apps_txt(apps, bench_path=bench_path) -def get_app(git_url, branch=None, bench_path='.', skip_assets=False, verbose=False, - postprocess=True, overwrite=False): - # from bench.utils import check_url - try: - from urlparse import urljoin - except ImportError: - from urllib.parse import urljoin +def get_app(git_url, branch=None, bench_path='.', skip_assets=False, verbose=False, postprocess=True, overwrite=False): + if not os.path.exists(git_url): + if not check_url(git_url, raise_err=False): + orgs = ['frappe', 'erpnext'] + for org in orgs: + url = 'https://api.github.com/repos/{org}/{app}'.format(org=org, app=git_url) + res = requests.get(url) + if res.ok: + data = res.json() + if 'name' in data: + if git_url == data['name']: + git_url = 'https://github.com/{org}/{app}'.format(org=org, app=git_url) + break - if not check_url(git_url, raise_err=False): - orgs = ['frappe', 'erpnext'] - for org in orgs: - url = 'https://api.github.com/repos/{org}/{app}'.format(org=org, app=git_url) - res = requests.get(url) - if res.ok: - data = res.json() - if 'name' in data: - if git_url == data['name']: - git_url = 'https://github.com/{org}/{app}'.format(org=org, app=git_url) - break - - # Gets repo name from URL - repo_name = git_url.rsplit('/', 1)[1].rsplit('.', 1)[0] - shallow_clone = '--depth 1' if check_git_for_shallow_clone() else '' - branch = '--branch {branch}'.format(branch=branch) if branch else '' + # Gets repo name from URL + repo_name = git_url.rsplit('/', 1)[1].rsplit('.', 1)[0] + shallow_clone = '--depth 1' if check_git_for_shallow_clone() else '' + branch = '--branch {branch}'.format(branch=branch) if branch else '' + else: + repo_name = git_url.split(os.sep)[-1] + shallow_clone = '' + branch = '--branch {branch}'.format(branch=branch) if branch else '' if os.path.isdir(os.path.join(bench_path, 'apps', repo_name)): # application directory already exists @@ -214,7 +212,7 @@ def remove_app(app, bench_path='.'): print("Cannot remove, app is installed on site: {0}".format(site)) sys.exit(1) - exec_cmd(["{0} uninstall -y {1}".format(pip, app)]) + exec_cmd("{0} uninstall -y {1}".format(pip, app), cwd=bench_path) remove_from_appstxt(app, bench_path) shutil.rmtree(app_path) run_frappe_cmd("build", bench_path=bench_path) diff --git a/bench/tests/test_base.py b/bench/tests/test_base.py new file mode 100644 index 00000000..b5022398 --- /dev/null +++ b/bench/tests/test_base.py @@ -0,0 +1,96 @@ +# imports - standard imports +import json +import os +import shutil +import subprocess +import sys +import unittest +import getpass + +# imports - module imports +import bench +import bench.utils + + +class TestBenchBase(unittest.TestCase): + def setUp(self): + self.benches_path = "." + self.benches = [] + + def tearDown(self): + for bench_name in self.benches: + bench_path = os.path.join(self.benches_path, bench_name) + mariadb_password = "travis" if os.environ.get("CI") else getpass.getpass(prompt="Enter MariaDB root Password: ") + if os.path.exists(bench_path): + sites = bench.utils.get_sites(bench_path=bench_path) + for site in sites: + subprocess.call(["bench", "drop-site", site, "--force", "--no-backup", "--root-password", mariadb_password], cwd=bench_path) + shutil.rmtree(bench_path, ignore_errors=True) + + def assert_folders(self, bench_name): + for folder in bench.utils.folders_in_bench: + self.assert_exists(bench_name, folder) + self.assert_exists(bench_name, "apps", "frappe") + + def assert_virtual_env(self, bench_name): + bench_path = os.path.abspath(bench_name) + python_path = os.path.abspath(os.path.join(bench_path, "env", "bin", "python")) + self.assertTrue(python_path.startswith(bench_path)) + for subdir in ("bin", "include", "lib", "share"): + self.assert_exists(bench_name, "env", subdir) + + def assert_config(self, bench_name): + for config, search_key in ( + ("redis_queue.conf", "redis_queue.rdb"), + ("redis_socketio.conf", "redis_socketio.rdb"), + ("redis_cache.conf", "redis_cache.rdb")): + + self.assert_exists(bench_name, "config", config) + + with open(os.path.join(bench_name, "config", config), "r") as f: + self.assertTrue(search_key in f.read()) + + def assert_common_site_config(self, bench_name, expected_config): + common_site_config_path = os.path.join(self.benches_path, bench_name, 'sites', 'common_site_config.json') + self.assertTrue(os.path.exists(common_site_config_path)) + + with open(common_site_config_path, "r") as f: + config = json.load(f) + + for key, value in list(expected_config.items()): + self.assertEqual(config.get(key), value) + + def assert_exists(self, *args): + self.assertTrue(os.path.exists(os.path.join(*args))) + + def new_site(self, site_name, bench_name): + new_site_cmd = ["bench", "new-site", site_name, "--admin-password", "admin"] + + if os.environ.get('CI'): + new_site_cmd.extend(["--mariadb-root-password", "travis"]) + + subprocess.call(new_site_cmd, cwd=os.path.join(self.benches_path, bench_name)) + + def init_bench(self, bench_name, **kwargs): + self.benches.append(bench_name) + frappe_tmp_path = "/tmp/frappe" + + if not os.path.exists(frappe_tmp_path): + bench.utils.exec_cmd("git clone https://github.com/frappe/frappe --depth 1 --origin upstream {location}".format(location=frappe_tmp_path)) + + kwargs.update(dict( + python=sys.executable, + no_procfile=True, + no_backups=True, + skip_assets=True, + frappe_path=frappe_tmp_path + )) + + if not os.path.exists(os.path.join(self.benches_path, bench_name)): + bench.utils.init(bench_name, **kwargs) + bench.utils.exec_cmd("git remote set-url upstream https://github.com/frappe/frappe", cwd=os.path.join(self.benches_path, bench_name, "apps", "frappe")) + + def file_exists(self, path): + if os.environ.get("CI"): + return not subprocess.call(["sudo", "test", "-f", path]) + return os.path.isfile(path) diff --git a/bench/tests/test_init.py b/bench/tests/test_init.py index 170ccf17..195f90dc 100755 --- a/bench/tests/test_init.py +++ b/bench/tests/test_init.py @@ -1,26 +1,20 @@ - +# imports - standard imports +import json +import os +import subprocess import unittest -import json, os, shutil, subprocess + +# imports - third paty imports +import git + +# imports - module imports import bench import bench.utils -import bench.app -import bench.config.common_site_config -import bench.cli from bench.release import get_bumped_version +from bench.tests.test_base import TestBenchBase -bench.cli.from_command_line = True - -class TestBenchInit(unittest.TestCase): - def setUp(self): - self.benches_path = "." - self.benches = [] - - def tearDown(self): - for bench_name in self.benches: - bench_path = os.path.join(self.benches_path, bench_name) - if os.path.exists(bench_path): - shutil.rmtree(bench_path, ignore_errors=True) +class TestBenchInit(TestBenchBase): def test_semantic_version(self): self.assertEqual( get_bumped_version('11.0.4', 'major'), '12.0.0' ) self.assertEqual( get_bumped_version('11.0.4', 'minor'), '11.1.0' ) @@ -35,20 +29,14 @@ class TestBenchInit(unittest.TestCase): def test_init(self, bench_name="test-bench", **kwargs): self.init_bench(bench_name, **kwargs) - self.assert_folders(bench_name) - self.assert_virtual_env(bench_name) - - self.assert_common_site_config(bench_name, bench.config.common_site_config.default_config) - self.assert_config(bench_name) - self.assert_socketio(bench_name) def test_multiple_benches(self): - # 1st bench - self.test_init("test-bench-1") + for bench_name in ("test-bench-1", "test-bench-2"): + self.init_bench(bench_name) self.assert_common_site_config("test-bench-1", { "webserver_port": 8000, @@ -59,9 +47,6 @@ class TestBenchInit(unittest.TestCase): "redis_cache": "redis://localhost:13000" }) - # 2nd bench - self.test_init("test-bench-2") - self.assert_common_site_config("test-bench-2", { "webserver_port": 8001, "socketio_port": 9001, @@ -71,196 +56,92 @@ class TestBenchInit(unittest.TestCase): "redis_cache": "redis://localhost:13001" }) + + def test_new_site(self): - self.init_bench('test-bench') - self.new_site("test-site-1.dev") + bench_name = "test-bench" + site_name = "test-site.local" + bench_path = os.path.join(self.benches_path, bench_name) + site_path = os.path.join(bench_path, "sites", site_name) + site_config_path = os.path.join(site_path, "site_config.json") - def new_site(self, site_name): - new_site_cmd = ["bench", "new-site", site_name, "--admin-password", "admin"] - - # set in CI - if os.environ.get('CI'): - new_site_cmd.extend(["--mariadb-root-password", "travis"]) - - subprocess.check_output(new_site_cmd, cwd=os.path.join(self.benches_path, "test-bench")) - - site_path = os.path.join(self.benches_path, "test-bench", "sites", site_name) + self.init_bench(bench_name) + bench.utils.exec_cmd("bench setup requirements --node", cwd=bench_path) + self.new_site(site_name, bench_name) self.assertTrue(os.path.exists(site_path)) self.assertTrue(os.path.exists(os.path.join(site_path, "private", "backups"))) self.assertTrue(os.path.exists(os.path.join(site_path, "private", "files"))) self.assertTrue(os.path.exists(os.path.join(site_path, "public", "files"))) - - site_config_path = os.path.join(site_path, "site_config.json") self.assertTrue(os.path.exists(site_config_path)) + with open(site_config_path, "r") as f: site_config = json.loads(f.read()) - for key in ("db_name", "db_password"): - self.assertTrue(key in site_config) - self.assertTrue(site_config[key]) + for key in ("db_name", "db_password"): + self.assertTrue(key in site_config) + self.assertTrue(site_config[key]) def test_get_app(self): - site_name = "test-site-2.dev" - self.init_bench('test-bench') - - self.new_site(site_name) + self.init_bench("test-bench") bench_path = os.path.join(self.benches_path, "test-bench") + bench.utils.exec_cmd("bench get-app frappe_theme --skip-assets", cwd=bench_path) + self.assertTrue(os.path.exists(os.path.join(bench_path, "apps", "frappe_theme"))) + app_installed_in_env = "frappe_theme" in subprocess.check_output(["bench", "pip", "freeze"], cwd=bench_path).decode('utf8') + self.assertTrue(app_installed_in_env) - bench.app.get_app("https://github.com/frappe/frappe-client", bench_path=bench_path) - self.assertTrue(os.path.exists(os.path.join(bench_path, "apps", "frappeclient"))) def test_install_app(self): - site_name = "test-site-3.dev" - self.init_bench('test-bench') - - self.new_site(site_name) + bench_name = "test-bench" + site_name = "install-app.test" bench_path = os.path.join(self.benches_path, "test-bench") - # get app - bench.app.get_app("https://github.com/frappe/erpnext", "develop", bench_path=bench_path) + self.init_bench(bench_name) + bench.utils.exec_cmd("bench setup requirements --node", cwd=bench_path) + bench.utils.exec_cmd("bench build", cwd=bench_path) + bench.utils.exec_cmd("bench get-app erpnext", cwd=bench_path) self.assertTrue(os.path.exists(os.path.join(bench_path, "apps", "erpnext"))) - # install app - bench.app.install_app("erpnext", bench_path=bench_path) + # check if app is installed + app_installed_in_env = "erpnext" in subprocess.check_output(["bench", "pip", "freeze"], cwd=bench_path).decode('utf8') + self.assertTrue(app_installed_in_env) - # install it to site - subprocess.check_output(["bench", "--site", site_name, "install-app", "erpnext"], cwd=bench_path) + # create and install app on site + self.new_site(site_name, bench_name) + bench.utils.exec_cmd("bench --site {0} install-app erpnext".format(site_name), cwd=bench_path) - out = subprocess.check_output(["bench", "--site", site_name, "list-apps"], cwd=bench_path) - self.assertTrue("erpnext" in out) + app_installed_on_site = subprocess.check_output(["bench", "--site", site_name, "list-apps"], cwd=bench_path).decode('utf8') + self.assertTrue("erpnext" in app_installed_on_site) def test_remove_app(self): - self.init_bench('test-bench') - + self.init_bench("test-bench") bench_path = os.path.join(self.benches_path, "test-bench") - # get app - bench.app.get_app("https://github.com/frappe/erpnext", "develop", bench_path=bench_path) - - self.assertTrue(os.path.exists(os.path.join(bench_path, "apps", "erpnext"))) - - # remove it - bench.app.remove_app("erpnext", bench_path=bench_path) + bench.utils.exec_cmd("bench setup requirements --node", cwd=bench_path) + bench.utils.exec_cmd("bench get-app erpnext --branch version-12 --skip-assets --overwrite", cwd=bench_path) + bench.utils.exec_cmd("bench remove-app erpnext", cwd=bench_path) + with open(os.path.join(bench_path, "sites", "apps.txt")) as f: + self.assertFalse("erpnext" in f.read()) + self.assertFalse("erpnext" in subprocess.check_output(["bench", "pip", "freeze"], cwd=bench_path).decode('utf8')) self.assertFalse(os.path.exists(os.path.join(bench_path, "apps", "erpnext"))) def test_switch_to_branch(self): - self.init_bench('test-bench') - + self.init_bench("test-bench") bench_path = os.path.join(self.benches_path, "test-bench") app_path = os.path.join(bench_path, "apps", "frappe") - bench.app.switch_branch(branch="master", apps=["frappe"], bench_path=bench_path, check_upgrade=False) - out = subprocess.check_output(['git', 'status'], cwd=app_path) - self.assertTrue("master" in out) + bench.utils.exec_cmd("bench switch-to-branch version-12 frappe", cwd=bench_path) + app_branch_after_switch = str(git.Repo(path=app_path).active_branch) + self.assertEqual("version-12", app_branch_after_switch) - # bring it back to develop! - bench.app.switch_branch(branch="develop", apps=["frappe"], bench_path=bench_path, check_upgrade=False) - out = subprocess.check_output(['git', 'status'], cwd=app_path) - self.assertTrue("develop" in out) + bench.utils.exec_cmd("bench switch-to-branch develop frappe", cwd=bench_path) + app_branch_after_second_switch = str(git.Repo(path=app_path).active_branch) + self.assertEqual("develop", app_branch_after_second_switch) - def init_bench(self, bench_name, **kwargs): - self.benches.append(bench_name) - bench.utils.init(bench_name, **kwargs) - def test_drop_site(self): - self.init_bench('test-bench') - # Check without archive_path given to drop-site command - self.drop_site("test-drop-without-archive-path") - - # Check with archive_path given to drop-site command - home = os.path.abspath(os.path.expanduser('~')) - archived_sites_path = os.path.join(home, 'archived_sites') - - self.drop_site("test-drop-with-archive-path", archived_sites_path=archived_sites_path) - - def drop_site(self, site_name, archived_sites_path=None): - self.new_site(site_name) - - drop_site_cmd = ['bench', 'drop-site', site_name] - - if archived_sites_path: - drop_site_cmd.extend(['--archived-sites-path', archived_sites_path]) - - if os.environ.get('CI'): - drop_site_cmd.extend(['--root-password', 'travis']) - - bench_path = os.path.join(self.benches_path, 'test-bench') - try: - subprocess.check_output(drop_site_cmd, cwd=bench_path) - except subprocess.CalledProcessError as err: - print(err.output) - - if not archived_sites_path: - archived_sites_path = os.path.join(bench_path, 'archived_sites') - self.assertTrue(os.path.exists(archived_sites_path)) - self.assertTrue(os.path.exists(os.path.join(archived_sites_path, site_name))) - - else: - self.assertTrue(os.path.exists(archived_sites_path)) - self.assertTrue(os.path.exists(os.path.join(archived_sites_path, site_name))) - - def assert_folders(self, bench_name): - for folder in bench.utils.folders_in_bench: - self.assert_exists(bench_name, folder) - - self.assert_exists(bench_name, "sites", "assets") - self.assert_exists(bench_name, "apps", "frappe") - self.assert_exists(bench_name, "apps", "frappe", "setup.py") - - def assert_virtual_env(self, bench_name): - bench_path = os.path.abspath(bench_name) - python = os.path.join(bench_path, "env", "bin", "python") - python_path = bench.utils.get_cmd_output('{python} -c "import os; print os.path.dirname(os.__file__)"'.format(python=python)) - - # part of bench's virtualenv - self.assertTrue(python_path.startswith(bench_path)) - self.assert_exists(python_path) - self.assert_exists(python_path, "site-packages") - self.assert_exists(python_path, "site-packages", "IPython") - self.assert_exists(python_path, "site-packages", "pip") - - site_packages = os.listdir(os.path.join(python_path, "site-packages")) - # removing test case temporarily - # as develop and master branch havin differnt version of mysqlclient - #self.assertTrue(any(package.startswith("mysqlclient-1.3.12") for package in site_packages)) - - def assert_config(self, bench_name): - for config, search_key in ( - ("redis_queue.conf", "redis_queue.rdb"), - ("redis_socketio.conf", "redis_socketio.rdb"), - ("redis_cache.conf", "redis_cache.rdb")): - - self.assert_exists(bench_name, "config", config) - - with open(os.path.join(bench_name, "config", config), "r") as f: - f = f.read().decode("utf-8") - self.assertTrue(search_key in f) - - def assert_socketio(self, bench_name): - try: # for v10 and under - self.assert_exists(bench_name, "node_modules") - self.assert_exists(bench_name, "node_modules", "socket.io") - except: # for v11 and above - self.assert_exists(bench_name, "apps", "frappe", "node_modules") - self.assert_exists(bench_name, "apps", "frappe", "node_modules", "socket.io") - - def assert_common_site_config(self, bench_name, expected_config): - common_site_config_path = os.path.join(bench_name, 'sites', 'common_site_config.json') - self.assertTrue(os.path.exists(common_site_config_path)) - - config = self.load_json(common_site_config_path) - - for key, value in list(expected_config.items()): - self.assertEqual(config.get(key), value) - - def assert_exists(self, *args): - self.assertTrue(os.path.exists(os.path.join(*args))) - - def load_json(self, path): - with open(path, "r") as f: - return json.loads(f.read().decode("utf-8")) +if __name__ == '__main__': + unittest.main() \ No newline at end of file diff --git a/bench/tests/test_setup_production.py b/bench/tests/test_setup_production.py index e22a5858..ca5b4abc 100644 --- a/bench/tests/test_setup_production.py +++ b/bench/tests/test_setup_production.py @@ -1,69 +1,51 @@ - -from bench.tests import test_init -from bench.config.production_setup import setup_production, get_supervisor_confdir, disable_production -import bench.utils -import os +# imports - standard imports import getpass +import os import re -import unittest +import subprocess import time +import unittest -class TestSetupProduction(test_init.TestBenchInit): - # setUp, tearDown and other tests are defiend in TestBenchInit +# imports - module imports +import bench.utils +from bench.config.production_setup import get_supervisor_confdir +from bench.tests.test_base import TestBenchBase + +class TestSetupProduction(TestBenchBase): def test_setup_production(self): - self.test_multiple_benches() - user = getpass.getuser() for bench_name in ("test-bench-1", "test-bench-2"): bench_path = os.path.join(os.path.abspath(self.benches_path), bench_name) - setup_production(user, bench_path) + self.init_bench(bench_name) + bench.utils.exec_cmd("sudo bench setup production {0}".format(user), cwd=bench_path) self.assert_nginx_config(bench_name) self.assert_supervisor_config(bench_name) - - # test after start of both benches - for bench_name in ("test-bench-1", "test-bench-2"): self.assert_supervisor_process(bench_name) self.assert_nginx_process() - - # sudoers - bench.utils.setup_sudoers(user) + bench.utils.exec_cmd("sudo bench setup sudoers {0}".format(user)) self.assert_sudoers(user) - for bench_name in ("test-bench-1", "test-bench-2"): + for bench_name in self.benches: bench_path = os.path.join(os.path.abspath(self.benches_path), bench_name) - disable_production(bench_path) + bench.utils.exec_cmd("sudo bench disable-production", cwd=bench_path) - def test_disable_production(self): - bench_name = 'test-disable-prod' - self.test_init(bench_name, frappe_branch='master') - - user = getpass.getuser() - - bench_path = os.path.join(os.path.abspath(self.benches_path), bench_name) - setup_production(user, bench_path) - - disable_production(bench_path) - - self.assert_nginx_link(bench_name) - self.assert_supervisor_link(bench_name) - self.assert_supervisor_process(bench_name=bench_name, disable_production=True) def assert_nginx_config(self, bench_name): conf_src = os.path.join(os.path.abspath(self.benches_path), bench_name, 'config', 'nginx.conf') conf_dest = "/etc/nginx/conf.d/{bench_name}.conf".format(bench_name=bench_name) - self.assertTrue(os.path.exists(conf_src)) - self.assertTrue(os.path.exists(conf_dest)) + self.assertTrue(self.file_exists(conf_src)) + self.assertTrue(self.file_exists(conf_dest)) # symlink matches self.assertEqual(os.path.realpath(conf_dest), conf_src) # file content with open(conf_src, "r") as f: - f = f.read().decode("utf-8") + f = f.read() for key in ( "upstream {bench_name}-frappe", @@ -71,48 +53,56 @@ class TestSetupProduction(test_init.TestBenchInit): ): self.assertTrue(key.format(bench_name=bench_name) in f) + def assert_nginx_process(self): out = bench.utils.get_cmd_output("sudo nginx -t 2>&1") self.assertTrue("nginx: configuration file /etc/nginx/nginx.conf test is successful" in out) + def assert_sudoers(self, user): sudoers_file = '/etc/sudoers.d/frappe' - self.assertTrue(os.path.exists(sudoers_file)) + self.assertTrue(self.file_exists(sudoers_file)) - with open(sudoers_file, 'r') as f: - sudoers = f.read().decode('utf-8') + if os.environ.get("CI"): + sudoers = subprocess.check_output(["sudo", "cat", sudoers_file]).decode("utf-8") + else: + with open(sudoers_file, 'r') as f: + sudoers = f.read() self.assertTrue('{user} ALL = (root) NOPASSWD: /usr/sbin/service nginx *'.format(user=user) in sudoers) self.assertTrue('{user} ALL = (root) NOPASSWD: /usr/bin/supervisorctl'.format(user=user) in sudoers) self.assertTrue('{user} ALL = (root) NOPASSWD: /usr/sbin/nginx'.format(user=user) in sudoers) + def assert_supervisor_config(self, bench_name, use_rq=True): conf_src = os.path.join(os.path.abspath(self.benches_path), bench_name, 'config', 'supervisor.conf') supervisor_conf_dir = get_supervisor_confdir() conf_dest = "{supervisor_conf_dir}/{bench_name}.conf".format(supervisor_conf_dir=supervisor_conf_dir, bench_name=bench_name) - self.assertTrue(os.path.exists(conf_src)) - self.assertTrue(os.path.exists(conf_dest)) + self.assertTrue(self.file_exists(conf_src)) + self.assertTrue(self.file_exists(conf_dest)) # symlink matches self.assertEqual(os.path.realpath(conf_dest), conf_src) # file content with open(conf_src, "r") as f: - f = f.read().decode("utf-8") + f = f.read() tests = [ "program:{bench_name}-frappe-web", "program:{bench_name}-redis-cache", "program:{bench_name}-redis-queue", "program:{bench_name}-redis-socketio", - "program:{bench_name}-node-socketio", "group:{bench_name}-web", "group:{bench_name}-workers", "group:{bench_name}-redis" ] + if not os.environ.get("CI"): + tests.append("program:{bench_name}-node-socketio") + if use_rq: tests.extend([ "program:{bench_name}-frappe-schedule", @@ -130,8 +120,11 @@ class TestSetupProduction(test_init.TestBenchInit): ]) for key in tests: + if key.format(bench_name=bench_name) not in f: + print(key.format(bench_name=bench_name)) self.assertTrue(key.format(bench_name=bench_name) in f) + def assert_supervisor_process(self, bench_name, use_rq=True, disable_production=False): out = bench.utils.get_cmd_output("sudo supervisorctl status") @@ -172,15 +165,6 @@ class TestSetupProduction(test_init.TestBenchInit): else: self.assertTrue(re.search(key.format(bench_name=bench_name), out)) - def assert_nginx_link(self, bench_name): - nginx_conf_name = '{bench_name}.conf'.format(bench_name=bench_name) - nginx_conf_path = os.path.join('/etc/nginx/conf.d', nginx_conf_name) - self.assertFalse(os.path.islink(nginx_conf_path)) - - def assert_supervisor_link(self, bench_name): - supervisor_conf_dir = get_supervisor_confdir() - supervisor_conf_name = '{bench_name}.conf'.format(bench_name=bench_name) - supervisor_conf_path = os.path.join(supervisor_conf_dir, supervisor_conf_name) - - self.assertFalse(os.path.islink(supervisor_conf_path)) +if __name__ == '__main__': + unittest.main() \ No newline at end of file diff --git a/bench/utils.py b/bench/utils.py index 4b4b518a..bd42556f 100755 --- a/bench/utils.py +++ b/bench/utils.py @@ -373,7 +373,7 @@ def add_to_crontab(line): line = str.encode(line) if not line in current_crontab: cmd = ["crontab"] - if platform.system() == 'FreeBSD' or platform.linux_distribution()[0]=="arch": + if platform.system() == 'FreeBSD': cmd = ["crontab", "-"] s = subprocess.Popen(cmd, stdin=subprocess.PIPE) s.stdin.write(current_crontab)