accounting/bench
2
0
Fork 0
mirror of https://github.com/frappe/bench.git synced 2024-11-12 08:16:28 +00:00
Code Issues Releases Activity

Merge pull request #1157 from gavindsouza/b-update-breaking-patch

Browse Source 
fix: Misc fixes
This commit is contained in:
gavin 2021-04-23 12:21:08 +05:30 committed by GitHub
commit 0480b898ae
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 76 additions and 79 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
bench/app.py
View File

@ -443,9 +443,6 @@ def switch_branch(branch, apps=None, bench_path='.', upgrade=False, check_upgrad
def switch_to_branch(branch=None, apps=None, bench_path='.', upgrade=False):
switch_branch(branch, apps=apps, bench_path=bench_path, upgrade=upgrade)
def switch_to_master(apps=None, bench_path='.', upgrade=True):
switch_branch('master', apps=apps, bench_path=bench_path, upgrade=upgrade)
def switch_to_develop(apps=None, bench_path='.', upgrade=True):
switch_branch('develop', apps=apps, bench_path=bench_path, upgrade=upgrade)

3
bench/commands/__init__.py
View File

@ -27,11 +27,10 @@ bench_command.add_command(include_app_for_update)
bench_command.add_command(pip)
from bench.commands.update import update, retry_upgrade, switch_to_branch, switch_to_master, switch_to_develop
from bench.commands.update import update, retry_upgrade, switch_to_branch, switch_to_develop
bench_command.add_command(update)
bench_command.add_command(retry_upgrade)
bench_command.add_command(switch_to_branch)
bench_command.add_command(switch_to_master)
bench_command.add_command(switch_to_develop)

6
bench/commands/update.py
View File

@ -41,12 +41,6 @@ def switch_to_branch(branch, apps, upgrade=False):
switch_to_branch(branch=branch, apps=list(apps), upgrade=upgrade)
@click.command('switch-to-master', help="[DEPRECATED]: Switch frappe and erpnext to master branch")
def switch_to_master():
from bench.utils import log
log("`switch-to-master` has been deprecated as master branches were renamed to version-11")
@click.command('switch-to-develop')
def switch_to_develop(upgrade=False):
"Switch frappe and erpnext to develop branch"

4
bench/config/procfile.py
View File

@ -8,7 +8,7 @@ import click
import bench
from bench.app import use_rq
from bench.config.common_site_config import get_config
from bench.utils import find_executable
from bench.utils import which
def setup_procfile(bench_path, yes=False, skip_redis=False):
@ -19,7 +19,7 @@ def setup_procfile(bench_path, yes=False, skip_redis=False):
abort=True)
procfile = bench.config.env().get_template('Procfile').render(
node=find_executable("node") or find_executable("nodejs"),
node=which("node") or which("nodejs"),
use_rq=use_rq(bench_path),
webserver_port=config.get('webserver_port'),
CI=os.environ.get('CI'),

18
bench/config/production_setup.py
View File

@ -9,7 +9,7 @@ from bench.config.common_site_config import get_config
from bench.config.nginx import make_nginx_conf
from bench.config.supervisor import generate_supervisor_config, update_supervisord_config
from bench.config.systemd import generate_systemd_config
from bench.utils import CommandFailedError, exec_cmd, find_executable, fix_prod_setup_perms, get_bench_name, get_cmd_output, log
from bench.utils import CommandFailedError, exec_cmd, which, fix_prod_setup_perms, get_bench_name, get_cmd_output, log
logger = logging.getLogger(bench.PROJECT_NAME)
@ -17,13 +17,13 @@ logger = logging.getLogger(bench.PROJECT_NAME)
def setup_production_prerequisites():
"""Installs ansible, fail2banc, NGINX and supervisor"""
if not find_executable("ansible"):
if not which("ansible"):
exec_cmd("sudo {0} -m pip install ansible".format(sys.executable))
if not find_executable("fail2ban-client"):
if not which("fail2ban-client"):
exec_cmd("bench setup role fail2ban")
if not find_executable("nginx"):
if not which("nginx"):
exec_cmd("bench setup role nginx")
if not find_executable("supervisord"):
if not which("supervisord"):
exec_cmd("bench setup role supervisor")
@ -95,11 +95,11 @@ def disable_production(bench_path='.'):
def service(service_name, service_option):
if os.path.basename(find_executable('systemctl') or '') == 'systemctl' and is_running_systemd():
if os.path.basename(which('systemctl') or '') == 'systemctl' and is_running_systemd():
systemctl_cmd = "sudo {service_manager} {service_option} {service_name}"
exec_cmd(systemctl_cmd.format(service_manager='systemctl', service_option=service_option, service_name=service_name))
elif os.path.basename(find_executable('service') or '') == 'service':
elif os.path.basename(which('service') or '') == 'service':
service_cmd = "sudo {service_manager} {service_name} {service_option}"
exec_cmd(service_cmd.format(service_manager='service', service_name=service_name, service_option=service_option))
@ -145,7 +145,7 @@ def is_running_systemd():
def reload_supervisor():
supervisorctl = find_executable('supervisorctl')
supervisorctl = which('supervisorctl')
try:
# first try reread/update
@ -178,7 +178,7 @@ def reload_supervisor():
def reload_nginx():
try:
exec_cmd('sudo {0} -t'.format(find_executable('nginx')))
exec_cmd('sudo {0} -t'.format(which('nginx')))
except:
raise

8
bench/config/supervisor.py
View File

@ -6,7 +6,7 @@ import os
# imports - module imports
import bench
from bench.app import use_rq
from bench.utils import get_bench_name, find_executable
from bench.utils import get_bench_name, which
from bench.config.common_site_config import get_config, update_config, get_gunicorn_workers
# imports - third party imports
@ -31,8 +31,8 @@ def generate_supervisor_config(bench_path, user=None, yes=False, skip_redis=Fals
"user": user,
"use_rq": use_rq(bench_path),
"http_timeout": config.get("http_timeout", 120),
"redis_server": find_executable('redis-server'),
"node": find_executable('node') or find_executable('nodejs'),
"redis_server": which('redis-server'),
"node": which('node') or which('nodejs'),
"redis_cache_config": os.path.join(bench_dir, 'config', 'redis_cache.conf'),
"redis_socketio_config": os.path.join(bench_dir, 'config', 'redis_socketio.conf'),
"redis_queue_config": os.path.join(bench_dir, 'config', 'redis_queue.conf'),
@ -40,7 +40,7 @@ def generate_supervisor_config(bench_path, user=None, yes=False, skip_redis=Fals
"gunicorn_workers": config.get('gunicorn_workers', get_gunicorn_workers()["gunicorn_workers"]),
"bench_name": get_bench_name(bench_path),
"background_workers": config.get('background_workers') or 1,
"bench_cmd": find_executable('bench'),
"bench_cmd": which('bench'),
"skip_redis": skip_redis,
})

8
bench/config/systemd.py
View File

@ -9,7 +9,7 @@ import click
import bench
from bench.app import use_rq
from bench.config.common_site_config import get_config, get_gunicorn_workers, update_config
from bench.utils import exec_cmd, find_executable, get_bench_name
from bench.utils import exec_cmd, which, get_bench_name
def generate_systemd_config(bench_path, user=None, yes=False,
@ -53,8 +53,8 @@ def generate_systemd_config(bench_path, user=None, yes=False,
"user": user,
"use_rq": use_rq(bench_path),
"http_timeout": config.get("http_timeout", 120),
"redis_server": find_executable('redis-server'),
"node": find_executable('node') or find_executable('nodejs'),
"redis_server": which('redis-server'),
"node": which('node') or which('nodejs'),
"redis_cache_config": os.path.join(bench_dir, 'config', 'redis_cache.conf'),
"redis_socketio_config": os.path.join(bench_dir, 'config', 'redis_socketio.conf'),
"redis_queue_config": os.path.join(bench_dir, 'config', 'redis_queue.conf'),
@ -62,7 +62,7 @@ def generate_systemd_config(bench_path, user=None, yes=False,
"gunicorn_workers": config.get('gunicorn_workers', get_gunicorn_workers()["gunicorn_workers"]),
"bench_name": get_bench_name(bench_path),
"worker_target_wants": " ".join(background_workers),
"bench_cmd": find_executable('bench')
"bench_cmd": which('bench')
}
if not yes:

1
bench/tests/test_base.py
View File

@ -90,7 +90,6 @@ class TestBenchBase(unittest.TestCase):
python=sys.executable,
no_procfile=True,
no_backups=True,
skip_assets=True,
frappe_path=frappe_tmp_path
))

24
bench/tests/test_init.py
View File

@ -97,7 +97,7 @@ class TestBenchInit(TestBenchBase):
def test_get_app(self):
self.init_bench("test-bench")
bench_path = os.path.join(self.benches_path, "test-bench")
bench.utils.exec_cmd("bench get-app frappe_theme --skip-assets", cwd=bench_path)
bench.utils.exec_cmd("bench get-app frappe_theme", cwd=bench_path)
self.assertTrue(os.path.exists(os.path.join(bench_path, "apps", "frappe_theme")))
app_installed_in_env = "frappe_theme" in subprocess.check_output(["bench", "pip", "freeze"], cwd=bench_path).decode('utf8')
self.assertTrue(app_installed_in_env)
@ -111,22 +111,22 @@ class TestBenchInit(TestBenchBase):
self.init_bench(bench_name)
bench.utils.exec_cmd("bench setup requirements --node", cwd=bench_path)
bench.utils.exec_cmd("bench build", cwd=bench_path)
bench.utils.exec_cmd("bench get-app erpnext --branch {0}".format(FRAPPE_BRANCH), cwd=bench_path)
bench.utils.exec_cmd("bench get-app frappe_theme --branch master", cwd=bench_path)
self.assertTrue(os.path.exists(os.path.join(bench_path, "apps", "erpnext")))
self.assertTrue(os.path.exists(os.path.join(bench_path, "apps", "frappe_theme")))
# check if app is installed
app_installed_in_env = "erpnext" in subprocess.check_output(["bench", "pip", "freeze"], cwd=bench_path).decode('utf8')
app_installed_in_env = "frappe_theme" in subprocess.check_output(["bench", "pip", "freeze"], cwd=bench_path).decode('utf8')
self.assertTrue(app_installed_in_env)
# create and install app on site
self.new_site(site_name, bench_name)
installed_erpnext = not bench.utils.exec_cmd("bench --site {0} install-app erpnext".format(site_name), cwd=bench_path)
installed_app = not bench.utils.exec_cmd("bench --site {0} install-app frappe_theme".format(site_name), cwd=bench_path)
app_installed_on_site = subprocess.check_output(["bench", "--site", site_name, "list-apps"], cwd=bench_path).decode('utf8')
if installed_erpnext:
self.assertTrue("erpnext" in app_installed_on_site)
if installed_app:
self.assertTrue("frappe_theme" in app_installed_on_site)
def test_remove_app(self):
@ -134,13 +134,13 @@ class TestBenchInit(TestBenchBase):
bench_path = os.path.join(self.benches_path, "test-bench")
bench.utils.exec_cmd("bench setup requirements --node", cwd=bench_path)
bench.utils.exec_cmd("bench get-app erpnext --branch version-12 --skip-assets --overwrite", cwd=bench_path)
bench.utils.exec_cmd("bench remove-app erpnext", cwd=bench_path)
bench.utils.exec_cmd("bench get-app frappe_theme --branch master --overwrite", cwd=bench_path)
bench.utils.exec_cmd("bench remove-app frappe_theme", cwd=bench_path)
with open(os.path.join(bench_path, "sites", "apps.txt")) as f:
self.assertFalse("erpnext" in f.read())
self.assertFalse("erpnext" in subprocess.check_output(["bench", "pip", "freeze"], cwd=bench_path).decode('utf8'))
self.assertFalse(os.path.exists(os.path.join(bench_path, "apps", "erpnext")))
self.assertFalse("frappe_theme" in f.read())
self.assertFalse("frappe_theme" in subprocess.check_output(["bench", "pip", "freeze"], cwd=bench_path).decode('utf8'))
self.assertFalse(os.path.exists(os.path.join(bench_path, "apps", "frappe_theme")))
def test_switch_to_branch(self):

80
bench/utils.py
View File

@ -2,26 +2,17 @@
# -*- coding: utf-8 -*-
# imports - standard imports
import compileall
import errno
import glob
import grp
import itertools
import json
import logging
import os
import pwd
import re
import select
import site
import subprocess
import sys
from datetime import datetime
from distutils.spawn import find_executable
# imports - third party imports
import click
from six import iteritems
# imports - module imports
import bench
@ -143,6 +134,8 @@ def init(path, apps_path=None, no_procfile=False, no_backups=False,
try:
os.makedirs(os.path.join(path, dirname))
except OSError as e:
import errno
if e.errno == errno.EEXIST:
pass
@ -184,6 +177,7 @@ def init(path, apps_path=None, no_procfile=False, no_backups=False,
def update(pull=False, apps=None, patch=False, build=False, requirements=False, backup=True, compile=True,
force=False, reset=False, restart_supervisor=False, restart_systemd=False):
"""command: bench update"""
import re
from bench import patches
from bench.app import is_version_upgrade, pull_apps, validate_branch
from bench.config.common_site_config import get_config, update_config
@ -244,9 +238,11 @@ def update(pull=False, apps=None, patch=False, build=False, requirements=False,
post_upgrade(version_upgrade[1], version_upgrade[2], bench_path=bench_path)
if pull and compile:
from compileall import compile_dir
print('Compiling Python files...')
apps_dir = os.path.join(bench_path, 'apps')
compileall.compile_dir(apps_dir, quiet=1, rx=re.compile('.*node_modules.*'))
compile_dir(apps_dir, quiet=1, rx=re.compile('.*node_modules.*'))
if restart_supervisor or conf.get('restart_supervisor_on_update'):
restart_supervisor_processes(bench_path=bench_path)
@ -268,7 +264,7 @@ def copy_patches_txt(bench_path):
def clone_apps_from(bench_path, clone_from, update_app=True):
from .app import install_app
from bench.app import install_app
print('Copying apps from {0}...'.format(clone_from))
subprocess.check_output(['cp', '-R', os.path.join(clone_from, 'apps'), bench_path])
@ -316,7 +312,9 @@ def exec_cmd(cmd, cwd='.'):
logger.warning("{0} executed with exit code {1}".format(cmd_log, return_code))
def which(executable, raise_err = False):
def which(executable, raise_err=False):
from distutils.spawn import find_executable
exec_ = find_executable(executable)
if not exec_ and raise_err:
@ -356,10 +354,11 @@ def setup_socketio(bench_path='.'):
def patch_sites(bench_path='.'):
try:
run_frappe_cmd('--site', 'all', 'migrate', bench_path=bench_path)
except subprocess.CalledProcessError:
raise PatchError
for site in get_sites(bench_path=bench_path):
try:
migrate_site(site, bench_path=bench_path)
except subprocess.CalledProcessError:
raise PatchError
def build_assets(bench_path='.', app=None):
@ -410,9 +409,9 @@ def setup_sudoers(user):
template = bench.config.env().get_template('frappe_sudoers')
frappe_sudoers = template.render(**{
'user': user,
'service': find_executable('service'),
'systemctl': find_executable('systemctl'),
'nginx': find_executable('nginx'),
'service': which('service'),
'systemctl': which('systemctl'),
'nginx': which('nginx'),
})
frappe_sudoers = safe_decode(frappe_sudoers)
@ -448,7 +447,7 @@ def setup_logging(bench_path='.'):
def get_process_manager():
for proc_man in ['honcho', 'foreman', 'forego']:
proc_man_path = find_executable(proc_man)
proc_man_path = which(proc_man)
if proc_man_path:
return proc_man_path
@ -485,7 +484,7 @@ def get_git_version():
def check_git_for_shallow_clone():
from .config.common_site_config import get_config
from bench.config.common_site_config import get_config
config = get_config('.')
if config:
@ -513,7 +512,7 @@ def get_cmd_output(cmd, cwd='.', _raise=True):
def restart_supervisor_processes(bench_path='.', web_workers=False):
from .config.common_site_config import get_config
from bench.config.common_site_config import get_config
conf = get_config(bench_path=bench_path)
bench_name = get_bench_name(bench_path)
@ -599,7 +598,7 @@ def update_node_packages(bench_path='.'):
def update_yarn_packages(bench_path='.'):
apps_dir = os.path.join(bench_path, 'apps')
if not find_executable('yarn'):
if not which('yarn'):
print("Please install yarn using below command and try again.")
print("`npm install -g yarn`")
return
@ -620,6 +619,8 @@ def update_npm_packages(bench_path='.'):
if os.path.exists(package_json_path):
with open(package_json_path, "r") as f:
from six import iteritems
app_package_json = json.loads(f.read())
# package.json is usually a dict in a dict
for key, value in iteritems(app_package_json):
@ -643,6 +644,10 @@ def update_npm_packages(bench_path='.'):
exec_cmd('npm install', cwd=bench_path)
def migrate_site(site, bench_path='.'):
run_frappe_cmd('--site', site, 'migrate', bench_path=bench_path)
def backup_site(site, bench_path='.'):
run_frappe_cmd('--site', site, 'backup', bench_path=bench_path)
@ -653,9 +658,7 @@ def backup_all_sites(bench_path='.'):
def is_root():
if os.getuid() == 0:
return True
return False
return os.getuid() == 0
def set_mariadb_host(host, bench_path='.'):
@ -713,7 +716,8 @@ def drop_privileges(uid_name='nobody', gid_name='nogroup'):
def fix_prod_setup_perms(bench_path='.', frappe_user=None):
from .config.common_site_config import get_config
from glob import glob
from bench.config.common_site_config import get_config
if not frappe_user:
frappe_user = get_config(bench_path).get('frappe_user')
@ -724,14 +728,14 @@ def fix_prod_setup_perms(bench_path='.', frappe_user=None):
globs = ["logs/*", "config/*"]
for glob_name in globs:
for path in glob.glob(glob_name):
for path in glob(glob_name):
uid = pwd.getpwnam(frappe_user).pw_uid
gid = grp.getgrnam(frappe_user).gr_gid
os.chown(path, uid, gid)
def run_frappe_cmd(*args, **kwargs):
from .cli import from_command_line
from bench.cli import from_command_line
bench_path = kwargs.get('bench_path', '.')
f = get_env_cmd('python', bench_path=bench_path)
@ -757,15 +761,15 @@ def run_frappe_cmd(*args, **kwargs):
def validate_upgrade(from_ver, to_ver, bench_path='.'):
if to_ver >= 6:
if not find_executable('npm') and not (find_executable('node') or find_executable('nodejs')):
if not which('npm') and not (which('node') or which('nodejs')):
raise Exception("Please install nodejs and npm")
def post_upgrade(from_ver, to_ver, bench_path='.'):
from .config.common_site_config import get_config
from .config import redis
from .config.supervisor import generate_supervisor_config
from .config.nginx import make_nginx_conf
from bench.config.common_site_config import get_config
from bench.config import redis
from bench.config.supervisor import generate_supervisor_config
from bench.config.nginx import make_nginx_conf
conf = get_config(bench_path=bench_path)
print("-" * 80 + "Your bench was upgraded to version {0}".format(to_ver))
@ -843,8 +847,10 @@ def update_translations(app, lang):
def print_output(p):
from select import select
while p.poll() is None:
readx = select.select([p.stdout.fileno(), p.stderr.fileno()], [], [])[0]
readx = select([p.stdout.fileno(), p.stderr.fileno()], [], [])[0]
send_buffer = []
for fd in readx:
if fd == p.stdout.fileno():
@ -905,7 +911,7 @@ def set_git_remote_url(git_url, bench_path='.'):
def run_playbook(playbook_name, extra_vars=None, tag=None):
if not find_executable('ansible'):
if not which('ansible'):
print("Ansible is needed to run this command, please install it using 'pip install ansible'")
sys.exit(1)
args = ['ansible-playbook', '-c', 'local', playbook_name, '-vvvv']
@ -977,6 +983,8 @@ def migrate_env(python, backup=False):
# Backup venv: restore using `virtualenv --relocatable` if needed
if backup:
from datetime import datetime
parch = os.path.join(path, 'archived_envs')
if not os.path.exists(parch):
os.mkdir(parch)