diff --git a/MANIFEST.in b/MANIFEST.in index 5e899217..898efefa 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -3,3 +3,5 @@ include *.py recursive-include bench *.conf recursive-include bench *.py recursive-include bench *.txt +recursive-include bench *.json +recursive-include bench/templates * diff --git a/README.md b/README.md index adcf593c..517a0fc1 100755 --- a/README.md +++ b/README.md @@ -1,4 +1,7 @@ -# Bench +
+ +

Frappe Bench

+
[![Build Status](https://travis-ci.org/frappe/bench.svg?branch=master)](https://travis-ci.org/frappe/bench) diff --git a/bench/__init__.py b/bench/__init__.py index b5c69ef0..e120c37b 100644 --- a/bench/__init__.py +++ b/bench/__init__.py @@ -10,4 +10,4 @@ def set_frappe_version(bench_path='.'): from .app import get_current_frappe_version global FRAPPE_VERSION if not FRAPPE_VERSION: - FRAPPE_VERSION = get_current_frappe_version(bench_path=bench_path) + FRAPPE_VERSION = get_current_frappe_version(bench_path=bench_path) \ No newline at end of file diff --git a/bench/app.py b/bench/app.py index 64bac73c..87a8681f 100755 --- a/bench/app.py +++ b/bench/app.py @@ -48,10 +48,40 @@ def write_appstxt(apps, bench_path='.'): with open(os.path.join(bench_path, 'sites', 'apps.txt'), 'w') as f: return f.write('\n'.join(apps)) +def check_url(url, raise_err = True): + try: + from urlparse import urlparse + except ImportError: + from urllib.parse import urlparse + + parsed = urlparse(url) + if not parsed.scheme: + if raise_err: + raise TypeError('{url} Not a valid URL'.format(url = url)) + else: + return False + + return True + def get_app(git_url, branch=None, bench_path='.', build_asset_files=True, verbose=False): - #less verbose app install - if '/' not in git_url: - git_url = 'https://github.com/frappe/' + git_url + # from bench.utils import check_url + try: + from urlparse import urljoin + except ImportError: + from urllib.parse import urljoin + + if not check_url(git_url, raise_err = False): + orgs = ['frappe', 'erpnext'] + for org in orgs: + url = 'https://api.github.com/repos/{org}/{app}'.format(org = org, app = git_url) + res = requests.get(url) + if res.ok: + data = res.json() + if 'name' in data: + if git_url == data['name']: + git_url = 'https://github.com/{org}/{app}'.format(org = org, app = git_url) + break + #Gets repo name from URL repo_name = git_url.rsplit('/', 1)[1].rsplit('.', 1)[0] logger.info('getting app {}'.format(repo_name)) @@ -239,7 +269,7 @@ def get_upstream_version(app, branch=None, bench_path='.'): try: contents = subprocess.check_output(['git', 'show', 'upstream/{branch}:{app}/__init__.py'.format(branch=branch, app=app)], cwd=repo_dir, stderr=subprocess.STDOUT) except subprocess.CalledProcessError as e: - if "Invalid object" in e.output: + if b"Invalid object" in e.output: return None else: raise diff --git a/bench/cli.py b/bench/cli.py index 830e144a..8e05c9f0 100755 --- a/bench/cli.py +++ b/bench/cli.py @@ -95,7 +95,9 @@ def get_frappe_commands(bench_path='.'): if not os.path.exists(sites_path): return [] try: - return json.loads(get_cmd_output("{python} -m frappe.utils.bench_helper get-frappe-commands".format(python=python), cwd=sites_path)) + output = get_cmd_output("{python} -m frappe.utils.bench_helper get-frappe-commands".format(python=python), cwd=sites_path) + output = output.decode('utf-8') + return json.loads(output) except subprocess.CalledProcessError: return [] diff --git a/bench/commands/__init__.py b/bench/commands/__init__.py index 0907f12d..e4ecb8c3 100755 --- a/bench/commands/__init__.py +++ b/bench/commands/__init__.py @@ -1,5 +1,16 @@ import click +import os, shutil +import os.path as osp +import logging + +from datetime import datetime + +from bench.utils import which, exec_cmd + +log = logging.getLogger(__name__) +log.setLevel(logging.ERROR) + def print_bench_version(ctx, param, value): """Prints current bench version""" if not value or ctx.resilient_parsing: @@ -14,7 +25,6 @@ def print_bench_version(ctx, param, value): def bench_command(bench_path='.'): """Bench manager for Frappe""" import bench - from bench.app import get_current_frappe_version from bench.utils import setup_logging bench.set_frappe_version(bench_path=bench_path) @@ -70,4 +80,81 @@ bench_command.add_command(remote_reset_url) bench_command.add_command(remote_urls) from bench.commands.install import install -bench_command.add_command(install) \ No newline at end of file +bench_command.add_command(install) + +@click.command('migrate-env') +@click.argument('python', type = str) +@click.option('--no-backup', default = False, help = 'Do not backup the existing Virtual Environment') +def migrate_env(python, no_backup = False): + """ + Migrate Virtual Environment to desired Python Version. + """ + try: + # This is with the assumption that a bench is set-up within path. + path = os.getcwd() + + # I know, bad name for a flag. Thanks, Ameya! :| - + if not no_backup: + # Back, the f*ck up. + parch = osp.join(path, 'archived_envs') + if not osp.exists(parch): + os.mkdir(parch) + + # Simply moving. Thanks, Ameya. + # I'm keen to zip. + source = osp.join(path, 'env') + target = parch + + log.debug('Backing up Virtual Environment') + stamp = datetime.now().strftime('%Y%m%d_%H%M%S') + dest = osp.join(path, str(stamp)) + + # WARNING: This is an archive, you might have to use virtualenv --relocate + # That's because virtualenv creates symlinks with shebangs pointing to executables. + + # ...and shutil.copytree is a f*cking mess. + os.rename(source, dest) + shutil.move(dest, target) + + log.debug('Setting up a New Virtual {python} Environment'.format( + python = python + )) + + # Path to Python Executable (Basically $PYTHONPTH) + python = which(python) + + + virtualenv = which('virtualenv') + + nvenv = 'env' + pvenv = osp.join(path, nvenv) + + exec_cmd('{virtualenv} --python {python} {pvenv}'.format( + virtualenv = virtualenv, + python = python, + pvenv = pvenv + ), cwd = path) + + pip = osp.join(pvenv, 'bin', 'pip') + exec_cmd('{pip} install --upgrade pip'.format(pip=pip)) + exec_cmd('{pip} install --upgrade setuptools'.format(pip=pip)) + # TODO: Options + + papps = osp.join(path, 'apps') + apps = ['frappe'] + [app for app in os.listdir(papps) if app != 'frappe'] + + for app in apps: + papp = osp.join(papps, app) + if osp.isdir(papp) and osp.exists(osp.join(papp, 'setup.py')): + exec_cmd('{pip} install -e {app}'.format( + pip = pip, app = papp + )) + + log.debug('Migration Successful to {python}'.format( + python = python + )) + except: + log.debug('Migration Error') + raise + +bench_command.add_command(migrate_env) \ No newline at end of file diff --git a/bench/commands/make.py b/bench/commands/make.py index 18a06c92..639868ed 100755 --- a/bench/commands/make.py +++ b/bench/commands/make.py @@ -2,6 +2,8 @@ import click @click.command() @click.argument('path') +@click.option('--python', type = str, default = 'python', help = 'Path to Python Executable.') +@click.option('--ignore-exist', is_flag = True, default = False, help = "Ignore if Bench instance exists.") @click.option('--apps_path', default=None, help="path to json files with apps to install after init") @click.option('--frappe-path', default=None, help="path to frappe repo") @click.option('--frappe-branch', default=None, help="path to frappe repo") @@ -9,19 +11,23 @@ import click @click.option('--no-procfile', is_flag=True, help="Pull changes in all the apps in bench") @click.option('--no-backups',is_flag=True, help="Run migrations for all sites in the bench") @click.option('--no-auto-update',is_flag=True, help="Build JS and CSS artifacts for the bench") -@click.option('--verbose',is_flag=True, help="Verbose output during install") -@click.option('--skip-bench-mkdir', is_flag=True, help="Skip mkdir frappe-bench") @click.option('--skip-redis-config-generation', is_flag=True, help="Skip redis config generation if already specifying the common-site-config file") +@click.option('--verbose',is_flag=True, help="Verbose output during install") def init(path, apps_path, frappe_path, frappe_branch, no_procfile, no_backups, - no_auto_update, clone_from, verbose, skip_bench_mkdir, skip_redis_config_generation): - "Create a new bench" + no_auto_update, clone_from, verbose, skip_redis_config_generation, + ignore_exist = False, + python = 'python'): # Let's change we're ready. - + ''' + Create a New Bench Instance. + ''' from bench.utils import init init(path, apps_path=apps_path, no_procfile=no_procfile, no_backups=no_backups, no_auto_update=no_auto_update, frappe_path=frappe_path, frappe_branch=frappe_branch, - verbose=verbose, clone_from=clone_from, skip_bench_mkdir=skip_bench_mkdir, skip_redis_config_generation=skip_redis_config_generation) + verbose=verbose, clone_from=clone_from, skip_redis_config_generation=skip_redis_config_generation, + ignore_exist = ignore_exist, + python = python) click.echo('Bench {} initialized'.format(path)) - @click.command('get-app') @click.argument('name', nargs=-1) # Dummy argument for backward compatibility @click.argument('git-url') diff --git a/bench/commands/setup.py b/bench/commands/setup.py index adff7ecb..053e6357 100755 --- a/bench/commands/setup.py +++ b/bench/commands/setup.py @@ -4,8 +4,7 @@ import click, sys, json def setup(): "Setup bench" pass - - + @click.command('sudoers') @click.argument('user') def setup_sudoers(user): @@ -13,7 +12,6 @@ def setup_sudoers(user): from bench.utils import setup_sudoers setup_sudoers(user) - @click.command('nginx') @click.option('--yes', help='Yes to regeneration of nginx config file', default=False, is_flag=True) def setup_nginx(yes=False): @@ -47,7 +45,6 @@ def setup_fonts(): from bench.utils import setup_fonts setup_fonts() - @click.command('production') @click.argument('user') @click.option('--yes', help='Yes to regeneration config', is_flag=True, default=False) @@ -71,10 +68,11 @@ def setup_backups(): setup_backups() @click.command('env') -def setup_env(): +@click.option('--python', type = str, default = 'python', help = 'Path to Python Executable.') +def setup_env(python='python'): "Setup virtualenv for bench" from bench.utils import setup_env - setup_env() + setup_env(python=python) @click.command('firewall') @click.option('--ssh_port') diff --git a/bench/config/procfile.py b/bench/config/procfile.py index 3e4183e9..3dd7cece 100755 --- a/bench/config/procfile.py +++ b/bench/config/procfile.py @@ -13,7 +13,8 @@ def setup_procfile(bench_path, yes=False): procfile = bench.env.get_template('Procfile').render( node=find_executable("node") or find_executable("nodejs"), use_rq=use_rq(bench_path), - webserver_port=config.get('webserver_port')) + webserver_port=config.get('webserver_port'), + CI=os.environ.get('CI')) with open(procfile_path, 'w') as f: f.write(procfile) diff --git a/bench/config/redis.py b/bench/config/redis.py index 43423c8e..fe102135 100644 --- a/bench/config/redis.py +++ b/bench/config/redis.py @@ -56,8 +56,8 @@ def write_redis_config(template_name, context, bench_path): f.write(template.render(**context)) def get_redis_version(): - version_string = subprocess.check_output('redis-server --version', shell=True).strip() - + version_string = subprocess.check_output('redis-server --version', shell=True) + version_string = version_string.decode('utf-8').strip() # extract version number from string version = re.findall("\d+\.\d+", version_string) if not version: diff --git a/bench/config/templates/Procfile b/bench/config/templates/Procfile index 9a1e3ce2..c465bbac 100644 --- a/bench/config/templates/Procfile +++ b/bench/config/templates/Procfile @@ -4,7 +4,9 @@ redis_queue: redis-server config/redis_queue.conf web: bench serve {% if webserver_port -%} --port {{ webserver_port }} {%- endif %} socketio: {{ node }} apps/frappe/socketio.js +{% if not CI %} watch: bench watch +{% endif %} {% if use_rq -%} schedule: bench schedule worker_short: bench worker --queue short diff --git a/bench/package.json b/bench/package.json deleted file mode 100644 index 31121283..00000000 --- a/bench/package.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "name": "frappe", - "description": "Default package.json for frappe apps", - "dependencies": { - "babel-core": "^6.24.1", - "babel-preset-babili": "0.0.12", - "babel-preset-es2015": "^6.24.1", - "babel-preset-es2016": "^6.24.1", - "babel-preset-es2017": "^6.24.1", - "chokidar": "^1.7.0", - "cookie": "^0.3.1", - "express": "^4.15.3", - "less": "^2.7.2", - "redis": "^2.7.1", - "socket.io": "^2.0.1", - "superagent": "^3.5.2" - } -} diff --git a/bench/patches/patches.txt b/bench/patches/patches.txt index e979b0d6..754068c5 100644 --- a/bench/patches/patches.txt +++ b/bench/patches/patches.txt @@ -3,4 +3,4 @@ bench.patches.v3.celery_to_rq bench.patches.v3.redis_bind_ip bench.patches.v4.update_node bench.patches.v4.update_socketio - +bench.patches.v4.install_yarn #2 diff --git a/bench/patches/v4/install_yarn.py b/bench/patches/v4/install_yarn.py new file mode 100644 index 00000000..e29868e9 --- /dev/null +++ b/bench/patches/v4/install_yarn.py @@ -0,0 +1,5 @@ +import os +from bench.utils import exec_cmd + +def execute(bench_path): + exec_cmd('npm install yarn', os.path.join(bench_path, 'apps/frappe')) diff --git a/bench/utils.py b/bench/utils.py index caa55445..db869760 100755 --- a/bench/utils.py +++ b/bench/utils.py @@ -27,31 +27,33 @@ def get_env_cmd(cmd, bench_path='.'): def init(path, apps_path=None, no_procfile=False, no_backups=False, no_auto_update=False, frappe_path=None, frappe_branch=None, wheel_cache_dir=None, - verbose=False, clone_from=None, skip_bench_mkdir=False, skip_redis_config_generation=False): + verbose=False, clone_from=None, skip_redis_config_generation=False, + ignore_exist = False, + python = 'python'): # Let's change when we're ready. - from .app import get_app, install_apps_from_path from .config.common_site_config import make_config from .config import redis from .config.procfile import setup_procfile from bench.patches import set_all_patches_executed - if(skip_bench_mkdir): - pass + import os.path as osp + + if osp.exists(path): + if not ignore_exist: + raise ValueError('Bench Instance {path} already exists.'.format(path = path)) else: - if os.path.exists(path): - print('Directory {} already exists!'.format(path)) - raise Exception("Site directory already exists") os.makedirs(path) for dirname in folders_in_bench: try: os.makedirs(os.path.join(path, dirname)) - except OSError, e: + except OSError as e: if e.errno != os.errno.EEXIST: pass setup_logging() - setup_env(bench_path=path) + setup_env(bench_path=path, python = python) make_config(path) @@ -139,8 +141,21 @@ def exec_cmd(cmd, cwd='.'): if return_code > 0: raise CommandFailedError(cmd) -def setup_env(bench_path='.'): - exec_cmd('virtualenv -q {} -p {}'.format('env', sys.executable), cwd=bench_path) +def which(executable, raise_err = False): + from distutils.spawn import find_executable + exec_ = find_executable(executable) + + if not exec_ and raise_err: + raise ValueError('{executable} not found.'.format( + executable = executable + )) + + return exec_ + +def setup_env(bench_path='.', python = 'python'): + python = which(python, raise_err = True) + + exec_cmd('virtualenv -q {} -p {}'.format('env', python), cwd=bench_path) exec_cmd('./env/bin/pip -q install --upgrade pip', cwd=bench_path) exec_cmd('./env/bin/pip -q install wheel', cwd=bench_path) # exec_cmd('./env/bin/pip -q install https://github.com/frappe/MySQLdb1/archive/MySQLdb-1.2.5-patched.tar.gz', cwd=bench_path) @@ -232,13 +247,14 @@ def setup_backups(bench_path='.'): def add_to_crontab(line): current_crontab = read_crontab() + line = str.encode(line) if not line in current_crontab: cmd = ["crontab"] if platform.system() == 'FreeBSD': cmd = ["crontab", "-"] s = subprocess.Popen(cmd, stdin=subprocess.PIPE) s.stdin.write(current_crontab) - s.stdin.write(line + '\n') + s.stdin.write(line + b'\n') s.stdin.close() def read_crontab(): @@ -331,7 +347,9 @@ def check_cmd(cmd, cwd='.'): def get_git_version(): '''returns git version from `git --version` extracts version number from string `get version 1.9.1` etc''' - version = get_cmd_output("git --version").strip().split()[2] + version = get_cmd_output("git --version") + version = version.decode('utf-8') + version = version.strip().split()[2] version = '.'.join(version.split('.')[0:2]) return float(version) @@ -412,34 +430,17 @@ def update_requirements(bench_path='.'): def update_npm_packages(bench_path='.'): print('Updating node libraries...') apps_dir = os.path.join(bench_path, 'apps') - package_json = {} + + if not find_executable('yarn'): + print("Please install yarn using below command and try again.") + print("`npm install -g yarn`") + return for app in os.listdir(apps_dir): - package_json_path = os.path.join(apps_dir, app, 'package.json') + app_path = os.path.join(apps_dir, app) + if os.path.exists(os.path.join(app_path, 'package.json')): + exec_cmd('yarn install', cwd=app_path) - if os.path.exists(package_json_path): - with open(package_json_path, "r") as f: - app_package_json = json.loads(f.read()) - # package.json is usually a dict in a dict - for key, value in iteritems(app_package_json): - if not key in package_json: - package_json[key] = value - else: - if isinstance(value, dict): - package_json[key].update(value) - elif isinstance(value, list): - package_json[key].extend(value) - else: - package_json[key] = value - - if package_json is {}: - with open(os.path.join(os.path.dirname(__file__), 'package.json'), 'r') as f: - package_json = json.loads(f.read()) - - with open(os.path.join(bench_path, 'package.json'), 'w') as f: - f.write(json.dumps(package_json, indent=1, sort_keys=True)) - - exec_cmd('npm install', cwd=bench_path) def install_requirements(pip, req_file): if os.path.exists(req_file): @@ -766,11 +767,11 @@ def run_playbook(playbook_name, extra_vars=None, tag=None): print("Ansible is needed to run this command, please install it using 'pip install ansible'") sys.exit(1) args = ['ansible-playbook', '-c', 'local', playbook_name] - + if extra_vars: args.extend(['-e', json.dumps(extra_vars)]) - + if tag: args.extend(['-t', tag]) - + subprocess.check_call(args, cwd=os.path.join(os.path.dirname(bench.__path__[0]), 'playbooks')) diff --git a/playbooks/roles/nodejs/tasks/main.yml b/playbooks/roles/nodejs/tasks/main.yml index a72750a3..e521c27d 100644 --- a/playbooks/roles/nodejs/tasks/main.yml +++ b/playbooks/roles/nodejs/tasks/main.yml @@ -37,3 +37,8 @@ update_cache: yes force: yes when: ansible_os_family == 'Debian' or ansible_distribution == 'Ubuntu' + +- name: Install yarn + command: npm install -g yarn + become: yes + become_user: root \ No newline at end of file