mirror of
https://github.com/frappe/bench.git
synced 2024-09-23 04:29:02 +00:00
Merge branch 'master' into clean_code
This commit is contained in:
Ameya Shenoy
commit
2055616af2
@ -3,3 +3,5 @@ include *.py
|
||||
recursive-include bench *.conf
|
||||
recursive-include bench *.py
|
||||
recursive-include bench *.txt
|
||||
recursive-include bench *.json
|
||||
recursive-include bench/templates *
|
||||
|
||||
@ -1,4 +1,7 @@
|
||||
# Bench
|
||||
<div align="center">
|
||||
<img src="https://github.com/frappe/design/blob/master/logos/bench-logo.svg" height="128">
|
||||
<h2>Frappe Bench</h2>
|
||||
</div>
|
||||
|
||||
[](https://travis-ci.org/frappe/bench)
|
||||
|
||||
|
||||
@ -10,4 +10,4 @@ def set_frappe_version(bench_path='.'):
|
||||
from .app import get_current_frappe_version
|
||||
global FRAPPE_VERSION
|
||||
if not FRAPPE_VERSION:
|
||||
FRAPPE_VERSION = get_current_frappe_version(bench_path=bench_path)
|
||||
FRAPPE_VERSION = get_current_frappe_version(bench_path=bench_path)
|
||||
38
bench/app.py
38
bench/app.py
@ -48,10 +48,40 @@ def write_appstxt(apps, bench_path='.'):
|
||||
with open(os.path.join(bench_path, 'sites', 'apps.txt'), 'w') as f:
|
||||
return f.write('\n'.join(apps))
|
||||
|
||||
def check_url(url, raise_err = True):
|
||||
try:
|
||||
from urlparse import urlparse
|
||||
except ImportError:
|
||||
from urllib.parse import urlparse
|
||||
|
||||
parsed = urlparse(url)
|
||||
if not parsed.scheme:
|
||||
if raise_err:
|
||||
raise TypeError('{url} Not a valid URL'.format(url = url))
|
||||
else:
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
def get_app(git_url, branch=None, bench_path='.', build_asset_files=True, verbose=False):
|
||||
#less verbose app install
|
||||
if '/' not in git_url:
|
||||
git_url = 'https://github.com/frappe/' + git_url
|
||||
# from bench.utils import check_url
|
||||
try:
|
||||
from urlparse import urljoin
|
||||
except ImportError:
|
||||
from urllib.parse import urljoin
|
||||
|
||||
if not check_url(git_url, raise_err = False):
|
||||
orgs = ['frappe', 'erpnext']
|
||||
for org in orgs:
|
||||
url = 'https://api.github.com/repos/{org}/{app}'.format(org = org, app = git_url)
|
||||
res = requests.get(url)
|
||||
if res.ok:
|
||||
data = res.json()
|
||||
if 'name' in data:
|
||||
if git_url == data['name']:
|
||||
git_url = 'https://github.com/{org}/{app}'.format(org = org, app = git_url)
|
||||
break
|
||||
|
||||
#Gets repo name from URL
|
||||
repo_name = git_url.rsplit('/', 1)[1].rsplit('.', 1)[0]
|
||||
logger.info('getting app {}'.format(repo_name))
|
||||
@ -239,7 +269,7 @@ def get_upstream_version(app, branch=None, bench_path='.'):
|
||||
try:
|
||||
contents = subprocess.check_output(['git', 'show', 'upstream/{branch}:{app}/__init__.py'.format(branch=branch, app=app)], cwd=repo_dir, stderr=subprocess.STDOUT)
|
||||
except subprocess.CalledProcessError as e:
|
||||
if "Invalid object" in e.output:
|
||||
if b"Invalid object" in e.output:
|
||||
return None
|
||||
else:
|
||||
raise
|
||||
|
||||
@ -95,7 +95,9 @@ def get_frappe_commands(bench_path='.'):
|
||||
if not os.path.exists(sites_path):
|
||||
return []
|
||||
try:
|
||||
return json.loads(get_cmd_output("{python} -m frappe.utils.bench_helper get-frappe-commands".format(python=python), cwd=sites_path))
|
||||
output = get_cmd_output("{python} -m frappe.utils.bench_helper get-frappe-commands".format(python=python), cwd=sites_path)
|
||||
output = output.decode('utf-8')
|
||||
return json.loads(output)
|
||||
except subprocess.CalledProcessError:
|
||||
return []
|
||||
|
||||
|
||||
@ -1,5 +1,16 @@
|
||||
import click
|
||||
|
||||
import os, shutil
|
||||
import os.path as osp
|
||||
import logging
|
||||
|
||||
from datetime import datetime
|
||||
|
||||
from bench.utils import which, exec_cmd
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
log.setLevel(logging.ERROR)
|
||||
|
||||
def print_bench_version(ctx, param, value):
|
||||
"""Prints current bench version"""
|
||||
if not value or ctx.resilient_parsing:
|
||||
@ -14,7 +25,6 @@ def print_bench_version(ctx, param, value):
|
||||
def bench_command(bench_path='.'):
|
||||
"""Bench manager for Frappe"""
|
||||
import bench
|
||||
from bench.app import get_current_frappe_version
|
||||
from bench.utils import setup_logging
|
||||
|
||||
bench.set_frappe_version(bench_path=bench_path)
|
||||
@ -70,4 +80,81 @@ bench_command.add_command(remote_reset_url)
|
||||
bench_command.add_command(remote_urls)
|
||||
|
||||
from bench.commands.install import install
|
||||
bench_command.add_command(install)
|
||||
bench_command.add_command(install)
|
||||
|
||||
@click.command('migrate-env')
|
||||
@click.argument('python', type = str)
|
||||
@click.option('--no-backup', default = False, help = 'Do not backup the existing Virtual Environment')
|
||||
def migrate_env(python, no_backup = False):
|
||||
"""
|
||||
Migrate Virtual Environment to desired Python Version.
|
||||
"""
|
||||
try:
|
||||
# This is with the assumption that a bench is set-up within path.
|
||||
path = os.getcwd()
|
||||
|
||||
# I know, bad name for a flag. Thanks, Ameya! :| - <achilles@frappe.io>
|
||||
if not no_backup:
|
||||
# Back, the f*ck up.
|
||||
parch = osp.join(path, 'archived_envs')
|
||||
if not osp.exists(parch):
|
||||
os.mkdir(parch)
|
||||
|
||||
# Simply moving. Thanks, Ameya.
|
||||
# I'm keen to zip.
|
||||
source = osp.join(path, 'env')
|
||||
target = parch
|
||||
|
||||
log.debug('Backing up Virtual Environment')
|
||||
stamp = datetime.now().strftime('%Y%m%d_%H%M%S')
|
||||
dest = osp.join(path, str(stamp))
|
||||
|
||||
# WARNING: This is an archive, you might have to use virtualenv --relocate
|
||||
# That's because virtualenv creates symlinks with shebangs pointing to executables.
|
||||
|
||||
# ...and shutil.copytree is a f*cking mess.
|
||||
os.rename(source, dest)
|
||||
shutil.move(dest, target)
|
||||
|
||||
log.debug('Setting up a New Virtual {python} Environment'.format(
|
||||
python = python
|
||||
))
|
||||
|
||||
# Path to Python Executable (Basically $PYTHONPTH)
|
||||
python = which(python)
|
||||
|
||||
|
||||
virtualenv = which('virtualenv')
|
||||
|
||||
nvenv = 'env'
|
||||
pvenv = osp.join(path, nvenv)
|
||||
|
||||
exec_cmd('{virtualenv} --python {python} {pvenv}'.format(
|
||||
virtualenv = virtualenv,
|
||||
python = python,
|
||||
pvenv = pvenv
|
||||
), cwd = path)
|
||||
|
||||
pip = osp.join(pvenv, 'bin', 'pip')
|
||||
exec_cmd('{pip} install --upgrade pip'.format(pip=pip))
|
||||
exec_cmd('{pip} install --upgrade setuptools'.format(pip=pip))
|
||||
# TODO: Options
|
||||
|
||||
papps = osp.join(path, 'apps')
|
||||
apps = ['frappe'] + [app for app in os.listdir(papps) if app != 'frappe']
|
||||
|
||||
for app in apps:
|
||||
papp = osp.join(papps, app)
|
||||
if osp.isdir(papp) and osp.exists(osp.join(papp, 'setup.py')):
|
||||
exec_cmd('{pip} install -e {app}'.format(
|
||||
pip = pip, app = papp
|
||||
))
|
||||
|
||||
log.debug('Migration Successful to {python}'.format(
|
||||
python = python
|
||||
))
|
||||
except:
|
||||
log.debug('Migration Error')
|
||||
raise
|
||||
|
||||
bench_command.add_command(migrate_env)
|
||||
@ -2,6 +2,8 @@ import click
|
||||
|
||||
@click.command()
|
||||
@click.argument('path')
|
||||
@click.option('--python', type = str, default = 'python', help = 'Path to Python Executable.')
|
||||
@click.option('--ignore-exist', is_flag = True, default = False, help = "Ignore if Bench instance exists.")
|
||||
@click.option('--apps_path', default=None, help="path to json files with apps to install after init")
|
||||
@click.option('--frappe-path', default=None, help="path to frappe repo")
|
||||
@click.option('--frappe-branch', default=None, help="path to frappe repo")
|
||||
@ -9,19 +11,23 @@ import click
|
||||
@click.option('--no-procfile', is_flag=True, help="Pull changes in all the apps in bench")
|
||||
@click.option('--no-backups',is_flag=True, help="Run migrations for all sites in the bench")
|
||||
@click.option('--no-auto-update',is_flag=True, help="Build JS and CSS artifacts for the bench")
|
||||
@click.option('--verbose',is_flag=True, help="Verbose output during install")
|
||||
@click.option('--skip-bench-mkdir', is_flag=True, help="Skip mkdir frappe-bench")
|
||||
@click.option('--skip-redis-config-generation', is_flag=True, help="Skip redis config generation if already specifying the common-site-config file")
|
||||
@click.option('--verbose',is_flag=True, help="Verbose output during install")
|
||||
def init(path, apps_path, frappe_path, frappe_branch, no_procfile, no_backups,
|
||||
no_auto_update, clone_from, verbose, skip_bench_mkdir, skip_redis_config_generation):
|
||||
"Create a new bench"
|
||||
no_auto_update, clone_from, verbose, skip_redis_config_generation,
|
||||
ignore_exist = False,
|
||||
python = 'python'): # Let's change we're ready. - <achilles@frappe.io>
|
||||
'''
|
||||
Create a New Bench Instance.
|
||||
'''
|
||||
from bench.utils import init
|
||||
init(path, apps_path=apps_path, no_procfile=no_procfile, no_backups=no_backups,
|
||||
no_auto_update=no_auto_update, frappe_path=frappe_path, frappe_branch=frappe_branch,
|
||||
verbose=verbose, clone_from=clone_from, skip_bench_mkdir=skip_bench_mkdir, skip_redis_config_generation=skip_redis_config_generation)
|
||||
verbose=verbose, clone_from=clone_from, skip_redis_config_generation=skip_redis_config_generation,
|
||||
ignore_exist = ignore_exist,
|
||||
python = python)
|
||||
click.echo('Bench {} initialized'.format(path))
|
||||
|
||||
|
||||
@click.command('get-app')
|
||||
@click.argument('name', nargs=-1) # Dummy argument for backward compatibility
|
||||
@click.argument('git-url')
|
||||
|
||||
@ -4,8 +4,7 @@ import click, sys, json
|
||||
def setup():
|
||||
"Setup bench"
|
||||
pass
|
||||
|
||||
|
||||
|
||||
@click.command('sudoers')
|
||||
@click.argument('user')
|
||||
def setup_sudoers(user):
|
||||
@ -13,7 +12,6 @@ def setup_sudoers(user):
|
||||
from bench.utils import setup_sudoers
|
||||
setup_sudoers(user)
|
||||
|
||||
|
||||
@click.command('nginx')
|
||||
@click.option('--yes', help='Yes to regeneration of nginx config file', default=False, is_flag=True)
|
||||
def setup_nginx(yes=False):
|
||||
@ -47,7 +45,6 @@ def setup_fonts():
|
||||
from bench.utils import setup_fonts
|
||||
setup_fonts()
|
||||
|
||||
|
||||
@click.command('production')
|
||||
@click.argument('user')
|
||||
@click.option('--yes', help='Yes to regeneration config', is_flag=True, default=False)
|
||||
@ -71,10 +68,11 @@ def setup_backups():
|
||||
setup_backups()
|
||||
|
||||
@click.command('env')
|
||||
def setup_env():
|
||||
@click.option('--python', type = str, default = 'python', help = 'Path to Python Executable.')
|
||||
def setup_env(python='python'):
|
||||
"Setup virtualenv for bench"
|
||||
from bench.utils import setup_env
|
||||
setup_env()
|
||||
setup_env(python=python)
|
||||
|
||||
@click.command('firewall')
|
||||
@click.option('--ssh_port')
|
||||
|
||||
@ -13,7 +13,8 @@ def setup_procfile(bench_path, yes=False):
|
||||
procfile = bench.env.get_template('Procfile').render(
|
||||
node=find_executable("node") or find_executable("nodejs"),
|
||||
use_rq=use_rq(bench_path),
|
||||
webserver_port=config.get('webserver_port'))
|
||||
webserver_port=config.get('webserver_port'),
|
||||
CI=os.environ.get('CI'))
|
||||
|
||||
with open(procfile_path, 'w') as f:
|
||||
f.write(procfile)
|
||||
|
||||
@ -56,8 +56,8 @@ def write_redis_config(template_name, context, bench_path):
|
||||
f.write(template.render(**context))
|
||||
|
||||
def get_redis_version():
|
||||
version_string = subprocess.check_output('redis-server --version', shell=True).strip()
|
||||
|
||||
version_string = subprocess.check_output('redis-server --version', shell=True)
|
||||
version_string = version_string.decode('utf-8').strip()
|
||||
# extract version number from string
|
||||
version = re.findall("\d+\.\d+", version_string)
|
||||
if not version:
|
||||
|
||||
@ -4,7 +4,9 @@ redis_queue: redis-server config/redis_queue.conf
|
||||
web: bench serve {% if webserver_port -%} --port {{ webserver_port }} {%- endif %}
|
||||
|
||||
socketio: {{ node }} apps/frappe/socketio.js
|
||||
{% if not CI %}
|
||||
watch: bench watch
|
||||
{% endif %}
|
||||
{% if use_rq -%}
|
||||
schedule: bench schedule
|
||||
worker_short: bench worker --queue short
|
||||
|
||||
@ -1,18 +0,0 @@
|
||||
{
|
||||
"name": "frappe",
|
||||
"description": "Default package.json for frappe apps",
|
||||
"dependencies": {
|
||||
"babel-core": "^6.24.1",
|
||||
"babel-preset-babili": "0.0.12",
|
||||
"babel-preset-es2015": "^6.24.1",
|
||||
"babel-preset-es2016": "^6.24.1",
|
||||
"babel-preset-es2017": "^6.24.1",
|
||||
"chokidar": "^1.7.0",
|
||||
"cookie": "^0.3.1",
|
||||
"express": "^4.15.3",
|
||||
"less": "^2.7.2",
|
||||
"redis": "^2.7.1",
|
||||
"socket.io": "^2.0.1",
|
||||
"superagent": "^3.5.2"
|
||||
}
|
||||
}
|
||||
@ -3,4 +3,4 @@ bench.patches.v3.celery_to_rq
|
||||
bench.patches.v3.redis_bind_ip
|
||||
bench.patches.v4.update_node
|
||||
bench.patches.v4.update_socketio
|
||||
|
||||
bench.patches.v4.install_yarn #2
|
||||
|
||||
5
bench/patches/v4/install_yarn.py
Normal file
5
bench/patches/v4/install_yarn.py
Normal file
@ -0,0 +1,5 @@
|
||||
import os
|
||||
from bench.utils import exec_cmd
|
||||
|
||||
def execute(bench_path):
|
||||
exec_cmd('npm install yarn', os.path.join(bench_path, 'apps/frappe'))
|
||||
@ -27,31 +27,33 @@ def get_env_cmd(cmd, bench_path='.'):
|
||||
|
||||
def init(path, apps_path=None, no_procfile=False, no_backups=False,
|
||||
no_auto_update=False, frappe_path=None, frappe_branch=None, wheel_cache_dir=None,
|
||||
verbose=False, clone_from=None, skip_bench_mkdir=False, skip_redis_config_generation=False):
|
||||
verbose=False, clone_from=None, skip_redis_config_generation=False,
|
||||
ignore_exist = False,
|
||||
python = 'python'): # Let's change when we're ready. - <achilles@frappe.io>
|
||||
from .app import get_app, install_apps_from_path
|
||||
from .config.common_site_config import make_config
|
||||
from .config import redis
|
||||
from .config.procfile import setup_procfile
|
||||
from bench.patches import set_all_patches_executed
|
||||
|
||||
if(skip_bench_mkdir):
|
||||
pass
|
||||
import os.path as osp
|
||||
|
||||
if osp.exists(path):
|
||||
if not ignore_exist:
|
||||
raise ValueError('Bench Instance {path} already exists.'.format(path = path))
|
||||
else:
|
||||
if os.path.exists(path):
|
||||
print('Directory {} already exists!'.format(path))
|
||||
raise Exception("Site directory already exists")
|
||||
os.makedirs(path)
|
||||
|
||||
for dirname in folders_in_bench:
|
||||
try:
|
||||
os.makedirs(os.path.join(path, dirname))
|
||||
except OSError, e:
|
||||
except OSError as e:
|
||||
if e.errno != os.errno.EEXIST:
|
||||
pass
|
||||
|
||||
setup_logging()
|
||||
|
||||
setup_env(bench_path=path)
|
||||
setup_env(bench_path=path, python = python)
|
||||
|
||||
make_config(path)
|
||||
|
||||
@ -139,8 +141,21 @@ def exec_cmd(cmd, cwd='.'):
|
||||
if return_code > 0:
|
||||
raise CommandFailedError(cmd)
|
||||
|
||||
def setup_env(bench_path='.'):
|
||||
exec_cmd('virtualenv -q {} -p {}'.format('env', sys.executable), cwd=bench_path)
|
||||
def which(executable, raise_err = False):
|
||||
from distutils.spawn import find_executable
|
||||
exec_ = find_executable(executable)
|
||||
|
||||
if not exec_ and raise_err:
|
||||
raise ValueError('{executable} not found.'.format(
|
||||
executable = executable
|
||||
))
|
||||
|
||||
return exec_
|
||||
|
||||
def setup_env(bench_path='.', python = 'python'):
|
||||
python = which(python, raise_err = True)
|
||||
|
||||
exec_cmd('virtualenv -q {} -p {}'.format('env', python), cwd=bench_path)
|
||||
exec_cmd('./env/bin/pip -q install --upgrade pip', cwd=bench_path)
|
||||
exec_cmd('./env/bin/pip -q install wheel', cwd=bench_path)
|
||||
# exec_cmd('./env/bin/pip -q install https://github.com/frappe/MySQLdb1/archive/MySQLdb-1.2.5-patched.tar.gz', cwd=bench_path)
|
||||
@ -232,13 +247,14 @@ def setup_backups(bench_path='.'):
|
||||
|
||||
def add_to_crontab(line):
|
||||
current_crontab = read_crontab()
|
||||
line = str.encode(line)
|
||||
if not line in current_crontab:
|
||||
cmd = ["crontab"]
|
||||
if platform.system() == 'FreeBSD':
|
||||
cmd = ["crontab", "-"]
|
||||
s = subprocess.Popen(cmd, stdin=subprocess.PIPE)
|
||||
s.stdin.write(current_crontab)
|
||||
s.stdin.write(line + '\n')
|
||||
s.stdin.write(line + b'\n')
|
||||
s.stdin.close()
|
||||
|
||||
def read_crontab():
|
||||
@ -331,7 +347,9 @@ def check_cmd(cmd, cwd='.'):
|
||||
def get_git_version():
|
||||
'''returns git version from `git --version`
|
||||
extracts version number from string `get version 1.9.1` etc'''
|
||||
version = get_cmd_output("git --version").strip().split()[2]
|
||||
version = get_cmd_output("git --version")
|
||||
version = version.decode('utf-8')
|
||||
version = version.strip().split()[2]
|
||||
version = '.'.join(version.split('.')[0:2])
|
||||
return float(version)
|
||||
|
||||
@ -412,34 +430,17 @@ def update_requirements(bench_path='.'):
|
||||
def update_npm_packages(bench_path='.'):
|
||||
print('Updating node libraries...')
|
||||
apps_dir = os.path.join(bench_path, 'apps')
|
||||
package_json = {}
|
||||
|
||||
if not find_executable('yarn'):
|
||||
print("Please install yarn using below command and try again.")
|
||||
print("`npm install -g yarn`")
|
||||
return
|
||||
|
||||
for app in os.listdir(apps_dir):
|
||||
package_json_path = os.path.join(apps_dir, app, 'package.json')
|
||||
app_path = os.path.join(apps_dir, app)
|
||||
if os.path.exists(os.path.join(app_path, 'package.json')):
|
||||
exec_cmd('yarn install', cwd=app_path)
|
||||
|
||||
if os.path.exists(package_json_path):
|
||||
with open(package_json_path, "r") as f:
|
||||
app_package_json = json.loads(f.read())
|
||||
# package.json is usually a dict in a dict
|
||||
for key, value in iteritems(app_package_json):
|
||||
if not key in package_json:
|
||||
package_json[key] = value
|
||||
else:
|
||||
if isinstance(value, dict):
|
||||
package_json[key].update(value)
|
||||
elif isinstance(value, list):
|
||||
package_json[key].extend(value)
|
||||
else:
|
||||
package_json[key] = value
|
||||
|
||||
if package_json is {}:
|
||||
with open(os.path.join(os.path.dirname(__file__), 'package.json'), 'r') as f:
|
||||
package_json = json.loads(f.read())
|
||||
|
||||
with open(os.path.join(bench_path, 'package.json'), 'w') as f:
|
||||
f.write(json.dumps(package_json, indent=1, sort_keys=True))
|
||||
|
||||
exec_cmd('npm install', cwd=bench_path)
|
||||
|
||||
def install_requirements(pip, req_file):
|
||||
if os.path.exists(req_file):
|
||||
@ -766,11 +767,11 @@ def run_playbook(playbook_name, extra_vars=None, tag=None):
|
||||
print("Ansible is needed to run this command, please install it using 'pip install ansible'")
|
||||
sys.exit(1)
|
||||
args = ['ansible-playbook', '-c', 'local', playbook_name]
|
||||
|
||||
|
||||
if extra_vars:
|
||||
args.extend(['-e', json.dumps(extra_vars)])
|
||||
|
||||
|
||||
if tag:
|
||||
args.extend(['-t', tag])
|
||||
|
||||
|
||||
subprocess.check_call(args, cwd=os.path.join(os.path.dirname(bench.__path__[0]), 'playbooks'))
|
||||
|
||||
@ -37,3 +37,8 @@
|
||||
update_cache: yes
|
||||
force: yes
|
||||
when: ansible_os_family == 'Debian' or ansible_distribution == 'Ubuntu'
|
||||
|
||||
- name: Install yarn
|
||||
command: npm install -g yarn
|
||||
become: yes
|
||||
become_user: root
|
||||
Loading…
Reference in New Issue
Block a user