Merge branch 'master' into clean_code

MANIFEST.in

@@ -3,3 +3,5 @@ include *.py
 recursive-include bench *.conf
 recursive-include bench *.py
 recursive-include bench *.txt
+recursive-include bench *.json
+recursive-include bench/templates *

README.md

@@ -1,4 +1,7 @@
-# Bench
+<div align="center">
+    <img src="https://github.com/frappe/design/blob/master/logos/bench-logo.svg" height="128">
+    <h2>Frappe Bench</h2>
+</div>
 
 [![Build Status](https://travis-ci.org/frappe/bench.svg?branch=master)](https://travis-ci.org/frappe/bench)
 

bench/__init__.py

@@ -10,4 +10,4 @@ def set_frappe_version(bench_path='.'):
 	from .app import get_current_frappe_version
 	global FRAPPE_VERSION
 	if not FRAPPE_VERSION:
-		FRAPPE_VERSION = get_current_frappe_version(bench_path=bench_path)
+		FRAPPE_VERSION = get_current_frappe_version(bench_path=bench_path)

bench/app.py

@@ -48,10 +48,40 @@ def write_appstxt(apps, bench_path='.'):
 	with open(os.path.join(bench_path, 'sites', 'apps.txt'), 'w') as f:
 		return f.write('\n'.join(apps))
 
+def check_url(url, raise_err = True):
+	try:
+		from urlparse import urlparse
+	except ImportError:
+		from urllib.parse import urlparse
+
+	parsed = urlparse(url)
+	if not parsed.scheme:
+		if raise_err:
+			raise TypeError('{url} Not a valid URL'.format(url = url))
+		else:
+			return False
+	
+	return True
+	
 def get_app(git_url, branch=None, bench_path='.', build_asset_files=True, verbose=False):
-	#less verbose app install
+	# from bench.utils import check_url
-	if '/' not in git_url:
+	try:
-		git_url = 'https://github.com/frappe/' + git_url
+		from urlparse import urljoin
+	except ImportError:
+		from urllib.parse import urljoin
+
+	if not check_url(git_url, raise_err = False):
+		orgs = ['frappe', 'erpnext']
+		for org in orgs:
+			url = 'https://api.github.com/repos/{org}/{app}'.format(org = org, app = git_url)
+			res = requests.get(url)
+			if res.ok:
+				data    = res.json()
+				if 'name' in data:
+					if git_url == data['name']:
+						git_url = 'https://github.com/{org}/{app}'.format(org = org, app = git_url)
+						break
+
 	#Gets repo name from URL
 	repo_name = git_url.rsplit('/', 1)[1].rsplit('.', 1)[0]
 	logger.info('getting app {}'.format(repo_name))
@@ -239,7 +269,7 @@ def get_upstream_version(app, branch=None, bench_path='.'):
 	try:
 		contents = subprocess.check_output(['git', 'show', 'upstream/{branch}:{app}/__init__.py'.format(branch=branch, app=app)], cwd=repo_dir, stderr=subprocess.STDOUT)
 	except subprocess.CalledProcessError as e:
-		if "Invalid object" in e.output:
+		if b"Invalid object" in e.output:
 			return None
 		else:
 			raise

bench/cli.py

@@ -95,7 +95,9 @@ def get_frappe_commands(bench_path='.'):
 	if not os.path.exists(sites_path):
 		return []
 	try:
-		return json.loads(get_cmd_output("{python} -m frappe.utils.bench_helper get-frappe-commands".format(python=python), cwd=sites_path))
+		output = get_cmd_output("{python} -m frappe.utils.bench_helper get-frappe-commands".format(python=python), cwd=sites_path)
+		output = output.decode('utf-8')
+		return json.loads(output)
 	except subprocess.CalledProcessError:
 		return []
 

bench/commands/__init__.py

@@ -1,5 +1,16 @@
 import click
 
+import os, shutil
+import os.path as osp
+import logging
+
+from datetime import datetime
+
+from bench.utils import which, exec_cmd
+
+log = logging.getLogger(__name__)
+log.setLevel(logging.ERROR)
+
 def print_bench_version(ctx, param, value):
 	"""Prints current bench version"""
 	if not value or ctx.resilient_parsing:
@@ -14,7 +25,6 @@ def print_bench_version(ctx, param, value):
 def bench_command(bench_path='.'):
 	"""Bench manager for Frappe"""
 	import bench
-	from bench.app import get_current_frappe_version
 	from bench.utils import setup_logging
 
 	bench.set_frappe_version(bench_path=bench_path)
@@ -70,4 +80,81 @@ bench_command.add_command(remote_reset_url)
 bench_command.add_command(remote_urls)
 
 from bench.commands.install import install
-bench_command.add_command(install)
+bench_command.add_command(install)
+
+@click.command('migrate-env')
+@click.argument('python', type = str)
+@click.option('--no-backup', default = False, help = 'Do not backup the existing Virtual Environment')
+def migrate_env(python, no_backup = False):
+	"""
+	Migrate Virtual Environment to desired Python Version.
+	"""
+	try:
+		# This is with the assumption that a bench is set-up within path.
+		path       = os.getcwd()
+
+		# I know, bad name for a flag. Thanks, Ameya! :| - <achilles@frappe.io>
+		if not no_backup:
+			# Back, the f*ck up.
+			parch = osp.join(path, 'archived_envs')
+			if not osp.exists(parch):
+				os.mkdir(parch)
+
+			# Simply moving. Thanks, Ameya.
+			# I'm keen to zip.
+			source = osp.join(path, 'env')
+			target = parch
+
+			log.debug('Backing up Virtual Environment')
+			stamp  = datetime.now().strftime('%Y%m%d_%H%M%S')
+			dest   = osp.join(path, str(stamp))
+
+			# WARNING: This is an archive, you might have to use virtualenv --relocate
+			# That's because virtualenv creates symlinks with shebangs pointing to executables.
+
+			# ...and shutil.copytree is a f*cking mess.
+			os.rename(source, dest)
+			shutil.move(dest, target)
+
+		log.debug('Setting up a New Virtual {python} Environment'.format(
+			python = python
+		))
+
+		# Path to Python Executable (Basically $PYTHONPTH)
+		python     = which(python)
+
+
+		virtualenv = which('virtualenv')
+
+		nvenv      = 'env'
+		pvenv      = osp.join(path, nvenv)
+
+		exec_cmd('{virtualenv} --python {python} {pvenv}'.format(
+			virtualenv = virtualenv,
+			python     = python,
+			pvenv      = pvenv
+		), cwd = path)
+
+		pip = osp.join(pvenv, 'bin', 'pip')
+		exec_cmd('{pip} install --upgrade pip'.format(pip=pip))
+		exec_cmd('{pip} install --upgrade setuptools'.format(pip=pip))
+		# TODO: Options
+
+		papps  = osp.join(path, 'apps')
+		apps   = ['frappe'] + [app for app in os.listdir(papps) if app != 'frappe']
+
+		for app in apps:
+			papp = osp.join(papps, app)
+			if osp.isdir(papp) and osp.exists(osp.join(papp, 'setup.py')):
+				exec_cmd('{pip} install -e {app}'.format(
+					pip = pip, app = papp
+				))
+
+		log.debug('Migration Successful to {python}'.format(
+			python = python
+		))
+	except:
+		log.debug('Migration Error')
+		raise
+
+bench_command.add_command(migrate_env)

bench/commands/make.py

@@ -2,6 +2,8 @@ import click
 
 @click.command()
 @click.argument('path')
+@click.option('--python', type = str, default = 'python', help = 'Path to Python Executable.')
+@click.option('--ignore-exist', is_flag = True, default = False, help = "Ignore if Bench instance exists.")
 @click.option('--apps_path', default=None, help="path to json files with apps to install after init")
 @click.option('--frappe-path', default=None, help="path to frappe repo")
 @click.option('--frappe-branch', default=None, help="path to frappe repo")
@@ -9,19 +11,23 @@ import click
 @click.option('--no-procfile', is_flag=True, help="Pull changes in all the apps in bench")
 @click.option('--no-backups',is_flag=True, help="Run migrations for all sites in the bench")
 @click.option('--no-auto-update',is_flag=True, help="Build JS and CSS artifacts for the bench")
-@click.option('--verbose',is_flag=True, help="Verbose output during install")
-@click.option('--skip-bench-mkdir', is_flag=True, help="Skip mkdir frappe-bench")
 @click.option('--skip-redis-config-generation', is_flag=True, help="Skip redis config generation if already specifying the common-site-config file")
+@click.option('--verbose',is_flag=True, help="Verbose output during install")
 def init(path, apps_path, frappe_path, frappe_branch, no_procfile, no_backups,
-		no_auto_update, clone_from, verbose, skip_bench_mkdir, skip_redis_config_generation):
+		no_auto_update, clone_from, verbose, skip_redis_config_generation,
-	"Create a new bench"
+		ignore_exist = False,
+		python 		 = 'python'): # Let's change we're ready. - <achilles@frappe.io>
+	'''
+	Create a New Bench Instance.
+	'''
 	from bench.utils import init
 	init(path, apps_path=apps_path, no_procfile=no_procfile, no_backups=no_backups,
 			no_auto_update=no_auto_update, frappe_path=frappe_path, frappe_branch=frappe_branch,
-			verbose=verbose, clone_from=clone_from, skip_bench_mkdir=skip_bench_mkdir, skip_redis_config_generation=skip_redis_config_generation)
+			verbose=verbose, clone_from=clone_from, skip_redis_config_generation=skip_redis_config_generation,
+			ignore_exist = ignore_exist,
+			python 		 = python)
 	click.echo('Bench {} initialized'.format(path))
 
-
 @click.command('get-app')
 @click.argument('name', nargs=-1) # Dummy argument for backward compatibility
 @click.argument('git-url')

bench/commands/setup.py

@@ -4,8 +4,7 @@ import click, sys, json
 def setup():
 	"Setup bench"
 	pass
-
+	
-
 @click.command('sudoers')
 @click.argument('user')
 def setup_sudoers(user):
@@ -13,7 +12,6 @@ def setup_sudoers(user):
 	from bench.utils import setup_sudoers
 	setup_sudoers(user)
 
-
 @click.command('nginx')
 @click.option('--yes', help='Yes to regeneration of nginx config file', default=False, is_flag=True)
 def setup_nginx(yes=False):
@@ -47,7 +45,6 @@ def setup_fonts():
 	from bench.utils import setup_fonts
 	setup_fonts()
 
-
 @click.command('production')
 @click.argument('user')
 @click.option('--yes', help='Yes to regeneration config', is_flag=True, default=False)
@@ -71,10 +68,11 @@ def setup_backups():
 	setup_backups()
 
 @click.command('env')
-def setup_env():
+@click.option('--python', type = str, default = 'python', help = 'Path to Python Executable.')
+def setup_env(python='python'):
 	"Setup virtualenv for bench"
 	from bench.utils import setup_env
-	setup_env()
+	setup_env(python=python)
 
 @click.command('firewall')
 @click.option('--ssh_port')

bench/config/procfile.py

@@ -13,7 +13,8 @@ def setup_procfile(bench_path, yes=False):
 	procfile = bench.env.get_template('Procfile').render(
 		node=find_executable("node") or find_executable("nodejs"),
 		use_rq=use_rq(bench_path),
-		webserver_port=config.get('webserver_port'))
+		webserver_port=config.get('webserver_port'),
+		CI=os.environ.get('CI'))
 
 	with open(procfile_path, 'w') as f:
 		f.write(procfile)

bench/config/redis.py

@@ -56,8 +56,8 @@ def write_redis_config(template_name, context, bench_path):
 		f.write(template.render(**context))
 
 def get_redis_version():
-	version_string = subprocess.check_output('redis-server --version', shell=True).strip()
+	version_string = subprocess.check_output('redis-server --version', shell=True)
-
+	version_string = version_string.decode('utf-8').strip()
 	# extract version number from string
 	version = re.findall("\d+\.\d+", version_string)
 	if not version:

bench/config/templates/Procfile

@@ -4,7 +4,9 @@ redis_queue: redis-server config/redis_queue.conf
 web: bench serve {% if webserver_port -%} --port {{ webserver_port }} {%- endif %}
 
 socketio: {{ node }} apps/frappe/socketio.js
+{% if not CI %}
 watch: bench watch
+{% endif %}
 {% if use_rq -%}
 schedule: bench schedule
 worker_short: bench worker --queue short

bench/package.json

@@ -1,18 +0,0 @@
-{
-  "name": "frappe",
-  "description": "Default package.json for frappe apps",
-  "dependencies": {
-    "babel-core": "^6.24.1",
-    "babel-preset-babili": "0.0.12",
-    "babel-preset-es2015": "^6.24.1",
-    "babel-preset-es2016": "^6.24.1",
-    "babel-preset-es2017": "^6.24.1",
-    "chokidar": "^1.7.0",
-    "cookie": "^0.3.1",
-    "express": "^4.15.3",
-    "less": "^2.7.2",
-    "redis": "^2.7.1",
-    "socket.io": "^2.0.1",
-    "superagent": "^3.5.2"
-  }
-}

bench/patches/patches.txt

@@ -3,4 +3,4 @@ bench.patches.v3.celery_to_rq
 bench.patches.v3.redis_bind_ip
 bench.patches.v4.update_node
 bench.patches.v4.update_socketio
-
+bench.patches.v4.install_yarn #2

bench/patches/v4/install_yarn.py

@@ -0,0 +1,5 @@
+import os
+from bench.utils import exec_cmd
+
+def execute(bench_path):
+	exec_cmd('npm install yarn', os.path.join(bench_path, 'apps/frappe'))

bench/utils.py

@@ -27,31 +27,33 @@ def get_env_cmd(cmd, bench_path='.'):
 
 def init(path, apps_path=None, no_procfile=False, no_backups=False,
 		no_auto_update=False, frappe_path=None, frappe_branch=None, wheel_cache_dir=None,
-		verbose=False, clone_from=None, skip_bench_mkdir=False, skip_redis_config_generation=False):
+		verbose=False, clone_from=None, skip_redis_config_generation=False,
+		ignore_exist = False,
+		python		 = 'python'): # Let's change when we're ready. - <achilles@frappe.io>
 	from .app import get_app, install_apps_from_path
 	from .config.common_site_config import make_config
 	from .config import redis
 	from .config.procfile import setup_procfile
 	from bench.patches import set_all_patches_executed
 
-	if(skip_bench_mkdir):
+	import os.path as osp
-		pass
+
+	if osp.exists(path):
+		if not ignore_exist:
+			raise ValueError('Bench Instance {path} already exists.'.format(path = path))
 	else:
-		if os.path.exists(path):
-			print('Directory {} already exists!'.format(path))
-			raise Exception("Site directory already exists")
 		os.makedirs(path)
 
 	for dirname in folders_in_bench:
 		try:
 			os.makedirs(os.path.join(path, dirname))
-		except OSError, e:
+		except OSError as e:
 			if e.errno != os.errno.EEXIST:
 				pass
 
 	setup_logging()
 
-	setup_env(bench_path=path)
+	setup_env(bench_path=path, python = python)
 
 	make_config(path)
 
@@ -139,8 +141,21 @@ def exec_cmd(cmd, cwd='.'):
 	if return_code > 0:
 		raise CommandFailedError(cmd)
 
-def setup_env(bench_path='.'):
+def which(executable, raise_err = False):
-	exec_cmd('virtualenv -q {} -p {}'.format('env', sys.executable), cwd=bench_path)
+	from distutils.spawn import find_executable
+	exec_ = find_executable(executable)
+
+	if not exec_ and raise_err:
+		raise ValueError('{executable} not found.'.format(
+			executable = executable
+		))
+
+	return exec_
+
+def setup_env(bench_path='.', python = 'python'):
+	python = which(python, raise_err = True)
+
+	exec_cmd('virtualenv -q {} -p {}'.format('env', python), cwd=bench_path)
 	exec_cmd('./env/bin/pip -q install --upgrade pip', cwd=bench_path)
 	exec_cmd('./env/bin/pip -q install wheel', cwd=bench_path)
 	# exec_cmd('./env/bin/pip -q install https://github.com/frappe/MySQLdb1/archive/MySQLdb-1.2.5-patched.tar.gz', cwd=bench_path)
@@ -232,13 +247,14 @@ def setup_backups(bench_path='.'):
 
 def add_to_crontab(line):
 	current_crontab = read_crontab()
+	line = str.encode(line)
 	if not line in current_crontab:
 		cmd = ["crontab"]
 		if platform.system() == 'FreeBSD':
 			cmd = ["crontab", "-"]
 		s = subprocess.Popen(cmd, stdin=subprocess.PIPE)
 		s.stdin.write(current_crontab)
-		s.stdin.write(line + '\n')
+		s.stdin.write(line + b'\n')
 		s.stdin.close()
 
 def read_crontab():
@@ -331,7 +347,9 @@ def check_cmd(cmd, cwd='.'):
 def get_git_version():
 	'''returns git version from `git --version`
 	extracts version number from string `get version 1.9.1` etc'''
-	version = get_cmd_output("git --version").strip().split()[2]
+	version = get_cmd_output("git --version")
+	version = version.decode('utf-8')
+	version = version.strip().split()[2]
 	version = '.'.join(version.split('.')[0:2])
 	return float(version)
 
@@ -412,34 +430,17 @@ def update_requirements(bench_path='.'):
 def update_npm_packages(bench_path='.'):
 	print('Updating node libraries...')
 	apps_dir = os.path.join(bench_path, 'apps')
-	package_json = {}
+
+	if not find_executable('yarn'):
+		print("Please install yarn using below command and try again.")
+		print("`npm install -g yarn`")
+		return
 
 	for app in os.listdir(apps_dir):
-		package_json_path = os.path.join(apps_dir, app, 'package.json')
+		app_path = os.path.join(apps_dir, app)
+		if os.path.exists(os.path.join(app_path, 'package.json')):
+			exec_cmd('yarn install', cwd=app_path)
 
-		if os.path.exists(package_json_path):
-			with open(package_json_path, "r") as f:
-				app_package_json = json.loads(f.read())
-				# package.json is usually a dict in a dict
-				for key, value in iteritems(app_package_json):
-					if not key in package_json:
-						package_json[key] = value
-					else:
-						if isinstance(value, dict):
-							package_json[key].update(value)
-						elif isinstance(value, list):
-							package_json[key].extend(value)
-						else:
-							package_json[key] = value
-
-	if package_json is {}:
-		with open(os.path.join(os.path.dirname(__file__), 'package.json'), 'r') as f:
-			package_json = json.loads(f.read())
-
-	with open(os.path.join(bench_path, 'package.json'), 'w') as f:
-		f.write(json.dumps(package_json, indent=1, sort_keys=True))
-
-	exec_cmd('npm install', cwd=bench_path)
 
 def install_requirements(pip, req_file):
 	if os.path.exists(req_file):
@@ -766,11 +767,11 @@ def run_playbook(playbook_name, extra_vars=None, tag=None):
 		print("Ansible is needed to run this command, please install it using 'pip install ansible'")
 		sys.exit(1)
 	args = ['ansible-playbook', '-c', 'local', playbook_name]
-	
+
 	if extra_vars:
 		args.extend(['-e', json.dumps(extra_vars)])
-	
+
 	if tag:
 		args.extend(['-t', tag])
-	
+
 	subprocess.check_call(args, cwd=os.path.join(os.path.dirname(bench.__path__[0]), 'playbooks'))

playbooks/roles/nodejs/tasks/main.yml

@@ -37,3 +37,8 @@
     update_cache: yes
     force: yes
   when: ansible_os_family == 'Debian' or ansible_distribution == 'Ubuntu'
+
+- name: Install yarn
+  command: npm install -g yarn
+  become: yes
+  become_user: root