Merge branch 'develop' into referrer-policy

.github/ISSUE_TEMPLATE/bug-report.md

@@ -6,8 +6,14 @@ labels: bug
 
 Issue: Bug report
 
+Please make sure your issue is reproducible on the latest bench version. The currently supported branches are:
+ - PyPI [latest](https://pypi.org/project/frappe-bench/) 
+ - master         (minor bug fixes)
+ - v5.x           (Merged with develop on every release)
+ - develop        (all updates)
+
 **Do the checklist before filing an issue:**
- - [ ] Can you replicate the issue?
+ - [ ] Can you replicate the issue on the supported bench versions?
  - [ ] Is this something you can debug and fix? Send a pull request! Bug fixes and documentation fixes are welcome
 
 **Describe the bug** :chart_with_downwards_trend:
@@ -34,9 +40,8 @@ If applicable, add screenshots to help explain your problem.
 
 **Version Information**
 
-Can be found out by running `bench version` in your respective bench folder
-
-- Bench Branch: `master` _(Only master is supported)_
+Can be found out by running `bench version` in your respective bench folder.
+- Bench Branch:
 - Frappe Version:
 - ERPNext Version:
 

.travis.yml

@@ -18,45 +18,55 @@ matrix:
   - name: "Python 2.7 Basic Setup"
     python: 2.7
     env: TEST=bench
-    script: python -m unittest -v bench.tests.test_init
+    script: python bench/tests/test_init.py TestBenchInit.basic
 
   - name: "Python 3.6 Basic Setup"
     python: 3.6
     env: TEST=bench
-    script: python -m unittest -v bench.tests.test_init
+    script: python bench/tests/test_init.py TestBenchInit.basic
 
   - name: "Python 3.7 Basic Setup"
     python: 3.7
     env: TEST=bench
-    script: python -m unittest -v bench.tests.test_init
+    script: python bench/tests/test_init.py TestBenchInit.basic
 
   - name: "Python 3.8 Production Setup"
     python: 3.8
     env: TEST=bench
-    script: python -m unittest -v bench.tests.test_setup_production
+    script: python bench/tests/test_setup_production.py TestSetupProduction.production
 
   - name: "Python 2.7 Production Setup"
     python: 2.7
     env: TEST=bench
-    script: python -m unittest -v bench.tests.test_setup_production
+    script: python bench/tests/test_setup_production.py TestSetupProduction.production
 
   - name: "Python 3.6 Production Setup"
     python: 3.6
     env: TEST=bench
-    script: python -m unittest -v bench.tests.test_setup_production
+    script: python bench/tests/test_setup_production.py TestSetupProduction.production
 
   - name: "Python 3.7 Production Setup"
     python: 3.7
     env: TEST=bench
-    script: python -m unittest -v bench.tests.test_setup_production
+    script: python bench/tests/test_setup_production.py TestSetupProduction.production
 
   - name: "Python 3.8 Production Setup"
     python: 3.8
     env: TEST=bench
-    script: python -m unittest -v bench.tests.test_setup_production
+    script: python bench/tests/test_setup_production.py TestSetupProduction.production
 
-  - name: "Python 3.6 Easy Install"
-    python: 3.6
+  - name: "Python 2.7 Tests"
+    python: 2.7
+    env: TEST=bench
+    script: python -m unittest -v bench.tests.test_init
+
+  - name: "Python 3.7 Tests"
+    python: 3.7
+    env: TEST=bench
+    script: python -m unittest -v bench.tests.test_init
+
+  - name: "Python 3.5 Easy Install"
+    python: 3.5
     env: TEST=easy_install
     script: sudo python $TRAVIS_BUILD_DIR/install.py --user travis --run-travis --production --verbose
 

README.md

@@ -158,7 +158,7 @@ $ pip install frappe-bench
 For more extensive distribution-dependent documentation, check out the following guides:
 
  - [Hitchhiker's Guide to Installing Frappe on Linux](https://github.com/frappe/frappe/wiki/The-Hitchhiker%27s-Guide-to-Installing-Frappe-on-Linux)
- - [Hitchhiker's Guide to Installing Frappe on MacOS](https://github.com/frappe/frappe/wiki/The-Hitchhiker%27s-Guide-to-Installing-Frappe-on-Mac-OS-X)
+ - [Hitchhiker's Guide to Installing Frappe on MacOS](https://github.com/frappe/bench/wiki/Setting-up-a-Mac-for-Frappe-ERPNext-Development)
 
 
 ## Basic Usage

bench/cli.py

@@ -32,7 +32,7 @@ def cli():
 	change_dir()
 	change_uid()
 
-	if is_dist_editable(bench.PROJECT_NAME) and len(sys.argv) > 1 and sys.argv[1] != "src":
+	if is_dist_editable(bench.PROJECT_NAME) and len(sys.argv) > 1 and sys.argv[1] != "src" and not get_config(".").get("developer_mode"):
 		log("bench is installed in editable mode!\n\nThis is not the recommended mode of installation for production. Instead, install the package from PyPI with: `pip install frappe-bench`\n", level=3)
 
 	if not is_bench_directory() and not cmd_requires_root() and len(sys.argv) > 1 and sys.argv[1] not in ("init", "find", "src"):

bench/commands/setup.py

@@ -43,6 +43,7 @@ def reload_nginx():
 @click.option("--user", help="optional user argument")
 @click.option("--yes", help="Yes to regeneration of supervisor config", is_flag=True, default=False)
 def setup_supervisor(user=None, yes=False):
+	bench.config.supervisor.update_supervisord_config(user=user, yes=yes)
 	bench.config.supervisor.generate_supervisor_config(bench_path=".", user=user, yes=yes)
 
 

bench/commands/update.py

@@ -6,7 +6,7 @@ from bench.app import pull_apps
 from bench.utils import post_upgrade, patch_sites, build_assets
 
 
-@click.command('update', help="Updates bench tool and if executed in a bench directory, without any flags will backup, pull, setup requirements, build, run patches and restart bench. Using specific flags will only do certain tasks instead of all")
+@click.command('update', help="Performs an update operation on current bench. Without any flags will backup, pull, setup requirements, build, run patches and restart bench. Using specific flags will only do certain tasks instead of all")
 @click.option('--pull', is_flag=True, help="Pull updates for all the apps in bench")
 @click.option('--apps', type=str)
 @click.option('--patch', is_flag=True, help="Run migrations for all sites in the bench")

bench/commands/utils.py

@@ -8,11 +8,12 @@ import click
 
 @click.command('start', help="Start Frappe development processes")
 @click.option('--no-dev', is_flag=True, default=False)
+@click.option('--no-prefix', is_flag=True, default=False, help="Hide process name from bench start log")
 @click.option('--concurrency', '-c', type=str)
 @click.option('--procfile', '-p', type=str)
-def start(no_dev, concurrency, procfile):
+def start(no_dev, concurrency, procfile, no_prefix):
 	from bench.utils import start
-	start(no_dev=no_dev, concurrency=concurrency, procfile=procfile)
+	start(no_dev=no_dev, concurrency=concurrency, procfile=procfile, no_prefix=no_prefix)
 
 
 @click.command('restart', help="Restart supervisor processes or systemd units")

bench/config/production_setup.py

@@ -1,13 +1,18 @@
 # imports - standard imports
 import os
+import logging
 import sys
 
 # imports - module imports
+import bench
 from bench.config.common_site_config import get_config
 from bench.config.nginx import make_nginx_conf
-from bench.config.supervisor import generate_supervisor_config
+from bench.config.supervisor import generate_supervisor_config, update_supervisord_config
 from bench.config.systemd import generate_systemd_config
-from bench.utils import CommandFailedError, exec_cmd, find_executable, fix_prod_setup_perms, get_bench_name, get_cmd_output
+from bench.utils import CommandFailedError, exec_cmd, find_executable, fix_prod_setup_perms, get_bench_name, get_cmd_output, log
+
+
+logger = logging.getLogger(bench.PROJECT_NAME)
 
 
 def setup_production_prerequisites():
@@ -23,14 +28,20 @@ def setup_production_prerequisites():
 
 
 def setup_production(user, bench_path='.', yes=False):
+	print("Setting Up prerequisites...")
 	setup_production_prerequisites()
 	if get_config(bench_path).get('restart_supervisor_on_update') and get_config(bench_path).get('restart_systemd_on_update'):
 		raise Exception("You cannot use supervisor and systemd at the same time. Modify your common_site_config accordingly." )
 
 	if get_config(bench_path).get('restart_systemd_on_update'):
+		print("Setting Up systemd...")
 		generate_systemd_config(bench_path=bench_path, user=user, yes=yes)
 	else:
+		print("Setting Up supervisor...")
+		update_supervisord_config(user=user, yes=yes)
 		generate_supervisor_config(bench_path=bench_path, user=user, yes=yes)
+
+	print("Setting Up NGINX...")
 	make_nginx_conf(bench_path=bench_path, yes=yes)
 	fix_prod_setup_perms(bench_path, frappe_user=user)
 	remove_default_nginx_configs()
@@ -38,6 +49,7 @@ def setup_production(user, bench_path='.', yes=False):
 	bench_name = get_bench_name(bench_path)
 	nginx_conf = '/etc/nginx/conf.d/{bench_name}.conf'.format(bench_name=bench_name)
 
+	print("Setting Up symlinks and reloading services...")
 	if get_config(bench_path).get('restart_supervisor_on_update'):
 		supervisor_conf_extn = "ini" if is_centos7() else "conf"
 		supervisor_conf = os.path.join(get_supervisor_confdir(), '{bench_name}.{extn}'.format(
@@ -100,7 +112,7 @@ def service(service_name, service_option):
 			exec_cmd(service_manager_command)
 
 		else:
-			raise Exception('No service manager found')
+			log("No service manager found: '{0} {1}' failed to execute".format(service_name, service_option), level=2)
 
 
 def get_supervisor_confdir():

bench/config/supervisor.py

@@ -1,5 +1,6 @@
 # imports - standard imports
 import getpass
+import logging
 import os
 
 # imports - module imports
@@ -13,13 +14,14 @@ import click
 from six.moves import configparser
 
 
+logger = logging.getLogger(bench.PROJECT_NAME)
+
+
 def generate_supervisor_config(bench_path, user=None, yes=False):
 	"""Generate supervisor config for respective bench path"""
 	if not user:
 		user = getpass.getuser()
 
-	update_supervisord_conf(user=user)
-
 	template = bench.config.env.get_template('supervisor.conf')
 	config = get_config(bench_path=bench_path)
 	bench_dir = os.path.abspath(bench_path)
@@ -64,14 +66,23 @@ def get_supervisord_conf():
 			return possibility
 
 
-def update_supervisord_conf(user):
-	"""From bench v5.0, we're moving to supervisor running as user"""
+def update_supervisord_config(user=None, yes=False):
+	"""From bench v5.x, we're moving to supervisor running as user"""
 	from bench.config.production_setup import service
 
+	if not user:
+		user = getpass.getuser()
+
 	supervisord_conf = get_supervisord_conf()
 	section = "unix_http_server"
+	updated_values = {
+		"chmod": "0760",
+		"chown": "{user}:{user}".format(user=user)
+	}
+	supervisord_conf_changes = ""
 
 	if not supervisord_conf:
+		logger.log("supervisord.conf not found")
 		return
 
 	config = configparser.ConfigParser()
@@ -79,12 +90,35 @@ def update_supervisord_conf(user):
 
 	if section not in config.sections():
 		config.add_section(section)
+		action = "Section {0} Added".format(section)
+		logger.log(action)
+		supervisord_conf_changes += '\n' + action
 
-	config.set(section, "chmod", "0760")
-	config.set(section, "chown", "{user}:{user}".format(user=user))
+	for key, value in updated_values.items():
+		try:
+			current_value = config.get(section, key)
+		except configparser.NoOptionError:
+			current_value = ""
 
-	with open(supervisord_conf, "w") as f:
-		config.write(f)
+		if current_value.strip() != value:
+			config.set(section, key, value)
+			action = "Updated supervisord.conf: '{0}' changed from '{1}' to '{2}'".format(key, current_value, value)
+			logger.log(action)
+			supervisord_conf_changes += '\n' + action
 
-	# restart supervisor to take new changes into effect
-	service('supervisor', 'restart')
+	if not supervisord_conf_changes:
+		logger.log("supervisord.conf not updated")
+		return
+
+	if not yes:
+		click.confirm("{0} will be updated with the following values:\n{1}\nDo you want to continue?".format(supervisord_conf, supervisord_conf_changes), abort=True)
+
+	try:
+		with open(supervisord_conf, "w") as f:
+			config.write(f)
+			logger.log("Updated supervisord.conf at '{0}'".format(supervisord_conf))
+	except Exception as e:
+		logger.log("Updating supervisord.conf failed due to '{0}'".format(e))
+
+	# Reread supervisor configuration, reload supervisord and supervisorctl, restart services that were started
+	service('supervisor', 'reload')

bench/config/templates/nginx.conf

@@ -86,7 +86,7 @@ server {
 	}
 
 	location @webserver {
-		proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+		proxy_set_header X-Forwarded-For $remote_addr;
 		proxy_set_header X-Forwarded-Proto $scheme;
 		proxy_set_header X-Frappe-Site-Name {{ site_name }};
 		proxy_set_header Host $host;

bench/patches/patches.txt

@@ -5,3 +5,4 @@ bench.patches.v4.update_node
 bench.patches.v4.update_socketio
 bench.patches.v4.install_yarn #2
 bench.patches.v5.fix_user_permissions
+bench.patches.v5.fix_backup_cronjob

bench/patches/v5/fix_backup_cronjob.py

@@ -0,0 +1,15 @@
+from bench.config.common_site_config import get_config
+from crontab import CronTab
+
+
+def execute(bench_path):
+	"""
+		This patch fixes a cron job that would backup sites every minute per 6 hours
+	"""
+
+	user = get_config(bench_path=bench_path).get('frappe_user')
+	user_crontab = CronTab(user=user)
+
+	for job in user_crontab.find_comment("bench auto backups set for every 6 hours"):
+		job.every(6).hours()
+		user_crontab.write()

bench/playbooks/roles/common/tasks/ubuntu.yml

@@ -33,9 +33,9 @@
 - name: install pdf prerequisites for Ubuntu >= 18.04
   apt:
     pkg:
-      - libssl1.0-dev
+      - libssl1.1
     state: present
     force: yes
   when: ansible_distribution_version is version_compare('18.04', 'ge')
 
-...
+...

bench/playbooks/roles/dns_caching/handlers/main.yml

@@ -1,4 +1,4 @@
 ---
-- name: Restart network manager
+- name: restart network manager
   service: name=NetworkManager state=restarted
 ...

bench/playbooks/roles/nodejs/defaults/main.yml

@@ -1,3 +1,3 @@
 ---
-node_version: 8
+node_version: 12
 ...

bench/playbooks/roles/wkhtmltopdf/tasks/main.yml

@@ -20,6 +20,12 @@
     force: yes
   when: ansible_os_family == 'Debian'
 
+- name: download wkthmltox Ubuntu 20
+  get_url:
+    url: https://github.com/wkhtmltopdf/wkhtmltopdf/releases/download/0.12.5/wkhtmltox_0.12.5-1.focal_amd64.deb
+    dest: /tmp/wkhtmltox.deb
+  when: ansible_distribution == 'Ubuntu' and ansible_distribution_major_version == '20'
+    
 - name: download wkthmltox Ubuntu 18
   get_url:
     url: https://github.com/wkhtmltopdf/wkhtmltopdf/releases/download/0.12.5/wkhtmltox_0.12.5-1.bionic_{{ "amd64" if ansible_architecture == "x86_64" else "i386"}}.deb

bench/tests/test_base.py

@@ -1,19 +1,19 @@
 # imports - standard imports
+import getpass
 import json
 import os
 import shutil
 import subprocess
 import sys
+import traceback
 import unittest
-import getpass
-
-# imports - module imports
-import bench
-import bench.utils
 
 # imports - third party imports
 from six import PY2
 
+# imports - module imports
+import bench
+import bench.utils
 
 if PY2:
 	FRAPPE_BRANCH = "version-12"
@@ -102,3 +102,9 @@ class TestBenchBase(unittest.TestCase):
 		if os.environ.get("CI"):
 			return not subprocess.call(["sudo", "test", "-f", path])
 		return os.path.isfile(path)
+
+	def get_traceback(self):
+		exc_type, exc_value, exc_tb = sys.exc_info()
+		trace_list = traceback.format_exception(exc_type, exc_value, exc_tb)
+		body = "".join(str(t) for t in trace_list)
+		return body

bench/tests/test_init.py

@@ -11,7 +11,7 @@ import git
 import bench
 import bench.utils
 from bench.release import get_bumped_version
-from bench.tests.test_base import TestBenchBase, FRAPPE_BRANCH
+from bench.tests.test_base import FRAPPE_BRANCH, TestBenchBase
 
 
 class TestBenchInit(TestBenchBase):
@@ -34,6 +34,13 @@ class TestBenchInit(TestBenchBase):
 		self.assert_config(bench_name)
 
 
+	def basic(self):
+		try:
+			self.test_init()
+		except Exception:
+			print(self.get_traceback())
+
+
 	def test_multiple_benches(self):
 		for bench_name in ("test-bench-1", "test-bench-2"):
 			self.init_bench(bench_name)
@@ -148,4 +155,4 @@ class TestBenchInit(TestBenchBase):
 
 
 if __name__ == '__main__':
-	unittest.main()
+	unittest.main()

bench/tests/test_setup_production.py

@@ -19,7 +19,7 @@ class TestSetupProduction(TestBenchBase):
 		for bench_name in ("test-bench-1", "test-bench-2"):
 			bench_path = os.path.join(os.path.abspath(self.benches_path), bench_name)
 			self.init_bench(bench_name)
-			bench.utils.exec_cmd("sudo bench setup production {0}".format(user), cwd=bench_path)
+			bench.utils.exec_cmd("sudo bench setup production {0} --yes".format(user), cwd=bench_path)
 			self.assert_nginx_config(bench_name)
 			self.assert_supervisor_config(bench_name)
 			self.assert_supervisor_process(bench_name)
@@ -33,6 +33,13 @@ class TestSetupProduction(TestBenchBase):
 			bench.utils.exec_cmd("sudo bench disable-production", cwd=bench_path)
 
 
+	def production(self):
+		try:
+			self.test_setup_production()
+		except Exception:
+			print(self.get_traceback())
+
+
 	def assert_nginx_config(self, bench_name):
 		conf_src = os.path.join(os.path.abspath(self.benches_path), bench_name, 'config', 'nginx.conf')
 		conf_dest = "/etc/nginx/conf.d/{bench_name}.conf".format(bench_name=bench_name)
@@ -169,4 +176,4 @@ class TestSetupProduction(TestBenchBase):
 
 
 if __name__ == '__main__':
-	unittest.main()
+	unittest.main()

bench/utils.py

@@ -70,9 +70,16 @@ def log(message, level=0):
 		2: color.red + 'ERROR',			# fail
 		3: color.yellow + 'WARN'		# warn/suggest
 	}
+	loggers = {
+		2: logger.error,
+		3: logger.warning
+	}
+
 	start_line = (levels.get(level) + ': ') if level in levels else ''
+	level_logger = loggers.get(level, logger.info)
 	end_line = '\033[0m'
 
+	level_logger(message)
 	print(start_line + message + end_line)
 
 
@@ -393,7 +400,7 @@ def setup_backups(bench_path='.'):
 
 	if job_command not in str(system_crontab):
 		job = system_crontab.new(command=job_command, comment="bench auto backups set for every 6 hours")
-		job.hour.every(6)
+		job.every(6).hours()
 		system_crontab.write()
 
 
@@ -454,7 +461,7 @@ def get_process_manager():
 			return proc_man_path
 
 
-def start(no_dev=False, concurrency=None, procfile=None):
+def start(no_dev=False, concurrency=None, procfile=None, no_prefix=False):
 	program = get_process_manager()
 	if not program:
 		raise Exception("No process manager found")
@@ -469,6 +476,9 @@ def start(no_dev=False, concurrency=None, procfile=None):
 	if procfile:
 		command.extend(['-f', procfile])
 
+	if no_prefix:
+		command.extend(['--no-prefix'])
+		
 	os.execv(program, command)
 
 

docs/easy_install.md

@@ -88,4 +88,4 @@ TLDR; Save the logs!
 
 3. A lot of things can go wrong in setting up the environment due to prior settings, company protocols or even breaking changes in system packages and their dependencies.
 
-4. Sharing your logfile in any issues opened related to this can help us find solutions to it faster and make the sript better!
+4. Sharing your logfile in any issues opened related to this can help us find solutions to it faster and make the script better!

install.py

@@ -20,6 +20,7 @@ execution_time = "{:%H:%M}".format(execution_timestamp)
 log_file_name = "easy-install__{0}__{1}.log".format(execution_day, execution_time.replace(':', '-'))
 log_path = os.path.join(tmp_log_folder, log_file_name)
 log_stream = sys.stdout
+distro_required = not ((sys.version_info.major < 3) or (sys.version_info.major == 3 and sys.version_info.minor < 5))
 
 
 def log(message, level=0):
@@ -78,8 +79,8 @@ def check_distribution_compatibility():
 	dist_name, dist_version = get_distribution_info()
 	supported_dists = {
 		'macos': [10.9, 10.10, 10.11, 10.12],
-		'ubuntu': [14, 15, 16, 18, 19],
-		'debian': [8, 9],
+		'ubuntu': [14, 15, 16, 18, 19, 20],
+		'debian': [8, 9, 10],
 		'centos': [7]
 	}
 
@@ -94,10 +95,32 @@ def check_distribution_compatibility():
 		log("Sorry, the installer doesn't support {0}. Aborting installation!".format(dist_name), level=2)
 
 
+def import_with_install(package):
+	# copied from https://discuss.erpnext.com/u/nikunj_patel
+	# https://discuss.erpnext.com/t/easy-install-setup-guide-for-erpnext-installation-on-ubuntu-20-04-lts-with-some-modification-of-course/62375/5
+	# need to move to top said v13 for fully python3 era
+	import importlib
+
+	try:
+		importlib.import_module(package)
+	except ImportError:
+		# caveat : pip3 must be installed
+
+		import pip
+
+		pip.main(['install', package])
+	finally:
+		globals()[package] = importlib.import_module(package)
+
+
 def get_distribution_info():
 	# return distribution name and major version
 	if platform.system() == "Linux":
-		current_dist = platform.dist()
+		if distro_required:
+			current_dist = distro.linux_distribution(full_distribution_name=True)
+		else:
+			current_dist = platform.dist()
+
 		return current_dist[0].lower(), current_dist[1].rsplit('.')[0]
 
 	elif platform.system() == "Darwin":
@@ -366,6 +389,12 @@ def run_playbook(playbook_name, sudo=False, extra_vars=None):
 	return success
 
 
+def setup_script_requirements():
+	if distro_required:
+		install_package('pip3', 'python3-pip')
+		import_with_install('distro')
+
+
 def parse_commandline_args():
 	import argparse
 
@@ -396,17 +425,19 @@ def parse_commandline_args():
 	parser.add_argument('--overwrite', dest='overwrite', action='store_true', default=False, help='Whether to overwrite an existing bench')
 	# set passwords
 	parser.add_argument('--mysql-root-password', dest='mysql_root_password', help='Set mysql root password')
-	parser.add_argument('--mariadb-version', dest='mariadb_version', default='10.2', help='Specify mariadb version')
+	parser.add_argument('--mariadb-version', dest='mariadb_version', default='10.4', help='Specify mariadb version')
 	parser.add_argument('--admin-password', dest='admin_password', help='Set admin password')
 	parser.add_argument('--bench-name', dest='bench_name', help='Create bench with specified name. Default name is frappe-bench')
 	# Python interpreter to be used
 	parser.add_argument('--python', dest='python', default='python3', help=argparse.SUPPRESS)
 	# LXC Support
 	parser.add_argument('--container', dest='container', default=False, action='store_true', help='Use if you\'re creating inside LXC')
+
 	args = parser.parse_args()
 
 	return args
 
+
 if __name__ == '__main__':
 	if sys.version[0] == '2':
 		if not os.environ.get('CI'):
@@ -433,6 +464,7 @@ if __name__ == '__main__':
 	with warnings.catch_warnings():
 		warnings.simplefilter("ignore")
 		setup_log_stream(args)
+		setup_script_requirements()
 		check_distribution_compatibility()
 		check_system_package_managers()
 		check_environment()