Merge branch 'develop' into semantic-release-bench

.github/semantic.yml

@@ -0,0 +1,13 @@
+# Always validate the PR title AND all the commits
+titleAndCommits: true
+
+# Allow use of Merge commits (eg on github: "Merge branch 'master' into feature/ride-unicorns")
+# this is only relevant when using commitsOnly: true (or titleAndCommits: true)
+allowMergeCommits: true
+
+# Allow use of Revert commits (eg on github: "Revert "feat: ride unicorns"")
+# this is only relevant when using commitsOnly: true (or titleAndCommits: true)
+allowRevertCommits: true
+
+# For allowed PR types: https://github.com/commitizen/conventional-commit-types/blob/v3.0.0/index.json
+# Tool Reference: https://github.com/zeke/semantic-pull-requests

README.md

@@ -18,6 +18,7 @@ Bench is a command-line utility that helps you to install, update, and manage mu
  - [Bench Manager](#bench-manager)
  - [Guides](#guides)
  - [Resources](#resources)
+ - [Development](#development)
  - [License](#license)
 
 
@@ -254,6 +255,36 @@ For an exhaustive list of guides, check out [Bench Guides](https://frappe.io/doc
 For an exhaustive list of resources, check out [Bench Resources](https://frappe.io/docs/user/en/bench/resources).
 
 
+## Development
+
+To contribute and develop on the bench CLI tool, clone this repo and create an editable install. In editable mode, you may get the following warning everytime you run a bench command:
+
+	WARN: bench is installed in editable mode!
+
+	This is not the recommended mode of installation for production. Instead, install the package from PyPI with: `pip install frappe-bench`
+
+
+```sh
+$ git clone https://github.com/frappe/bench ~/bench-repo
+$ pip3 install -e ~/bench-repo
+$ bench src
+/Users/frappe/bench-repo
+```
+
+To clear up the editable install and switch to a stable version of bench, uninstall via pip and delete the corresponding egg file from the python path.
+
+
+```sh
+# Delete bench installed in editable install
+$ rm -r $(find ~ -name '*.egg-info')
+$ pip3 uninstall frappe-bench
+
+# Install latest released version of bench
+$ pip3 install -U frappe-bench
+```
+
+To confirm the switch, check the output of `bench src`. It should change from something like `$HOME/bench-repo` to `/usr/local/lib/python3.6/dist-packages` and stop the editable install warnings from getting triggered at every command.
+
 ## License
 
 This repository has been released under the [GNU GPLv3 License](LICENSE).

bench/cli.py

@@ -27,11 +27,13 @@ def cli():
 	command = " ".join(sys.argv)
 
 	change_working_directory()
-	logger = setup_logging() or logging.getLogger(bench.PROJECT_NAME)
+	logger = setup_logging()
 	logger.info(command)
+
+	if len(sys.argv) > 1 and sys.argv[1] not in ("src", ):
 		check_uid()
-	change_dir()
 		change_uid()
+		change_dir()
 
 	if is_dist_editable(bench.PROJECT_NAME) and len(sys.argv) > 1 and sys.argv[1] != "src" and not get_config(".").get("developer_mode"):
 		log("bench is installed in editable mode!\n\nThis is not the recommended mode of installation for production. Instead, install the package from PyPI with: `pip install frappe-bench`\n", level=3)

bench/commands/make.py

@@ -11,8 +11,8 @@ import click
 @click.option('--frappe-branch', default=None, help="path to frappe repo")
 @click.option('--clone-from', default=None, help="copy repos from path")
 @click.option('--clone-without-update', is_flag=True, help="copy repos from path without update")
-@click.option('--no-procfile', is_flag=True, help="Pull changes in all the apps in bench")
-@click.option('--no-backups',is_flag=True, help="Run migrations for all sites in the bench")
+@click.option('--no-procfile', is_flag=True, help="Do not create a Procfile")
+@click.option('--no-backups',is_flag=True, help="Do not set up automatic periodic backups for all sites on this bench")
 @click.option('--skip-redis-config-generation', is_flag=True, help="Skip redis config generation if already specifying the common-site-config file")
 @click.option('--skip-assets',is_flag=True, default=False, help="Do not build assets")
 @click.option('--verbose',is_flag=True, help="Verbose output during install")

bench/commands/update.py

@@ -15,11 +15,12 @@ from bench.utils import post_upgrade, patch_sites, build_assets
 @click.option('--restart-supervisor', is_flag=True, help="Restart supervisor processes after update")
 @click.option('--restart-systemd', is_flag=True, help="Restart systemd units after update")
 @click.option('--no-backup', is_flag=True, help="If this flag is set, sites won't be backed up prior to updates. Note: This is not recommended in production.")
+@click.option('--no-compile', is_flag=True, help="If set, Python bytecode won't be compiled before restarting the processes")
 @click.option('--force', is_flag=True, help="Forces major version upgrades")
 @click.option('--reset', is_flag=True, help="Hard resets git branch's to their new states overriding any changes and overriding rebase on pull")
-def update(pull, apps, patch, build, requirements, restart_supervisor, restart_systemd, no_backup, force, reset):
+def update(pull, apps, patch, build, requirements, restart_supervisor, restart_systemd, no_backup, no_compile, force, reset):
 	from bench.utils import update
-	update(pull=pull, apps=apps, patch=patch, build=build, requirements=requirements, restart_supervisor=restart_supervisor, restart_systemd=restart_systemd, backup=not no_backup, force=force, reset=reset)
+	update(pull=pull, apps=apps, patch=patch, build=build, requirements=requirements, restart_supervisor=restart_supervisor, restart_systemd=restart_systemd, backup=not no_backup, compile=not no_compile, force=force, reset=reset)
 
 
 @click.command('retry-upgrade', help="Retry a failed upgrade")

bench/commands/utils.py

@@ -111,7 +111,7 @@ def download_translations():
 	download_translations_p()
 
 
-@click.command('renew-lets-encrypt', help="Renew Let's Encrypt certificate")
+@click.command('renew-lets-encrypt', help="Sets Up latest cron and Renew Let's Encrypt certificate")
 def renew_lets_encrypt():
 	from bench.config.lets_encrypt import renew_certs
 	renew_certs()

bench/config/common_site_config.py

@@ -55,7 +55,7 @@ def get_gunicorn_workers():
 	'''This function will return the maximum workers that can be started depending upon
 	number of cpu's present on the machine'''
 	return {
-		"gunicorn_workers": multiprocessing.cpu_count()
+		"gunicorn_workers": multiprocessing.cpu_count() * 2 + 1
 	}
 
 def update_config_for_frappe(config, bench_path):

bench/config/lets_encrypt.py

@@ -87,10 +87,16 @@ def run_certbot_and_setup_ssl(site, custom_domain, bench_path, interactive=True)
 
 def setup_crontab():
 	job_command = '/opt/certbot-auto renew -a nginx --post-hook "systemctl reload nginx"'
+	job_comment = 'Renew lets-encrypt every month'
+	print("Setting Up cron job to {0}".format(job_comment))
+
 	system_crontab = CronTab(user='root')
-	if job_command not in str(system_crontab):
-		job = system_crontab.new(command=job_command, comment="Renew lets-encrypt every month")
-		job.day.on(1)
+
+	for job in system_crontab.find_comment(comment=job_comment): # Removes older entries
+		system_crontab.remove(job)
+
+	job = system_crontab.new(command=job_command, comment=job_comment)
+	job.setall('0 0 */1 * *') # Run at 00:00 every day-of-month
 	system_crontab.write()
 
 
@@ -113,10 +119,13 @@ def get_certbot_path():
 
 
 def renew_certs():
+	# Needs to be run with sudo
 	click.confirm('Running this will stop the nginx service temporarily causing your sites to go offline\n'
 		'Do you want to continue?',
 		abort=True)
 
+	setup_crontab()
+
 	service('nginx', 'stop')
 	exec_cmd("{path} renew".format(path=get_certbot_path()))
 	service('nginx', 'start')

bench/playbooks/roles/bench/tasks/setup_erpnext.yml

@@ -13,16 +13,17 @@
   - name: Check whether the site already exists
     stat: path="{{ bench_path }}/sites/{{ site }}"
     register: site_folder
+    when: not without_site
 
   - name: Create a new site
     command: "bench new-site {{ site }} --admin-password '{{ admin_password }}' --mariadb-root-password '{{ mysql_root_password }}'"
     args:
       chdir: "{{ bench_path }}"
-    when: not site_folder.stat.exists
+    when: not without_site and not site_folder.stat.exists
 
   - name: Install ERPNext to default site
     command: "bench --site {{ site }} install-app erpnext"
     args:
       chdir: "{{ bench_path }}"
-    when: not without_erpnext
+    when: not without_site and not without_erpnext
 ...

bench/playbooks/roles/mariadb/tasks/main.yml

@@ -48,6 +48,10 @@
       [mysqld]
       pid-file        = /var/run/mysqld/mysqld.pid
       socket          = /var/run/mysqld/mysqld.sock
+
+      # setting appeared inside mysql but overwritten by mariadb inside mariadb.conf.d/xx-server.cnf valued as utf8mb4_general_ci
+
+      collation-server = utf8mb4_unicode_ci
     create: yes
   become: yes
   become_user: root

bench/playbooks/roles/wkhtmltopdf/tasks/main.yml

@@ -24,7 +24,14 @@
   get_url:
     url: https://github.com/wkhtmltopdf/wkhtmltopdf/releases/download/0.12.5/wkhtmltox_0.12.5-1.focal_amd64.deb
     dest: /tmp/wkhtmltox.deb
-  when: ansible_distribution == 'Ubuntu' and ansible_distribution_major_version == '20'
+  when: ansible_distribution == 'Ubuntu' and ansible_distribution_major_version == '20' and ansible_architecture != 'aarch64'
+
+- name: download wkthmltox Ubuntu 20 arm64
+  get_url:
+    # wkhtmltox supports arm64 starting from 0.12.6
+    url: https://github.com/wkhtmltopdf/packaging/releases/download/0.12.6-1/wkhtmltox_0.12.6-1.focal_arm64.deb
+    dest: /tmp/wkhtmltox.deb
+  when: ansible_distribution == 'Ubuntu' and ansible_distribution_major_version == '20' and ansible_architecture == 'aarch64'
     
 - name: download wkthmltox Ubuntu 18
   get_url:

bench/tests/test_init.py

@@ -9,6 +9,7 @@ import git
 
 # imports - module imports
 import bench
+import bench.cli
 import bench.utils
 from bench.release import get_bumped_version
 from bench.tests.test_base import FRAPPE_BRANCH, TestBenchBase
@@ -27,6 +28,10 @@ class TestBenchInit(TestBenchBase):
 		self.assertEqual( get_bumped_version('11.0.5-beta.22', 'prerelease'), '11.0.5-beta.23' )
 
 
+	def test_utils(self):
+		self.assertEqual(subprocess.call("bench"), 0)
+
+
 	def test_init(self, bench_name="test-bench", **kwargs):
 		self.init_bench(bench_name, **kwargs)
 		self.assert_folders(bench_name)

bench/utils.py

@@ -2,6 +2,7 @@
 # -*- coding: utf-8 -*-
 
 # imports - standard imports
+import compileall
 import errno
 import glob
 import grp
@@ -183,8 +184,8 @@ def init(path, apps_path=None, no_procfile=False, no_backups=False,
 	copy_patches_txt(path)
 
 
-def update(pull=False, apps=None, patch=False, build=False, requirements=False, backup=True, force=False, reset=False,
-	restart_supervisor=False, restart_systemd=False):
+def update(pull=False, apps=None, patch=False, build=False, requirements=False, backup=True, compile=True,
+	force=False, reset=False, restart_supervisor=False, restart_systemd=False):
 	"""command: bench update"""
 	from bench import patches
 	from bench.app import is_version_upgrade, pull_apps, validate_branch
@@ -218,7 +219,6 @@ def update(pull=False, apps=None, patch=False, build=False, requirements=False,
 
 	if version_upgrade[0] or (not version_upgrade[0] and force):
 		validate_upgrade(version_upgrade[1], version_upgrade[2], bench_path=bench_path)
-
 	conf.update({ "maintenance_mode": 1, "pause_scheduler": 1 })
 	update_config(conf, bench_path=bench_path)
 
@@ -246,6 +246,10 @@ def update(pull=False, apps=None, patch=False, build=False, requirements=False,
 	if version_upgrade[0] or (not version_upgrade[0] and force):
 		post_upgrade(version_upgrade[1], version_upgrade[2], bench_path=bench_path)
 
+	if pull and compile:
+		print("Compiling Python files...")
+		compileall.compile_dir('../apps', quiet=1, rx=re.compile('.*node_modules.*'))
+
 	if restart_supervisor or conf.get('restart_supervisor_on_update'):
 		restart_supervisor_processes(bench_path=bench_path)
 
@@ -443,10 +447,13 @@ def setup_logging(bench_path='.'):
 	logging.Logger.log = logv
 
 	if os.path.exists(os.path.join(bench_path, 'logs')):
-		logger = logging.getLogger(bench.PROJECT_NAME)
 		log_file = os.path.join(bench_path, 'logs', 'bench.log')
-		formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s')
 		hdlr = logging.FileHandler(log_file)
+	else:
+		hdlr = logging.NullHandler()
+
+	logger = logging.getLogger(bench.PROJECT_NAME)
+	formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s')
 	hdlr.setFormatter(formatter)
 	logger.addHandler(hdlr)
 	logger.setLevel(logging.DEBUG)

docs/bench_usage.md

@@ -72,7 +72,7 @@ These commands belong directly to the bench group so they can be invoked directl
 
  - **init**: Initialize a new bench instance in the specified path. This sets up a complete bench folder with an `apps` folder which contains all the Frappe apps available in the current bench, `sites` folder that stores all site data seperated by individual site folders, `config` folder that contains your redis, NGINX and supervisor configuration files. The `env` folder consists of all python dependencies the current bench and installed Frappe applications have.
  - **restart**: Restart web, supervisor, systemd processes units. Used in production setup.
- - **update**: Updates bench tool and if executed in a bench directory, without any flags will backup, pull, setup requirements, build, run patches and restart bench. Using specific flags will only do certain tasks instead of all.
+ - **update**: If executed in a bench directory, without any flags will backup, pull, setup requirements, build, run patches and restart bench. Using specific flags will only do certain tasks instead of all.
  - **migrate-env**: Migrate Virtual Environment to desired Python version. This regenerates the `env` folder with the specified Python version.
  - **retry-upgrade**: Retry a failed upgrade
  - **disable-production**: Disables production environment for the bench.

docs/commands_and_usage.md

@@ -2,7 +2,12 @@
 
 * Updating
 
-	Currently, `bench update` can be run from any directory however the context of the command changes. If run from a bench directory, the vanilla command itself updates all apps, runs migrations and backs up all sites.
+To update the bench CLI tool, depending on your method of installation, you may use 
+
+	pip3 install -U frappe-bench
+
+
+To backup, update all apps and sites on your bench, you may use
 
 	bench update
 

install.py

@@ -157,13 +157,21 @@ def install_prerequisites():
 		]
 	})
 
+	# until psycopg2-binary is available for aarch64 (Arm 64-bit), we'll need libpq and libssl dev packages to build psycopg2 from source
+	if platform.machine() == 'aarch64':
+		log("Installing libpq and libssl dev packages to build psycopg2 for aarch64...")
+		run_os_command({
+			'apt-get': ['sudo apt-get install -y libpq-dev libssl-dev'],
+			'yum': ['sudo yum install -y libpq-devel openssl-devel']
+		})
+
 	install_package('curl')
 	install_package('wget')
 	install_package('git')
 	install_package('pip3', 'python3-pip')
 
 	success = run_os_command({
-		'python3': "sudo -H python3 -m pip install --upgrade setuptools cryptography ansible==2.8.5 pip"
+		'python3': "sudo -H python3 -m pip install --upgrade setuptools wheel cryptography ansible~=2.8.15 pip"
 	})
 
 	if not (success or shutil.which('ansible')):
@@ -247,10 +255,9 @@ def install_bench(args):
 		else:
 			frappe_branch = "version-{0}".format(args.version)
 			erpnext_branch = "version-{0}".format(args.version)
-	else:
+	# Allow override of frappe_branch and erpnext_branch, regardless of args.version (which always has a default set)
 	if args.frappe_branch:
 		frappe_branch = args.frappe_branch
-
 	if args.erpnext_branch:
 		erpnext_branch = args.erpnext_branch
 
@@ -261,6 +268,10 @@ def install_bench(args):
 	extra_vars.update(bench_name=bench_name)
 
 	# Will install ERPNext production setup by default
+	if args.without_erpnext:
+		log("Initializing bench {bench_name}:\n\tFrappe Branch: {frappe_branch}\n\tERPNext will not be installed due to --without-erpnext".format(bench_name=bench_name, frappe_branch=frappe_branch))
+	else:
+		log("Initializing bench {bench_name}:\n\tFrappe Branch: {frappe_branch}\n\tERPNext Branch: {erpnext_branch}".format(bench_name=bench_name, frappe_branch=frappe_branch, erpnext_branch=erpnext_branch))
 	run_playbook('site.yml', sudo=True, extra_vars=extra_vars)
 
 	if os.path.exists(tmp_bench_repo):
@@ -269,15 +280,19 @@ def install_bench(args):
 
 def clone_bench_repo(args):
 	'''Clones the bench repository in the user folder'''
-	branch = args.bench_branch or 'master'
+	branch = args.bench_branch or 'develop'
 	repo_url = args.repo_url or 'https://github.com/frappe/bench'
 
 	if os.path.exists(tmp_bench_repo):
+		log('Not cloning already existing Bench repository at {tmp_bench_repo}'.format(tmp_bench_repo=tmp_bench_repo))
 		return 0
 	elif args.without_bench_setup:
 		clone_path = os.path.join(os.path.expanduser('~'), 'bench')
+		log('--without-bench-setup specified, clone path is: {clone_path}'.format(clone_path=clone_path))
 	else:
 		clone_path = tmp_bench_repo
+		# Not logging repo_url to avoid accidental credential leak in case credential is embedded in URL
+		log('Cloning bench repository branch {branch} into {clone_path}'.format(branch=branch, clone_path=clone_path))
 
 	success = run_os_command(
 		{'git': 'git clone --quiet {repo_url} {bench_repo} --depth 1 --branch {branch}'.format(
@@ -327,8 +342,8 @@ def get_passwords(args):
 					mysql_root_password = ''
 					continue
 
-			# admin password
-			if not admin_password:
+			# admin password, only needed if we're also creating a site
+			if not admin_password and not args.without_site:
 				admin_password = getpass.unix_getpass(prompt='Please enter the default Administrator user password: ')
 				conf_admin_passswd = getpass.unix_getpass(prompt='Re-enter Administrator password: ')
 
@@ -336,6 +351,8 @@ def get_passwords(args):
 					passwords_didnt_match("Administrator")
 					admin_password = ''
 					continue
+			elif args.without_site:
+				log("Not creating a new site due to --without-site")
 
 			pass_set = False
 	else:
@@ -405,8 +422,8 @@ def parse_commandline_args():
 
 	args_group.add_argument('--develop', dest='develop', action='store_true', default=False, help='Install developer setup')
 	args_group.add_argument('--production', dest='production', action='store_true', default=False, help='Setup Production environment for bench')
-	parser.add_argument('--site', dest='site', action='store', default='site1.local', help='Specifiy name for your first ERPNext site')
-	parser.add_argument('--without-site', dest='without_site', action='store_true', default=False)
+	parser.add_argument('--site', dest='site', action='store', default='site1.local', help='Specify name for your first ERPNext site')
+	parser.add_argument('--without-site', dest='without_site', action='store_true', default=False, help='Do not create a new site')
 	parser.add_argument('--verbose', dest='verbose', action='store_true', default=False, help='Run the script in verbose mode')
 	parser.add_argument('--user', dest='user', help='Install frappe-bench for this user')
 	parser.add_argument('--bench-branch', dest='bench_branch', help='Clone a particular branch of bench repository')
@@ -464,11 +481,11 @@ if __name__ == '__main__':
 	with warnings.catch_warnings():
 		warnings.simplefilter("ignore")
 		setup_log_stream(args)
+		install_prerequisites()
 		setup_script_requirements()
 		check_distribution_compatibility()
 		check_system_package_managers()
 		check_environment()
-		install_prerequisites()
 		install_bench(args)
 
 	log("Bench + Frappe + ERPNext has been successfully installed!")

requirements.txt

@@ -5,6 +5,6 @@ Jinja2==2.10.3
 python-crontab==2.4.0
 requests==2.22.0
 semantic-version==2.8.2
-setuptools==40.8.0
-six==1.12.0
-virtualenv==16.6.0
+setuptools
+six
+virtualenv

vm/scripts/set_message_production.sh

@@ -6,7 +6,7 @@ message="
  Please access ERPNext by going to http://localhost:8080 on the host system.
  The username is \"Administrator\" and password is \"admin\"
 
- Do consider donating at https://frappe.io/buy
+ Consider buying professional support from us at https://erpnext.com/support 
 
  To update, login as
  username: frappe