diff --git a/.github/semantic.yml b/.github/semantic.yml new file mode 100644 index 00000000..e1e53bc1 --- /dev/null +++ b/.github/semantic.yml @@ -0,0 +1,13 @@ +# Always validate the PR title AND all the commits +titleAndCommits: true + +# Allow use of Merge commits (eg on github: "Merge branch 'master' into feature/ride-unicorns") +# this is only relevant when using commitsOnly: true (or titleAndCommits: true) +allowMergeCommits: true + +# Allow use of Revert commits (eg on github: "Revert "feat: ride unicorns"") +# this is only relevant when using commitsOnly: true (or titleAndCommits: true) +allowRevertCommits: true + +# For allowed PR types: https://github.com/commitizen/conventional-commit-types/blob/v3.0.0/index.json +# Tool Reference: https://github.com/zeke/semantic-pull-requests diff --git a/README.md b/README.md index 8bdc2af4..abbbb7f4 100755 --- a/README.md +++ b/README.md @@ -18,6 +18,7 @@ Bench is a command-line utility that helps you to install, update, and manage mu - [Bench Manager](#bench-manager) - [Guides](#guides) - [Resources](#resources) + - [Development](#development) - [License](#license) @@ -254,6 +255,36 @@ For an exhaustive list of guides, check out [Bench Guides](https://frappe.io/doc For an exhaustive list of resources, check out [Bench Resources](https://frappe.io/docs/user/en/bench/resources). +## Development + +To contribute and develop on the bench CLI tool, clone this repo and create an editable install. In editable mode, you may get the following warning everytime you run a bench command: + + WARN: bench is installed in editable mode! + + This is not the recommended mode of installation for production. Instead, install the package from PyPI with: `pip install frappe-bench` + + +```sh +$ git clone https://github.com/frappe/bench ~/bench-repo +$ pip3 install -e ~/bench-repo +$ bench src +/Users/frappe/bench-repo +``` + +To clear up the editable install and switch to a stable version of bench, uninstall via pip and delete the corresponding egg file from the python path. + + +```sh +# Delete bench installed in editable install +$ rm -r $(find ~ -name '*.egg-info') +$ pip3 uninstall frappe-bench + +# Install latest released version of bench +$ pip3 install -U frappe-bench +``` + +To confirm the switch, check the output of `bench src`. It should change from something like `$HOME/bench-repo` to `/usr/local/lib/python3.6/dist-packages` and stop the editable install warnings from getting triggered at every command. + ## License This repository has been released under the [GNU GPLv3 License](LICENSE). diff --git a/bench/cli.py b/bench/cli.py index de949aba..05c74d86 100755 --- a/bench/cli.py +++ b/bench/cli.py @@ -27,11 +27,13 @@ def cli(): command = " ".join(sys.argv) change_working_directory() - logger = setup_logging() or logging.getLogger(bench.PROJECT_NAME) + logger = setup_logging() logger.info(command) - check_uid() - change_dir() - change_uid() + + if len(sys.argv) > 1 and sys.argv[1] not in ("src", ): + check_uid() + change_uid() + change_dir() if is_dist_editable(bench.PROJECT_NAME) and len(sys.argv) > 1 and sys.argv[1] != "src" and not get_config(".").get("developer_mode"): log("bench is installed in editable mode!\n\nThis is not the recommended mode of installation for production. Instead, install the package from PyPI with: `pip install frappe-bench`\n", level=3) diff --git a/bench/commands/make.py b/bench/commands/make.py index 7fc799e6..554b0603 100755 --- a/bench/commands/make.py +++ b/bench/commands/make.py @@ -11,8 +11,8 @@ import click @click.option('--frappe-branch', default=None, help="path to frappe repo") @click.option('--clone-from', default=None, help="copy repos from path") @click.option('--clone-without-update', is_flag=True, help="copy repos from path without update") -@click.option('--no-procfile', is_flag=True, help="Pull changes in all the apps in bench") -@click.option('--no-backups',is_flag=True, help="Run migrations for all sites in the bench") +@click.option('--no-procfile', is_flag=True, help="Do not create a Procfile") +@click.option('--no-backups',is_flag=True, help="Do not set up automatic periodic backups for all sites on this bench") @click.option('--skip-redis-config-generation', is_flag=True, help="Skip redis config generation if already specifying the common-site-config file") @click.option('--skip-assets',is_flag=True, default=False, help="Do not build assets") @click.option('--verbose',is_flag=True, help="Verbose output during install") diff --git a/bench/commands/update.py b/bench/commands/update.py index dee57d28..e1113b40 100755 --- a/bench/commands/update.py +++ b/bench/commands/update.py @@ -15,11 +15,12 @@ from bench.utils import post_upgrade, patch_sites, build_assets @click.option('--restart-supervisor', is_flag=True, help="Restart supervisor processes after update") @click.option('--restart-systemd', is_flag=True, help="Restart systemd units after update") @click.option('--no-backup', is_flag=True, help="If this flag is set, sites won't be backed up prior to updates. Note: This is not recommended in production.") +@click.option('--no-compile', is_flag=True, help="If set, Python bytecode won't be compiled before restarting the processes") @click.option('--force', is_flag=True, help="Forces major version upgrades") @click.option('--reset', is_flag=True, help="Hard resets git branch's to their new states overriding any changes and overriding rebase on pull") -def update(pull, apps, patch, build, requirements, restart_supervisor, restart_systemd, no_backup, force, reset): +def update(pull, apps, patch, build, requirements, restart_supervisor, restart_systemd, no_backup, no_compile, force, reset): from bench.utils import update - update(pull=pull, apps=apps, patch=patch, build=build, requirements=requirements, restart_supervisor=restart_supervisor, restart_systemd=restart_systemd, backup=not no_backup, force=force, reset=reset) + update(pull=pull, apps=apps, patch=patch, build=build, requirements=requirements, restart_supervisor=restart_supervisor, restart_systemd=restart_systemd, backup=not no_backup, compile=not no_compile, force=force, reset=reset) @click.command('retry-upgrade', help="Retry a failed upgrade") diff --git a/bench/commands/utils.py b/bench/commands/utils.py index 78899d19..5d9ac6d5 100644 --- a/bench/commands/utils.py +++ b/bench/commands/utils.py @@ -111,7 +111,7 @@ def download_translations(): download_translations_p() -@click.command('renew-lets-encrypt', help="Renew Let's Encrypt certificate") +@click.command('renew-lets-encrypt', help="Sets Up latest cron and Renew Let's Encrypt certificate") def renew_lets_encrypt(): from bench.config.lets_encrypt import renew_certs renew_certs() diff --git a/bench/config/common_site_config.py b/bench/config/common_site_config.py index 02c6778a..d71d9efd 100644 --- a/bench/config/common_site_config.py +++ b/bench/config/common_site_config.py @@ -55,7 +55,7 @@ def get_gunicorn_workers(): '''This function will return the maximum workers that can be started depending upon number of cpu's present on the machine''' return { - "gunicorn_workers": multiprocessing.cpu_count() + "gunicorn_workers": multiprocessing.cpu_count() * 2 + 1 } def update_config_for_frappe(config, bench_path): diff --git a/bench/config/lets_encrypt.py b/bench/config/lets_encrypt.py index 5f0396b9..32ca3380 100755 --- a/bench/config/lets_encrypt.py +++ b/bench/config/lets_encrypt.py @@ -87,11 +87,17 @@ def run_certbot_and_setup_ssl(site, custom_domain, bench_path, interactive=True) def setup_crontab(): job_command = '/opt/certbot-auto renew -a nginx --post-hook "systemctl reload nginx"' + job_comment = 'Renew lets-encrypt every month' + print("Setting Up cron job to {0}".format(job_comment)) + system_crontab = CronTab(user='root') - if job_command not in str(system_crontab): - job = system_crontab.new(command=job_command, comment="Renew lets-encrypt every month") - job.day.on(1) - system_crontab.write() + + for job in system_crontab.find_comment(comment=job_comment): # Removes older entries + system_crontab.remove(job) + + job = system_crontab.new(command=job_command, comment=job_comment) + job.setall('0 0 */1 * *') # Run at 00:00 every day-of-month + system_crontab.write() def create_dir_if_missing(path): @@ -113,10 +119,13 @@ def get_certbot_path(): def renew_certs(): + # Needs to be run with sudo click.confirm('Running this will stop the nginx service temporarily causing your sites to go offline\n' 'Do you want to continue?', abort=True) + setup_crontab() + service('nginx', 'stop') exec_cmd("{path} renew".format(path=get_certbot_path())) service('nginx', 'start') diff --git a/bench/playbooks/roles/bench/tasks/setup_erpnext.yml b/bench/playbooks/roles/bench/tasks/setup_erpnext.yml index 32e498d1..9db153ab 100644 --- a/bench/playbooks/roles/bench/tasks/setup_erpnext.yml +++ b/bench/playbooks/roles/bench/tasks/setup_erpnext.yml @@ -13,16 +13,17 @@ - name: Check whether the site already exists stat: path="{{ bench_path }}/sites/{{ site }}" register: site_folder + when: not without_site - name: Create a new site command: "bench new-site {{ site }} --admin-password '{{ admin_password }}' --mariadb-root-password '{{ mysql_root_password }}'" args: chdir: "{{ bench_path }}" - when: not site_folder.stat.exists + when: not without_site and not site_folder.stat.exists - name: Install ERPNext to default site command: "bench --site {{ site }} install-app erpnext" args: chdir: "{{ bench_path }}" - when: not without_erpnext + when: not without_site and not without_erpnext ... \ No newline at end of file diff --git a/bench/playbooks/roles/mariadb/tasks/main.yml b/bench/playbooks/roles/mariadb/tasks/main.yml index 1f119005..8079583d 100644 --- a/bench/playbooks/roles/mariadb/tasks/main.yml +++ b/bench/playbooks/roles/mariadb/tasks/main.yml @@ -48,6 +48,10 @@ [mysqld] pid-file = /var/run/mysqld/mysqld.pid socket = /var/run/mysqld/mysqld.sock + + # setting appeared inside mysql but overwritten by mariadb inside mariadb.conf.d/xx-server.cnf valued as utf8mb4_general_ci + + collation-server = utf8mb4_unicode_ci create: yes become: yes become_user: root diff --git a/bench/playbooks/roles/wkhtmltopdf/tasks/main.yml b/bench/playbooks/roles/wkhtmltopdf/tasks/main.yml index aea2f55f..2a6a89ea 100644 --- a/bench/playbooks/roles/wkhtmltopdf/tasks/main.yml +++ b/bench/playbooks/roles/wkhtmltopdf/tasks/main.yml @@ -24,7 +24,14 @@ get_url: url: https://github.com/wkhtmltopdf/wkhtmltopdf/releases/download/0.12.5/wkhtmltox_0.12.5-1.focal_amd64.deb dest: /tmp/wkhtmltox.deb - when: ansible_distribution == 'Ubuntu' and ansible_distribution_major_version == '20' + when: ansible_distribution == 'Ubuntu' and ansible_distribution_major_version == '20' and ansible_architecture != 'aarch64' + +- name: download wkthmltox Ubuntu 20 arm64 + get_url: + # wkhtmltox supports arm64 starting from 0.12.6 + url: https://github.com/wkhtmltopdf/packaging/releases/download/0.12.6-1/wkhtmltox_0.12.6-1.focal_arm64.deb + dest: /tmp/wkhtmltox.deb + when: ansible_distribution == 'Ubuntu' and ansible_distribution_major_version == '20' and ansible_architecture == 'aarch64' - name: download wkthmltox Ubuntu 18 get_url: diff --git a/bench/tests/test_init.py b/bench/tests/test_init.py index 69d49a45..2f409899 100755 --- a/bench/tests/test_init.py +++ b/bench/tests/test_init.py @@ -9,6 +9,7 @@ import git # imports - module imports import bench +import bench.cli import bench.utils from bench.release import get_bumped_version from bench.tests.test_base import FRAPPE_BRANCH, TestBenchBase @@ -27,6 +28,10 @@ class TestBenchInit(TestBenchBase): self.assertEqual( get_bumped_version('11.0.5-beta.22', 'prerelease'), '11.0.5-beta.23' ) + def test_utils(self): + self.assertEqual(subprocess.call("bench"), 0) + + def test_init(self, bench_name="test-bench", **kwargs): self.init_bench(bench_name, **kwargs) self.assert_folders(bench_name) diff --git a/bench/utils.py b/bench/utils.py index 4a4fa813..bb1b55e4 100755 --- a/bench/utils.py +++ b/bench/utils.py @@ -2,6 +2,7 @@ # -*- coding: utf-8 -*- # imports - standard imports +import compileall import errno import glob import grp @@ -183,8 +184,8 @@ def init(path, apps_path=None, no_procfile=False, no_backups=False, copy_patches_txt(path) -def update(pull=False, apps=None, patch=False, build=False, requirements=False, backup=True, force=False, reset=False, - restart_supervisor=False, restart_systemd=False): +def update(pull=False, apps=None, patch=False, build=False, requirements=False, backup=True, compile=True, + force=False, reset=False, restart_supervisor=False, restart_systemd=False): """command: bench update""" from bench import patches from bench.app import is_version_upgrade, pull_apps, validate_branch @@ -218,7 +219,6 @@ def update(pull=False, apps=None, patch=False, build=False, requirements=False, if version_upgrade[0] or (not version_upgrade[0] and force): validate_upgrade(version_upgrade[1], version_upgrade[2], bench_path=bench_path) - conf.update({ "maintenance_mode": 1, "pause_scheduler": 1 }) update_config(conf, bench_path=bench_path) @@ -246,6 +246,10 @@ def update(pull=False, apps=None, patch=False, build=False, requirements=False, if version_upgrade[0] or (not version_upgrade[0] and force): post_upgrade(version_upgrade[1], version_upgrade[2], bench_path=bench_path) + if pull and compile: + print("Compiling Python files...") + compileall.compile_dir('../apps', quiet=1, rx=re.compile('.*node_modules.*')) + if restart_supervisor or conf.get('restart_supervisor_on_update'): restart_supervisor_processes(bench_path=bench_path) @@ -443,15 +447,18 @@ def setup_logging(bench_path='.'): logging.Logger.log = logv if os.path.exists(os.path.join(bench_path, 'logs')): - logger = logging.getLogger(bench.PROJECT_NAME) log_file = os.path.join(bench_path, 'logs', 'bench.log') - formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s') hdlr = logging.FileHandler(log_file) - hdlr.setFormatter(formatter) - logger.addHandler(hdlr) - logger.setLevel(logging.DEBUG) + else: + hdlr = logging.NullHandler() - return logger + logger = logging.getLogger(bench.PROJECT_NAME) + formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s') + hdlr.setFormatter(formatter) + logger.addHandler(hdlr) + logger.setLevel(logging.DEBUG) + + return logger def get_process_manager(): @@ -478,7 +485,7 @@ def start(no_dev=False, concurrency=None, procfile=None, no_prefix=False): if no_prefix: command.extend(['--no-prefix']) - + os.execv(program, command) diff --git a/docs/bench_usage.md b/docs/bench_usage.md index cf9a0364..807d021f 100644 --- a/docs/bench_usage.md +++ b/docs/bench_usage.md @@ -72,7 +72,7 @@ These commands belong directly to the bench group so they can be invoked directl - **init**: Initialize a new bench instance in the specified path. This sets up a complete bench folder with an `apps` folder which contains all the Frappe apps available in the current bench, `sites` folder that stores all site data seperated by individual site folders, `config` folder that contains your redis, NGINX and supervisor configuration files. The `env` folder consists of all python dependencies the current bench and installed Frappe applications have. - **restart**: Restart web, supervisor, systemd processes units. Used in production setup. - - **update**: Updates bench tool and if executed in a bench directory, without any flags will backup, pull, setup requirements, build, run patches and restart bench. Using specific flags will only do certain tasks instead of all. + - **update**: If executed in a bench directory, without any flags will backup, pull, setup requirements, build, run patches and restart bench. Using specific flags will only do certain tasks instead of all. - **migrate-env**: Migrate Virtual Environment to desired Python version. This regenerates the `env` folder with the specified Python version. - **retry-upgrade**: Retry a failed upgrade - **disable-production**: Disables production environment for the bench. diff --git a/docs/commands_and_usage.md b/docs/commands_and_usage.md index 92ad33bc..ed71cc59 100644 --- a/docs/commands_and_usage.md +++ b/docs/commands_and_usage.md @@ -2,9 +2,14 @@ * Updating - Currently, `bench update` can be run from any directory however the context of the command changes. If run from a bench directory, the vanilla command itself updates all apps, runs migrations and backs up all sites. +To update the bench CLI tool, depending on your method of installation, you may use - bench update + pip3 install -U frappe-bench + + +To backup, update all apps and sites on your bench, you may use + + bench update To manually update the bench, run `bench update` to update all the apps, run diff --git a/install.py b/install.py index 2b70d7f4..71647824 100644 --- a/install.py +++ b/install.py @@ -157,13 +157,21 @@ def install_prerequisites(): ] }) + # until psycopg2-binary is available for aarch64 (Arm 64-bit), we'll need libpq and libssl dev packages to build psycopg2 from source + if platform.machine() == 'aarch64': + log("Installing libpq and libssl dev packages to build psycopg2 for aarch64...") + run_os_command({ + 'apt-get': ['sudo apt-get install -y libpq-dev libssl-dev'], + 'yum': ['sudo yum install -y libpq-devel openssl-devel'] + }) + install_package('curl') install_package('wget') install_package('git') install_package('pip3', 'python3-pip') success = run_os_command({ - 'python3': "sudo -H python3 -m pip install --upgrade setuptools cryptography ansible==2.8.5 pip" + 'python3': "sudo -H python3 -m pip install --upgrade setuptools wheel cryptography ansible~=2.8.15 pip" }) if not (success or shutil.which('ansible')): @@ -247,12 +255,11 @@ def install_bench(args): else: frappe_branch = "version-{0}".format(args.version) erpnext_branch = "version-{0}".format(args.version) - else: - if args.frappe_branch: - frappe_branch = args.frappe_branch - - if args.erpnext_branch: - erpnext_branch = args.erpnext_branch + # Allow override of frappe_branch and erpnext_branch, regardless of args.version (which always has a default set) + if args.frappe_branch: + frappe_branch = args.frappe_branch + if args.erpnext_branch: + erpnext_branch = args.erpnext_branch extra_vars.update(frappe_branch=frappe_branch) extra_vars.update(erpnext_branch=erpnext_branch) @@ -261,6 +268,10 @@ def install_bench(args): extra_vars.update(bench_name=bench_name) # Will install ERPNext production setup by default + if args.without_erpnext: + log("Initializing bench {bench_name}:\n\tFrappe Branch: {frappe_branch}\n\tERPNext will not be installed due to --without-erpnext".format(bench_name=bench_name, frappe_branch=frappe_branch)) + else: + log("Initializing bench {bench_name}:\n\tFrappe Branch: {frappe_branch}\n\tERPNext Branch: {erpnext_branch}".format(bench_name=bench_name, frappe_branch=frappe_branch, erpnext_branch=erpnext_branch)) run_playbook('site.yml', sudo=True, extra_vars=extra_vars) if os.path.exists(tmp_bench_repo): @@ -269,15 +280,19 @@ def install_bench(args): def clone_bench_repo(args): '''Clones the bench repository in the user folder''' - branch = args.bench_branch or 'master' + branch = args.bench_branch or 'develop' repo_url = args.repo_url or 'https://github.com/frappe/bench' if os.path.exists(tmp_bench_repo): + log('Not cloning already existing Bench repository at {tmp_bench_repo}'.format(tmp_bench_repo=tmp_bench_repo)) return 0 elif args.without_bench_setup: clone_path = os.path.join(os.path.expanduser('~'), 'bench') + log('--without-bench-setup specified, clone path is: {clone_path}'.format(clone_path=clone_path)) else: clone_path = tmp_bench_repo + # Not logging repo_url to avoid accidental credential leak in case credential is embedded in URL + log('Cloning bench repository branch {branch} into {clone_path}'.format(branch=branch, clone_path=clone_path)) success = run_os_command( {'git': 'git clone --quiet {repo_url} {bench_repo} --depth 1 --branch {branch}'.format( @@ -327,8 +342,8 @@ def get_passwords(args): mysql_root_password = '' continue - # admin password - if not admin_password: + # admin password, only needed if we're also creating a site + if not admin_password and not args.without_site: admin_password = getpass.unix_getpass(prompt='Please enter the default Administrator user password: ') conf_admin_passswd = getpass.unix_getpass(prompt='Re-enter Administrator password: ') @@ -336,6 +351,8 @@ def get_passwords(args): passwords_didnt_match("Administrator") admin_password = '' continue + elif args.without_site: + log("Not creating a new site due to --without-site") pass_set = False else: @@ -405,8 +422,8 @@ def parse_commandline_args(): args_group.add_argument('--develop', dest='develop', action='store_true', default=False, help='Install developer setup') args_group.add_argument('--production', dest='production', action='store_true', default=False, help='Setup Production environment for bench') - parser.add_argument('--site', dest='site', action='store', default='site1.local', help='Specifiy name for your first ERPNext site') - parser.add_argument('--without-site', dest='without_site', action='store_true', default=False) + parser.add_argument('--site', dest='site', action='store', default='site1.local', help='Specify name for your first ERPNext site') + parser.add_argument('--without-site', dest='without_site', action='store_true', default=False, help='Do not create a new site') parser.add_argument('--verbose', dest='verbose', action='store_true', default=False, help='Run the script in verbose mode') parser.add_argument('--user', dest='user', help='Install frappe-bench for this user') parser.add_argument('--bench-branch', dest='bench_branch', help='Clone a particular branch of bench repository') @@ -464,11 +481,11 @@ if __name__ == '__main__': with warnings.catch_warnings(): warnings.simplefilter("ignore") setup_log_stream(args) + install_prerequisites() setup_script_requirements() check_distribution_compatibility() check_system_package_managers() check_environment() - install_prerequisites() install_bench(args) log("Bench + Frappe + ERPNext has been successfully installed!") diff --git a/requirements.txt b/requirements.txt index e00b90ea..ea1bdd08 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,6 +5,6 @@ Jinja2==2.10.3 python-crontab==2.4.0 requests==2.22.0 semantic-version==2.8.2 -setuptools==40.8.0 -six==1.12.0 -virtualenv==16.6.0 +setuptools +six +virtualenv diff --git a/vm/scripts/set_message_production.sh b/vm/scripts/set_message_production.sh index 74182ca6..2fddcb77 100644 --- a/vm/scripts/set_message_production.sh +++ b/vm/scripts/set_message_production.sh @@ -6,7 +6,7 @@ message=" Please access ERPNext by going to http://localhost:8080 on the host system. The username is \"Administrator\" and password is \"admin\" - Do consider donating at https://frappe.io/buy + Consider buying professional support from us at https://erpnext.com/support To update, login as username: frappe