From cd711b04dc995cbe1a4d97df3a32c01f2630a3aa Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Thu, 28 Nov 2019 10:11:03 +0530 Subject: [PATCH 1/8] fix: bench update compatible venv installed pkg --- bench/utils.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/bench/utils.py b/bench/utils.py index 852200f0..1536cb9c 100755 --- a/bench/utils.py +++ b/bench/utils.py @@ -503,6 +503,10 @@ def update_npm_packages(bench_path='.'): def install_requirements(pip, req_file, user=False): if os.path.exists(req_file): + # sys.real_prefix exists only in a virtualenv + if hasattr(sys, 'real_prefix'): + user = False + user_flag = "--user" if user else "" exec_cmd("{pip} install {user_flag} -q -U -r {req_file}".format(pip=pip, user_flag=user_flag, req_file=req_file)) From 17b447591f20cd9025519d020cc358e663131976 Mon Sep 17 00:00:00 2001 From: "Chinmay D. Pai" Date: Thu, 5 Dec 2019 15:23:13 +0530 Subject: [PATCH 2/8] fix(cron): change cron command for renewing ssl Signed-off-by: Chinmay D. Pai --- bench/config/lets_encrypt.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/bench/config/lets_encrypt.py b/bench/config/lets_encrypt.py index 4ec0f851..40f3450a 100755 --- a/bench/config/lets_encrypt.py +++ b/bench/config/lets_encrypt.py @@ -81,7 +81,7 @@ def run_certbot_and_setup_ssl(site, custom_domain, bench_path, interactive=True) def setup_crontab(): - job_command = 'sudo service nginx stop && /opt/certbot-auto renew && sudo service nginx start' + job_command = '/opt/certbot-auto renew -a nginx --post-hook "systemctl reload nginx"' system_crontab = CronTab(tabfile='/etc/crontab', user=True) if job_command not in str(system_crontab): job = system_crontab.new(command=job_command, comment="Renew lets-encrypt every month") @@ -148,9 +148,9 @@ def setup_wildcard_ssl(domain, email, bench_path, exclude_base_domain): try: exec_cmd("{path} certonly --manual --preferred-challenges=dns {email_param} \ - --server https://acme-v02.api.letsencrypt.org/directory \ - --agree-tos -d {domain}".format(path=get_certbot_path(), domain=' -d '.join(domain_list), - email_param=email_param)) + --server https://acme-v02.api.letsencrypt.org/directory \ + --agree-tos -d {domain}".format(path=get_certbot_path(), domain=' -d '.join(domain_list), + email_param=email_param)) except CommandFailedError: print("There was a problem trying to setup SSL") @@ -161,7 +161,7 @@ def setup_wildcard_ssl(domain, email, bench_path, exclude_base_domain): "wildcard": { "domain": domain, "ssl_certificate": os.path.join(ssl_path, "fullchain.pem"), - "ssl_certificate_key": os.path.join(ssl_path, "privkey.pem") + "ssl_certificate_key": os.path.join(ssl_path, "privkey.pem") } } @@ -171,4 +171,4 @@ def setup_wildcard_ssl(domain, email, bench_path, exclude_base_domain): make_nginx_conf(bench_path) print("Restrting Nginx service") service('nginx', 'restart') - \ No newline at end of file + From 7051aa3d4eb4eceb3d245b7b22a75be5444d6190 Mon Sep 17 00:00:00 2001 From: Development for People <47140294+developmentforpeople@users.noreply.github.com> Date: Sat, 7 Dec 2019 07:05:21 +0100 Subject: [PATCH 3/8] fix(nginx): add redirect rules to avoid duplicate content --- bench/config/templates/nginx.conf | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/bench/config/templates/nginx.conf b/bench/config/templates/nginx.conf index 557b97ae..b6cb8c9b 100644 --- a/bench/config/templates/nginx.conf +++ b/bench/config/templates/nginx.conf @@ -59,6 +59,10 @@ server { location / { + rewrite ^(.+)/$ $1 permanent; + rewrite ^(.+)/index\.html$ $1 permanent; + rewrite ^(.+)\.html$ $1 permanent; + location ~ ^/files/.*.(htm|html|svg|xml) { add_header Content-disposition "attachment"; try_files /{{ site_name }}/public/$uri @webserver; From 75ffa0325e5a4d2c223c2db6148f51b6d2f0cab1 Mon Sep 17 00:00:00 2001 From: Chinmay Pai Date: Sun, 15 Dec 2019 10:16:21 +0530 Subject: [PATCH 4/8] fix: use six to check for string types (#874) fixes issue where unicode keyword does not exist for python3: Traceback (most recent call last): File "/usr/local/bin/bench", line 11, in load_entry_point('bench', 'console_scripts', 'bench')() File "/home/frappe/.bench/bench/cli.py", line 40, in cli bench_command() File "/usr/local/lib/python3.6/dist-packages/click/core.py", line 764, in __call__ return self.main(*args, **kwargs) File "/usr/local/lib/python3.6/dist-packages/click/core.py", line 717, in main rv = self.invoke(ctx) File "/usr/local/lib/python3.6/dist-packages/click/core.py", line 1137, in invoke return _process_result(sub_ctx.command.invoke(sub_ctx)) File "/usr/local/lib/python3.6/dist-packages/click/core.py", line 1137, in invoke return _process_result(sub_ctx.command.invoke(sub_ctx)) File "/usr/local/lib/python3.6/dist-packages/click/core.py", line 956, in invoke return ctx.invoke(self.callback, **ctx.params) File "/usr/local/lib/python3.6/dist-packages/click/core.py", line 555, in invoke return callback(*args, **kwargs) File "/home/frappe/.bench/bench/commands/setup.py", line 22, in setup_nginx make_nginx_conf(bench_path=".", yes=yes) File "/home/frappe/.bench/bench/config/nginx.py", line 13, in make_nginx_conf sites = prepare_sites(config, bench_path) File "/home/frappe/.bench/bench/config/nginx.py", line 100, in prepare_sites sites_configs = get_sites_with_config(bench_path=bench_path) File "/home/frappe/.bench/bench/config/nginx.py", line 217, in get_sites_with_config if isinstance(domain, str) or isinstance(domain, unicode): NameError: name 'unicode' is not defined Signed-off-by: Chinmay D. Pai --- bench/config/nginx.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/bench/config/nginx.py b/bench/config/nginx.py index 4d60529d..7e584c3a 100644 --- a/bench/config/nginx.py +++ b/bench/config/nginx.py @@ -1,5 +1,6 @@ import os, json, click, random, string, hashlib from bench.utils import get_sites, get_bench_name, exec_cmd +from six import string_types def make_nginx_conf(bench_path, yes=False): from bench import env @@ -214,7 +215,7 @@ def get_sites_with_config(bench_path): if dns_multitenant and site_config.get('domains'): for domain in site_config.get('domains'): # domain can be a string or a dict with 'domain', 'ssl_certificate', 'ssl_certificate_key' - if isinstance(domain, str) or isinstance(domain, unicode): + if isinstance(domain, string_types): domain = { 'domain': domain } domain['name'] = site @@ -227,7 +228,7 @@ def get_sites_with_config(bench_path): def use_wildcard_certificate(bench_path, ret): ''' stored in common_site_config.json as: - "wildcard": { + "wildcard": { "domain": "*.erpnext.com", "ssl_certificate": "/path/to/erpnext.com.cert", "ssl_certificate_key": "/path/to/erpnext.com.key" From 72c70b23bf4b4bdc01761dc6be66e26343dc4dda Mon Sep 17 00:00:00 2001 From: Marco Gabriel Date: Sun, 15 Dec 2019 05:49:34 +0100 Subject: [PATCH 5/8] feat(easy_install): add support for debian 10 * Easy Installer did not support Debian 10 Buster, so I changed the playbooks to support Debian 10. * Related Issue: #831 * Tested in a newly deployed Debian 10 Buster running on KVM and in a brand new deployed LXC Container (--container) also running fine. --- playbooks/install.py | 4 +-- playbooks/roles/common/tasks/debian.yml | 25 ++++++++++++++++--- .../roles/nodejs/tasks/debian_family.yml | 6 +++-- 3 files changed, 28 insertions(+), 7 deletions(-) diff --git a/playbooks/install.py b/playbooks/install.py index 7c23a6bb..e2905c6e 100644 --- a/playbooks/install.py +++ b/playbooks/install.py @@ -142,7 +142,7 @@ def install_bench(args): shutil.rmtree(tmp_bench_repo) def check_distribution_compatibility(): - supported_dists = {'ubuntu': [14, 15, 16, 18, 19], 'debian': [8, 9], + supported_dists = {'ubuntu': [14, 15, 16, 18, 19], 'debian': [8, 9, 10], 'centos': [7], 'macos': [10.9, 10.10, 10.11, 10.12]} dist_name, dist_version = get_distribution_info() @@ -389,7 +389,7 @@ def parse_commandline_args(): # set passwords parser.add_argument('--mysql-root-password', dest='mysql_root_password', help='Set mysql root password') - parser.add_argument('--mariadb-version', dest='mariadb_version', default='10.2', help='Specify mariadb version') + parser.add_argument('--mariadb-version', dest='mariadb_version', default='10.4', help='Specify mariadb version') parser.add_argument('--admin-password', dest='admin_password', help='Set admin password') parser.add_argument('--bench-name', dest='bench_name', help='Create bench with specified name. Default name is frappe-bench') diff --git a/playbooks/roles/common/tasks/debian.yml b/playbooks/roles/common/tasks/debian.yml index 8aa2555c..4327e3b9 100644 --- a/playbooks/roles/common/tasks/debian.yml +++ b/playbooks/roles/common/tasks/debian.yml @@ -12,18 +12,37 @@ - tk8.5-dev when: ansible_distribution_version | version_compare('8', 'lt') -- name: install pillow prerequisites for Debian >= 8 +- name: install pillow prerequisites for Debian 8 apt: pkg={{ item }} state=present with_items: - libjpeg62-turbo-dev - libtiff5-dev - tcl8.5-dev - tk8.5-dev - when: ansible_distribution_version | version_compare('8', 'ge') + when: ansible_distribution_version | version_compare('8', 'eq') + +- name: install pillow prerequisites for Debian 9 + apt: pkg={{ item }} state=present + with_items: + - libjpeg62-turbo-dev + - libtiff5-dev + - tcl8.5-dev + - tk8.5-dev + when: ansible_distribution_version | version_compare('9', 'eq') + + +- name: install pillow prerequisites for Debian >= 10 + apt: pkg={{ item }} state=present + with_items: + - libjpeg62-turbo-dev + - libtiff5-dev + - tcl8.6-dev + - tk8.6-dev + when: ansible_distribution_version | version_compare('10', 'ge') - name: install pdf prerequisites debian apt: pkg={{ item }} state=present force=yes with_items: - libssl-dev -... \ No newline at end of file +... diff --git a/playbooks/roles/nodejs/tasks/debian_family.yml b/playbooks/roles/nodejs/tasks/debian_family.yml index b2985729..0067e465 100644 --- a/playbooks/roles/nodejs/tasks/debian_family.yml +++ b/playbooks/roles/nodejs/tasks/debian_family.yml @@ -7,8 +7,10 @@ - name: Install nodejs {{ node_version }} apt: - name: nodejs + name: + - nodejs + - npm state: present update_cache: yes force: yes -... \ No newline at end of file +... From d2d577201ddb1aa6927cf91652688111cd11d371 Mon Sep 17 00:00:00 2001 From: Chinmay Pai Date: Sun, 15 Dec 2019 10:41:27 +0530 Subject: [PATCH 6/8] fix: remove unused method and use six to reload module (#875) * remove download_chart_of_accounts method as it is not being used anywhere * use reload_module from six for flake8 fixes Signed-off-by: Chinmay D. Pai --- bench/commands/update.py | 13 +++++-------- bench/utils.py | 6 ------ 2 files changed, 5 insertions(+), 14 deletions(-) diff --git a/bench/commands/update.py b/bench/commands/update.py index af32b424..278506f7 100755 --- a/bench/commands/update.py +++ b/bench/commands/update.py @@ -1,11 +1,13 @@ import click -import sys, os +import sys +import os from bench.config.common_site_config import get_config, update_config from bench.app import pull_all_apps, is_version_upgrade, validate_branch from bench.utils import (update_bench, validate_upgrade, pre_upgrade, post_upgrade, before_update, update_requirements, update_node_packages, backup_all_sites, patch_sites, build_assets, restart_supervisor_processes, restart_systemd_processes) from bench import patches +from six import reload_module @click.command('update') @@ -88,13 +90,8 @@ def _update(pull=False, patch=False, build=False, update_bench=False, auto=False pre_upgrade(version_upgrade[1], version_upgrade[2], bench_path=bench_path) import bench.utils, bench.app print('Reloading bench...') - if sys.version_info >= (3, 4): - import importlib - importlib.reload(bench.utils) - importlib.reload(bench.app) - else: - reload(bench.utils) - reload(bench.app) + reload_module(bench.utils) + reload_module(bench.app) if patch: print('Patching sites...') diff --git a/bench/utils.py b/bench/utils.py index 1536cb9c..bd665fa7 100755 --- a/bench/utils.py +++ b/bench/utils.py @@ -727,12 +727,6 @@ def update_translations(app, lang): print('downloaded for', app, lang) -def download_chart_of_accounts(): - charts_dir = os.path.join('apps', "erpnext", "erpnext", 'accounts', 'chart_of_accounts', "submitted") - csv_file = os.path.join(translations_dir, lang + '.csv') - url = "https://translate.erpnext.com/files/{}-{}.csv".format(app, lang) - r = requests.get(url, stream=True) - r.raise_for_status() def print_output(p): while p.poll() is None: From 459432b1956b0f0651dc9303fcad8f300555ffe6 Mon Sep 17 00:00:00 2001 From: Chinmay Pai Date: Sun, 15 Dec 2019 13:45:17 +0530 Subject: [PATCH 7/8] fix: incorrect import statement use six.moves instead of six --- bench/commands/update.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bench/commands/update.py b/bench/commands/update.py index 278506f7..f87cda95 100755 --- a/bench/commands/update.py +++ b/bench/commands/update.py @@ -7,7 +7,7 @@ from bench.utils import (update_bench, validate_upgrade, pre_upgrade, post_upgra update_requirements, update_node_packages, backup_all_sites, patch_sites, build_assets, restart_supervisor_processes, restart_systemd_processes) from bench import patches -from six import reload_module +from six.moves import reload_module @click.command('update') From d1ebb4c3b2821379382106905d4560200f3480b9 Mon Sep 17 00:00:00 2001 From: Chinmay Pai Date: Mon, 16 Dec 2019 16:36:55 +0530 Subject: [PATCH 8/8] security(nginx): use only strong ciphers over ssl (#869) * security(nginx): use only strong ciphers over ssl * drop support for TLSv1 and TLSv1.1, and add support for TLSv1.3 * disable all ciphers else for EECDH+AESGCM and EDH+AESGCM * disable session ticketing * use secp384r1 as certificate curve * enable strict transport security with preloading * enable xss-protection Signed-off-by: Chinmay D. Pai * chore: explicitly state ssl on listening socket Signed-off-by: Chinmay D. Pai --- bench/config/templates/nginx.conf | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/bench/config/templates/nginx.conf b/bench/config/templates/nginx.conf index b6cb8c9b..9df365f3 100644 --- a/bench/config/templates/nginx.conf +++ b/bench/config/templates/nginx.conf @@ -12,7 +12,12 @@ map {{ from_variable }} {{ to_variable }} { {%- macro server_block(bench_name, port, server_names, site_name, sites_path, ssl_certificate, ssl_certificate_key) %} server { + {% if ssl_certificate and ssl_certificate_key %} + listen {{ port }} ssl; + {% else %} listen {{ port }}; + {% endif %} + server_name {% for name in server_names -%} {{ name }} @@ -30,12 +35,20 @@ server { ssl_certificate {{ ssl_certificate }}; ssl_certificate_key {{ ssl_certificate_key }}; ssl_session_timeout 5m; - ssl_protocols TLSv1 TLSv1.1 TLSv1.2; - ssl_ciphers "EECDH+ECDSA+AESGCM EECDH+aRSA+AESGCM EECDH+ECDSA+SHA384 EECDH+ECDSA+SHA256 EECDH+aRSA+SHA384 EECDH+aRSA+SHA256 EECDH+aRSA+RC4 EECDH EDH+aRSA RC4 !aNULL !eNULL !LOW !3DES !MD5 !EXP !PSK !SRP !DSS"; - ssl_prefer_server_ciphers on; + ssl_session_cache shared:SSL:10m; + ssl_session_tickets off; + ssl_stapling on; + ssl_stapling_verify on; + ssl_protocols TLSv1.2 TLSv1.3; + ssl_ciphers EECDH+AESGCM:EDH+AESGCM; + ssl_ecdh_curve secp384r1; + ssl_prefer_server_ciphers on; {% endif %} add_header X-Frame-Options "SAMEORIGIN"; + add_header Strict-Transport-Security "max-age=63072000; includeSubDomains; preload"; + add_header X-Content-Type-Options nosniff; + add_header X-XSS-Protection "1; mode=block"; location /assets { try_files $uri =404;