From 3dc8b48da76ec8ad0575dc48fd6ee8f5ba689e90 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Wed, 4 Dec 2019 18:24:56 +0530 Subject: [PATCH 01/66] fix(ansible): python api deprecated warnings --- playbooks/macosx.yml | 30 ++++---- .../roles/bench/tasks/setup_firewall.yml | 10 ++- playbooks/roles/common/tasks/debian.yml | 34 ++++---- .../roles/common/tasks/debian_family.yml | 60 ++++++++------- playbooks/roles/common/tasks/macos.yml | 20 ++--- .../roles/common/tasks/redhat_family.yml | 77 ++++++++++--------- playbooks/roles/common/tasks/ubuntu.yml | 45 ++++++----- playbooks/roles/frappe_selinux/tasks/main.yml | 9 ++- playbooks/roles/mariadb/tasks/centos.yml | 15 ++-- playbooks/roles/mariadb/tasks/debian.yml | 16 ++-- .../roles/mariadb/tasks/ubuntu-trusty.yml | 16 ++-- .../mariadb/tasks/ubuntu-xenial_bionic.yml | 16 ++-- .../roles/nodejs/tasks/debian_family.yml | 2 +- playbooks/roles/ntpd/tasks/main.yml | 18 +++-- .../roles/packer/tasks/debian_family.yml | 8 +- .../roles/packer/tasks/redhat_family.yml | 7 +- playbooks/roles/redis/tasks/main.yml | 22 +++--- .../roles/virtualbox/tasks/debian_family.yml | 17 ++-- .../roles/virtualbox/tasks/redhat_family.yml | 9 ++- playbooks/roles/wkhtmltopdf/tasks/main.yml | 27 ++++--- 20 files changed, 255 insertions(+), 203 deletions(-) diff --git a/playbooks/macosx.yml b/playbooks/macosx.yml index 2ca0e174..12be8f04 100644 --- a/playbooks/macosx.yml +++ b/playbooks/macosx.yml @@ -2,24 +2,26 @@ - hosts: localhost become: yes become_user: root + vars: bench_repo_path: "/Users/{{ ansible_user_id }}/.bench" bench_path: "/Users/{{ ansible_user_id }}/frappe-bench" - tasks: - # install pre-requisites - - name: install prequisites - homebrew: name={{ item }} state=present - with_items: - - cmake - - redis - - mariadb - - nodejs - # install wkhtmltopdf - - name: cask installs - homebrew_cask: name={{ item }} state=present - with_items: - - wkhtmltopdf + tasks: + - name: install prequisites + homebrew: + name: + - cmake + - redis + - mariadb + - nodejs + state: present + + - name: install wkhtmltopdf + homebrew_cask: + name: + - wkhtmltopdf + state: present - name: configure mariadb include: roles/mariadb/tasks/main.yml diff --git a/playbooks/roles/bench/tasks/setup_firewall.yml b/playbooks/roles/bench/tasks/setup_firewall.yml index f1fc9fa8..f3186b42 100644 --- a/playbooks/roles/bench/tasks/setup_firewall.yml +++ b/playbooks/roles/bench/tasks/setup_firewall.yml @@ -31,10 +31,12 @@ # For Ubuntu / Debian - name: Install ufw - apt: name={{ item }} state=present force=yes - with_items: - - python-selinux - - ufw + apt: + state: present + force: yes + pkg: + - python-selinux + - ufw when: ansible_distribution == 'Ubuntu' or ansible_distribution == 'Debian' - name: Enable Firewall diff --git a/playbooks/roles/common/tasks/debian.yml b/playbooks/roles/common/tasks/debian.yml index 8aa2555c..b057706c 100644 --- a/playbooks/roles/common/tasks/debian.yml +++ b/playbooks/roles/common/tasks/debian.yml @@ -4,26 +4,30 @@ pip: name=pyOpenSSL version=16.2.0 - name: install pillow prerequisites for Debian < 8 - apt: pkg={{ item }} state=present - with_items: - - libjpeg8-dev - - libtiff4-dev - - tcl8.5-dev - - tk8.5-dev + apt: + pkg: + - libjpeg8-dev + - libtiff4-dev + - tcl8.5-dev + - tk8.5-dev + state: present when: ansible_distribution_version | version_compare('8', 'lt') - name: install pillow prerequisites for Debian >= 8 - apt: pkg={{ item }} state=present - with_items: - - libjpeg62-turbo-dev - - libtiff5-dev - - tcl8.5-dev - - tk8.5-dev + apt: + pkg: + - libjpeg62-turbo-dev + - libtiff5-dev + - tcl8.5-dev + - tk8.5-dev + state: present when: ansible_distribution_version | version_compare('8', 'ge') - name: install pdf prerequisites debian - apt: pkg={{ item }} state=present force=yes - with_items: - - libssl-dev + apt: + pkg: + - libssl-dev + state: present + force: yes ... \ No newline at end of file diff --git a/playbooks/roles/common/tasks/debian_family.yml b/playbooks/roles/common/tasks/debian_family.yml index 1fd90fcd..64eb8485 100644 --- a/playbooks/roles/common/tasks/debian_family.yml +++ b/playbooks/roles/common/tasks/debian_family.yml @@ -3,35 +3,37 @@ - name: Install prerequisites using apt-get become: yes become_user: root - apt: pkg={{ item }} state=present force=yes - with_items: - - dnsmasq - - fontconfig - - git # Version control - - htop # Server stats - - libcrypto++-dev - - libfreetype6-dev - - liblcms2-dev - - libwebp-dev - - libxext6 - - libxrender1 - - libxslt1-dev - - libxslt1.1 - - libffi-dev - - ntp # Clock synchronization - - postfix # Mail Server - - python3-dev # Installing python developer suite - - python-tk - - screen # To aid ssh sessions with connectivity problems - - vim # Is that supposed to be a question!? - - xfonts-75dpi - - xfonts-base - - zlib1g-dev - - apt-transport-https - - libsasl2-dev - - libldap2-dev - - libcups2-dev - - pv # Show progress during database restore + apt: + pkg: + - dnsmasq + - fontconfig + - git # Version control + - htop # Server stats + - libcrypto++-dev + - libfreetype6-dev + - liblcms2-dev + - libwebp-dev + - libxext6 + - libxrender1 + - libxslt1-dev + - libxslt1.1 + - libffi-dev + - ntp # Clock synchronization + - postfix # Mail Server + - python3-dev # Installing python developer suite + - python-tk + - screen # To aid ssh sessions with connectivity problems + - vim # Is that supposed to be a question!? + - xfonts-75dpi + - xfonts-base + - zlib1g-dev + - apt-transport-https + - libsasl2-dev + - libldap2-dev + - libcups2-dev + - pv # Show progress during database restore + state: present + force: yes - include_tasks: debian.yml when: ansible_distribution == 'Debian' diff --git a/playbooks/roles/common/tasks/macos.yml b/playbooks/roles/common/tasks/macos.yml index a5fbd0bf..3fed1da6 100644 --- a/playbooks/roles/common/tasks/macos.yml +++ b/playbooks/roles/common/tasks/macos.yml @@ -9,18 +9,20 @@ tasks: # install pre-requisites - name: install prequisites - homebrew: name={{ item }} state=present - with_items: - - cmake - - redis - - mariadb - - nodejs + homebrew: + name: + - cmake + - redis + - mariadb + - nodejs + state: present # install wkhtmltopdf - name: cask installs - homebrew_cask: name={{ item }} state=present - with_items: - - wkhtmltopdf + homebrew_cask: + name: + - wkhtmltopdf + state: present - name: configure mariadb include_tasks: roles/mariadb/tasks/main.yml diff --git a/playbooks/roles/common/tasks/redhat_family.yml b/playbooks/roles/common/tasks/redhat_family.yml index c2ae15ef..d41097c6 100644 --- a/playbooks/roles/common/tasks/redhat_family.yml +++ b/playbooks/roles/common/tasks/redhat_family.yml @@ -10,42 +10,43 @@ - name: "Setup prerequisites using yum" become: yes become_user: root - yum: name={{ item }} state=present - with_items: - - bzip2-devel - - cronie - - dnsmasq - - freetype-devel - - git - - htop - - lcms2-devel - - libjpeg-devel - - libtiff-devel - - libffi-devel - - libwebp-devel - - libXext - - libXrender - - libzip-devel - - libffi-devel - - ntp - - openssl-devel - - postfix - - python36u - - python-devel - - python-setuptools - - python-pip - - redis - - screen - - sudo - - tcl-devel - - tk-devel - - vim - - which - - xorg-x11-fonts-75dpi - - xorg-x11-fonts-Type1 - - zlib-devel - - openssl-devel - - openldap-devel - - libselinux-python - - cups-libs + yum: + name: + - bzip2-devel + - cronie + - dnsmasq + - freetype-devel + - git + - htop + - lcms2-devel + - libjpeg-devel + - libtiff-devel + - libffi-devel + - libwebp-devel + - libXext + - libXrender + - libzip-devel + - libffi-devel + - ntp + - openssl-devel + - postfix + - python36u + - python-devel + - python-setuptools + - python-pip + - redis + - screen + - sudo + - tcl-devel + - tk-devel + - vim + - which + - xorg-x11-fonts-75dpi + - xorg-x11-fonts-Type1 + - zlib-devel + - openssl-devel + - openldap-devel + - libselinux-python + - cups-libs + state: present ... diff --git a/playbooks/roles/common/tasks/ubuntu.yml b/playbooks/roles/common/tasks/ubuntu.yml index dae12063..20c76a0b 100644 --- a/playbooks/roles/common/tasks/ubuntu.yml +++ b/playbooks/roles/common/tasks/ubuntu.yml @@ -1,34 +1,41 @@ --- - name: install pillow prerequisites for Ubuntu < 14.04 - apt: pkg={{ item }} state=present force=yes - with_items: - - libjpeg8-dev - - libtiff4-dev - - tcl8.5-dev - - tk8.5-dev + apt: + pkg: + - libjpeg8-dev + - libtiff4-dev + - tcl8.5-dev + - tk8.5-dev + state: present + force: yes when: ansible_distribution_version | version_compare('14.04', 'lt') - name: install pillow prerequisites for Ubuntu >= 14.04 - apt: pkg={{ item }} state=present force=yes - with_items: - - libjpeg8-dev - - libtiff5-dev - - tcl8.6-dev - - tk8.6-dev + apt: + pkg: + - libjpeg8-dev + - libtiff5-dev + - tcl8.6-dev + - tk8.6-dev + state: present + force: yes when: ansible_distribution_version | version_compare('14.04', 'ge') - - name: install pdf prerequisites for Ubuntu < 18.04 - apt: pkg={{ item }} state=present force=yes - with_items: - - libssl-dev + apt: + pkg: + - libssl-dev + state: present + force: yes when: ansible_distribution_version | version_compare('18.04', 'lt') - name: install pdf prerequisites for Ubuntu >= 18.04 - apt: pkg={{ item }} state=present force=yes - with_items: - - libssl1.0-dev + apt: + pkg: + - libssl1.0-dev + state: present + force: yes when: ansible_distribution_version | version_compare('18.04', 'ge') ... \ No newline at end of file diff --git a/playbooks/roles/frappe_selinux/tasks/main.yml b/playbooks/roles/frappe_selinux/tasks/main.yml index 43602163..085ad59f 100644 --- a/playbooks/roles/frappe_selinux/tasks/main.yml +++ b/playbooks/roles/frappe_selinux/tasks/main.yml @@ -1,9 +1,10 @@ --- - name: Install deps - yum: name="{{item}}" state=present - with_items: - - policycoreutils-python - - selinux-policy-devel + yum: + name: + - policycoreutils-python + - selinux-policy-devel + state: present when: ansible_distribution == 'CentOS' - name: Check enabled SELinux modules diff --git a/playbooks/roles/mariadb/tasks/centos.yml b/playbooks/roles/mariadb/tasks/centos.yml index c069aacd..7dbe69d1 100644 --- a/playbooks/roles/mariadb/tasks/centos.yml +++ b/playbooks/roles/mariadb/tasks/centos.yml @@ -3,11 +3,16 @@ template: src=mariadb_centos.repo.j2 dest=/etc/yum.repos.d/mariadb.repo owner=root group=root mode=0644 - name: Install MariaDB - yum: name={{ item }} enablerepo=mariadb state=present - with_items: - - MariaDB-server - - MariaDB-client + yum: + name: + - MariaDB-server + - MariaDB-client + enablerepo: mariadb + state: present - name: Install MySQLdb Python package for secure installations. - yum: name=MySQL-python state=present + yum: + name: + - MySQL-python + state: present when: mysql_secure_installation and mysql_root_password is defined diff --git a/playbooks/roles/mariadb/tasks/debian.yml b/playbooks/roles/mariadb/tasks/debian.yml index 8e9596c5..246aff59 100644 --- a/playbooks/roles/mariadb/tasks/debian.yml +++ b/playbooks/roles/mariadb/tasks/debian.yml @@ -21,11 +21,13 @@ shell: export DEBIAN_FRONTEND=noninteractive - name: apt-get install - apt: pkg={{ item }} update_cache=yes state=present - with_items: - - mariadb-server - - mariadb-client - - mariadb-common - - libmariadbclient18 - - python3-mysqldb + apt: + pkg: + - mariadb-server + - mariadb-client + - mariadb-common + - libmariadbclient18 + - python3-mysqldb + update_cache: yes + state: present ... \ No newline at end of file diff --git a/playbooks/roles/mariadb/tasks/ubuntu-trusty.yml b/playbooks/roles/mariadb/tasks/ubuntu-trusty.yml index 6493b985..45839b48 100644 --- a/playbooks/roles/mariadb/tasks/ubuntu-trusty.yml +++ b/playbooks/roles/mariadb/tasks/ubuntu-trusty.yml @@ -16,12 +16,16 @@ changed_when: false - name: Install MariaDB - apt: pkg={{ item }} state=present - with_items: - - mariadb-server - - mariadb-client - - libmariadbclient18 + apt: + pkg: + - mariadb-server + - mariadb-client + - libmariadbclient18 + state: present - name: Install MySQLdb Python package for secure installations. - apt: pkg=python3-mysqldb state=present + apt: + pkg: + - python3-mysqldb + state: present when: mysql_secure_installation and mysql_root_password is defined diff --git a/playbooks/roles/mariadb/tasks/ubuntu-xenial_bionic.yml b/playbooks/roles/mariadb/tasks/ubuntu-xenial_bionic.yml index 2163cc00..a96dff1b 100644 --- a/playbooks/roles/mariadb/tasks/ubuntu-xenial_bionic.yml +++ b/playbooks/roles/mariadb/tasks/ubuntu-xenial_bionic.yml @@ -16,12 +16,16 @@ changed_when: false - name: Install MariaDB - apt: pkg={{ item }} state=present - with_items: - - mariadb-server - - mariadb-client - - libmariadbclient18 + apt: + pkg: + - mariadb-server + - mariadb-client + - libmariadbclient18 + state: present - name: Install MySQLdb Python package for secure installations. - apt: pkg=python3-mysqldb state=present + apt: + pkg: + - python3-mysqldb + state: present when: mysql_secure_installation and mysql_root_password is defined diff --git a/playbooks/roles/nodejs/tasks/debian_family.yml b/playbooks/roles/nodejs/tasks/debian_family.yml index b2985729..989e18cb 100644 --- a/playbooks/roles/nodejs/tasks/debian_family.yml +++ b/playbooks/roles/nodejs/tasks/debian_family.yml @@ -7,7 +7,7 @@ - name: Install nodejs {{ node_version }} apt: - name: nodejs + pkg: nodejs state: present update_cache: yes force: yes diff --git a/playbooks/roles/ntpd/tasks/main.yml b/playbooks/roles/ntpd/tasks/main.yml index b2cfb190..46a16768 100644 --- a/playbooks/roles/ntpd/tasks/main.yml +++ b/playbooks/roles/ntpd/tasks/main.yml @@ -1,9 +1,10 @@ --- - name: Install ntpd - yum: name="{{item}}" state=installed - with_items: - - ntp - - ntpdate + yum: + name: + - ntp + - ntpdate + state: installed when: ansible_distribution == 'CentOS' - name: Enable ntpd @@ -11,10 +12,11 @@ when: ansible_distribution == 'CentOS' - name: Install ntpd - apt: name="{{item}}" state=installed - with_items: - - ntp - - ntpdate + apt: + pkg: + - ntp + - ntpdate + state: installed when: ansible_distribution == 'Debian' or ansible_distribution == 'Ubuntu' - name: Enable ntpd diff --git a/playbooks/roles/packer/tasks/debian_family.yml b/playbooks/roles/packer/tasks/debian_family.yml index 52d34486..6c3c1496 100644 --- a/playbooks/roles/packer/tasks/debian_family.yml +++ b/playbooks/roles/packer/tasks/debian_family.yml @@ -1,6 +1,8 @@ --- - name: Install unzip - apt: pkg={{ item }} update_cache=yes state=present - with_items: - - unzip + apt: + pkg: + - unzip + update_cache: yes + state: present ... \ No newline at end of file diff --git a/playbooks/roles/packer/tasks/redhat_family.yml b/playbooks/roles/packer/tasks/redhat_family.yml index 937adfc0..8cb1071d 100644 --- a/playbooks/roles/packer/tasks/redhat_family.yml +++ b/playbooks/roles/packer/tasks/redhat_family.yml @@ -1,7 +1,8 @@ --- - name: Install unzip - yum: name={{ item }} state=present - with_items: - - unzip + yum: + name: + - unzip + state: present ... diff --git a/playbooks/roles/redis/tasks/main.yml b/playbooks/roles/redis/tasks/main.yml index 472c65c4..4c009dcc 100644 --- a/playbooks/roles/redis/tasks/main.yml +++ b/playbooks/roles/redis/tasks/main.yml @@ -1,21 +1,25 @@ --- - name: Install yum packages - yum: name={{ item }} state=present - with_items: - - redis + yum: + name: + - redis + state: present when: ansible_os_family == 'RedHat' # Prerequisite for Debian and Ubuntu - name: Install apt packages - apt: pkg={{ item }} state=present force=yes - with_items: - - redis-server + apt: + pkg: + - redis-server + state: present + force: yes when: ansible_distribution == 'Debian' or ansible_distribution == 'Ubuntu' # Prerequisite for MACOS - name: install prequisites for macos - homebrew: name={{ item }} state=present - with_items: - - redis + homebrew: + name: + - redis + state: present when: ansible_distribution == 'MacOSX' ... \ No newline at end of file diff --git a/playbooks/roles/virtualbox/tasks/debian_family.yml b/playbooks/roles/virtualbox/tasks/debian_family.yml index 8dbcab79..9367c9b7 100644 --- a/playbooks/roles/virtualbox/tasks/debian_family.yml +++ b/playbooks/roles/virtualbox/tasks/debian_family.yml @@ -1,9 +1,10 @@ --- - name: Install dependencies - apt: pkg={{ item }} state=present - with_items: - - apt-transport-https - - ca-certificates + apt: + pkg: + - apt-transport-https + - ca-certificates + state: present - name: Add VirtualBox to sources.list apt_repository: @@ -23,7 +24,9 @@ when: (ansible_distribution == "Debian" and ansible_distribution_major_version < "8") or (ansible_distribution == "Ubuntu" and ansible_distribution_major_version < "16") - name: Install VirtualBox - apt: pkg={{ item }} update_cache=yes state=present - with_items: - - virtualbox-{{ virtualbox_version }} + apt: + pkg: + - virtualbox-{{ virtualbox_version }} + update_cache: yes + state: present ... diff --git a/playbooks/roles/virtualbox/tasks/redhat_family.yml b/playbooks/roles/virtualbox/tasks/redhat_family.yml index 76bd56ab..899de175 100644 --- a/playbooks/roles/virtualbox/tasks/redhat_family.yml +++ b/playbooks/roles/virtualbox/tasks/redhat_family.yml @@ -5,10 +5,11 @@ state: present - name: Install dependencies - yum: name={{ item }} state=present - with_items: - - kernel-devel - - deltarpm + yum: + name: + - kernel-devel + - deltarpm + state: present - copy: src=virtualbox_centos.repo dest=/etc/yum.repos.d/virtualbox.repo owner=root group=root mode=0644 force=no diff --git a/playbooks/roles/wkhtmltopdf/tasks/main.yml b/playbooks/roles/wkhtmltopdf/tasks/main.yml index 18ea0016..59f5c736 100644 --- a/playbooks/roles/wkhtmltopdf/tasks/main.yml +++ b/playbooks/roles/wkhtmltopdf/tasks/main.yml @@ -1,20 +1,23 @@ --- - name: install base fonts - yum: name={{ item }} state=present - with_items: - - libXrender - - libXext - - xorg-x11-fonts-75dpi - - xorg-x11-fonts-Type1 + yum: + name: + - libXrender + - libXext + - xorg-x11-fonts-75dpi + - xorg-x11-fonts-Type1 + state: present when: ansible_os_family == 'RedHat' - name: install base fonts - apt: name={{ item }} state=present force=yes - with_items: - - libxrender1 - - libxext6 - - xfonts-75dpi - - xfonts-base + apt: + pkg: + - libxrender1 + - libxext6 + - xfonts-75dpi + - xfonts-base + state: present + force: yes when: ansible_os_family == 'Debian' # wkhtmltopdf has been locked down to 0.12.3 intentionally since 0.12.4 has problems. From fd7d59d2be27c49c4bda45c2595395524ecdb546 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Wed, 4 Dec 2019 18:24:56 +0530 Subject: [PATCH 02/66] fix(ansible): deprecated warnings install loops --- playbooks/macosx.yml | 30 ++++---- .../roles/bench/tasks/setup_firewall.yml | 10 ++- playbooks/roles/common/tasks/debian.yml | 34 ++++---- .../roles/common/tasks/debian_family.yml | 60 ++++++++------- playbooks/roles/common/tasks/macos.yml | 20 ++--- .../roles/common/tasks/redhat_family.yml | 77 ++++++++++--------- playbooks/roles/common/tasks/ubuntu.yml | 45 ++++++----- playbooks/roles/frappe_selinux/tasks/main.yml | 9 ++- playbooks/roles/mariadb/tasks/centos.yml | 15 ++-- playbooks/roles/mariadb/tasks/debian.yml | 16 ++-- .../roles/mariadb/tasks/ubuntu-trusty.yml | 16 ++-- .../mariadb/tasks/ubuntu-xenial_bionic.yml | 16 ++-- .../roles/nodejs/tasks/debian_family.yml | 2 +- playbooks/roles/ntpd/tasks/main.yml | 18 +++-- .../roles/packer/tasks/debian_family.yml | 8 +- .../roles/packer/tasks/redhat_family.yml | 7 +- playbooks/roles/redis/tasks/main.yml | 22 +++--- .../roles/virtualbox/tasks/debian_family.yml | 17 ++-- .../roles/virtualbox/tasks/redhat_family.yml | 9 ++- playbooks/roles/wkhtmltopdf/tasks/main.yml | 27 ++++--- 20 files changed, 255 insertions(+), 203 deletions(-) diff --git a/playbooks/macosx.yml b/playbooks/macosx.yml index 2ca0e174..12be8f04 100644 --- a/playbooks/macosx.yml +++ b/playbooks/macosx.yml @@ -2,24 +2,26 @@ - hosts: localhost become: yes become_user: root + vars: bench_repo_path: "/Users/{{ ansible_user_id }}/.bench" bench_path: "/Users/{{ ansible_user_id }}/frappe-bench" - tasks: - # install pre-requisites - - name: install prequisites - homebrew: name={{ item }} state=present - with_items: - - cmake - - redis - - mariadb - - nodejs - # install wkhtmltopdf - - name: cask installs - homebrew_cask: name={{ item }} state=present - with_items: - - wkhtmltopdf + tasks: + - name: install prequisites + homebrew: + name: + - cmake + - redis + - mariadb + - nodejs + state: present + + - name: install wkhtmltopdf + homebrew_cask: + name: + - wkhtmltopdf + state: present - name: configure mariadb include: roles/mariadb/tasks/main.yml diff --git a/playbooks/roles/bench/tasks/setup_firewall.yml b/playbooks/roles/bench/tasks/setup_firewall.yml index f1fc9fa8..f3186b42 100644 --- a/playbooks/roles/bench/tasks/setup_firewall.yml +++ b/playbooks/roles/bench/tasks/setup_firewall.yml @@ -31,10 +31,12 @@ # For Ubuntu / Debian - name: Install ufw - apt: name={{ item }} state=present force=yes - with_items: - - python-selinux - - ufw + apt: + state: present + force: yes + pkg: + - python-selinux + - ufw when: ansible_distribution == 'Ubuntu' or ansible_distribution == 'Debian' - name: Enable Firewall diff --git a/playbooks/roles/common/tasks/debian.yml b/playbooks/roles/common/tasks/debian.yml index 8aa2555c..b057706c 100644 --- a/playbooks/roles/common/tasks/debian.yml +++ b/playbooks/roles/common/tasks/debian.yml @@ -4,26 +4,30 @@ pip: name=pyOpenSSL version=16.2.0 - name: install pillow prerequisites for Debian < 8 - apt: pkg={{ item }} state=present - with_items: - - libjpeg8-dev - - libtiff4-dev - - tcl8.5-dev - - tk8.5-dev + apt: + pkg: + - libjpeg8-dev + - libtiff4-dev + - tcl8.5-dev + - tk8.5-dev + state: present when: ansible_distribution_version | version_compare('8', 'lt') - name: install pillow prerequisites for Debian >= 8 - apt: pkg={{ item }} state=present - with_items: - - libjpeg62-turbo-dev - - libtiff5-dev - - tcl8.5-dev - - tk8.5-dev + apt: + pkg: + - libjpeg62-turbo-dev + - libtiff5-dev + - tcl8.5-dev + - tk8.5-dev + state: present when: ansible_distribution_version | version_compare('8', 'ge') - name: install pdf prerequisites debian - apt: pkg={{ item }} state=present force=yes - with_items: - - libssl-dev + apt: + pkg: + - libssl-dev + state: present + force: yes ... \ No newline at end of file diff --git a/playbooks/roles/common/tasks/debian_family.yml b/playbooks/roles/common/tasks/debian_family.yml index 1fd90fcd..64eb8485 100644 --- a/playbooks/roles/common/tasks/debian_family.yml +++ b/playbooks/roles/common/tasks/debian_family.yml @@ -3,35 +3,37 @@ - name: Install prerequisites using apt-get become: yes become_user: root - apt: pkg={{ item }} state=present force=yes - with_items: - - dnsmasq - - fontconfig - - git # Version control - - htop # Server stats - - libcrypto++-dev - - libfreetype6-dev - - liblcms2-dev - - libwebp-dev - - libxext6 - - libxrender1 - - libxslt1-dev - - libxslt1.1 - - libffi-dev - - ntp # Clock synchronization - - postfix # Mail Server - - python3-dev # Installing python developer suite - - python-tk - - screen # To aid ssh sessions with connectivity problems - - vim # Is that supposed to be a question!? - - xfonts-75dpi - - xfonts-base - - zlib1g-dev - - apt-transport-https - - libsasl2-dev - - libldap2-dev - - libcups2-dev - - pv # Show progress during database restore + apt: + pkg: + - dnsmasq + - fontconfig + - git # Version control + - htop # Server stats + - libcrypto++-dev + - libfreetype6-dev + - liblcms2-dev + - libwebp-dev + - libxext6 + - libxrender1 + - libxslt1-dev + - libxslt1.1 + - libffi-dev + - ntp # Clock synchronization + - postfix # Mail Server + - python3-dev # Installing python developer suite + - python-tk + - screen # To aid ssh sessions with connectivity problems + - vim # Is that supposed to be a question!? + - xfonts-75dpi + - xfonts-base + - zlib1g-dev + - apt-transport-https + - libsasl2-dev + - libldap2-dev + - libcups2-dev + - pv # Show progress during database restore + state: present + force: yes - include_tasks: debian.yml when: ansible_distribution == 'Debian' diff --git a/playbooks/roles/common/tasks/macos.yml b/playbooks/roles/common/tasks/macos.yml index a5fbd0bf..3fed1da6 100644 --- a/playbooks/roles/common/tasks/macos.yml +++ b/playbooks/roles/common/tasks/macos.yml @@ -9,18 +9,20 @@ tasks: # install pre-requisites - name: install prequisites - homebrew: name={{ item }} state=present - with_items: - - cmake - - redis - - mariadb - - nodejs + homebrew: + name: + - cmake + - redis + - mariadb + - nodejs + state: present # install wkhtmltopdf - name: cask installs - homebrew_cask: name={{ item }} state=present - with_items: - - wkhtmltopdf + homebrew_cask: + name: + - wkhtmltopdf + state: present - name: configure mariadb include_tasks: roles/mariadb/tasks/main.yml diff --git a/playbooks/roles/common/tasks/redhat_family.yml b/playbooks/roles/common/tasks/redhat_family.yml index c2ae15ef..d41097c6 100644 --- a/playbooks/roles/common/tasks/redhat_family.yml +++ b/playbooks/roles/common/tasks/redhat_family.yml @@ -10,42 +10,43 @@ - name: "Setup prerequisites using yum" become: yes become_user: root - yum: name={{ item }} state=present - with_items: - - bzip2-devel - - cronie - - dnsmasq - - freetype-devel - - git - - htop - - lcms2-devel - - libjpeg-devel - - libtiff-devel - - libffi-devel - - libwebp-devel - - libXext - - libXrender - - libzip-devel - - libffi-devel - - ntp - - openssl-devel - - postfix - - python36u - - python-devel - - python-setuptools - - python-pip - - redis - - screen - - sudo - - tcl-devel - - tk-devel - - vim - - which - - xorg-x11-fonts-75dpi - - xorg-x11-fonts-Type1 - - zlib-devel - - openssl-devel - - openldap-devel - - libselinux-python - - cups-libs + yum: + name: + - bzip2-devel + - cronie + - dnsmasq + - freetype-devel + - git + - htop + - lcms2-devel + - libjpeg-devel + - libtiff-devel + - libffi-devel + - libwebp-devel + - libXext + - libXrender + - libzip-devel + - libffi-devel + - ntp + - openssl-devel + - postfix + - python36u + - python-devel + - python-setuptools + - python-pip + - redis + - screen + - sudo + - tcl-devel + - tk-devel + - vim + - which + - xorg-x11-fonts-75dpi + - xorg-x11-fonts-Type1 + - zlib-devel + - openssl-devel + - openldap-devel + - libselinux-python + - cups-libs + state: present ... diff --git a/playbooks/roles/common/tasks/ubuntu.yml b/playbooks/roles/common/tasks/ubuntu.yml index dae12063..20c76a0b 100644 --- a/playbooks/roles/common/tasks/ubuntu.yml +++ b/playbooks/roles/common/tasks/ubuntu.yml @@ -1,34 +1,41 @@ --- - name: install pillow prerequisites for Ubuntu < 14.04 - apt: pkg={{ item }} state=present force=yes - with_items: - - libjpeg8-dev - - libtiff4-dev - - tcl8.5-dev - - tk8.5-dev + apt: + pkg: + - libjpeg8-dev + - libtiff4-dev + - tcl8.5-dev + - tk8.5-dev + state: present + force: yes when: ansible_distribution_version | version_compare('14.04', 'lt') - name: install pillow prerequisites for Ubuntu >= 14.04 - apt: pkg={{ item }} state=present force=yes - with_items: - - libjpeg8-dev - - libtiff5-dev - - tcl8.6-dev - - tk8.6-dev + apt: + pkg: + - libjpeg8-dev + - libtiff5-dev + - tcl8.6-dev + - tk8.6-dev + state: present + force: yes when: ansible_distribution_version | version_compare('14.04', 'ge') - - name: install pdf prerequisites for Ubuntu < 18.04 - apt: pkg={{ item }} state=present force=yes - with_items: - - libssl-dev + apt: + pkg: + - libssl-dev + state: present + force: yes when: ansible_distribution_version | version_compare('18.04', 'lt') - name: install pdf prerequisites for Ubuntu >= 18.04 - apt: pkg={{ item }} state=present force=yes - with_items: - - libssl1.0-dev + apt: + pkg: + - libssl1.0-dev + state: present + force: yes when: ansible_distribution_version | version_compare('18.04', 'ge') ... \ No newline at end of file diff --git a/playbooks/roles/frappe_selinux/tasks/main.yml b/playbooks/roles/frappe_selinux/tasks/main.yml index 43602163..085ad59f 100644 --- a/playbooks/roles/frappe_selinux/tasks/main.yml +++ b/playbooks/roles/frappe_selinux/tasks/main.yml @@ -1,9 +1,10 @@ --- - name: Install deps - yum: name="{{item}}" state=present - with_items: - - policycoreutils-python - - selinux-policy-devel + yum: + name: + - policycoreutils-python + - selinux-policy-devel + state: present when: ansible_distribution == 'CentOS' - name: Check enabled SELinux modules diff --git a/playbooks/roles/mariadb/tasks/centos.yml b/playbooks/roles/mariadb/tasks/centos.yml index c069aacd..7dbe69d1 100644 --- a/playbooks/roles/mariadb/tasks/centos.yml +++ b/playbooks/roles/mariadb/tasks/centos.yml @@ -3,11 +3,16 @@ template: src=mariadb_centos.repo.j2 dest=/etc/yum.repos.d/mariadb.repo owner=root group=root mode=0644 - name: Install MariaDB - yum: name={{ item }} enablerepo=mariadb state=present - with_items: - - MariaDB-server - - MariaDB-client + yum: + name: + - MariaDB-server + - MariaDB-client + enablerepo: mariadb + state: present - name: Install MySQLdb Python package for secure installations. - yum: name=MySQL-python state=present + yum: + name: + - MySQL-python + state: present when: mysql_secure_installation and mysql_root_password is defined diff --git a/playbooks/roles/mariadb/tasks/debian.yml b/playbooks/roles/mariadb/tasks/debian.yml index 8e9596c5..246aff59 100644 --- a/playbooks/roles/mariadb/tasks/debian.yml +++ b/playbooks/roles/mariadb/tasks/debian.yml @@ -21,11 +21,13 @@ shell: export DEBIAN_FRONTEND=noninteractive - name: apt-get install - apt: pkg={{ item }} update_cache=yes state=present - with_items: - - mariadb-server - - mariadb-client - - mariadb-common - - libmariadbclient18 - - python3-mysqldb + apt: + pkg: + - mariadb-server + - mariadb-client + - mariadb-common + - libmariadbclient18 + - python3-mysqldb + update_cache: yes + state: present ... \ No newline at end of file diff --git a/playbooks/roles/mariadb/tasks/ubuntu-trusty.yml b/playbooks/roles/mariadb/tasks/ubuntu-trusty.yml index 6493b985..45839b48 100644 --- a/playbooks/roles/mariadb/tasks/ubuntu-trusty.yml +++ b/playbooks/roles/mariadb/tasks/ubuntu-trusty.yml @@ -16,12 +16,16 @@ changed_when: false - name: Install MariaDB - apt: pkg={{ item }} state=present - with_items: - - mariadb-server - - mariadb-client - - libmariadbclient18 + apt: + pkg: + - mariadb-server + - mariadb-client + - libmariadbclient18 + state: present - name: Install MySQLdb Python package for secure installations. - apt: pkg=python3-mysqldb state=present + apt: + pkg: + - python3-mysqldb + state: present when: mysql_secure_installation and mysql_root_password is defined diff --git a/playbooks/roles/mariadb/tasks/ubuntu-xenial_bionic.yml b/playbooks/roles/mariadb/tasks/ubuntu-xenial_bionic.yml index 2163cc00..a96dff1b 100644 --- a/playbooks/roles/mariadb/tasks/ubuntu-xenial_bionic.yml +++ b/playbooks/roles/mariadb/tasks/ubuntu-xenial_bionic.yml @@ -16,12 +16,16 @@ changed_when: false - name: Install MariaDB - apt: pkg={{ item }} state=present - with_items: - - mariadb-server - - mariadb-client - - libmariadbclient18 + apt: + pkg: + - mariadb-server + - mariadb-client + - libmariadbclient18 + state: present - name: Install MySQLdb Python package for secure installations. - apt: pkg=python3-mysqldb state=present + apt: + pkg: + - python3-mysqldb + state: present when: mysql_secure_installation and mysql_root_password is defined diff --git a/playbooks/roles/nodejs/tasks/debian_family.yml b/playbooks/roles/nodejs/tasks/debian_family.yml index b2985729..989e18cb 100644 --- a/playbooks/roles/nodejs/tasks/debian_family.yml +++ b/playbooks/roles/nodejs/tasks/debian_family.yml @@ -7,7 +7,7 @@ - name: Install nodejs {{ node_version }} apt: - name: nodejs + pkg: nodejs state: present update_cache: yes force: yes diff --git a/playbooks/roles/ntpd/tasks/main.yml b/playbooks/roles/ntpd/tasks/main.yml index b2cfb190..46a16768 100644 --- a/playbooks/roles/ntpd/tasks/main.yml +++ b/playbooks/roles/ntpd/tasks/main.yml @@ -1,9 +1,10 @@ --- - name: Install ntpd - yum: name="{{item}}" state=installed - with_items: - - ntp - - ntpdate + yum: + name: + - ntp + - ntpdate + state: installed when: ansible_distribution == 'CentOS' - name: Enable ntpd @@ -11,10 +12,11 @@ when: ansible_distribution == 'CentOS' - name: Install ntpd - apt: name="{{item}}" state=installed - with_items: - - ntp - - ntpdate + apt: + pkg: + - ntp + - ntpdate + state: installed when: ansible_distribution == 'Debian' or ansible_distribution == 'Ubuntu' - name: Enable ntpd diff --git a/playbooks/roles/packer/tasks/debian_family.yml b/playbooks/roles/packer/tasks/debian_family.yml index 52d34486..6c3c1496 100644 --- a/playbooks/roles/packer/tasks/debian_family.yml +++ b/playbooks/roles/packer/tasks/debian_family.yml @@ -1,6 +1,8 @@ --- - name: Install unzip - apt: pkg={{ item }} update_cache=yes state=present - with_items: - - unzip + apt: + pkg: + - unzip + update_cache: yes + state: present ... \ No newline at end of file diff --git a/playbooks/roles/packer/tasks/redhat_family.yml b/playbooks/roles/packer/tasks/redhat_family.yml index 937adfc0..8cb1071d 100644 --- a/playbooks/roles/packer/tasks/redhat_family.yml +++ b/playbooks/roles/packer/tasks/redhat_family.yml @@ -1,7 +1,8 @@ --- - name: Install unzip - yum: name={{ item }} state=present - with_items: - - unzip + yum: + name: + - unzip + state: present ... diff --git a/playbooks/roles/redis/tasks/main.yml b/playbooks/roles/redis/tasks/main.yml index 472c65c4..4c009dcc 100644 --- a/playbooks/roles/redis/tasks/main.yml +++ b/playbooks/roles/redis/tasks/main.yml @@ -1,21 +1,25 @@ --- - name: Install yum packages - yum: name={{ item }} state=present - with_items: - - redis + yum: + name: + - redis + state: present when: ansible_os_family == 'RedHat' # Prerequisite for Debian and Ubuntu - name: Install apt packages - apt: pkg={{ item }} state=present force=yes - with_items: - - redis-server + apt: + pkg: + - redis-server + state: present + force: yes when: ansible_distribution == 'Debian' or ansible_distribution == 'Ubuntu' # Prerequisite for MACOS - name: install prequisites for macos - homebrew: name={{ item }} state=present - with_items: - - redis + homebrew: + name: + - redis + state: present when: ansible_distribution == 'MacOSX' ... \ No newline at end of file diff --git a/playbooks/roles/virtualbox/tasks/debian_family.yml b/playbooks/roles/virtualbox/tasks/debian_family.yml index 8dbcab79..9367c9b7 100644 --- a/playbooks/roles/virtualbox/tasks/debian_family.yml +++ b/playbooks/roles/virtualbox/tasks/debian_family.yml @@ -1,9 +1,10 @@ --- - name: Install dependencies - apt: pkg={{ item }} state=present - with_items: - - apt-transport-https - - ca-certificates + apt: + pkg: + - apt-transport-https + - ca-certificates + state: present - name: Add VirtualBox to sources.list apt_repository: @@ -23,7 +24,9 @@ when: (ansible_distribution == "Debian" and ansible_distribution_major_version < "8") or (ansible_distribution == "Ubuntu" and ansible_distribution_major_version < "16") - name: Install VirtualBox - apt: pkg={{ item }} update_cache=yes state=present - with_items: - - virtualbox-{{ virtualbox_version }} + apt: + pkg: + - virtualbox-{{ virtualbox_version }} + update_cache: yes + state: present ... diff --git a/playbooks/roles/virtualbox/tasks/redhat_family.yml b/playbooks/roles/virtualbox/tasks/redhat_family.yml index 76bd56ab..899de175 100644 --- a/playbooks/roles/virtualbox/tasks/redhat_family.yml +++ b/playbooks/roles/virtualbox/tasks/redhat_family.yml @@ -5,10 +5,11 @@ state: present - name: Install dependencies - yum: name={{ item }} state=present - with_items: - - kernel-devel - - deltarpm + yum: + name: + - kernel-devel + - deltarpm + state: present - copy: src=virtualbox_centos.repo dest=/etc/yum.repos.d/virtualbox.repo owner=root group=root mode=0644 force=no diff --git a/playbooks/roles/wkhtmltopdf/tasks/main.yml b/playbooks/roles/wkhtmltopdf/tasks/main.yml index 18ea0016..59f5c736 100644 --- a/playbooks/roles/wkhtmltopdf/tasks/main.yml +++ b/playbooks/roles/wkhtmltopdf/tasks/main.yml @@ -1,20 +1,23 @@ --- - name: install base fonts - yum: name={{ item }} state=present - with_items: - - libXrender - - libXext - - xorg-x11-fonts-75dpi - - xorg-x11-fonts-Type1 + yum: + name: + - libXrender + - libXext + - xorg-x11-fonts-75dpi + - xorg-x11-fonts-Type1 + state: present when: ansible_os_family == 'RedHat' - name: install base fonts - apt: name={{ item }} state=present force=yes - with_items: - - libxrender1 - - libxext6 - - xfonts-75dpi - - xfonts-base + apt: + pkg: + - libxrender1 + - libxext6 + - xfonts-75dpi + - xfonts-base + state: present + force: yes when: ansible_os_family == 'Debian' # wkhtmltopdf has been locked down to 0.12.3 intentionally since 0.12.4 has problems. From 498e777b5745b3e939e42c0536ccb431c3f25804 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Wed, 4 Dec 2019 18:40:07 +0530 Subject: [PATCH 03/66] fix(ansible): deprecated warnings for | expression --- playbooks/roles/common/tasks/debian.yml | 4 ++-- playbooks/roles/common/tasks/ubuntu.yml | 8 ++++---- playbooks/roles/mariadb/tasks/debian.yml | 4 ++-- playbooks/roles/nginx/tasks/setup-Debian.yml | 6 +++--- playbooks/roles/packer/tasks/main.yml | 12 ++++++------ 5 files changed, 17 insertions(+), 17 deletions(-) diff --git a/playbooks/roles/common/tasks/debian.yml b/playbooks/roles/common/tasks/debian.yml index b057706c..14de785e 100644 --- a/playbooks/roles/common/tasks/debian.yml +++ b/playbooks/roles/common/tasks/debian.yml @@ -11,7 +11,7 @@ - tcl8.5-dev - tk8.5-dev state: present - when: ansible_distribution_version | version_compare('8', 'lt') + when: ansible_distribution_version is version_compare('8', 'lt') - name: install pillow prerequisites for Debian >= 8 apt: @@ -21,7 +21,7 @@ - tcl8.5-dev - tk8.5-dev state: present - when: ansible_distribution_version | version_compare('8', 'ge') + when: ansible_distribution_version is version_compare('8', 'ge') - name: install pdf prerequisites debian apt: diff --git a/playbooks/roles/common/tasks/ubuntu.yml b/playbooks/roles/common/tasks/ubuntu.yml index 20c76a0b..527e42c2 100644 --- a/playbooks/roles/common/tasks/ubuntu.yml +++ b/playbooks/roles/common/tasks/ubuntu.yml @@ -9,7 +9,7 @@ - tk8.5-dev state: present force: yes - when: ansible_distribution_version | version_compare('14.04', 'lt') + when: ansible_distribution_version is version_compare('14.04', 'lt') - name: install pillow prerequisites for Ubuntu >= 14.04 apt: @@ -20,7 +20,7 @@ - tk8.6-dev state: present force: yes - when: ansible_distribution_version | version_compare('14.04', 'ge') + when: ansible_distribution_version is version_compare('14.04', 'ge') - name: install pdf prerequisites for Ubuntu < 18.04 apt: @@ -28,7 +28,7 @@ - libssl-dev state: present force: yes - when: ansible_distribution_version | version_compare('18.04', 'lt') + when: ansible_distribution_version is version_compare('18.04', 'lt') - name: install pdf prerequisites for Ubuntu >= 18.04 apt: @@ -36,6 +36,6 @@ - libssl1.0-dev state: present force: yes - when: ansible_distribution_version | version_compare('18.04', 'ge') + when: ansible_distribution_version is version_compare('18.04', 'ge') ... \ No newline at end of file diff --git a/playbooks/roles/mariadb/tasks/debian.yml b/playbooks/roles/mariadb/tasks/debian.yml index 246aff59..6694f9e8 100644 --- a/playbooks/roles/mariadb/tasks/debian.yml +++ b/playbooks/roles/mariadb/tasks/debian.yml @@ -1,11 +1,11 @@ --- - name: Add apt key for mariadb for Debian <= 8 apt_key: keyserver=hkp://keyserver.ubuntu.com:80 id=0xcbcb082a1bb943db state=present - when: ansible_distribution_major_version | version_compare('8', 'le') + when: ansible_distribution_major_version is version_compare('8', 'le') - name: Add apt key for mariadb for Debian > 8 apt_key: keyserver=hkp://keyserver.ubuntu.com:80 id=0xF1656F24C74CD1D8 state=present - when: ansible_distribution_major_version | version_compare('8', 'gt') + when: ansible_distribution_major_version is version_compare('8', 'gt') - name: Add apt repository apt_repository: diff --git a/playbooks/roles/nginx/tasks/setup-Debian.yml b/playbooks/roles/nginx/tasks/setup-Debian.yml index 2f50fb37..375d1c3b 100644 --- a/playbooks/roles/nginx/tasks/setup-Debian.yml +++ b/playbooks/roles/nginx/tasks/setup-Debian.yml @@ -3,14 +3,14 @@ apt_key: url: http://nginx.org/keys/nginx_signing.key state: present - when: ansible_distribution == 'Debian' and ansible_distribution_version | version_compare('8', 'lt') + when: ansible_distribution == 'Debian' and ansible_distribution_version is version_compare('8', 'lt') - name: Add nginx apt repository for Debian < 8 apt_repository: repo: 'deb [arch=amd64,i386] http://nginx.org/packages/debian/ {{ ansible_distribution_release }} nginx' state: present - when: ansible_distribution == 'Debian' and ansible_distribution_version | version_compare('8', 'lt') - + when: ansible_distribution == 'Debian' and ansible_distribution_version is version_compare('8', 'lt') + - name: Ensure nginx is installed. apt: pkg: nginx diff --git a/playbooks/roles/packer/tasks/main.yml b/playbooks/roles/packer/tasks/main.yml index e31041ca..5b990583 100644 --- a/playbooks/roles/packer/tasks/main.yml +++ b/playbooks/roles/packer/tasks/main.yml @@ -10,28 +10,28 @@ when: packer.stat.exists - include_tasks: debian_family.yml - when: ansible_os_family == 'Debian' and packer.stat.exists == False + when: ansible_os_family == 'Debian' and packer.stat.exists == False - include_tasks: redhat_family.yml - when: ansible_os_family == "RedHat" and packer.stat.exists == False + when: ansible_os_family == "RedHat" and packer.stat.exists == False - name: Delete packer if < 1.2.1 file: state: absent path: /opt/packer - when: (packer.stat.exists) and (packer_version | version_compare('1.2.1', '<')) + when: (packer.stat.exists) and (packer_version is version_compare('1.2.1', '<')) - name: Download packer zip file command: chdir=/opt/ wget https://releases.hashicorp.com/packer/1.2.1/packer_1.2.1_linux_amd64.zip - when: (packer.stat.exists == False) or (packer_version | version_compare('1.2.1', '<')) + when: (packer.stat.exists == False) or (packer_version is version_compare('1.2.1', '<')) - name: Unzip the packer binary in /opt command: chdir=/opt/ unzip packer_1.2.1_linux_amd64.zip - when: (packer.stat.exists == False) or (packer_version | version_compare('1.2.1', '<')) + when: (packer.stat.exists == False) or (packer_version is version_compare('1.2.1', '<')) - name: Remove the downloaded packer zip file file: state: absent path: /opt/packer_1.2.1_linux_amd64.zip - when: (packer.stat.exists == False) or (packer_version | version_compare('1.2.1', '<')) + when: (packer.stat.exists == False) or (packer_version is version_compare('1.2.1', '<')) ... \ No newline at end of file From 459432b1956b0f0651dc9303fcad8f300555ffe6 Mon Sep 17 00:00:00 2001 From: Chinmay Pai Date: Sun, 15 Dec 2019 13:45:17 +0530 Subject: [PATCH 04/66] fix: incorrect import statement use six.moves instead of six --- bench/commands/update.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bench/commands/update.py b/bench/commands/update.py index 278506f7..f87cda95 100755 --- a/bench/commands/update.py +++ b/bench/commands/update.py @@ -7,7 +7,7 @@ from bench.utils import (update_bench, validate_upgrade, pre_upgrade, post_upgra update_requirements, update_node_packages, backup_all_sites, patch_sites, build_assets, restart_supervisor_processes, restart_systemd_processes) from bench import patches -from six import reload_module +from six.moves import reload_module @click.command('update') From d1ebb4c3b2821379382106905d4560200f3480b9 Mon Sep 17 00:00:00 2001 From: Chinmay Pai Date: Mon, 16 Dec 2019 16:36:55 +0530 Subject: [PATCH 05/66] security(nginx): use only strong ciphers over ssl (#869) * security(nginx): use only strong ciphers over ssl * drop support for TLSv1 and TLSv1.1, and add support for TLSv1.3 * disable all ciphers else for EECDH+AESGCM and EDH+AESGCM * disable session ticketing * use secp384r1 as certificate curve * enable strict transport security with preloading * enable xss-protection Signed-off-by: Chinmay D. Pai * chore: explicitly state ssl on listening socket Signed-off-by: Chinmay D. Pai --- bench/config/templates/nginx.conf | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/bench/config/templates/nginx.conf b/bench/config/templates/nginx.conf index b6cb8c9b..9df365f3 100644 --- a/bench/config/templates/nginx.conf +++ b/bench/config/templates/nginx.conf @@ -12,7 +12,12 @@ map {{ from_variable }} {{ to_variable }} { {%- macro server_block(bench_name, port, server_names, site_name, sites_path, ssl_certificate, ssl_certificate_key) %} server { + {% if ssl_certificate and ssl_certificate_key %} + listen {{ port }} ssl; + {% else %} listen {{ port }}; + {% endif %} + server_name {% for name in server_names -%} {{ name }} @@ -30,12 +35,20 @@ server { ssl_certificate {{ ssl_certificate }}; ssl_certificate_key {{ ssl_certificate_key }}; ssl_session_timeout 5m; - ssl_protocols TLSv1 TLSv1.1 TLSv1.2; - ssl_ciphers "EECDH+ECDSA+AESGCM EECDH+aRSA+AESGCM EECDH+ECDSA+SHA384 EECDH+ECDSA+SHA256 EECDH+aRSA+SHA384 EECDH+aRSA+SHA256 EECDH+aRSA+RC4 EECDH EDH+aRSA RC4 !aNULL !eNULL !LOW !3DES !MD5 !EXP !PSK !SRP !DSS"; - ssl_prefer_server_ciphers on; + ssl_session_cache shared:SSL:10m; + ssl_session_tickets off; + ssl_stapling on; + ssl_stapling_verify on; + ssl_protocols TLSv1.2 TLSv1.3; + ssl_ciphers EECDH+AESGCM:EDH+AESGCM; + ssl_ecdh_curve secp384r1; + ssl_prefer_server_ciphers on; {% endif %} add_header X-Frame-Options "SAMEORIGIN"; + add_header Strict-Transport-Security "max-age=63072000; includeSubDomains; preload"; + add_header X-Content-Type-Options nosniff; + add_header X-XSS-Protection "1; mode=block"; location /assets { try_files $uri =404; From 0d4ace95a66f1f373a151ea48367325b1520970c Mon Sep 17 00:00:00 2001 From: Chinmay Pai Date: Tue, 17 Dec 2019 12:51:16 +0530 Subject: [PATCH 06/66] fix(fonts): remove unused setup_fonts method (#878) this method replaces existing system fonts directory under /etc/fonts and /usr/share/fonts with fonts that are probably obsolete by now and are no longer used. removing this method so that it does not mess with anybody's system fonts Signed-off-by: Chinmay D. Pai --- bench/commands/setup.py | 7 ------- bench/utils.py | 14 -------------- 2 files changed, 21 deletions(-) diff --git a/bench/commands/setup.py b/bench/commands/setup.py index bb59a857..22e96591 100755 --- a/bench/commands/setup.py +++ b/bench/commands/setup.py @@ -41,12 +41,6 @@ def setup_redis(): generate_config('.') -@click.command('fonts') -def setup_fonts(): - "Add frappe fonts to system" - from bench.utils import setup_fonts - setup_fonts() - @click.command('production') @click.argument('user') @click.option('--yes', help='Yes to regeneration config', is_flag=True, default=False) @@ -324,7 +318,6 @@ setup.add_command(setup_socketio) setup.add_command(setup_requirements) setup.add_command(setup_manager) setup.add_command(setup_config) -setup.add_command(setup_fonts) setup.add_command(add_domain) setup.add_command(remove_domain) setup.add_command(sync_domains) diff --git a/bench/utils.py b/bench/utils.py index bd665fa7..e3f2df82 100755 --- a/bench/utils.py +++ b/bench/utils.py @@ -794,20 +794,6 @@ def validate_pillow_dependencies(bench_path, requirements): def get_bench_name(bench_path): return os.path.basename(os.path.abspath(bench_path)) -def setup_fonts(): - fonts_path = os.path.join('/tmp', 'fonts') - - if os.path.exists('/etc/fonts_backup'): - return - - exec_cmd("git clone https://github.com/frappe/fonts.git", cwd='/tmp') - os.rename('/etc/fonts', '/etc/fonts_backup') - os.rename('/usr/share/fonts', '/usr/share/fonts_backup') - os.rename(os.path.join(fonts_path, 'etc_fonts'), '/etc/fonts') - os.rename(os.path.join(fonts_path, 'usr_share_fonts'), '/usr/share/fonts') - shutil.rmtree(fonts_path) - exec_cmd("fc-cache -fv") - def set_git_remote_url(git_url, bench_path='.'): "Set app remote git url" app = git_url.rsplit('/', 1)[1].rsplit('.', 1)[0] From fb13dfb0c28fbff6141d605c15ed86605fb61df7 Mon Sep 17 00:00:00 2001 From: sahil28297 <37302950+sahil28297@users.noreply.github.com> Date: Tue, 17 Dec 2019 15:05:19 +0530 Subject: [PATCH 07/66] Revert "fix(fonts): remove unused setup_fonts method" (#881) This reverts commit 0d4ace95a66f1f373a151ea48367325b1520970c. --- bench/commands/setup.py | 7 +++++++ bench/utils.py | 14 ++++++++++++++ 2 files changed, 21 insertions(+) diff --git a/bench/commands/setup.py b/bench/commands/setup.py index 22e96591..bb59a857 100755 --- a/bench/commands/setup.py +++ b/bench/commands/setup.py @@ -41,6 +41,12 @@ def setup_redis(): generate_config('.') +@click.command('fonts') +def setup_fonts(): + "Add frappe fonts to system" + from bench.utils import setup_fonts + setup_fonts() + @click.command('production') @click.argument('user') @click.option('--yes', help='Yes to regeneration config', is_flag=True, default=False) @@ -318,6 +324,7 @@ setup.add_command(setup_socketio) setup.add_command(setup_requirements) setup.add_command(setup_manager) setup.add_command(setup_config) +setup.add_command(setup_fonts) setup.add_command(add_domain) setup.add_command(remove_domain) setup.add_command(sync_domains) diff --git a/bench/utils.py b/bench/utils.py index e3f2df82..bd665fa7 100755 --- a/bench/utils.py +++ b/bench/utils.py @@ -794,6 +794,20 @@ def validate_pillow_dependencies(bench_path, requirements): def get_bench_name(bench_path): return os.path.basename(os.path.abspath(bench_path)) +def setup_fonts(): + fonts_path = os.path.join('/tmp', 'fonts') + + if os.path.exists('/etc/fonts_backup'): + return + + exec_cmd("git clone https://github.com/frappe/fonts.git", cwd='/tmp') + os.rename('/etc/fonts', '/etc/fonts_backup') + os.rename('/usr/share/fonts', '/usr/share/fonts_backup') + os.rename(os.path.join(fonts_path, 'etc_fonts'), '/etc/fonts') + os.rename(os.path.join(fonts_path, 'usr_share_fonts'), '/usr/share/fonts') + shutil.rmtree(fonts_path) + exec_cmd("fc-cache -fv") + def set_git_remote_url(git_url, bench_path='.'): "Set app remote git url" app = git_url.rsplit('/', 1)[1].rsplit('.', 1)[0] From e9205b5032ea1302ebea0f853004340df035eb85 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Wed, 18 Dec 2019 10:49:38 +0530 Subject: [PATCH 08/66] fix: remove npm to fix ubuntu 18 install conflicts --- playbooks/roles/nodejs/tasks/debian_family.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/playbooks/roles/nodejs/tasks/debian_family.yml b/playbooks/roles/nodejs/tasks/debian_family.yml index 989e18cb..ac2dc039 100644 --- a/playbooks/roles/nodejs/tasks/debian_family.yml +++ b/playbooks/roles/nodejs/tasks/debian_family.yml @@ -6,9 +6,7 @@ shell: "curl --silent --location https://deb.nodesource.com/setup_{{ node_version }}.x | bash -" - name: Install nodejs {{ node_version }} - apt: - pkg: nodejs + package: + name: nodejs state: present - update_cache: yes - force: yes ... \ No newline at end of file From fce86fa896be6ed9a085d2fc73518737cf589857 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Wed, 18 Dec 2019 14:11:05 +0530 Subject: [PATCH 09/66] fix: fix permissions for python3 bench init --- playbooks/roles/bench/tasks/main.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/playbooks/roles/bench/tasks/main.yml b/playbooks/roles/bench/tasks/main.yml index 1294d054..142e41cc 100644 --- a/playbooks/roles/bench/tasks/main.yml +++ b/playbooks/roles/bench/tasks/main.yml @@ -28,6 +28,10 @@ stat: path="{{ bench_path }}" register: bench_stat + - name: Fix permissions + become_method: sudo + command: chown {{ frappe_user }} -R ~ + - name: python3 bench init for develop command: bench init {{ bench_path }} --frappe-path {{ frappe_repo_url }} --frappe-branch {{ frappe_branch }} --python {{ python }} args: From a922d198a27ba3e2e001c495d84f7eb4e3591735 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Wed, 11 Dec 2019 10:06:43 +0530 Subject: [PATCH 10/66] chore: update deprecated usages --- playbooks/roles/bench/tasks/main.yml | 7 +++++-- playbooks/roles/mariadb/tasks/main.yml | 22 ++++++++++++++++---- playbooks/roles/nginx/tasks/setup-Debian.yml | 2 +- playbooks/roles/ntpd/tasks/main.yml | 4 ++-- 4 files changed, 26 insertions(+), 9 deletions(-) diff --git a/playbooks/roles/bench/tasks/main.yml b/playbooks/roles/bench/tasks/main.yml index 142e41cc..fa0d9e93 100644 --- a/playbooks/roles/bench/tasks/main.yml +++ b/playbooks/roles/bench/tasks/main.yml @@ -14,7 +14,9 @@ when: tmp_bench.stat.exists and not bench_repo_register.stat.exists - name: install bench - pip: name={{ bench_repo_path }} extra_args='-e' + pip: + name: '{{ bench_repo_path }}' + extra_args: '-e' become: yes become_user: root @@ -25,7 +27,8 @@ when: overwrite - name: Check whether bench exists - stat: path="{{ bench_path }}" + stat: + path: "{{ bench_path }}" register: bench_stat - name: Fix permissions diff --git a/playbooks/roles/mariadb/tasks/main.yml b/playbooks/roles/mariadb/tasks/main.yml index 27f15f8d..1f119005 100644 --- a/playbooks/roles/mariadb/tasks/main.yml +++ b/playbooks/roles/mariadb/tasks/main.yml @@ -9,7 +9,12 @@ when: ansible_distribution == 'Ubuntu' and ansible_distribution_major_version|int >= 16 - name: Add configuration - template: src={{ mysql_conf_tpl }} dest={{ mysql_conf_dir[ansible_distribution] }}/{{ mysql_conf_file }} owner=root group=root mode=0644 + template: + src: '{{ mysql_conf_tpl }}' + dest: '{{ mysql_conf_dir[ansible_distribution] }}/{{ mysql_conf_file }}' + owner: root + group: root + mode: 0644 when: mysql_conf_tpl != 'change_me' and ansible_distribution != 'Debian' notify: restart mysql @@ -17,7 +22,12 @@ when: ansible_distribution == 'Debian' - name: Add configuration - template: src={{ mysql_conf_tpl }} dest={{ mysql_conf_dir[ansible_distribution] }}/{{ mysql_conf_file }} owner=root group=root mode=0644 + template: + src: '{{ mysql_conf_tpl }}' + dest: '{{ mysql_conf_dir[ansible_distribution] }}/{{ mysql_conf_file }}' + owner: root + group: root + mode: 0644 when: mysql_conf_tpl != 'change_me' and ansible_distribution == 'Debian' notify: restart mysql @@ -44,7 +54,10 @@ when: ansible_distribution == 'Ubuntu' or ansible_distribution == 'Debian' - name: Start and enable service - service: name=mysql state=started enabled=yes + service: + name: mysql + state: started + enabled: yes - debug: msg: "{{ mysql_root_password }}" @@ -52,7 +65,8 @@ - include_tasks: mysql_secure_installation.yml when: mysql_root_password is defined -- debug: var=mysql_secure_installation +- debug: + var: mysql_secure_installation when: mysql_secure_installation and mysql_root_password is defined ... diff --git a/playbooks/roles/nginx/tasks/setup-Debian.yml b/playbooks/roles/nginx/tasks/setup-Debian.yml index 375d1c3b..fd9ef24f 100644 --- a/playbooks/roles/nginx/tasks/setup-Debian.yml +++ b/playbooks/roles/nginx/tasks/setup-Debian.yml @@ -14,5 +14,5 @@ - name: Ensure nginx is installed. apt: pkg: nginx - state: installed + state: present default_release: "{{ nginx_default_release }}" diff --git a/playbooks/roles/ntpd/tasks/main.yml b/playbooks/roles/ntpd/tasks/main.yml index 46a16768..110ab50e 100644 --- a/playbooks/roles/ntpd/tasks/main.yml +++ b/playbooks/roles/ntpd/tasks/main.yml @@ -4,7 +4,7 @@ name: - ntp - ntpdate - state: installed + state: present when: ansible_distribution == 'CentOS' - name: Enable ntpd @@ -16,7 +16,7 @@ pkg: - ntp - ntpdate - state: installed + state: present when: ansible_distribution == 'Debian' or ansible_distribution == 'Ubuntu' - name: Enable ntpd From b3feaa255bef1e5dbb617f55a6bb085a2286049c Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Wed, 18 Dec 2019 14:33:21 +0530 Subject: [PATCH 11/66] feat(easy-install): seperate logging and cleaner UI --- playbooks/install.py | 299 +++++++++++++++------------ playbooks/roles/bench/tasks/main.yml | 2 +- 2 files changed, 164 insertions(+), 137 deletions(-) diff --git a/playbooks/install.py b/playbooks/install.py index e2905c6e..0b7e6734 100644 --- a/playbooks/install.py +++ b/playbooks/install.py @@ -1,10 +1,112 @@ -# wget setup_frappe.py | python3 -import os, sys, subprocess, getpass, json, multiprocessing, shutil, platform -from distutils.spawn import find_executable +#!/usr/bin/env python3 +import os, sys, subprocess, getpass, json, multiprocessing, shutil, platform, warnings +from builtins import print as _ tmp_bench_repo = '/tmp/.bench' +tmp_log_folder = '/tmp/logs/' +log_path = os.path.join(tmp_log_folder, 'install_bench.log') +log_stream = sys.stdout + + +def print(message, level=0): + levels = { + 0: '\033[94m', # normal + 1: '\033[92m', # success + 2: '\033[91m', # fail + 3: '\033[93m' # warn/suggest + } + start = levels.get(level) or '' + end = '\033[0m' + _(start + message + end) + + +def find_executable(executable, path=None): + """Tries to find 'executable' in the directories listed in 'path'. + A string listing directories separated by 'os.pathsep'; defaults to + os.environ['PATH']. Returns the complete filename or None if not found. + + source: https://github.com/python/cpython/blob/master/Lib/distutils/spawn.py + """ + _, ext = os.path.splitext(executable) + if (sys.platform == 'win32') and (ext != '.exe'): + executable = executable + '.exe' + + if os.path.isfile(executable): + return executable + + if path is None: + path = os.environ.get('PATH', None) + if path is None: + try: + path = os.confstr("CS_PATH") + except (AttributeError, ValueError): + # os.confstr() or CS_PATH is not available + path = os.defpath + # bpo-35755: Don't use os.defpath if the PATH environment variable is + # set to an empty string + + # PATH='' doesn't match, whereas PATH=':' looks in the current directory + if not path: + return None + + paths = path.split(os.pathsep) + for p in paths: + f = os.path.join(p, executable) + if os.path.isfile(f): + # the file exists, we have a shot at spawn working + return f + return None + + +def run_os_command(command_map): + '''command_map is a dictionary of {'executable': command}. For ex. {'apt-get': 'sudo apt-get install -y python2.7'}''' + success = True + + for executable, commands in command_map.items(): + if find_executable(executable): + if isinstance(commands, str): + commands = [commands] + + for command in commands: + returncode = subprocess.check_call(command, shell=True, stdout=log_stream, stderr=sys.stderr) + success = success and (returncode == 0) + + return success + + +def could_not_install(package): + raise Exception('Could not install {0}. Please install it manually.'.format(package)) + + +def is_sudo_user(): + return os.geteuid() == 0 + + +def install_package(package): + if find_executable(package): + print("{} already installed!".format(package), level=1) + else: + print("Installing {}...".format(package)) + success = run_os_command({ + 'apt-get': ['sudo apt-get install -y {0}'.format(package)], + 'yum': ['sudo yum install -y {0}'.format(package)] + }) + if success: + print("{} Installed!".format(package), level=1) + else: + could_not_install(package) + def install_bench(args): + global log_stream + sys.stderr = sys.stdout + + if not args.verbose: + if not os.path.exists(tmp_log_folder): + os.makedirs(tmp_log_folder) + log_stream = open(log_path, 'w') + print("Logs are saved under {}".format(log_path), level=3) + check_distribution_compatibility() check_brew_installed() # pre-requisites for bench repo cloning @@ -19,9 +121,7 @@ def install_bench(args): 'yum': [ 'sudo yum groupinstall -y "Development tools"', 'sudo yum install -y epel-release redhat-lsb-core git python-setuptools python-devel openssl-devel libffi-devel' - ], - # epel-release is required to install redis, so installing it before the playbook-run. - # redhat-lsb-core is required, so that ansible can set ansible_lsb variable + ] }) if not find_executable("git"): @@ -36,13 +136,13 @@ def install_bench(args): # secure pip installation if find_executable('pip'): run_os_command({ - 'pip': 'sudo pip install --upgrade setuptools cryptography pip' + 'pip': 'sudo -H pip install --upgrade setuptools cryptography pip' }) else: if not os.path.exists("get-pip.py"): run_os_command({ - 'wget': 'wget https://bootstrap.pypa.io/get-pip.py' + 'wget': 'wget -q https://bootstrap.pypa.io/get-pip.py' }) success = run_os_command({ @@ -53,15 +153,15 @@ def install_bench(args): dist_name, dist_version = get_distribution_info() if dist_name == 'centos': run_os_command({ - 'pip': 'sudo pip install --upgrade --ignore-installed requests' + 'pip': 'sudo -H pip install --upgrade --ignore-installed requests' }) else: run_os_command({ - 'pip': 'sudo pip install --upgrade requests' + 'pip': 'sudo -H pip install --upgrade requests' }) success = run_os_command({ - 'pip': "sudo pip install --upgrade setuptools cryptography ansible==2.8.5 pip" + 'pip': "sudo -H pip install --upgrade setuptools cryptography ansible==2.8.5 pip" }) if not success: @@ -134,107 +234,72 @@ def install_bench(args): # Will install ERPNext production setup by default run_playbook('site.yml', sudo=True, extra_vars=extra_vars) - # # Will do changes for production if --production flag is passed - # if args.production: - # run_playbook('production.yml', sudo=True, extra_vars=extra_vars) - if os.path.exists(tmp_bench_repo): shutil.rmtree(tmp_bench_repo) -def check_distribution_compatibility(): - supported_dists = {'ubuntu': [14, 15, 16, 18, 19], 'debian': [8, 9, 10], - 'centos': [7], 'macos': [10.9, 10.10, 10.11, 10.12]} +def check_distribution_compatibility(): dist_name, dist_version = get_distribution_info() + supported_dists = { + 'macos': [10.9, 10.10, 10.11, 10.12], + 'ubuntu': [14, 15, 16, 18, 19], + 'debian': [8, 9], + 'centos': [7] + } + + print("Checking System Compatibility...") if dist_name in supported_dists: if float(dist_version) in supported_dists[dist_name]: - return + print("{0} {1} is compatible!".format(dist_name, dist_version), level=1) + else: + print("Install on {0} {1} instead".format(dist_name, supported_dists[dist_name][-1]), level=3) + else: + print("Sorry, the installer doesn't support {0}. Aborting installation!".format(dist_name), level=2) - print("Sorry, the installer doesn't support {0} {1}. Aborting installation!".format(dist_name, dist_version)) - if dist_name in supported_dists: - print("Install on {0} {1} instead".format(dist_name, supported_dists[dist_name][-1])) - sys.exit(1) def get_distribution_info(): # return distribution name and major version if platform.system() == "Linux": current_dist = platform.dist() return current_dist[0].lower(), current_dist[1].rsplit('.')[0] + elif platform.system() == "Darwin": current_dist = platform.mac_ver() return "macos", current_dist[0].rsplit('.', 1)[0] -def install_package(package): - package_exec = find_executable(package) - - if not package_exec: - success = run_os_command({ - 'apt-get': ['sudo apt-get install -y {0}'.format(package)], - 'yum': ['sudo yum install -y {0}'.format(package)] - }) - else: - return - - if not success: - could_not_install(package) def check_brew_installed(): - if 'Darwin' not in os.uname(): - return + if 'Darwin' in os.uname(): + if not find_executable('brew'): + raise Exception(''' + Please install brew package manager before proceeding with bench setup. Please run following + to install brew package manager on your machine, - brew_exec = find_executable('brew') + /usr/bin/ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)" + ''') - if not brew_exec: - raise Exception(''' - Please install brew package manager before proceeding with bench setup. Please run following - to install brew package manager on your machine, - - /usr/bin/ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)" - ''') def clone_bench_repo(args): '''Clones the bench repository in the user folder''' - if os.path.exists(tmp_bench_repo): - return 0 - - elif args.without_bench_setup: - clone_path = os.path.join(os.path.expanduser('~'), 'bench') - - else: - clone_path = tmp_bench_repo - branch = args.bench_branch or 'master' repo_url = args.repo_url or 'https://github.com/frappe/bench' + if os.path.exists(tmp_bench_repo): + return 0 + elif args.without_bench_setup: + clone_path = os.path.join(os.path.expanduser('~'), 'bench') + else: + clone_path = tmp_bench_repo success = run_os_command( - {'git': 'git clone {repo_url} {bench_repo} --depth 1 --branch {branch}'.format( + {'git': 'git clone --quiet {repo_url} {bench_repo} --depth 1 --branch {branch}'.format( repo_url=repo_url, bench_repo=clone_path, branch=branch)} ) return success -def run_os_command(command_map): - '''command_map is a dictionary of {'executable': command}. For ex. {'apt-get': 'sudo apt-get install -y python2.7'} ''' - success = True - for executable, commands in list(command_map.items()): - if find_executable(executable): - if isinstance(commands, str): - commands = [commands] - for command in commands: - returncode = subprocess.check_call(command, shell=True) - success = success and ( returncode == 0 ) - break - - return success - -def could_not_install(package): - raise Exception('Could not install {0}. Please install it manually.'.format(package)) - -def is_sudo_user(): - return os.geteuid() == 0 def get_passwords(args): @@ -324,92 +389,54 @@ def run_playbook(playbook_name, sudo=False, extra_vars=None): else: cwd = os.path.join(os.path.expanduser('~'), 'bench') - success = subprocess.check_call(args, cwd=os.path.join(cwd, 'playbooks')) + success = subprocess.check_call(args, cwd=os.path.join(cwd, 'playbooks'), stdout=log_stream, stderr=sys.stderr) return success + def parse_commandline_args(): import argparse parser = argparse.ArgumentParser(description='Frappe Installer') - # Arguments develop and production are mutually exclusive both can't be specified together. # Hence, we need to create a group for discouraging use of both options at the same time. args_group = parser.add_mutually_exclusive_group() - args_group.add_argument('--develop', dest='develop', action='store_true', default=False, - help='Install developer setup') - - args_group.add_argument('--production', dest='production', action='store_true', - default=False, help='Setup Production environment for bench') - - parser.add_argument('--site', dest='site', action='store', default='site1.local', - help='Specifiy name for your first ERPNext site') - - parser.add_argument('--without-site', dest='without_site', action='store_true', - default=False) - - parser.add_argument('--verbose', dest='verbosity', action='store_true', default=False, - help='Run the script in verbose mode') - + args_group.add_argument('--develop', dest='develop', action='store_true', default=False, help='Install developer setup') + args_group.add_argument('--production', dest='production', action='store_true', default=False, help='Setup Production environment for bench') + parser.add_argument('--site', dest='site', action='store', default='site1.local', help='Specifiy name for your first ERPNext site') + parser.add_argument('--without-site', dest='without_site', action='store_true', default=False) + parser.add_argument('--verbose', dest='verbose', action='store_true', default=False, help='Run the script in verbose mode') parser.add_argument('--user', dest='user', help='Install frappe-bench for this user') - parser.add_argument('--bench-branch', dest='bench_branch', help='Clone a particular branch of bench repository') - parser.add_argument('--repo-url', dest='repo_url', help='Clone bench from the given url') - - parser.add_argument('--frappe-repo-url', dest='frappe_repo_url', action='store', default='https://github.com/frappe/frappe', - help='Clone frappe from the given url') - - parser.add_argument('--frappe-branch', dest='frappe_branch', action='store', - help='Clone a particular branch of frappe') - - parser.add_argument('--erpnext-repo-url', dest='erpnext_repo_url', action='store', default='https://github.com/frappe/erpnext', - help='Clone erpnext from the given url') - - parser.add_argument('--erpnext-branch', dest='erpnext_branch', action='store', - help='Clone a particular branch of erpnext') - - parser.add_argument('--without-erpnext', dest='without_erpnext', action='store_true', default=False, - help='Prevent fetching ERPNext') - + parser.add_argument('--frappe-repo-url', dest='frappe_repo_url', action='store', default='https://github.com/frappe/frappe', help='Clone frappe from the given url') + parser.add_argument('--frappe-branch', dest='frappe_branch', action='store', help='Clone a particular branch of frappe') + parser.add_argument('--erpnext-repo-url', dest='erpnext_repo_url', action='store', default='https://github.com/frappe/erpnext', help='Clone erpnext from the given url') + parser.add_argument('--erpnext-branch', dest='erpnext_branch', action='store', help='Clone a particular branch of erpnext') + parser.add_argument('--without-erpnext', dest='without_erpnext', action='store_true', default=False, help='Prevent fetching ERPNext') # direct provision to install versions - parser.add_argument('--version', dest='version', action='store', default='12', type=int, - help='Clone particular version of frappe and erpnext') - + parser.add_argument('--version', dest='version', action='store', default='12', type=int, help='Clone particular version of frappe and erpnext') # To enable testing of script using Travis, this should skip the prompt - parser.add_argument('--run-travis', dest='run_travis', action='store_true', default=False, - help=argparse.SUPPRESS) - - parser.add_argument('--without-bench-setup', dest='without_bench_setup', action='store_true', default=False, - help=argparse.SUPPRESS) - + parser.add_argument('--run-travis', dest='run_travis', action='store_true', default=False, help=argparse.SUPPRESS) + parser.add_argument('--without-bench-setup', dest='without_bench_setup', action='store_true', default=False, help=argparse.SUPPRESS) # whether to overwrite an existing bench - parser.add_argument('--overwrite', dest='overwrite', action='store_true', default=False, - help='Whether to overwrite an existing bench') - + parser.add_argument('--overwrite', dest='overwrite', action='store_true', default=False, help='Whether to overwrite an existing bench') # set passwords parser.add_argument('--mysql-root-password', dest='mysql_root_password', help='Set mysql root password') - parser.add_argument('--mariadb-version', dest='mariadb_version', default='10.4', help='Specify mariadb version') + parser.add_argument('--mariadb-version', dest='mariadb_version', default='10.2', help='Specify mariadb version') parser.add_argument('--admin-password', dest='admin_password', help='Set admin password') parser.add_argument('--bench-name', dest='bench_name', help='Create bench with specified name. Default name is frappe-bench') - # Python interpreter to be used - parser.add_argument('--python', dest='python', default='python3', - help=argparse.SUPPRESS - ) - + parser.add_argument('--python', dest='python', default='python3', help=argparse.SUPPRESS) # LXC Support - parser.add_argument('--container', dest='container', default=False, action='store_true', - help='Use if you\'re creating inside LXC' - ) - + parser.add_argument('--container', dest='container', default=False, action='store_true', help='Use if you\'re creating inside LXC') args = parser.parse_args() return args if __name__ == '__main__': args = parse_commandline_args() - - install_bench(args) - - print('''Frappe/ERPNext has been successfully installed!''') + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + install_bench(args) + print("Frappe/ERPNext has been successfully installed!") diff --git a/playbooks/roles/bench/tasks/main.yml b/playbooks/roles/bench/tasks/main.yml index fa0d9e93..a227e77d 100644 --- a/playbooks/roles/bench/tasks/main.yml +++ b/playbooks/roles/bench/tasks/main.yml @@ -32,7 +32,7 @@ register: bench_stat - name: Fix permissions - become_method: sudo + become_user: root command: chown {{ frappe_user }} -R ~ - name: python3 bench init for develop From 2b6260d63ec62570efba123f369587ead6b9dc8f Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Wed, 18 Dec 2019 18:19:17 +0530 Subject: [PATCH 12/66] fix: install missing dirmngr for debian installs --- playbooks/roles/mariadb/tasks/debian.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/playbooks/roles/mariadb/tasks/debian.yml b/playbooks/roles/mariadb/tasks/debian.yml index 6694f9e8..968aa394 100644 --- a/playbooks/roles/mariadb/tasks/debian.yml +++ b/playbooks/roles/mariadb/tasks/debian.yml @@ -3,6 +3,12 @@ apt_key: keyserver=hkp://keyserver.ubuntu.com:80 id=0xcbcb082a1bb943db state=present when: ansible_distribution_major_version is version_compare('8', 'le') +- name: Install dirmngr for apt key for mariadb for Debian > 8 + apt: + pkg: dirmngr + state: present + when: ansible_distribution_major_version is version_compare('8', 'gt') + - name: Add apt key for mariadb for Debian > 8 apt_key: keyserver=hkp://keyserver.ubuntu.com:80 id=0xF1656F24C74CD1D8 state=present when: ansible_distribution_major_version is version_compare('8', 'gt') From bc190a6d3b87dcab938cab3b4edbde290b4acfc3 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Thu, 19 Dec 2019 14:01:24 +0530 Subject: [PATCH 13/66] chore: check if locales are set to avoid install breaking for click setup + minor fixes --- playbooks/install.py | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/playbooks/install.py b/playbooks/install.py index 0b7e6734..ba3f8bd1 100644 --- a/playbooks/install.py +++ b/playbooks/install.py @@ -2,6 +2,7 @@ import os, sys, subprocess, getpass, json, multiprocessing, shutil, platform, warnings from builtins import print as _ + tmp_bench_repo = '/tmp/.bench' tmp_log_folder = '/tmp/logs/' log_path = os.path.join(tmp_log_folder, 'install_bench.log') @@ -58,6 +59,20 @@ def find_executable(executable, path=None): return None +def check_environment(): + needed_environ_vars = ['LANG', 'LC_ALL'] + message = '' + + for var in needed_environ_vars: + if var not in os.environ: + message += "\nexport {}=C.UTF-8".format(var) + + if message: + print("Bench's CLI needs these to be defined!", level=3) + print("Run the following commands in shell: {}".format(message), level=2) + sys.exit() + + def run_os_command(command_map): '''command_map is a dictionary of {'executable': command}. For ex. {'apt-get': 'sudo apt-get install -y python2.7'}''' success = True @@ -435,8 +450,14 @@ def parse_commandline_args(): return args if __name__ == '__main__': + if not is_sudo_user(): + print("Please run this script as a non-root user with sudo privileges", level=3) + sys.exit() + args = parse_commandline_args() with warnings.catch_warnings(): warnings.simplefilter("ignore") + check_environment() install_bench(args) - print("Frappe/ERPNext has been successfully installed!") + + print("Bench + Frappe + ERPNext has been successfully installed!") From 07d23d6911d222514b9b645d43891f6dce204860 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Thu, 19 Dec 2019 14:38:46 +0530 Subject: [PATCH 14/66] chore: rename print function to log --- playbooks/install.py | 33 ++++++++++++++++----------------- 1 file changed, 16 insertions(+), 17 deletions(-) diff --git a/playbooks/install.py b/playbooks/install.py index ba3f8bd1..102ffc77 100644 --- a/playbooks/install.py +++ b/playbooks/install.py @@ -1,6 +1,5 @@ #!/usr/bin/env python3 import os, sys, subprocess, getpass, json, multiprocessing, shutil, platform, warnings -from builtins import print as _ tmp_bench_repo = '/tmp/.bench' @@ -9,7 +8,7 @@ log_path = os.path.join(tmp_log_folder, 'install_bench.log') log_stream = sys.stdout -def print(message, level=0): +def log(message, level=0): levels = { 0: '\033[94m', # normal 1: '\033[92m', # success @@ -18,7 +17,7 @@ def print(message, level=0): } start = levels.get(level) or '' end = '\033[0m' - _(start + message + end) + print(start + message + end) def find_executable(executable, path=None): @@ -68,8 +67,8 @@ def check_environment(): message += "\nexport {}=C.UTF-8".format(var) if message: - print("Bench's CLI needs these to be defined!", level=3) - print("Run the following commands in shell: {}".format(message), level=2) + log("Bench's CLI needs these to be defined!", level=3) + log("Run the following commands in shell: {}".format(message), level=2) sys.exit() @@ -99,15 +98,15 @@ def is_sudo_user(): def install_package(package): if find_executable(package): - print("{} already installed!".format(package), level=1) + log("{} already installed!".format(package), level=1) else: - print("Installing {}...".format(package)) + log("Installing {}...".format(package)) success = run_os_command({ 'apt-get': ['sudo apt-get install -y {0}'.format(package)], 'yum': ['sudo yum install -y {0}'.format(package)] }) if success: - print("{} Installed!".format(package), level=1) + log("{} Installed!".format(package), level=1) else: could_not_install(package) @@ -120,7 +119,7 @@ def install_bench(args): if not os.path.exists(tmp_log_folder): os.makedirs(tmp_log_folder) log_stream = open(log_path, 'w') - print("Logs are saved under {}".format(log_path), level=3) + log("Logs are saved under {}".format(log_path), level=3) check_distribution_compatibility() check_brew_installed() @@ -145,7 +144,7 @@ def install_bench(args): }) if not success: - print('Could not install pre-requisites. Please check for errors or install them manually.') + log('Could not install pre-requisites. Please check for errors or install them manually.') return # secure pip installation @@ -262,14 +261,14 @@ def check_distribution_compatibility(): 'centos': [7] } - print("Checking System Compatibility...") + log("Checking System Compatibility...") if dist_name in supported_dists: if float(dist_version) in supported_dists[dist_name]: - print("{0} {1} is compatible!".format(dist_name, dist_version), level=1) + log("{0} {1} is compatible!".format(dist_name, dist_version), level=1) else: - print("Install on {0} {1} instead".format(dist_name, supported_dists[dist_name][-1]), level=3) + log("Install on {0} {1} instead".format(dist_name, supported_dists[dist_name][-1]), level=3) else: - print("Sorry, the installer doesn't support {0}. Aborting installation!".format(dist_name), level=2) + log("Sorry, the installer doesn't support {0}. Aborting installation!".format(dist_name), level=2) def get_distribution_info(): @@ -374,7 +373,7 @@ def get_passwords(args): with open(passwords_file_path, 'w') as f: json.dump(passwords, f, indent=1) - print('Passwords saved at ~/passwords.txt') + log('Passwords saved at ~/passwords.txt') return passwords @@ -451,7 +450,7 @@ def parse_commandline_args(): if __name__ == '__main__': if not is_sudo_user(): - print("Please run this script as a non-root user with sudo privileges", level=3) + log("Please run this script as a non-root user with sudo privileges", level=3) sys.exit() args = parse_commandline_args() @@ -460,4 +459,4 @@ if __name__ == '__main__': check_environment() install_bench(args) - print("Bench + Frappe + ERPNext has been successfully installed!") + log("Bench + Frappe + ERPNext has been successfully installed!") From b005720a2bec397653352955d00d03cbf0fd2fd2 Mon Sep 17 00:00:00 2001 From: Shridhar Patil Date: Thu, 19 Dec 2019 17:35:48 +0530 Subject: [PATCH 15/66] skip asstes while doing get-app --- bench/commands/make.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/bench/commands/make.py b/bench/commands/make.py index b3ca8341..5425130d 100755 --- a/bench/commands/make.py +++ b/bench/commands/make.py @@ -35,10 +35,11 @@ def init(path, apps_path, frappe_path, frappe_branch, no_procfile, no_backups, @click.argument('name', nargs=-1) # Dummy argument for backward compatibility @click.argument('git-url') @click.option('--branch', default=None, help="branch to checkout") -def get_app(git_url, branch, name=None): +@click.option('--skip-assets', is_flag=True, default=True, help="Do not build assets") +def get_app(git_url, branch, name=None, skip_assets=True): "clone an app from the internet and set it up in your bench" from bench.app import get_app - get_app(git_url, branch=branch) + get_app(git_url, branch=branch, build_asset_files=skip_assets) @click.command('new-app') From 9e3d5c8587a2f6471a656ee2eac8ce6a1a9e0140 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Fri, 20 Dec 2019 14:45:56 +0530 Subject: [PATCH 16/66] chore: use shutil.which instead of find_executable --- playbooks/install.py | 48 +++++--------------------------------------- 1 file changed, 5 insertions(+), 43 deletions(-) diff --git a/playbooks/install.py b/playbooks/install.py index 102ffc77..df967011 100644 --- a/playbooks/install.py +++ b/playbooks/install.py @@ -20,44 +20,6 @@ def log(message, level=0): print(start + message + end) -def find_executable(executable, path=None): - """Tries to find 'executable' in the directories listed in 'path'. - A string listing directories separated by 'os.pathsep'; defaults to - os.environ['PATH']. Returns the complete filename or None if not found. - - source: https://github.com/python/cpython/blob/master/Lib/distutils/spawn.py - """ - _, ext = os.path.splitext(executable) - if (sys.platform == 'win32') and (ext != '.exe'): - executable = executable + '.exe' - - if os.path.isfile(executable): - return executable - - if path is None: - path = os.environ.get('PATH', None) - if path is None: - try: - path = os.confstr("CS_PATH") - except (AttributeError, ValueError): - # os.confstr() or CS_PATH is not available - path = os.defpath - # bpo-35755: Don't use os.defpath if the PATH environment variable is - # set to an empty string - - # PATH='' doesn't match, whereas PATH=':' looks in the current directory - if not path: - return None - - paths = path.split(os.pathsep) - for p in paths: - f = os.path.join(p, executable) - if os.path.isfile(f): - # the file exists, we have a shot at spawn working - return f - return None - - def check_environment(): needed_environ_vars = ['LANG', 'LC_ALL'] message = '' @@ -77,7 +39,7 @@ def run_os_command(command_map): success = True for executable, commands in command_map.items(): - if find_executable(executable): + if shutil.which(executable): if isinstance(commands, str): commands = [commands] @@ -97,7 +59,7 @@ def is_sudo_user(): def install_package(package): - if find_executable(package): + if shutil.which(package): log("{} already installed!".format(package), level=1) else: log("Installing {}...".format(package)) @@ -138,7 +100,7 @@ def install_bench(args): ] }) - if not find_executable("git"): + if not shutil.which("git"): success = run_os_command({ 'brew': 'brew install git' }) @@ -148,7 +110,7 @@ def install_bench(args): return # secure pip installation - if find_executable('pip'): + if shutil.which('pip'): run_os_command({ 'pip': 'sudo -H pip install --upgrade setuptools cryptography pip' }) @@ -284,7 +246,7 @@ def get_distribution_info(): def check_brew_installed(): if 'Darwin' in os.uname(): - if not find_executable('brew'): + if not shutil.which('brew'): raise Exception(''' Please install brew package manager before proceeding with bench setup. Please run following to install brew package manager on your machine, From a65e9f09dc398a90d40c6ba547f9072f2a814aa5 Mon Sep 17 00:00:00 2001 From: Shridhar Patil Date: Fri, 20 Dec 2019 17:20:28 +0530 Subject: [PATCH 17/66] changed parameter name --- bench/app.py | 6 +++--- bench/commands/make.py | 6 +++--- bench/utils.py | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/bench/app.py b/bench/app.py index 5becd1eb..5ff210b9 100755 --- a/bench/app.py +++ b/bench/app.py @@ -92,7 +92,7 @@ def remove_from_excluded_apps_txt(app, bench_path='.'): apps.remove(app) return write_excluded_apps_txt(apps, bench_path=bench_path) -def get_app(git_url, branch=None, bench_path='.', build_asset_files=True, verbose=False, +def get_app(git_url, branch=None, bench_path='.', skip_assets=False, verbose=False, postprocess = True): # from bench.utils import check_url try: @@ -137,7 +137,7 @@ def get_app(git_url, branch=None, bench_path='.', build_asset_files=True, verbos if postprocess: - if build_asset_files: + if not skip_assets: build_assets(bench_path=bench_path, app=app_name) conf = get_config(bench_path=bench_path) @@ -412,7 +412,7 @@ def get_major_version(version): def install_apps_from_path(path, bench_path='.'): apps = get_apps_json(path) for app in apps: - get_app(app['url'], branch=app.get('branch'), bench_path=bench_path, build_asset_files=False) + get_app(app['url'], branch=app.get('branch'), bench_path=bench_path, skip_assets=True) def get_apps_json(path): if path.startswith('http'): diff --git a/bench/commands/make.py b/bench/commands/make.py index 5425130d..213d11ea 100755 --- a/bench/commands/make.py +++ b/bench/commands/make.py @@ -35,11 +35,11 @@ def init(path, apps_path, frappe_path, frappe_branch, no_procfile, no_backups, @click.argument('name', nargs=-1) # Dummy argument for backward compatibility @click.argument('git-url') @click.option('--branch', default=None, help="branch to checkout") -@click.option('--skip-assets', is_flag=True, default=True, help="Do not build assets") -def get_app(git_url, branch, name=None, skip_assets=True): +@click.option('--skip-assets', is_flag=True, default=False, help="Do not build assets") +def get_app(git_url, branch, name=None, skip_assets=False): "clone an app from the internet and set it up in your bench" from bench.app import get_app - get_app(git_url, branch=branch, build_asset_files=skip_assets) + get_app(git_url, branch=branch, skip_assets=skip_assets) @click.command('new-app') diff --git a/bench/utils.py b/bench/utils.py index bd665fa7..8e3dfd0b 100755 --- a/bench/utils.py +++ b/bench/utils.py @@ -72,7 +72,7 @@ def init(path, apps_path=None, no_procfile=False, no_backups=False, if not frappe_path: frappe_path = 'https://github.com/frappe/frappe.git' - get_app(frappe_path, branch=frappe_branch, bench_path=path, build_asset_files=False, verbose=verbose) + get_app(frappe_path, branch=frappe_branch, bench_path=path, skip_assets=True, verbose=verbose) if apps_path: install_apps_from_path(apps_path, bench_path=path) From c93a25d336499a06c45251eb59160d34f234d650 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Wed, 25 Dec 2019 19:23:14 +0530 Subject: [PATCH 18/66] feat: "bench update" from non-bench directory --- bench/commands/update.py | 9 ++++++-- bench/utils.py | 45 ++++++++++++++++++++++++++++++---------- 2 files changed, 41 insertions(+), 13 deletions(-) diff --git a/bench/commands/update.py b/bench/commands/update.py index f87cda95..82f857ba 100755 --- a/bench/commands/update.py +++ b/bench/commands/update.py @@ -5,7 +5,7 @@ from bench.config.common_site_config import get_config, update_config from bench.app import pull_all_apps, is_version_upgrade, validate_branch from bench.utils import (update_bench, validate_upgrade, pre_upgrade, post_upgrade, before_update, update_requirements, update_node_packages, backup_all_sites, patch_sites, build_assets, - restart_supervisor_processes, restart_systemd_processes) + restart_supervisor_processes, restart_systemd_processes, is_bench_directory) from bench import patches from six.moves import reload_module @@ -25,6 +25,11 @@ from six.moves import reload_module def update(pull=False, patch=False, build=False, bench=False, auto=False, restart_supervisor=False, restart_systemd=False, requirements=False, no_backup=False, force=False, reset=False): "Update bench" + if not is_bench_directory(): + """Update only bench if bench update called from outside a bench""" + update_bench(bench_repo=True, requirements=True) + sys.exit() + if not (pull or patch or build or bench or requirements): pull, patch, build, bench, requirements = True, True, True, True, True @@ -35,7 +40,7 @@ def update(pull=False, patch=False, build=False, bench=False, auto=False, restar conf = get_config(".") if bench and conf.get('update_bench_on_update'): - update_bench() + update_bench(bench_repo=True, requirements=False) restart_update({ 'pull': pull, 'patch': patch, diff --git a/bench/utils.py b/bench/utils.py index bd665fa7..192ea5a0 100755 --- a/bench/utils.py +++ b/bench/utils.py @@ -16,6 +16,18 @@ logger = logging.getLogger(__name__) folders_in_bench = ('apps', 'sites', 'config', 'logs', 'config/pids') + +def is_bench_directory(): + cur_dir = os.path.curdir + is_bench = True + + for folder in folders_in_bench: + path = os.path.join(cur_dir, folder) + is_bench = is_bench and os.path.exists(path) + + return is_bench + + def safe_decode(string, encoding = 'utf-8'): try: string = string.decode(encoding) @@ -256,13 +268,19 @@ def read_crontab(): s.stdout.close() return out -def update_bench(): - logger.info('updating bench') +def update_bench(bench_repo=True, requirements=True): + logger.info("Updating bench") # bench-repo folder cwd = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) - exec_cmd("git pull", cwd=cwd) + if bench_repo: + exec_cmd("git pull", cwd=cwd) + + if requirements: + update_bench_requirements() + + logger.info("Bench Updated!") def setup_sudoers(user): if not os.path.exists('/etc/sudoers.d'): @@ -424,19 +442,24 @@ def set_default_site(site, bench_path='.'): exec_cmd("{frappe} --use {site}".format(frappe=get_frappe(bench_path=bench_path), site=site), cwd=os.path.join(bench_path, 'sites')) -def update_requirements(bench_path='.'): - print('Updating Python libraries...') - - # update env pip - env_pip = os.path.join(bench_path, 'env', 'bin', 'pip') - exec_cmd("{pip} install -q -U pip".format(pip=env_pip)) - - # Update bench requirements (at user level) +def update_bench_requirements(): bench_req_file = os.path.join(os.path.dirname(bench.__path__[0]), 'requirements.txt') user_pip = which("pip" if PY2 else "pip3") install_requirements(user_pip, bench_req_file, user=True) +def update_env_pip(bench_path): + env_pip = os.path.join(bench_path, 'env', 'bin', 'pip') + exec_cmd("{pip} install -q -U pip".format(pip=env_pip)) + +def update_requirements(bench_path='.'): from bench.app import get_apps, install_app + print('Updating Python libraries...') + + # update env pip + update_env_pip(bench_path) + + # Update bench requirements (at user level) + update_bench_requirements() for app in get_apps(): install_app(app, bench_path=bench_path) From 186c0eafdac0adfc75905026368f8f04d6f8da05 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Wed, 25 Dec 2019 19:24:23 +0530 Subject: [PATCH 19/66] feat: auto-stash changes in bench (tool) on update --- bench/utils.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/bench/utils.py b/bench/utils.py index 192ea5a0..74aeeabc 100755 --- a/bench/utils.py +++ b/bench/utils.py @@ -275,7 +275,11 @@ def update_bench(bench_repo=True, requirements=True): cwd = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) if bench_repo: - exec_cmd("git pull", cwd=cwd) + try: + exec_cmd("git pull", cwd=cwd) + except bench.utils.CommandFailedError: + exec_cmd("git stash", cwd=cwd) + logger.info("Stashing changes made at {}\nUse git stash apply to recover changes after the successful update!".format(cwd)) if requirements: update_bench_requirements() From c7381defd3fcaf447710316db579f3aa7d16ae66 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Fri, 27 Dec 2019 13:14:33 +0530 Subject: [PATCH 20/66] fix: updated sequence of package updates chore: breakdown install_bench in smaller functions --- playbooks/install.py | 226 ++++++++++++++++++++++--------------------- 1 file changed, 116 insertions(+), 110 deletions(-) diff --git a/playbooks/install.py b/playbooks/install.py index df967011..9330009f 100644 --- a/playbooks/install.py +++ b/playbooks/install.py @@ -2,8 +2,8 @@ import os, sys, subprocess, getpass, json, multiprocessing, shutil, platform, warnings -tmp_bench_repo = '/tmp/.bench' -tmp_log_folder = '/tmp/logs/' +tmp_bench_repo = os.path.join('/', 'tmp', '.bench') +tmp_log_folder = os.path.join('/', 'tmp', 'logs') log_path = os.path.join(tmp_log_folder, 'install_bench.log') log_stream = sys.stdout @@ -20,20 +20,75 @@ def log(message, level=0): print(start + message + end) +def setup_log_stream(args): + global log_stream + sys.stderr = sys.stdout + + if not args.verbose: + if not os.path.exists(tmp_log_folder): + os.makedirs(tmp_log_folder) + log_stream = open(log_path, 'w') + log("Logs are saved under {0}".format(log_path), level=3) + + def check_environment(): needed_environ_vars = ['LANG', 'LC_ALL'] message = '' for var in needed_environ_vars: if var not in os.environ: - message += "\nexport {}=C.UTF-8".format(var) + message += "\nexport {0}=C.UTF-8".format(var) if message: log("Bench's CLI needs these to be defined!", level=3) - log("Run the following commands in shell: {}".format(message), level=2) + log("Run the following commands in shell: {0}".format(message), level=2) sys.exit() +def check_system_package_managers(): + if 'Darwin' in os.uname(): + if not shutil.which('brew'): + raise Exception(''' + Please install brew package manager before proceeding with bench setup. Please run following + to install brew package manager on your machine, + + /usr/bin/ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)" + ''') + if 'Linux' in os.uname(): + if not any([shutil.which(x) for x in ['apt-get', 'yum']]): + raise Exception('Cannot find any compatible package manager!') + + +def check_distribution_compatibility(): + dist_name, dist_version = get_distribution_info() + supported_dists = { + 'macos': [10.9, 10.10, 10.11, 10.12], + 'ubuntu': [14, 15, 16, 18, 19], + 'debian': [8, 9], + 'centos': [7] + } + + log("Checking System Compatibility...") + if dist_name in supported_dists: + if float(dist_version) in supported_dists[dist_name]: + log("{0} {1} is compatible!".format(dist_name, dist_version), level=1) + else: + log("Install on {0} {1} instead".format(dist_name, supported_dists[dist_name][-1]), level=3) + else: + log("Sorry, the installer doesn't support {0}. Aborting installation!".format(dist_name), level=2) + + +def get_distribution_info(): + # return distribution name and major version + if platform.system() == "Linux": + current_dist = platform.dist() + return current_dist[0].lower(), current_dist[1].rsplit('.')[0] + + elif platform.system() == "Darwin": + current_dist = platform.mac_ver() + return "macos", current_dist[0].rsplit('.', 1)[0] + + def run_os_command(command_map): '''command_map is a dictionary of {'executable': command}. For ex. {'apt-get': 'sudo apt-get install -y python2.7'}''' success = True @@ -50,66 +105,7 @@ def run_os_command(command_map): return success -def could_not_install(package): - raise Exception('Could not install {0}. Please install it manually.'.format(package)) - - -def is_sudo_user(): - return os.geteuid() == 0 - - -def install_package(package): - if shutil.which(package): - log("{} already installed!".format(package), level=1) - else: - log("Installing {}...".format(package)) - success = run_os_command({ - 'apt-get': ['sudo apt-get install -y {0}'.format(package)], - 'yum': ['sudo yum install -y {0}'.format(package)] - }) - if success: - log("{} Installed!".format(package), level=1) - else: - could_not_install(package) - - -def install_bench(args): - global log_stream - sys.stderr = sys.stdout - - if not args.verbose: - if not os.path.exists(tmp_log_folder): - os.makedirs(tmp_log_folder) - log_stream = open(log_path, 'w') - log("Logs are saved under {}".format(log_path), level=3) - - check_distribution_compatibility() - check_brew_installed() - # pre-requisites for bench repo cloning - install_package('curl') - install_package('wget') - - success = run_os_command({ - 'apt-get': [ - 'sudo apt-get update', - 'sudo apt-get install -y git build-essential python3-setuptools python3-dev libffi-dev' - ], - 'yum': [ - 'sudo yum groupinstall -y "Development tools"', - 'sudo yum install -y epel-release redhat-lsb-core git python-setuptools python-devel openssl-devel libffi-devel' - ] - }) - - if not shutil.which("git"): - success = run_os_command({ - 'brew': 'brew install git' - }) - - if not success: - log('Could not install pre-requisites. Please check for errors or install them manually.') - return - - # secure pip installation +def install_pip(): if shutil.which('pip'): run_os_command({ 'pip': 'sudo -H pip install --upgrade setuptools cryptography pip' @@ -136,6 +132,27 @@ def install_bench(args): 'pip': 'sudo -H pip install --upgrade requests' }) + log("pip installed!", level=1) + + +def install_prerequisites(): + # pre-requisites for bench repo cloning + run_os_command({ + 'apt-get': [ + 'sudo apt-get update', + 'sudo apt-get install -y git build-essential python3-setuptools python3-dev libffi-dev' + ], + 'yum': [ + 'sudo yum groupinstall -y "Development tools"', + 'sudo yum install -y epel-release redhat-lsb-core git python-setuptools python-devel openssl-devel libffi-devel' + ] + }) + + install_package('curl') + install_package('wget') + install_package('git') + install_pip() + success = run_os_command({ 'pip': "sudo -H pip install --upgrade setuptools cryptography ansible==2.8.5 pip" }) @@ -143,6 +160,32 @@ def install_bench(args): if not success: could_not_install('Ansible') + +def could_not_install(package): + raise Exception('Could not install {0}. Please install it manually.'.format(package)) + + +def is_sudo_user(): + return os.geteuid() == 0 + + +def install_package(package): + if shutil.which(package): + log("{0} already installed!".format(package), level=1) + else: + log("Installing {0}...".format(package)) + success = run_os_command({ + 'apt-get': ['sudo apt-get install -y {0}'.format(package)], + 'yum': ['sudo yum install -y {0}'.format(package)], + 'brew': ['brew install {0}'.format(package)] + }) + if success: + log("{0} installed!".format(package), level=1) + return success + could_not_install(package) + + +def install_bench(args): # clone bench repo if not args.run_travis: clone_bench_repo(args) @@ -214,47 +257,6 @@ def install_bench(args): shutil.rmtree(tmp_bench_repo) -def check_distribution_compatibility(): - dist_name, dist_version = get_distribution_info() - supported_dists = { - 'macos': [10.9, 10.10, 10.11, 10.12], - 'ubuntu': [14, 15, 16, 18, 19], - 'debian': [8, 9], - 'centos': [7] - } - - log("Checking System Compatibility...") - if dist_name in supported_dists: - if float(dist_version) in supported_dists[dist_name]: - log("{0} {1} is compatible!".format(dist_name, dist_version), level=1) - else: - log("Install on {0} {1} instead".format(dist_name, supported_dists[dist_name][-1]), level=3) - else: - log("Sorry, the installer doesn't support {0}. Aborting installation!".format(dist_name), level=2) - - -def get_distribution_info(): - # return distribution name and major version - if platform.system() == "Linux": - current_dist = platform.dist() - return current_dist[0].lower(), current_dist[1].rsplit('.')[0] - - elif platform.system() == "Darwin": - current_dist = platform.mac_ver() - return "macos", current_dist[0].rsplit('.', 1)[0] - - -def check_brew_installed(): - if 'Darwin' in os.uname(): - if not shutil.which('brew'): - raise Exception(''' - Please install brew package manager before proceeding with bench setup. Please run following - to install brew package manager on your machine, - - /usr/bin/ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)" - ''') - - def clone_bench_repo(args): '''Clones the bench repository in the user folder''' branch = args.bench_branch or 'master' @@ -275,15 +277,12 @@ def clone_bench_repo(args): return success - - - def get_passwords(args): """ Returns a dict of passwords for further use and creates passwords.txt in the bench user's home directory """ - + log("Input mySQL and Frappe Administrator passwords:") ignore_prompt = args.run_travis or args.without_bench_setup mysql_root_password, admin_password = '', '' passwords_file_path = os.path.join(os.path.expanduser('~' + args.user), 'passwords.txt') @@ -343,13 +342,15 @@ def get_passwords(args): def get_extra_vars_json(extra_args): # We need to pass production as extra_vars to the playbook to execute conditionals in the # playbook. Extra variables can passed as json or key=value pair. Here, we will use JSON. - json_path = os.path.join('/tmp', 'extra_vars.json') + json_path = os.path.join('/', 'tmp', 'extra_vars.json') extra_vars = dict(list(extra_args.items())) + with open(json_path, mode='w') as j: json.dump(extra_vars, j, indent=1, sort_keys=True) return ('@' + json_path) + def run_playbook(playbook_name, sudo=False, extra_vars=None): args = ['ansible-playbook', '-c', 'local', playbook_name , '-vvvv'] @@ -416,9 +417,14 @@ if __name__ == '__main__': sys.exit() args = parse_commandline_args() + with warnings.catch_warnings(): warnings.simplefilter("ignore") + setup_log_stream(args) + check_distribution_compatibility() + check_system_package_managers() check_environment() + install_prerequisites() install_bench(args) log("Bench + Frappe + ERPNext has been successfully installed!") From a7a8790de6f18bd2dd572a71ceaf4cb868d81b49 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Fri, 27 Dec 2019 15:44:03 +0530 Subject: [PATCH 21/66] fix: git stash requires user details pre git v2.21 --- bench/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bench/utils.py b/bench/utils.py index ab75dad0..82fe900f 100755 --- a/bench/utils.py +++ b/bench/utils.py @@ -278,7 +278,7 @@ def update_bench(bench_repo=True, requirements=True): try: exec_cmd("git pull", cwd=cwd) except bench.utils.CommandFailedError: - exec_cmd("git stash", cwd=cwd) + exec_cmd("git -c user.name=bench -c user.email=developers@frappe.io stash", cwd=cwd) logger.info("Stashing changes made at {}\nUse git stash apply to recover changes after the successful update!".format(cwd)) if requirements: From 69055a86426adf73e4962fdd20005261591b49d6 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Mon, 30 Dec 2019 16:46:35 +0530 Subject: [PATCH 22/66] chore: updated README --- README.md | 79 ++++++++++++++++++++++++++++++------------------------- 1 file changed, 43 insertions(+), 36 deletions(-) diff --git a/README.md b/README.md index 18baf8ad..a6faffe5 100755 --- a/README.md +++ b/README.md @@ -4,9 +4,8 @@

Frappe Bench

-[![Build Status](https://circleci.com/gh/frappe/bench.svg?style=svg)](https://circleci.com/gh/frappe/bench) -The bench is a command-line utility that helps you to install apps, manage multiple sites and update Frappe / ERPNext apps on */nix (CentOS, Debian, Ubuntu, etc) for development and production. Bench will also create nginx and supervisor config files, setup backups and much more. +The bench is a command-line utility that helps you to install apps, manage multiple sites and update Frappe / ERPNext apps on */nix (Ubuntu, Debian, CentOS, etc) for development and production. Bench will also create nginx and supervisor config files, setup backups and much more. If you are using on a VPS make sure it has >= 1Gb of RAM or has swap setup properly. @@ -26,7 +25,7 @@ You will need a computer/server. Options include: To install the Frappe/ERPNext server software, you will need an operating system on your normal computer which is not Windows. Note that the command line interface does work on Windows, and you can use Frappe/ERPNext from any operating system with a web browser. However, the server software does not run on Windows. It does run on other operating systems, so choose one of these instead: -- Linux: Debian, Ubuntu, CentOS are the preferred distros and are well tested. [Arch Linux](https://github.com/frappe/bench/wiki/Install-ERPNext-on-ArchLinux) can also be used +- Linux: Ubuntu, Debian, CentOS are the preferred distros and are tested. [Arch Linux](https://github.com/frappe/bench/wiki/Install-ERPNext-on-ArchLinux) can also be used - Mac OS X ### Manual Install @@ -43,14 +42,15 @@ To manually install frappe/erpnext, you can follow this [this wiki](https://gith • pip 15+ (py dependency manager) • cron (scheduled jobs) • wkhtmltopdf (version 0.12.5 with patched qt) (for pdf generation) -• Nginx (for production) +• Nginx (for production) + #### 2. Install Bench Install bench as a *non root* user, - git clone https://github.com/frappe/bench bench-repo - pip install --user -e bench-repo + git clone https://github.com/frappe/bench ~/.bench + pip install --user -e ~/.bench Note: Please do not remove the bench directory the above commands will create @@ -100,24 +100,17 @@ Note: Please do not remove the bench directory the above commands will create - This is an opinionated setup so it is best to setup on a blank server. - Works on Ubuntu 16.04+, CentOS 7+, Debian 8+ -- You may have to install Python 2.7 (eg on Ubuntu 16.04+) by running `apt-get install python-minimal` -- You may also have to install build-essential and python-setuptools by running `apt-get install build-essential python-setuptools` -- This script will install the pre-requisites, install bench and setup an ERPNext site -- Passwords for Frappe Administrator and MariaDB (root) will be asked +- You may have to install Python 3 and other essentials by running `apt-get install python3-minimal build-essential python3-setuptools` +- This script will install the pre-requisites, install bench and setup an ERPNext site `(site1.local under frappe-bench)` +- Passwords for Frappe Administrator and MariaDB (root) will be asked and saved under `~/passwoords.txt` - MariaDB (root) password may be `password` on a fresh server - You can then login as **Administrator** with the Administrator password +- The log file is saved under `/tmp/logs/install_bench.log` in case you run into any issues during the install. - If you find any problems, post them on the forum: [https://discuss.erpnext.com](https://discuss.erpnext.com) Open your Terminal and enter: -#### 1. Download the install script - -For Linux: - - wget https://raw.githubusercontent.com/frappe/bench/master/playbooks/install.py - - -#### 2. Run the install script +#### 0. Setup user & Download the install script If you are on a fresh server and logged in as root, at first create a dedicated user for frappe & equip this user with sudo privileges @@ -129,36 +122,47 @@ If you are on a fresh server and logged in as root, at first create a dedicated *(it is very common to use "frappe" as frappe-username, but this comes with the security flaw of ["frappe" ranking very high](https://www.reddit.com/r/dataisbeautiful/comments/b3sirt/i_deployed_over_a_dozen_cyber_honeypots_all_over/?st=JTJ0SC0Q&sh=76e05240) in as a username challenged in hacking attempts. So, for production sites it is highly recommended to use a custom username harder to guess)* -For developer setup: +Switch to `[frappe-user]` (using `su [frappe-user]`) and start the setup - sudo python install.py --develop + wget https://raw.githubusercontent.com/frappe/bench/master/playbooks/install.py -For production: - sudo python install.py --production --user [frappe-user] +#### 1. Run the install script -use --user flag to create a user and install using that user (By default, the script will create a user with the username `frappe` if the --user flag is not used) + sudo python3 install.py - python install.py --develop --user [frappe-user] +*Note: `user` flag to create a user and install using that user (By default, the script will create a user with the username `frappe` if the --user flag is not used)* -use --container flag to install inside a container (this will prevent the `/proc/sys/vm/swappiness: Read-only` file system error) +For production or development, append teh `--production` or `--develop` flag to the command respectively. - sudo python install.py --production --user [frappe-user] --container + sudo python3 install.py --production --user [frappe-user] + +or + + sudo python3 install.py --develop + sudo python3 install.py --develop --user [frappe-user] + + sudo python3 install.py --production --user [frappe-user] --container + +*Note: `container` flag to install inside a container (this will prevent the `/proc/sys/vm/swappiness: Read-only` file system error)* + + + python3 install.py --production --version 11 --user [frappe-user] use --version flag to install specific version - python install.py --develop --version 11 --user [frappe-user] + python3 install.py --production --version 11 --python python2.7 --user [frappe-user] -use --python flag to specify virtual environments python version, by default script setup python 3 +use --python flag to specify virtual environments python version, by default script setup python3 - python install.py --develop --version 11 --python python2.7 --user [frappe-user] #### What will this script do? - Install all the pre-requisites -- Install the command line `bench` -- Create a new bench (a folder that will contain your entire frappe/erpnext setup) -- Create a new ERPNext site on the bench +- Install the command line `bench` (under ~/.bench) +- Create a new bench (a folder that will contain your entire frappe/erpnext setup at ~/frappe-bench) +- Create a new ERPNext site on the bench (site1.local) + #### How do I start ERPNext @@ -170,7 +174,7 @@ use --python flag to specify virtual environments python version, by default scr ## Bench Manager (GUI for Bench) -Bench Manager is a graphical user interface to emulate the functionalities of Frappé Bench. Like the command line utility it helps you install apps, manage multiple sites, update apps and much more. +Bench Manager is a graphical user interface to emulate the functionalities of Frappe Bench. Like the command line utility it helps you install apps, manage multiple sites, update apps and much more. ### Installation @@ -185,10 +189,11 @@ What all it does: ## Docker Install - For Developers (beta) -1. For developer setup, you can also use the official [Frappé Docker](https://github.com/frappe/frappe_docker/). +1. For developer setup, you can also use the official [Frappe Docker](https://github.com/frappe/frappe_docker/). 2. The app, mariadb and redis run on individual containers 3. This setup supports multi-tenancy and exposes the frappe-bench volume as a external storage. -4. For more details, [read the instructions on the Frappé Docker README](https://github.com/frappe/frappe_docker/) +4. For more details, [read the instructions on the Frappe Docker README](https://github.com/frappe/frappe_docker/) + Help ==== @@ -197,6 +202,7 @@ For bench help, you can type bench --help + Updating ======== @@ -213,7 +219,8 @@ You can also run the parts of the bench selectively. `bench update --bench` will only update the bench utility (this project) -`bench update --requirements` will only update dependencies (python packages) for the apps installed +`bench update --requirements` will only update all dependencies (Python + Node) for the apps available in current bench + Guides ======= From bb677d17df08fc50d590aa6d0c598abdc5258c9b Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Tue, 31 Dec 2019 14:06:00 +0530 Subject: [PATCH 23/66] perf: Faster Commands execution by commands caching --- bench/cli.py | 36 ++++++++++++++++++++++----------- bench/commands/__init__.py | 5 ++++- bench/commands/update.py | 4 +++- bench/commands/utils.py | 20 +++++++++++++++++++ bench/utils.py | 41 ++++++++++++++++++++++++++++++++------ 5 files changed, 86 insertions(+), 20 deletions(-) diff --git a/bench/cli.py b/bench/cli.py index fd419d00..a51bf305 100755 --- a/bench/cli.py +++ b/bench/cli.py @@ -1,13 +1,24 @@ +import json +import logging +import os +import pwd +import subprocess +import sys + import click -import os, sys, logging, json, pwd, subprocess -from bench.utils import is_root, PatchError, drop_privileges, get_env_cmd, get_cmd_output, get_frappe + from bench.app import get_apps -from bench.config.common_site_config import get_config from bench.commands import bench_command +from bench.config.common_site_config import get_config +from bench.utils import (PatchError, clear_command_cache, drop_privileges, + frappe_commands_file, generate_command_cache, + get_cmd_output, get_env_cmd, get_frappe, is_root) + logger = logging.getLogger('bench') from_command_line = False + def cli(): global from_command_line from_command_line = True @@ -90,18 +101,19 @@ def frappe_cmd(bench_path='.'): os.execv(f, [f] + ['-m', 'frappe.utils.bench_helper', 'frappe'] + sys.argv[1:]) def get_frappe_commands(bench_path='.'): - python = get_env_cmd('python', bench_path=bench_path) sites_path = os.path.join(bench_path, 'sites') + if not os.path.exists(sites_path): return [] - try: - output = get_cmd_output("{python} -m frappe.utils.bench_helper get-frappe-commands".format(python=python), cwd=sites_path) - # output = output.decode('utf-8') - return json.loads(output) - except subprocess.CalledProcessError as e: - if hasattr(e, "stderr"): - print(e.stderr.decode('utf-8')) - return [] + + if os.path.exists(frappe_commands_file): + command_dump = open(frappe_commands_file, 'r').read() + output = json.loads(command_dump) if command_dump else [] + + else: + output = generate_command_cache() + + return output or [] def get_frappe_help(bench_path='.'): python = get_env_cmd('python', bench_path=bench_path) diff --git a/bench/commands/__init__.py b/bench/commands/__init__.py index 8772a0a2..184d6cfd 100755 --- a/bench/commands/__init__.py +++ b/bench/commands/__init__.py @@ -47,7 +47,8 @@ bench_command.add_command(switch_to_develop) from bench.commands.utils import (start, restart, set_nginx_port, set_ssl_certificate, set_ssl_certificate_key, set_url_root, set_mariadb_host, set_default_site, download_translations, shell, backup_site, backup_all_sites, release, renew_lets_encrypt, - disable_production, bench_src, prepare_beta_release, set_redis_cache_host, set_redis_queue_host, set_redis_socketio_host) + disable_production, bench_src, prepare_beta_release, set_redis_cache_host, set_redis_queue_host, set_redis_socketio_host, + generate_command_cache, clear_command_cache) bench_command.add_command(start) bench_command.add_command(restart) bench_command.add_command(set_nginx_port) @@ -68,6 +69,8 @@ bench_command.add_command(renew_lets_encrypt) bench_command.add_command(disable_production) bench_command.add_command(bench_src) bench_command.add_command(prepare_beta_release) +bench_command.add_command(generate_command_cache) +bench_command.add_command(clear_command_cache) from bench.commands.setup import setup bench_command.add_command(setup) diff --git a/bench/commands/update.py b/bench/commands/update.py index 82f857ba..68653dcf 100755 --- a/bench/commands/update.py +++ b/bench/commands/update.py @@ -5,7 +5,7 @@ from bench.config.common_site_config import get_config, update_config from bench.app import pull_all_apps, is_version_upgrade, validate_branch from bench.utils import (update_bench, validate_upgrade, pre_upgrade, post_upgrade, before_update, update_requirements, update_node_packages, backup_all_sites, patch_sites, build_assets, - restart_supervisor_processes, restart_systemd_processes, is_bench_directory) + restart_supervisor_processes, restart_systemd_processes, is_bench_directory, clear_command_cache) from bench import patches from six.moves import reload_module @@ -30,6 +30,8 @@ def update(pull=False, patch=False, build=False, bench=False, auto=False, restar update_bench(bench_repo=True, requirements=True) sys.exit() + clear_command_cache(bench_path='.') + if not (pull or patch or build or bench or requirements): pull, patch, build, bench, requirements = True, True, True, True, True diff --git a/bench/commands/utils.py b/bench/commands/utils.py index 9ba55bb8..0c289e8f 100644 --- a/bench/commands/utils.py +++ b/bench/commands/utils.py @@ -190,3 +190,23 @@ def bench_src(): """Prints bench source folder path, which can be used as: cd `bench src` """ import bench print(os.path.dirname(bench.__path__[0])) + + +@click.command('generate-command-cache') +def generate_command_cache(bench_path='.'): + """ + Caches Frappe commands (speeds up bench commands execution) + Default caching behaviour: generated the first time any command is run + """ + from bench.utils import generate_command_cache + return generate_command_cache(bench_path=bench_path) + + +@click.command('clear-command-cache') +def clear_command_cache(bench_path='.'): + """ + Clears commands cached + Default invalidation behaviour: destroyed on each run of `bench update` + """ + from bench.utils import clear_command_cache + return clear_command_cache(bench_path=bench_path) diff --git a/bench/utils.py b/bench/utils.py index 82fe900f..5ba3b7ba 100755 --- a/bench/utils.py +++ b/bench/utils.py @@ -1,7 +1,7 @@ -import os, sys, shutil, subprocess, logging, itertools, requests, json, platform, select, pwd, grp, multiprocessing, hashlib, glob +import os, sys, shutil, subprocess, logging, itertools, requests, json, platform, select, pwd, grp, multiprocessing, hashlib, glob, errno +import semantic_version from distutils.spawn import find_executable import bench -import semantic_version from bench import env from six import iteritems, PY2 @@ -13,7 +13,7 @@ class CommandFailedError(Exception): pass logger = logging.getLogger(__name__) - +frappe_commands_file = '.frappe-cmd' folders_in_bench = ('apps', 'sites', 'config', 'logs', 'config/pids') @@ -188,9 +188,7 @@ def setup_env(bench_path='.', python = 'python3'): pip = os.path.join('env', 'bin', 'pip') exec_cmd('virtualenv -q {} -p {}'.format('env', python), cwd=bench_path) - exec_cmd('{} -q install --upgrade pip'.format(pip), cwd=bench_path) - exec_cmd('{} -q install wheel'.format(pip), cwd=bench_path) - exec_cmd('{} -q install six'.format(pip), cwd=bench_path) + exec_cmd('{} -q install --upgrade pip wheel six'.format(pip), cwd=bench_path) exec_cmd('{} -q install -e git+https://github.com/frappe/python-pdfkit.git#egg=pdfkit'.format(pip), cwd=bench_path) def setup_socketio(bench_path='.'): @@ -860,3 +858,34 @@ def run_playbook(playbook_name, extra_vars=None, tag=None): args.extend(['-t', tag]) subprocess.check_call(args, cwd=os.path.join(os.path.dirname(bench.__path__[0]), 'playbooks')) + +def generate_command_cache(bench_path='.'): + """ + Caches Frappe commands (speeds up bench commands execution) + Default caching behaviour: generated the first time any command is run + """ + + python = get_env_cmd('python', bench_path=bench_path) + sites_path = os.path.join(bench_path, 'sites') + + if os.path.exists(frappe_commands_file): + os.remove(frappe_commands_file) + + try: + output = get_cmd_output("{python} -m frappe.utils.bench_helper get-frappe-commands".format(python=python), cwd=sites_path) + json.dump(eval(output), open(frappe_commands_file, 'w')) + return output + except subprocess.CalledProcessError as e: + if hasattr(e, "stderr"): + print(e.stderr.decode('utf-8')) + +def clear_command_cache(bench_path='.'): + """ + Clears commands cached + Default invalidation behaviour: destroyed on each run of `bench update` + """ + + if os.path.exists(frappe_commands_file): + os.remove(frappe_commands_file) + else: + print("Bench command cache doesn't exist in this folder!") \ No newline at end of file From 3b91d98d856b57aec3bb3c38eb8b09aa11d96ccc Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Tue, 31 Dec 2019 14:08:37 +0530 Subject: [PATCH 24/66] fix: compatibility fixes, better cpu utilization --- bench/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bench/utils.py b/bench/utils.py index 5ba3b7ba..4f046449 100755 --- a/bench/utils.py +++ b/bench/utils.py @@ -69,7 +69,7 @@ def init(path, apps_path=None, no_procfile=False, no_backups=False, try: os.makedirs(os.path.join(path, dirname)) except OSError as e: - if e.errno == os.errno.EEXIST: + if e.errno == errno.EEXIST: pass setup_logging() @@ -716,7 +716,7 @@ def update_translations_p(args): print('Download failed for', args[0], args[1]) def download_translations_p(): - pool = multiprocessing.Pool(4) + pool = multiprocessing.Pool(multiprocessing.cpu_count()) langs = get_langs() apps = ('frappe', 'erpnext') From f251feae8727d3608f0e570af2032d75fc3eebe5 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Thu, 2 Jan 2020 14:21:29 +0530 Subject: [PATCH 25/66] fix: install pip via pacman, minor fixes --- playbooks/install.py | 50 +++++++++++--------------------------------- 1 file changed, 12 insertions(+), 38 deletions(-) diff --git a/playbooks/install.py b/playbooks/install.py index 9330009f..e91a6e0a 100644 --- a/playbooks/install.py +++ b/playbooks/install.py @@ -1,7 +1,6 @@ #!/usr/bin/env python3 import os, sys, subprocess, getpass, json, multiprocessing, shutil, platform, warnings - tmp_bench_repo = os.path.join('/', 'tmp', '.bench') tmp_log_folder = os.path.join('/', 'tmp', 'logs') log_path = os.path.join(tmp_log_folder, 'install_bench.log') @@ -105,36 +104,6 @@ def run_os_command(command_map): return success -def install_pip(): - if shutil.which('pip'): - run_os_command({ - 'pip': 'sudo -H pip install --upgrade setuptools cryptography pip' - }) - - else: - if not os.path.exists("get-pip.py"): - run_os_command({ - 'wget': 'wget -q https://bootstrap.pypa.io/get-pip.py' - }) - - success = run_os_command({ - 'python3': 'sudo python3 get-pip.py --force-reinstall' - }) - - if success: - dist_name, dist_version = get_distribution_info() - if dist_name == 'centos': - run_os_command({ - 'pip': 'sudo -H pip install --upgrade --ignore-installed requests' - }) - else: - run_os_command({ - 'pip': 'sudo -H pip install --upgrade requests' - }) - - log("pip installed!", level=1) - - def install_prerequisites(): # pre-requisites for bench repo cloning run_os_command({ @@ -151,13 +120,13 @@ def install_prerequisites(): install_package('curl') install_package('wget') install_package('git') - install_pip() + install_package('pip3', 'python3-pip') success = run_os_command({ - 'pip': "sudo -H pip install --upgrade setuptools cryptography ansible==2.8.5 pip" + 'python3': "sudo -H python3 -m pip install --upgrade setuptools cryptography ansible==2.8.5 pip" }) - if not success: + if not (success or shutil.which('ansible')): could_not_install('Ansible') @@ -169,15 +138,16 @@ def is_sudo_user(): return os.geteuid() == 0 -def install_package(package): +def install_package(package, package_name=None): if shutil.which(package): log("{0} already installed!".format(package), level=1) else: log("Installing {0}...".format(package)) + package_name = package_name or package success = run_os_command({ - 'apt-get': ['sudo apt-get install -y {0}'.format(package)], - 'yum': ['sudo yum install -y {0}'.format(package)], - 'brew': ['brew install {0}'.format(package)] + 'apt-get': ['sudo apt-get install -y {0}'.format(package_name)], + 'yum': ['sudo yum install -y {0}'.format(package_name)], + 'brew': ['brew install {0}'.format(package_name)] }) if success: log("{0} installed!".format(package), level=1) @@ -412,6 +382,10 @@ def parse_commandline_args(): return args if __name__ == '__main__': + if sys.version[0] == '2': + if not raw_input("It is recommended to run this script with Python 3\nDo you still wish to continue? [Y/n]: ").lower() == "y": + sys.exit() + if not is_sudo_user(): log("Please run this script as a non-root user with sudo privileges", level=3) sys.exit() From a3dc9e69b77464dd1e855a52b4d7a3deafb262ef Mon Sep 17 00:00:00 2001 From: Development for People <47140294+developmentforpeople@users.noreply.github.com> Date: Sat, 9 Nov 2019 16:22:09 +0100 Subject: [PATCH 26/66] fix: bench.utils.get_sites updated to frappe.utils.get_sites avoids issues with non sites folder like 'jupyter_notebooks' --- bench/utils.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/bench/utils.py b/bench/utils.py index 51aae64a..fd87e702 100755 --- a/bench/utils.py +++ b/bench/utils.py @@ -208,10 +208,15 @@ def build_assets(bench_path='.', app=None): exec_cmd(command, cwd=bench_path) def get_sites(bench_path='.'): - sites_dir = os.path.join(bench_path, "sites") - sites = [site for site in os.listdir(sites_dir) - if os.path.isdir(os.path.join(sites_dir, site)) and site not in ('assets',)] - return sites + sites_path = os.path.join(bench_path, 'sites') + sites = [] + for site in os.listdir(sites_path): + path = os.path.join(sites_path, site) + if (os.path.isdir(path) + and not os.path.islink(path) + and os.path.exists(os.path.join(path, 'site_config.json'))): + sites.append(site) + return sorted(sites) def get_sites_dir(bench_path='.'): return os.path.abspath(os.path.join(bench_path, 'sites')) From 8764813121152cb80fde403a21ab5f541074c525 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Thu, 2 Jan 2020 18:30:31 +0530 Subject: [PATCH 27/66] fix: "faster" get_sites function chore: dropped unused get_sites_dir function style: pythonic "is not" usage --- bench/commands/utils.py | 2 +- bench/utils.py | 15 +++------------ 2 files changed, 4 insertions(+), 13 deletions(-) diff --git a/bench/commands/utils.py b/bench/commands/utils.py index 9ba55bb8..b93d2f59 100644 --- a/bench/commands/utils.py +++ b/bench/commands/utils.py @@ -139,7 +139,7 @@ def shell(bench_path='.'): def backup_site(site): "backup site" from bench.utils import get_sites, backup_site - if not site in get_sites(bench_path='.'): + if site not in get_sites(bench_path='.'): print('site not found') sys.exit(1) backup_site(site, bench_path='.') diff --git a/bench/utils.py b/bench/utils.py index fd87e702..c706ed89 100755 --- a/bench/utils.py +++ b/bench/utils.py @@ -209,17 +209,8 @@ def build_assets(bench_path='.', app=None): def get_sites(bench_path='.'): sites_path = os.path.join(bench_path, 'sites') - sites = [] - for site in os.listdir(sites_path): - path = os.path.join(sites_path, site) - if (os.path.isdir(path) - and not os.path.islink(path) - and os.path.exists(os.path.join(path, 'site_config.json'))): - sites.append(site) - return sorted(sites) - -def get_sites_dir(bench_path='.'): - return os.path.abspath(os.path.join(bench_path, 'sites')) + sites = (site for site in os.listdir(sites_path) if os.path.exists(os.path.join(sites_path, site, 'site_config.json'))) + return sites def get_bench_dir(bench_path='.'): return os.path.abspath(bench_path) @@ -425,7 +416,7 @@ def restart_systemd_processes(bench_path='.', web_workers=False): exec_cmd('sudo systemctl start -- $(systemctl show -p Requires {bench_name}.target | cut -d= -f2)'.format(bench_name=bench_name)) def set_default_site(site, bench_path='.'): - if not site in get_sites(bench_path=bench_path): + if site not in get_sites(bench_path=bench_path): raise Exception("Site not in bench") exec_cmd("{frappe} --use {site}".format(frappe=get_frappe(bench_path=bench_path), site=site), cwd=os.path.join(bench_path, 'sites')) From aebd75990a02e2991a963e4906ee068b59057b4b Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Mon, 6 Jan 2020 12:11:28 +0530 Subject: [PATCH 28/66] fix(easy_install): overwrite config files --- playbooks/roles/bench/tasks/setup_bench_production.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playbooks/roles/bench/tasks/setup_bench_production.yml b/playbooks/roles/bench/tasks/setup_bench_production.yml index 49cc44f3..b0725da1 100644 --- a/playbooks/roles/bench/tasks/setup_bench_production.yml +++ b/playbooks/roles/bench/tasks/setup_bench_production.yml @@ -2,7 +2,7 @@ - name: Setup production become: yes become_user: root - command: bench setup production {{ frappe_user }} + command: bench setup production {{ frappe_user }} --yes args: chdir: '{{ bench_path }}' From b80cecc917ce9ad8236701e378977c96d8172956 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Tue, 7 Jan 2020 15:34:34 +0530 Subject: [PATCH 29/66] fix(easy_install): log files saved with timestamp --- playbooks/install.py | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/playbooks/install.py b/playbooks/install.py index 9330009f..b6f4b4e5 100644 --- a/playbooks/install.py +++ b/playbooks/install.py @@ -1,10 +1,16 @@ #!/usr/bin/env python3 -import os, sys, subprocess, getpass, json, multiprocessing, shutil, platform, warnings +import os, sys, subprocess, getpass, json, multiprocessing, shutil, platform, warnings, datetime tmp_bench_repo = os.path.join('/', 'tmp', '.bench') tmp_log_folder = os.path.join('/', 'tmp', 'logs') -log_path = os.path.join(tmp_log_folder, 'install_bench.log') + +execution_day = "{:%Y-%m-%d}".format(datetime.datetime.utcnow()) +execution_time = "{:%H-%M}".format(datetime.datetime.utcnow()) +execution_ts = "{0}__{1}".format(execution_day, execution_time) + +log_file_name = "easy-install__{}.log".format(execution_ts) +log_path = os.path.join(tmp_log_folder, log_file_name) log_stream = sys.stdout @@ -29,6 +35,7 @@ def setup_log_stream(args): os.makedirs(tmp_log_folder) log_stream = open(log_path, 'w') log("Logs are saved under {0}".format(log_path), level=3) + print("Install script run at {}\n\n".format(execution_ts), file=log_stream) def check_environment(): @@ -73,6 +80,7 @@ def check_distribution_compatibility(): if float(dist_version) in supported_dists[dist_name]: log("{0} {1} is compatible!".format(dist_name, dist_version), level=1) else: + log("{0} {1} is detected".format(dist_name, dist_version), level=1) log("Install on {0} {1} instead".format(dist_name, supported_dists[dist_name][-1]), level=3) else: log("Sorry, the installer doesn't support {0}. Aborting installation!".format(dist_name), level=2) @@ -277,12 +285,16 @@ def clone_bench_repo(args): return success +def passwords_didnt_match(context=''): + log("{} passwords did not match!".format(context), level=3) + + def get_passwords(args): """ Returns a dict of passwords for further use and creates passwords.txt in the bench user's home directory """ - log("Input mySQL and Frappe Administrator passwords:") + log("Input MySQL and Frappe Administrator passwords:") ignore_prompt = args.run_travis or args.without_bench_setup mysql_root_password, admin_password = '', '' passwords_file_path = os.path.join(os.path.expanduser('~' + args.user), 'passwords.txt') @@ -309,6 +321,7 @@ def get_passwords(args): conf_mysql_passwd = getpass.unix_getpass(prompt='Re-enter mysql root password: ') if mysql_root_password != conf_mysql_passwd or mysql_root_password == '': + passwords_didnt_match("MySQL") mysql_root_password = '' continue @@ -318,6 +331,7 @@ def get_passwords(args): conf_admin_passswd = getpass.unix_getpass(prompt='Re-enter Administrator password: ') if admin_password != conf_admin_passswd or admin_password == '': + passwords_didnt_match("Administrator") admin_password = '' continue From 76d8b3d9f6285c24b8de5947f8b41438aa28b567 Mon Sep 17 00:00:00 2001 From: gavin Date: Tue, 7 Jan 2020 16:06:15 +0530 Subject: [PATCH 30/66] Revert "perf: faster bench" --- bench/cli.py | 36 ++++++++++-------------------- bench/commands/__init__.py | 5 +---- bench/commands/update.py | 4 +--- bench/commands/utils.py | 20 ----------------- bench/utils.py | 45 +++++++------------------------------- 5 files changed, 22 insertions(+), 88 deletions(-) diff --git a/bench/cli.py b/bench/cli.py index a51bf305..fd419d00 100755 --- a/bench/cli.py +++ b/bench/cli.py @@ -1,24 +1,13 @@ -import json -import logging -import os -import pwd -import subprocess -import sys - import click - +import os, sys, logging, json, pwd, subprocess +from bench.utils import is_root, PatchError, drop_privileges, get_env_cmd, get_cmd_output, get_frappe from bench.app import get_apps -from bench.commands import bench_command from bench.config.common_site_config import get_config -from bench.utils import (PatchError, clear_command_cache, drop_privileges, - frappe_commands_file, generate_command_cache, - get_cmd_output, get_env_cmd, get_frappe, is_root) - +from bench.commands import bench_command logger = logging.getLogger('bench') from_command_line = False - def cli(): global from_command_line from_command_line = True @@ -101,19 +90,18 @@ def frappe_cmd(bench_path='.'): os.execv(f, [f] + ['-m', 'frappe.utils.bench_helper', 'frappe'] + sys.argv[1:]) def get_frappe_commands(bench_path='.'): + python = get_env_cmd('python', bench_path=bench_path) sites_path = os.path.join(bench_path, 'sites') - if not os.path.exists(sites_path): return [] - - if os.path.exists(frappe_commands_file): - command_dump = open(frappe_commands_file, 'r').read() - output = json.loads(command_dump) if command_dump else [] - - else: - output = generate_command_cache() - - return output or [] + try: + output = get_cmd_output("{python} -m frappe.utils.bench_helper get-frappe-commands".format(python=python), cwd=sites_path) + # output = output.decode('utf-8') + return json.loads(output) + except subprocess.CalledProcessError as e: + if hasattr(e, "stderr"): + print(e.stderr.decode('utf-8')) + return [] def get_frappe_help(bench_path='.'): python = get_env_cmd('python', bench_path=bench_path) diff --git a/bench/commands/__init__.py b/bench/commands/__init__.py index 184d6cfd..8772a0a2 100755 --- a/bench/commands/__init__.py +++ b/bench/commands/__init__.py @@ -47,8 +47,7 @@ bench_command.add_command(switch_to_develop) from bench.commands.utils import (start, restart, set_nginx_port, set_ssl_certificate, set_ssl_certificate_key, set_url_root, set_mariadb_host, set_default_site, download_translations, shell, backup_site, backup_all_sites, release, renew_lets_encrypt, - disable_production, bench_src, prepare_beta_release, set_redis_cache_host, set_redis_queue_host, set_redis_socketio_host, - generate_command_cache, clear_command_cache) + disable_production, bench_src, prepare_beta_release, set_redis_cache_host, set_redis_queue_host, set_redis_socketio_host) bench_command.add_command(start) bench_command.add_command(restart) bench_command.add_command(set_nginx_port) @@ -69,8 +68,6 @@ bench_command.add_command(renew_lets_encrypt) bench_command.add_command(disable_production) bench_command.add_command(bench_src) bench_command.add_command(prepare_beta_release) -bench_command.add_command(generate_command_cache) -bench_command.add_command(clear_command_cache) from bench.commands.setup import setup bench_command.add_command(setup) diff --git a/bench/commands/update.py b/bench/commands/update.py index 68653dcf..82f857ba 100755 --- a/bench/commands/update.py +++ b/bench/commands/update.py @@ -5,7 +5,7 @@ from bench.config.common_site_config import get_config, update_config from bench.app import pull_all_apps, is_version_upgrade, validate_branch from bench.utils import (update_bench, validate_upgrade, pre_upgrade, post_upgrade, before_update, update_requirements, update_node_packages, backup_all_sites, patch_sites, build_assets, - restart_supervisor_processes, restart_systemd_processes, is_bench_directory, clear_command_cache) + restart_supervisor_processes, restart_systemd_processes, is_bench_directory) from bench import patches from six.moves import reload_module @@ -30,8 +30,6 @@ def update(pull=False, patch=False, build=False, bench=False, auto=False, restar update_bench(bench_repo=True, requirements=True) sys.exit() - clear_command_cache(bench_path='.') - if not (pull or patch or build or bench or requirements): pull, patch, build, bench, requirements = True, True, True, True, True diff --git a/bench/commands/utils.py b/bench/commands/utils.py index 13141c8e..b93d2f59 100644 --- a/bench/commands/utils.py +++ b/bench/commands/utils.py @@ -190,23 +190,3 @@ def bench_src(): """Prints bench source folder path, which can be used as: cd `bench src` """ import bench print(os.path.dirname(bench.__path__[0])) - - -@click.command('generate-command-cache') -def generate_command_cache(bench_path='.'): - """ - Caches Frappe commands (speeds up bench commands execution) - Default caching behaviour: generated the first time any command is run - """ - from bench.utils import generate_command_cache - return generate_command_cache(bench_path=bench_path) - - -@click.command('clear-command-cache') -def clear_command_cache(bench_path='.'): - """ - Clears commands cached - Default invalidation behaviour: destroyed on each run of `bench update` - """ - from bench.utils import clear_command_cache - return clear_command_cache(bench_path=bench_path) diff --git a/bench/utils.py b/bench/utils.py index ba9d2ea5..084e4e50 100755 --- a/bench/utils.py +++ b/bench/utils.py @@ -1,7 +1,7 @@ -import os, sys, shutil, subprocess, logging, itertools, requests, json, platform, select, pwd, grp, multiprocessing, hashlib, glob, errno -import semantic_version +import os, sys, shutil, subprocess, logging, itertools, requests, json, platform, select, pwd, grp, multiprocessing, hashlib, glob from distutils.spawn import find_executable import bench +import semantic_version from bench import env from six import iteritems, PY2 @@ -13,7 +13,7 @@ class CommandFailedError(Exception): pass logger = logging.getLogger(__name__) -frappe_commands_file = '.frappe-cmd' + folders_in_bench = ('apps', 'sites', 'config', 'logs', 'config/pids') @@ -69,7 +69,7 @@ def init(path, apps_path=None, no_procfile=False, no_backups=False, try: os.makedirs(os.path.join(path, dirname)) except OSError as e: - if e.errno == errno.EEXIST: + if e.errno == os.errno.EEXIST: pass setup_logging() @@ -188,7 +188,9 @@ def setup_env(bench_path='.', python = 'python3'): pip = os.path.join('env', 'bin', 'pip') exec_cmd('virtualenv -q {} -p {}'.format('env', python), cwd=bench_path) - exec_cmd('{} -q install --upgrade pip wheel six'.format(pip), cwd=bench_path) + exec_cmd('{} -q install --upgrade pip'.format(pip), cwd=bench_path) + exec_cmd('{} -q install wheel'.format(pip), cwd=bench_path) + exec_cmd('{} -q install six'.format(pip), cwd=bench_path) exec_cmd('{} -q install -e git+https://github.com/frappe/python-pdfkit.git#egg=pdfkit'.format(pip), cwd=bench_path) def setup_socketio(bench_path='.'): @@ -712,7 +714,7 @@ def update_translations_p(args): print('Download failed for', args[0], args[1]) def download_translations_p(): - pool = multiprocessing.Pool(multiprocessing.cpu_count()) + pool = multiprocessing.Pool(4) langs = get_langs() apps = ('frappe', 'erpnext') @@ -854,34 +856,3 @@ def run_playbook(playbook_name, extra_vars=None, tag=None): args.extend(['-t', tag]) subprocess.check_call(args, cwd=os.path.join(os.path.dirname(bench.__path__[0]), 'playbooks')) - -def generate_command_cache(bench_path='.'): - """ - Caches Frappe commands (speeds up bench commands execution) - Default caching behaviour: generated the first time any command is run - """ - - python = get_env_cmd('python', bench_path=bench_path) - sites_path = os.path.join(bench_path, 'sites') - - if os.path.exists(frappe_commands_file): - os.remove(frappe_commands_file) - - try: - output = get_cmd_output("{python} -m frappe.utils.bench_helper get-frappe-commands".format(python=python), cwd=sites_path) - json.dump(eval(output), open(frappe_commands_file, 'w')) - return output - except subprocess.CalledProcessError as e: - if hasattr(e, "stderr"): - print(e.stderr.decode('utf-8')) - -def clear_command_cache(bench_path='.'): - """ - Clears commands cached - Default invalidation behaviour: destroyed on each run of `bench update` - """ - - if os.path.exists(frappe_commands_file): - os.remove(frappe_commands_file) - else: - print("Bench command cache doesn't exist in this folder!") \ No newline at end of file From 9cc6e2e2a8859cff484097ae2b1122c866d3fc2b Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Thu, 9 Jan 2020 12:06:59 +0530 Subject: [PATCH 31/66] fix(easy_install): datetime formatting --- playbooks/install.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/playbooks/install.py b/playbooks/install.py index b6f4b4e5..f656c0b0 100644 --- a/playbooks/install.py +++ b/playbooks/install.py @@ -4,12 +4,10 @@ import os, sys, subprocess, getpass, json, multiprocessing, shutil, platform, wa tmp_bench_repo = os.path.join('/', 'tmp', '.bench') tmp_log_folder = os.path.join('/', 'tmp', 'logs') - -execution_day = "{:%Y-%m-%d}".format(datetime.datetime.utcnow()) -execution_time = "{:%H-%M}".format(datetime.datetime.utcnow()) -execution_ts = "{0}__{1}".format(execution_day, execution_time) - -log_file_name = "easy-install__{}.log".format(execution_ts) +execution_timestamp = datetime.datetime.utcnow() +execution_day = "{:%Y-%m-%d}".format(execution_timestamp) +execution_time = "{:%H:%M}".format(execution_timestamp) +log_file_name = "easy-install__{0}__{1}.log".format(execution_day, execution_time.replace(':', '-')) log_path = os.path.join(tmp_log_folder, log_file_name) log_stream = sys.stdout @@ -35,7 +33,7 @@ def setup_log_stream(args): os.makedirs(tmp_log_folder) log_stream = open(log_path, 'w') log("Logs are saved under {0}".format(log_path), level=3) - print("Install script run at {}\n\n".format(execution_ts), file=log_stream) + print("Install script run at {0} on {1}\n\n".format(execution_time, execution_day), file=log_stream) def check_environment(): From ef1a62455662ee670172dd3b555ad401951d14c0 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Fri, 10 Jan 2020 14:45:13 +0530 Subject: [PATCH 32/66] fix(cli): give warn that commands are not executed in a bench folder --- bench/cli.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/bench/cli.py b/bench/cli.py index fd419d00..21c8701b 100755 --- a/bench/cli.py +++ b/bench/cli.py @@ -1,6 +1,6 @@ import click import os, sys, logging, json, pwd, subprocess -from bench.utils import is_root, PatchError, drop_privileges, get_env_cmd, get_cmd_output, get_frappe +from bench.utils import is_root, PatchError, drop_privileges, get_env_cmd, get_cmd_output, get_frappe, log from bench.app import get_apps from bench.config.common_site_config import get_config from bench.commands import bench_command @@ -43,7 +43,7 @@ def cli(): def check_uid(): if cmd_requires_root() and not is_root(): - print('superuser privileges required for this command') + log('superuser privileges required for this command', level=3) sys.exit(1) def cmd_requires_root(): @@ -71,7 +71,7 @@ def change_uid(): drop_privileges(uid_name=frappe_user, gid_name=frappe_user) os.environ['HOME'] = pwd.getpwnam(frappe_user).pw_dir else: - print('You should not run this command as root') + log('You should not run this command as root', level=3) sys.exit(1) def old_frappe_cli(bench_path='.'): @@ -93,6 +93,7 @@ def get_frappe_commands(bench_path='.'): python = get_env_cmd('python', bench_path=bench_path) sites_path = os.path.join(bench_path, 'sites') if not os.path.exists(sites_path): + log("WARN: Command not being executed in bench directory", level=3) return [] try: output = get_cmd_output("{python} -m frappe.utils.bench_helper get-frappe-commands".format(python=python), cwd=sites_path) From e32281557f69af748573bfd02d4ccfe44e42bd3d Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Fri, 10 Jan 2020 14:48:09 +0530 Subject: [PATCH 33/66] feat: added bench.utils.log and updated is_bench_directory api --- bench/utils.py | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/bench/utils.py b/bench/utils.py index 82fe900f..d25668c2 100755 --- a/bench/utils.py +++ b/bench/utils.py @@ -17,17 +17,29 @@ logger = logging.getLogger(__name__) folders_in_bench = ('apps', 'sites', 'config', 'logs', 'config/pids') -def is_bench_directory(): - cur_dir = os.path.curdir +def is_bench_directory(directory=os.path.curdir): is_bench = True for folder in folders_in_bench: - path = os.path.join(cur_dir, folder) + path = os.path.abspath(os.path.join(directory, folder)) is_bench = is_bench and os.path.exists(path) return is_bench +def log(message, level=0): + levels = { + 0: '\033[94m', # normal + 1: '\033[92m', # success + 2: '\033[91m', # fail + 3: '\033[93m' # warn/suggest + } + start = levels.get(level) or '' + end = '\033[0m' + + print(start + message + end) + + def safe_decode(string, encoding = 'utf-8'): try: string = string.decode(encoding) From 963b2f9ce5ba5fb1426f00f01dd790ef60212a14 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Fri, 10 Jan 2020 14:49:16 +0530 Subject: [PATCH 34/66] feat: added bench find command --- bench/cli.py | 2 +- bench/commands/__init__.py | 3 ++- bench/commands/utils.py | 8 ++++++++ bench/utils.py | 32 ++++++++++++++++++++++++++++++-- 4 files changed, 41 insertions(+), 4 deletions(-) diff --git a/bench/cli.py b/bench/cli.py index 21c8701b..7eef2288 100755 --- a/bench/cli.py +++ b/bench/cli.py @@ -93,7 +93,7 @@ def get_frappe_commands(bench_path='.'): python = get_env_cmd('python', bench_path=bench_path) sites_path = os.path.join(bench_path, 'sites') if not os.path.exists(sites_path): - log("WARN: Command not being executed in bench directory", level=3) + log("Command not being executed in bench directory", level=3) return [] try: output = get_cmd_output("{python} -m frappe.utils.bench_helper get-frappe-commands".format(python=python), cwd=sites_path) diff --git a/bench/commands/__init__.py b/bench/commands/__init__.py index 8772a0a2..4572a42b 100755 --- a/bench/commands/__init__.py +++ b/bench/commands/__init__.py @@ -47,7 +47,7 @@ bench_command.add_command(switch_to_develop) from bench.commands.utils import (start, restart, set_nginx_port, set_ssl_certificate, set_ssl_certificate_key, set_url_root, set_mariadb_host, set_default_site, download_translations, shell, backup_site, backup_all_sites, release, renew_lets_encrypt, - disable_production, bench_src, prepare_beta_release, set_redis_cache_host, set_redis_queue_host, set_redis_socketio_host) + disable_production, bench_src, prepare_beta_release, set_redis_cache_host, set_redis_queue_host, set_redis_socketio_host, find_benches) bench_command.add_command(start) bench_command.add_command(restart) bench_command.add_command(set_nginx_port) @@ -68,6 +68,7 @@ bench_command.add_command(renew_lets_encrypt) bench_command.add_command(disable_production) bench_command.add_command(bench_src) bench_command.add_command(prepare_beta_release) +bench_command.add_command(find_benches) from bench.commands.setup import setup bench_command.add_command(setup) diff --git a/bench/commands/utils.py b/bench/commands/utils.py index 9ba55bb8..e71c7ea3 100644 --- a/bench/commands/utils.py +++ b/bench/commands/utils.py @@ -190,3 +190,11 @@ def bench_src(): """Prints bench source folder path, which can be used as: cd `bench src` """ import bench print(os.path.dirname(bench.__path__[0])) + + +@click.command('find') +@click.argument('location', default='') +def find_benches(location): + """Finds benches recursively from location""" + from bench.utils import find_benches + find_benches(directory=location) diff --git a/bench/utils.py b/bench/utils.py index d25668c2..f592beb4 100755 --- a/bench/utils.py +++ b/bench/utils.py @@ -31,8 +31,8 @@ def log(message, level=0): levels = { 0: '\033[94m', # normal 1: '\033[92m', # success - 2: '\033[91m', # fail - 3: '\033[93m' # warn/suggest + 2: '\033[91mERROR: ', # fail + 3: '\033[93mWARN: ' # warn/suggest } start = levels.get(level) or '' end = '\033[0m' @@ -872,3 +872,31 @@ def run_playbook(playbook_name, extra_vars=None, tag=None): args.extend(['-t', tag]) subprocess.check_call(args, cwd=os.path.join(os.path.dirname(bench.__path__[0]), 'playbooks')) + +def find_benches(directory=None): + if not directory: + directory = os.path.expanduser("~") + elif os.path.exists(directory): + directory = os.path.abspath(directory) + else: + log("Directory doesn't exist", level=2) + sys.exit(1) + + if is_bench_directory(directory): + if os.path.curdir == directory: + print("You are in a bench directory!") + else: + print("{0} is a bench directory!".format(directory)) + return + + benches = [] + for sub in os.listdir(directory): + sub = os.path.join(directory, sub) + if os.path.isdir(sub) and not os.path.islink(sub): + if is_bench_directory(sub): + print("{} found!".format(sub)) + benches.append(sub) + else: + benches.extend(find_benches(sub)) + + return benches From f0c189356534bd2ae6375722f390bc49dbccb40a Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Mon, 13 Jan 2020 02:36:27 +0530 Subject: [PATCH 35/66] feat: ability to rollback failed bench init --- bench/commands/make.py | 43 +++++++++++++++++++++++++++++++----------- 1 file changed, 32 insertions(+), 11 deletions(-) diff --git a/bench/commands/make.py b/bench/commands/make.py index 213d11ea..a3bbd2e4 100755 --- a/bench/commands/make.py +++ b/bench/commands/make.py @@ -15,21 +15,42 @@ import click @click.option('--skip-redis-config-generation', is_flag=True, help="Skip redis config generation if already specifying the common-site-config file") @click.option('--skip-assets',is_flag=True, default=False, help="Do not build assets") @click.option('--verbose',is_flag=True, help="Verbose output during install") -def init(path, apps_path, frappe_path, frappe_branch, no_procfile, no_backups, - no_auto_update, clone_from, verbose, skip_redis_config_generation, clone_without_update, - ignore_exist = False, skip_assets=False, - python = 'python3'): +def init(path, apps_path, frappe_path, frappe_branch, no_procfile, no_backups, no_auto_update, clone_from, verbose, skip_redis_config_generation, clone_without_update, ignore_exist=False, skip_assets=False, python='python3'): ''' Create a New Bench Instance. ''' - from bench.utils import init - init(path, apps_path=apps_path, no_procfile=no_procfile, no_backups=no_backups, - no_auto_update=no_auto_update, frappe_path=frappe_path, frappe_branch=frappe_branch, - verbose=verbose, clone_from=clone_from, skip_redis_config_generation=skip_redis_config_generation, + from bench.utils import init, log + + try: + init( + path, + apps_path=apps_path, + no_procfile=no_procfile, + no_backups=no_backups, + no_auto_update=no_auto_update, + frappe_path=frappe_path, + frappe_branch=frappe_branch, + verbose=verbose, + clone_from=clone_from, + skip_redis_config_generation=skip_redis_config_generation, clone_without_update=clone_without_update, - ignore_exist = ignore_exist, skip_assets=skip_assets, - python = python) - click.echo('Bench {} initialized'.format(path)) + ignore_exist=ignore_exist, + skip_assets=skip_assets, + python=python, + ) + log('Bench {} initialized'.format(path), level=1) + except SystemExit: + pass + except: + import os, shutil, time, six + # add a sleep here so that the traceback of other processes doesnt overlap with the prompts + time.sleep(1) + log("There was a problem while creating {}".format(path), level=2) + if six.moves.input("Do you want to rollback these changes? [Y/n]: ").lower() == "y": + print('Rolling back Bench "{}"'.format(path)) + if os.path.exists(path): + shutil.rmtree(path) + @click.command('get-app') @click.argument('name', nargs=-1) # Dummy argument for backward compatibility From 7fb1484b7f5dae561a098ab2a01c7f209664f626 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Mon, 13 Jan 2020 06:16:27 +0530 Subject: [PATCH 36/66] fix: seperate colors from log, change exception changes: thrown if bench dir exists on bench init style: fixed whitespaces and formatting --- bench/utils.py | 48 ++++++++++++++++++++++++++---------------------- 1 file changed, 26 insertions(+), 22 deletions(-) diff --git a/bench/utils.py b/bench/utils.py index 8204bfdf..4d2806f5 100755 --- a/bench/utils.py +++ b/bench/utils.py @@ -17,6 +17,14 @@ logger = logging.getLogger(__name__) folders_in_bench = ('apps', 'sites', 'config', 'logs', 'config/pids') +class color: + nc = '\033[0m' + blue = '\033[94m' + green = '\033[92m' + yellow = '\033[93m' + red = '\033[91m' + + def is_bench_directory(directory=os.path.curdir): is_bench = True @@ -29,12 +37,12 @@ def is_bench_directory(directory=os.path.curdir): def log(message, level=0): levels = { - 0: '\033[94m', # normal - 1: '\033[92m', # success - 2: '\033[91mERROR: ', # fail - 3: '\033[93mWARN: ' # warn/suggest + 0: color.blue + 'LOG', # normal + 1: color.green + 'SUCCESS', # success + 2: color.red + 'ERROR', # fail + 3: color.yellow + 'WARN' # warn/suggest } - start = levels.get(level) or '' + start = levels.get(level) + ': ' if level in levels else '' end = '\033[0m' print(start + message + end) @@ -57,23 +65,19 @@ def get_frappe(bench_path='.'): def get_env_cmd(cmd, bench_path='.'): return os.path.abspath(os.path.join(bench_path, 'env', 'bin', cmd)) -def init(path, apps_path=None, no_procfile=False, no_backups=False, - no_auto_update=False, frappe_path=None, frappe_branch=None, wheel_cache_dir=None, - verbose=False, clone_from=None, skip_redis_config_generation=False, - clone_without_update=False, - ignore_exist = False, skip_assets=False, - python = 'python3'): # Let's change when we're ready. - - from .app import get_app, install_apps_from_path - from .config.common_site_config import make_config - from .config import redis - from .config.procfile import setup_procfile +def init(path, apps_path=None, no_procfile=False, no_backups=False, no_auto_update=False, + frappe_path=None, frappe_branch=None, wheel_cache_dir=None, verbose=False, clone_from=None, + skip_redis_config_generation=False, clone_without_update=False, ignore_exist = False, skip_assets=False, python='python3'): + import os, errno + from bench.app import get_app, install_apps_from_path + from bench.config import redis + from bench.config.common_site_config import make_config + from bench.config.procfile import setup_procfile from bench.patches import set_all_patches_executed - import os.path as osp - - if osp.exists(path): - if not ignore_exist: - raise ValueError('Bench Instance {path} already exists.'.format(path = path)) + if os.path.exists(path) and not ignore_exist: + log('Path {path} already exists!'.format(path=path)) + sys.exit(0) else: os.makedirs(path) @@ -81,12 +85,12 @@ def init(path, apps_path=None, no_procfile=False, no_backups=False, try: os.makedirs(os.path.join(path, dirname)) except OSError as e: - if e.errno == os.errno.EEXIST: + if e.errno == errno.EEXIST: pass setup_logging() - setup_env(bench_path=path, python = python) + setup_env(bench_path=path, python=python) make_config(path) From 71af7320f29f52c43c284ef6ce92b6c7ec17e38d Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Wed, 15 Jan 2020 13:36:32 +0530 Subject: [PATCH 37/66] fix: ansible install on versioned pip --- bench/commands/setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bench/commands/setup.py b/bench/commands/setup.py index bb59a857..c027c0c4 100755 --- a/bench/commands/setup.py +++ b/bench/commands/setup.py @@ -1,4 +1,5 @@ from bench.utils import exec_cmd +from six import PY3 import click, sys, json import os @@ -57,7 +58,7 @@ def setup_production(user, yes=False): # Install prereqs for production from distutils.spawn import find_executable if not find_executable('ansible'): - exec_cmd("sudo pip install ansible") + exec_cmd("sudo {0} install ansible".format("pip3" if PY3 else "pip2")) if not find_executable('fail2ban-client'): exec_cmd("bench setup role fail2ban") if not find_executable('nginx'): From 1b0008d709ad15f379bc093ddec015169ba237c6 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Wed, 15 Jan 2020 14:24:03 +0530 Subject: [PATCH 38/66] fix: single process to pip install requirements, quiet flag added --- bench/app.py | 2 +- bench/utils.py | 4 +--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/bench/app.py b/bench/app.py index 5ff210b9..8b9ca2b0 100755 --- a/bench/app.py +++ b/bench/app.py @@ -118,7 +118,7 @@ def get_app(git_url, branch=None, bench_path='.', skip_assets=False, verbose=Fal shallow_clone = '--depth 1' if check_git_for_shallow_clone() else '' branch = '--branch {branch}'.format(branch=branch) if branch else '' - exec_cmd("git clone {git_url} {branch} {shallow_clone} --origin upstream".format( + exec_cmd("git clone -q {git_url} {branch} {shallow_clone} --origin upstream".format( git_url=git_url, shallow_clone=shallow_clone, branch=branch), diff --git a/bench/utils.py b/bench/utils.py index 4d2806f5..774363fd 100755 --- a/bench/utils.py +++ b/bench/utils.py @@ -204,9 +204,7 @@ def setup_env(bench_path='.', python = 'python3'): pip = os.path.join('env', 'bin', 'pip') exec_cmd('virtualenv -q {} -p {}'.format('env', python), cwd=bench_path) - exec_cmd('{} -q install --upgrade pip'.format(pip), cwd=bench_path) - exec_cmd('{} -q install wheel'.format(pip), cwd=bench_path) - exec_cmd('{} -q install six'.format(pip), cwd=bench_path) + exec_cmd('{} -q install -U pip wheel six'.format(pip), cwd=bench_path) exec_cmd('{} -q install -e git+https://github.com/frappe/python-pdfkit.git#egg=pdfkit'.format(pip), cwd=bench_path) def setup_socketio(bench_path='.'): From c64a7e90e694737519e8a568320badb00885a48a Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Wed, 15 Jan 2020 15:37:45 +0530 Subject: [PATCH 39/66] fix(utils): sort imports, add parenthesis on ambigupus expr --- bench/utils.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/bench/utils.py b/bench/utils.py index 774363fd..3e291c5c 100755 --- a/bench/utils.py +++ b/bench/utils.py @@ -1,4 +1,4 @@ -import os, sys, shutil, subprocess, logging, itertools, requests, json, platform, select, pwd, grp, multiprocessing, hashlib, glob +import os, sys, shutil, subprocess, logging, itertools, requests, json, platform, select, pwd, grp, multiprocessing, hashlib, glob, errno from distutils.spawn import find_executable import bench import semantic_version @@ -42,7 +42,7 @@ def log(message, level=0): 2: color.red + 'ERROR', # fail 3: color.yellow + 'WARN' # warn/suggest } - start = levels.get(level) + ': ' if level in levels else '' + start = (levels.get(level) + ': ') if level in levels else '' end = '\033[0m' print(start + message + end) @@ -68,7 +68,6 @@ def get_env_cmd(cmd, bench_path='.'): def init(path, apps_path=None, no_procfile=False, no_backups=False, no_auto_update=False, frappe_path=None, frappe_branch=None, wheel_cache_dir=None, verbose=False, clone_from=None, skip_redis_config_generation=False, clone_without_update=False, ignore_exist = False, skip_assets=False, python='python3'): - import os, errno from bench.app import get_app, install_apps_from_path from bench.config import redis from bench.config.common_site_config import make_config From 8b63e72443b4364fca1da0ca3adfd6689f972fdd Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Thu, 16 Jan 2020 14:22:29 +0530 Subject: [PATCH 40/66] chore: updated issue templates --- .github/ISSUE_TEMPLATE/bug-report.md | 47 +++++++++++++ .github/ISSUE_TEMPLATE/bug_report.md | 68 ------------------- .github/ISSUE_TEMPLATE/easy-install.md | 43 ++++++++++++ ...{feature_request.md => feature-request.md} | 2 +- .../questions-about-using-bench.md | 19 ++++++ 5 files changed, 110 insertions(+), 69 deletions(-) create mode 100644 .github/ISSUE_TEMPLATE/bug-report.md delete mode 100644 .github/ISSUE_TEMPLATE/bug_report.md create mode 100644 .github/ISSUE_TEMPLATE/easy-install.md rename .github/ISSUE_TEMPLATE/{feature_request.md => feature-request.md} (96%) create mode 100644 .github/ISSUE_TEMPLATE/questions-about-using-bench.md diff --git a/.github/ISSUE_TEMPLATE/bug-report.md b/.github/ISSUE_TEMPLATE/bug-report.md new file mode 100644 index 00000000..dbe2664f --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug-report.md @@ -0,0 +1,47 @@ +--- +name: Bug report +about: Report a bug encountered while using bench +labels: bug +--- + +Issue: Bug report + +**Do the checklist before filing an issue:** + - [ ] Can you replicate the issue? + - [ ] Is this something you can debug and fix? Send a pull request! Bug fixes and documentation fixes are welcome + +**Describe the bug** :chart_with_downwards_trend: +A clear and concise description of what the bug is. + +**To Reproduce** :page_with_curl: +Steps to reproduce the behavior: +1. Go to '...' +2. Click on '....' +3. Scroll down to '....' +4. See error + +**Expected behavior** :chart_with_upwards_trend: +A clear and concise description of what you expected to happen. + +**Screenshots** :crystal_ball: +If applicable, add screenshots to help explain your problem. + +**OS (please complete the following information):** :cyclone: +- [ ] Linux: `distro:version` +- [ ] macOS: `version` +- [ ] Windows `version` +- [ ] Others? `haros:distro:version` + +**Version Information** + +Can be found out by running `bench version` in your respective bench folder + +- Bench Branch: `master` _(Only master is supported)_ +- Frappe Version: +- ERPNext Version: + +**Additional context** :page_facing_up: +Add any other context about the problem here. + +**Possible Solution** :bookmark_tabs: +Any idea what might be causing the issue. Or if you have a proposed solution to the problem. diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md deleted file mode 100644 index aa3a104e..00000000 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ /dev/null @@ -1,68 +0,0 @@ ---- -name: Bug report -about: Create a report to help us improve - ---- - -Issue: Bug report - -Our project, as you've probably heard, is getting really popular and truth is we're getting a bit overwhelmed by the activity surrounding it. There are just too many issues for us to manage properly. - -**Do the checklist before filing an issue:** - - [ ] Have a usage question? Ask your question on [Discuss Forum](https://discuss.erpnext.com). We use [Discuss Forum](https://discuss.erpnext.com) for usage question and GitHub for bugs. - - [ ] Can you replicate the issue? - - [ ] Is this something you can debug and fix? Send a pull request! Bug fixes and documentation fixes are welcome - -**Describe the bug** :chart_with_downwards_trend: -A clear and concise description of what the bug is. - -**To Reproduce** :page_with_curl: -Steps to reproduce the behavior: -1. Go to '...' -2. Click on '....' -3. Scroll down to '....' -4. See error - -**Expected behavior** :chart_with_upwards_trend: -A clear and concise description of what you expected to happen. - -**Screenshots** :crystal_ball: -If applicable, add screenshots to help explain your problem. - -**Desktop (please complete the following information):** :cyclone: - - OS: - - [ ] Linux - - [ ] macOS - - [ ] Windows - - [ ] Others? Please mention: - - Browser: - - [ ] Safari - - [ ] Chrome - - [ ] Firefox - - [ ] Other? Please mention: - -**Smartphone (please complete the following information):** :iphone: :computer: - - Device: - - [ ] iPhone - - [ ] Android - - Browser: - - [ ] Safari - - [ ] Chrome - - [ ] Firefox - - [ ] Other? Please mention: - -**Version Information** -- Which branch are you on? - - [ ] `master` :star2: - - [ ] `develop` :fire: -- Frappe Version: -- ERPNext Version: - -**Additional context** :page_facing_up: -Add any other context about the problem here. - -**Possible Solution** :bookmark_tabs: -Any idea what might be causing the issue. Or if you have a proposed solution to the problem, - -**Please don't be intimidated by the long list of options you've fill. Try to fill out as much as you can. Remember, the more the information the easier it is for us to replicate and fix the issue** :grin: - diff --git a/.github/ISSUE_TEMPLATE/easy-install.md b/.github/ISSUE_TEMPLATE/easy-install.md new file mode 100644 index 00000000..6a067cf8 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/easy-install.md @@ -0,0 +1,43 @@ +--- +name: Easy Install +about: Report a issue encountered or a suggestion for improving experience while using easy install to setup a "Bench + Frappe + ERPNext" environment +labels: easy-install +--- + +Issue: Easy Install + +**Do the checklist before filing an issue:** + - [ ] Did you retain the logfile (path of logfile is shared while the script is run)? We definitely **need** the logfile to debug any easy install related issues. + - [ ] Is this something you can debug and fix? Send a pull request! Bug fixes and documentation fixes are welcome + +**Distro Information (Required)** + + +**Command Run (Required)** + + +**Log File (Required)** + + +**Screenshots** + + +**Additional context** + + +**Possible Solution** + diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature-request.md similarity index 96% rename from .github/ISSUE_TEMPLATE/feature_request.md rename to .github/ISSUE_TEMPLATE/feature-request.md index e09e5e95..d72fbd91 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.md +++ b/.github/ISSUE_TEMPLATE/feature-request.md @@ -1,7 +1,7 @@ --- name: Feature request about: Suggest an idea for this project - +label: feature-request --- Issue: Feature Request diff --git a/.github/ISSUE_TEMPLATE/questions-about-using-bench.md b/.github/ISSUE_TEMPLATE/questions-about-using-bench.md new file mode 100644 index 00000000..27942a06 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/questions-about-using-bench.md @@ -0,0 +1,19 @@ +--- +name: Question about using Bench/Frappe/Frappe Apps +about: This is not the appropriate channel +labels: invalid +--- + +Please post on our forums: + +for questions about using `bench`, probably the best place to start is the [bench repo](https://github.com/frappe/bench) or [bench intro](https://frappe.io/bench) or [bench docs](https://frappe.io/docs/bench) + +for questions about using the `Frappe Framework`: ~~https://discuss.frappe.io~~ => [stackoverflow](https://stackoverflow.com/questions/tagged/frappe) tagged under `frappe` + +for questions about using `ERPNext`: https://discuss.erpnext.com + +For documentation issues, use the [Frappe Framework Documentation](https://frappe.io/docs/user/en) or the [developer cheetsheet](https://github.com/frappe/frappe/wiki/Developer-Cheatsheet) + +For a slightly outdated yet informative developer guide: https://www.youtube.com/playlist?list=PL3lFfCEoMxvzHtsZHFJ4T3n5yMM3nGJ1W + +> **Posts that are not bug reports or feature requests will not be addressed on this issue tracker.** \ No newline at end of file From 869d27ff650fe78341afde97510f9a00d9719ee6 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Tue, 14 Jan 2020 08:34:17 +0530 Subject: [PATCH 41/66] chore: updated pull request template --- .github/PULL_REQUEST_TEMPLATE.md | 53 +++++++++++++++++++++++--------- 1 file changed, 39 insertions(+), 14 deletions(-) diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 8e2bd857..0dd1c8f8 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -1,4 +1,39 @@ -Pull-Request + + +What type of a PR is this? + +- [ ] Changes to Existing Features +- [ ] New Feature Submissions +- [ ] Bug Fix +- [ ] Breaking Change (include change in API behaviours, etc.) + +--- + +> Please provide enough information so that others can review your pull request: + + + +--- + +> Explain the **details** for making this change. What existing problem does the pull request solve? The following checklist could help - [ ] Have you followed the guidelines in our Contributing document? - [ ] Have you checked to ensure there aren't other open [Pull Requests](../pulls) for the same update/change? @@ -11,18 +46,8 @@ Pull-Request --- -What type of a PR is this? + -- [ ] Changes to Existing Features -- [ ] New Feature Submissions -- [ ] Bug Fix -- [ ] Breaking Change - ---- - -- Motivation and Context (What existing problem does the pull request solve): -- Related Issue: -- Screenshots (if applicable, remember, a picture tells a thousand words): - -**Please don't be intimidated by the long list of options you've fill. Try to fill out as much as you can. Remember, the more the information the easier it is for us to test and get your pull request merged** :grin: +> Screenshots/GIFs + \ No newline at end of file From f48bb5a1304fa20cb15c31dae9e90a11283b2009 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Thu, 16 Jan 2020 17:48:43 +0530 Subject: [PATCH 42/66] chore: updated and consolidated readme - added visual hierarchy and structure to text - added table of contents - breakdown additional information about each section in their respective files --- README.md | 279 ++++++------------ docs/commands_and_usage.md | 69 +++++ docs/contribution_guidelines.md | 4 +- docs/easy_install.md | 91 ++++++ docs/installation.md | 38 +++ ...ppe_erpext.md => releasing_frappe_apps.md} | 0 6 files changed, 286 insertions(+), 195 deletions(-) create mode 100644 docs/commands_and_usage.md create mode 100644 docs/easy_install.md create mode 100644 docs/installation.md rename docs/{releasing_frappe_erpext.md => releasing_frappe_apps.md} (100%) diff --git a/README.md b/README.md index a6faffe5..06837544 100755 --- a/README.md +++ b/README.md @@ -1,102 +1,79 @@ -
- -

Frappe Bench

+ +

bench

+bench is a command-line utility that helps you to install apps, manage multiple sites and update Frappe / ERPNext apps on */nix (macOS, Ubuntu, Debian, CentOS, etc) for development and production. -The bench is a command-line utility that helps you to install apps, manage multiple sites and update Frappe / ERPNext apps on */nix (Ubuntu, Debian, CentOS, etc) for development and production. Bench will also create nginx and supervisor config files, setup backups and much more. - -If you are using on a VPS make sure it has >= 1Gb of RAM or has swap setup properly. - -To do this install, you must have basic information on how Linux works and should be able to use the command-line. If you are looking easier ways to get started and evaluate ERPNext, [download the Virtual Machine](https://erpnext.com/download) or take [a free trial on erpnext.com](https://erpnext.com/pricing). - -If you have questions, please ask them on the [forum](https://discuss.erpnext.com/). - -## Installation - -### Installation Requirements - -You will need a computer/server. Options include: - -- A Normal Computer/VPS/Baremetal Server: This is strongly recommended. Frappe/ERPNext installs properly and works well on these -- A Raspberry Pi, SAN Appliance, Network Router, Gaming Console, etc.: Although you may be able to install Frappe/ERPNext on specialized hardware, it is unlikely to work well and will be difficult for us to support. Strongly consider using a normal computer/VPS/baremetal server instead. **We do not support specialized hardware**. -- A Toaster, Car, Firearm, Thermostat, etc.: Yes, many modern devices now have embedded computing capability. We live in interesting times. However, you should not install Frappe/ERPNext on these devices. Instead, install it on a normal computer/VPS/baremetal server. **We do not support installing on noncomputing devices**. - -To install the Frappe/ERPNext server software, you will need an operating system on your normal computer which is not Windows. Note that the command line interface does work on Windows, and you can use Frappe/ERPNext from any operating system with a web browser. However, the server software does not run on Windows. It does run on other operating systems, so choose one of these instead: - -- Linux: Ubuntu, Debian, CentOS are the preferred distros and are tested. [Arch Linux](https://github.com/frappe/bench/wiki/Install-ERPNext-on-ArchLinux) can also be used -- Mac OS X - -### Manual Install - -To manually install frappe/erpnext, you can follow this [this wiki](https://github.com/frappe/frappe/wiki/The-Hitchhiker%27s-Guide-to-Installing-Frappe-on-Linux) for Linux and [this wiki](https://github.com/frappe/frappe/wiki/The-Hitchhiker's-Guide-to-Installing-Frappe-on-Mac-OS-X) for MacOS. It gives an excellent explanation about the stack. You can also follow the steps mentioned below: - -#### 1. Install Pre-requisites -
-• Python 3.6+
-• Node.js 12
-• Redis 5					(caching and realtime updates)
-• MariaDB 10.3 / Postgres 9.5			(to run database driven apps)
-• yarn 1.12+					(js dependency manager)
-• pip 15+					(py dependency manager)
-• cron 						(scheduled jobs)
-• wkhtmltopdf (version 0.12.5 with patched qt) 	(for pdf generation)
-• Nginx 					(for production)
-
- -#### 2. Install Bench - -Install bench as a *non root* user, - - git clone https://github.com/frappe/bench ~/.bench - pip install --user -e ~/.bench - -Note: Please do not remove the bench directory the above commands will create - -#### Basic Usage - -* Create a new bench - - The init command will create a bench directory with frappe framework - installed. It will be setup for periodic backups and auto updates once - a day. - - bench init frappe-bench && cd frappe-bench - -* Add a site - - Frappe apps are run by frappe sites and you will have to create at least one - site. The new-site command allows you to do that. - - bench new-site site1.local - -* Add apps - - The get-app command gets remote frappe apps from a remote git repository and installs them. Example: [erpnext](https://github.com/frappe/erpnext) - - bench get-app erpnext https://github.com/frappe/erpnext - -* Install apps - - To install an app on your new site, use the bench `install-app` command. - - bench --site site1.local install-app erpnext - -* Start bench - - To start using the bench, use the `bench start` command - - bench start - - To login to Frappe / ERPNext, open your browser and go to `[your-external-ip]:8000`, probably `localhost:8000` - - The default username is "Administrator" and password is what you set when you created the new site. +_Note:_ If you are looking easier ways to get started and evaluate ERPNext, [download the Virtual Machine](https://erpnext.com/download) or take [a free trial on erpnext.com](https://erpnext.com/pricing). --- -## Easy Install +# Table of Contents + + - [bench CLI](#bench-cli) + - [Usage](#usage) + - [Installation](#installation) + - [Easy Install Script](#easy-install-script) + - [Release Bench](#release-bench) + - [Guides](#guides) + - [Resources](#resources) + +--- + +# bench CLI + +Bench is a command line tool that helps you install, setup, manage multiple sites and apps based on Frappe Framework. + +--- + +## Usage + +* Create a new bench + + bench init [bench-name] + +* Add a site under current bench + + bench new-site [site-name] + +* Add apps to bench + + bench get-app [app-name] [app-link] + +* Install apps on a particular site + + bench --site [site-name] install-app [app-name] + +* Start bench (only for development) + + bench start + +* Show bench help + + bench --help + +_Note:_ Apart from `bench init`, all other bench commands have to be run having the respective bench directory as the working directory. _(`bench update` may also be run, but it's behaviour is covered in depth in the docs)_ + +For more in depth information on commands and usage follow [here](https://github.com/frappe/bench/blob/master/docs/commands_and_usage.md). + +--- + +## Installation + +To do this install, you must have basic information on how Linux works and should be able to use the command-line. bench will also create nginx and supervisor config files, setup backups and much more. If you are using on a VPS make sure it has >= 1Gb of RAM or has swap setup properly. + + git clone https://github.com/frappe/bench ~/.bench + pip3 install --user -e ~/.bench + +As bench is a python application, its installation really depends on `python` + `pip` + `git`. The Frappe Framework, however has various other system dependencies like `nodejs`, `yarn`, `redis` and a database system like `mariadb` or `postgres`. Go through the [installation requirements](https://github.com/frappe/bench/blob/master/docs/installation.md) for an updated list. + +If you have questions, please ask them on the [forum](https://discuss.erpnext.com/c/bench). + +--- + +# Easy Install Script - This is an opinionated setup so it is best to setup on a blank server. - Works on Ubuntu 16.04+, CentOS 7+, Debian 8+ @@ -108,122 +85,38 @@ Note: Please do not remove the bench directory the above commands will create - The log file is saved under `/tmp/logs/install_bench.log` in case you run into any issues during the install. - If you find any problems, post them on the forum: [https://discuss.erpnext.com](https://discuss.erpnext.com) -Open your Terminal and enter: + wget https://raw.githubusercontent.com/frappe/bench/master/playbooks/install.py + python3 install.py --production -#### 0. Setup user & Download the install script - -If you are on a fresh server and logged in as root, at first create a dedicated user for frappe -& equip this user with sudo privileges - -``` - adduser [frappe-user] - usermod -aG sudo [frappe-user] -``` - -*(it is very common to use "frappe" as frappe-username, but this comes with the security flaw of ["frappe" ranking very high](https://www.reddit.com/r/dataisbeautiful/comments/b3sirt/i_deployed_over_a_dozen_cyber_honeypots_all_over/?st=JTJ0SC0Q&sh=76e05240) in as a username challenged in hacking attempts. So, for production sites it is highly recommended to use a custom username harder to guess)* - -Switch to `[frappe-user]` (using `su [frappe-user]`) and start the setup - - wget https://raw.githubusercontent.com/frappe/bench/master/playbooks/install.py - - -#### 1. Run the install script - - sudo python3 install.py - -*Note: `user` flag to create a user and install using that user (By default, the script will create a user with the username `frappe` if the --user flag is not used)* - -For production or development, append teh `--production` or `--develop` flag to the command respectively. - - sudo python3 install.py --production --user [frappe-user] - -or - - sudo python3 install.py --develop - sudo python3 install.py --develop --user [frappe-user] - - sudo python3 install.py --production --user [frappe-user] --container - -*Note: `container` flag to install inside a container (this will prevent the `/proc/sys/vm/swappiness: Read-only` file system error)* - - - python3 install.py --production --version 11 --user [frappe-user] - -use --version flag to install specific version - - python3 install.py --production --version 11 --python python2.7 --user [frappe-user] - -use --python flag to specify virtual environments python version, by default script setup python3 - - -#### What will this script do? - -- Install all the pre-requisites -- Install the command line `bench` (under ~/.bench) -- Create a new bench (a folder that will contain your entire frappe/erpnext setup at ~/frappe-bench) -- Create a new ERPNext site on the bench (site1.local) - - -#### How do I start ERPNext - -1. For development: Go to your bench folder (`frappe-bench` by default) and start the bench with `bench start` -2. For production: Your process will be setup and managed by `nginx` and `supervisor`. [Setup Production](https://frappe.io/docs/user/en/bench/guides/setup-production.html) +Follow [Easy Install Docs](https://github.com/frappe/bench/blob/master/docs/easy_install.md) for more information. --- +# Release Bench -## Bench Manager (GUI for Bench) +Releases can be made for [Frappe](https://github.com/frappe/frappe) apps using bench. More information about this process can be found [here](https://github.com/frappe/bench/blob/master/docs/releasing_frappe_apps.md). -Bench Manager is a graphical user interface to emulate the functionalities of Frappe Bench. Like the command line utility it helps you install apps, manage multiple sites, update apps and much more. +--- -### Installation +# Bench Manager (GUI for Bench) -``` -$ bench setup manager -``` +[Bench Manager](https://github.com/frappe/bench_manager) is a graphical user interface to emulate the functionalities of Frappe Bench. Like the command line utility it helps you install apps, manage multiple sites, update apps and much more. Install just by executing the following command in the respective bench directory. -What all it does: -1. Create new site bench-manager.local -2. Gets the `bench_manager` app from https://github.com/frappe/bench_manager if it doesn't exist already -3. Installs the bench_manager app on the site bench-manager.local + bench setup manager -## Docker Install - For Developers (beta) +--- + +# Docker Install 1. For developer setup, you can also use the official [Frappe Docker](https://github.com/frappe/frappe_docker/). -2. The app, mariadb and redis run on individual containers +2. The app, mariadb and redis run on individual containers. 3. This setup supports multi-tenancy and exposes the frappe-bench volume as a external storage. -4. For more details, [read the instructions on the Frappe Docker README](https://github.com/frappe/frappe_docker/) +4. For more details, [ead the instructions on the [Frappe Docker README](https://github.com/frappe/frappe_docker/) +--- -Help -==== +# Guides -For bench help, you can type - - bench --help - - -Updating -======== - -To manually update the bench, run `bench update` to update all the apps, run -patches, build JS and CSS files and restart supervisor (if configured to). - -You can also run the parts of the bench selectively. - -`bench update --pull` will only pull changes in the apps - -`bench update --patch` will only run database migrations in the apps - -`bench update --build` will only build JS and CSS files for the bench - -`bench update --bench` will only update the bench utility (this project) - -`bench update --requirements` will only update all dependencies (Python + Node) for the apps available in current bench - - -Guides -======= - [Configuring HTTPS](https://frappe.io/docs/user/en/bench/guides/configuring-https.html) - [Using Let's Encrypt to setup HTTPS](https://frappe.io/docs/user/en/bench/guides/lets-encrypt-ssl-setup.html) - [Diagnosing the Scheduler](https://frappe.io/docs/user/en/bench/guides/diagnosing-the-scheduler.html) @@ -233,9 +126,9 @@ Guides - [Setup Multitenancy](https://frappe.io/docs/user/en/bench/guides/setup-multitenancy.html) - [Stopping Production](https://github.com/frappe/bench/wiki/Stopping-Production-and-starting-Development) +--- -Resources -======= +# Resources - [Background Services](https://frappe.io/docs/user/en/bench/resources/background-services.html) - [Bench Commands Cheat Sheet](https://frappe.io/docs/user/en/bench/resources/bench-commands-cheatsheet.html) diff --git a/docs/commands_and_usage.md b/docs/commands_and_usage.md new file mode 100644 index 00000000..92ad33bc --- /dev/null +++ b/docs/commands_and_usage.md @@ -0,0 +1,69 @@ +## Usage + +* Updating + + Currently, `bench update` can be run from any directory however the context of the command changes. If run from a bench directory, the vanilla command itself updates all apps, runs migrations and backs up all sites. + + bench update + + +To manually update the bench, run `bench update` to update all the apps, run +patches, build JS and CSS files and restart supervisor (if configured to). + +You can also run the parts of the bench selectively. + +`bench update --pull` will only pull changes in the apps + +`bench update --patch` will only run database migrations in the apps + +`bench update --build` will only build JS and CSS files for the bench + +`bench update --bench` will only update the bench utility (this project) + +`bench update --requirements` will only update all dependencies (Python + Node) for the apps available in current bench + + +* Create a new bench + + The init command will create a bench directory with frappe framework installed. It will be setup for periodic backups and auto updates once a day. + + bench init frappe-bench && cd frappe-bench + +* Add a site + + Frappe apps are run by frappe sites and you will have to create at least one site. The new-site command allows you to do that. + + bench new-site site1.local + +* Add apps + + The get-app command gets remote frappe apps from a remote git repository and installs them. Example: [erpnext](https://github.com/frappe/erpnext) + + bench get-app erpnext https://github.com/frappe/erpnext + +* Install apps + + To install an app on your new site, use the bench `install-app` command. + + bench --site site1.local install-app erpnext + +* Start bench + + To start using the bench, use the `bench start` command + + bench start + + To login to Frappe / ERPNext, open your browser and go to `[your-external-ip]:8000`, probably `localhost:8000` + + The default username is "Administrator" and password is what you set when you created the new site. + +* Setup Manager + +## What it does + + bench setup manager + +1. Create new site bench-manager.local +2. Gets the `bench_manager` app from https://github.com/frappe/bench_manager if it doesn't exist already +3. Installs the bench_manager app on the site bench-manager.local + diff --git a/docs/contribution_guidelines.md b/docs/contribution_guidelines.md index 9d6ba368..874fa7db 100644 --- a/docs/contribution_guidelines.md +++ b/docs/contribution_guidelines.md @@ -2,9 +2,9 @@ ### Introduction (for first timers) -Thank you for your interesting in contributing to an open source project! Our world works on people taking initiative to contribute to the "commons" and contributing to open source means you are contributing to make things better for not only yourself, but everyone else too! So thank you for taking this initiative. +Thank you for your interest in contributing to our project! Our world works on people taking initiative to contribute to the "commons" and contributing to open source means you are contributing to make things better for not only yourself, but everyone else too! So kudos to you for taking this initiative. -Great projects also work because of great quality. Open source or not, the user really cares that things should work as they are advertised, and consistently. New features should follow the same pattern and so that users don't have to learn things again and again. +Great projects depend on good code quality and adhering to certain standards while making sure the goals of the project are met. New features should follow the same pattern and so that users don't have to learn things again and again. Developers who maintain open source also expect that you follow certain guidelines. These guidelines ensure that developers are able quickly give feedback on your contribution and how to make it better. Most probably you might have to go back and change a few things, but it will be in th interest of making this process better for everyone. So do be prepared for some back and forth. diff --git a/docs/easy_install.md b/docs/easy_install.md new file mode 100644 index 00000000..f83a8c4d --- /dev/null +++ b/docs/easy_install.md @@ -0,0 +1,91 @@ +# Easy Install Script + +- This script will install the pre-requisites, install bench and setup an ERPNext site `(site1.local under frappe-bench)` +- Passwords for Frappe Administrator and MariaDB (root) will be asked and saved under `~/passwoords.txt` +- MariaDB (root) password may be `password` on a fresh server +- You can then login as **Administrator** with the Administrator password +- The log file is saved under `/tmp/logs/install_bench.log` in case you run into any issues during the install. +- If you find any problems, post them on the forum: [https://discuss.erpnext.com](https://discuss.erpnext.com/tags/installation_problem) with the + +--- + +## What will this script do? + +- Install all the pre-requisites +- Install the command line `bench` (under ~/.bench) +- Create a new bench (a folder that will contain your entire frappe/erpnext setup at ~/frappe-bench) +- Create a new ERPNext site on the bench (site1.local) + +--- + +## Getting started with easy install... + +Open your Terminal and enter: + +#### 0. Setup user & Download the install script + +If you are on a fresh server and logged in as root, at first create a dedicated user for frappe +& equip this user with sudo privileges + +``` + adduser [frappe-user] + usermod -aG sudo [frappe-user] +``` + +*(it is very common to use "frappe" as frappe-username, but this comes with the security flaw of ["frappe" ranking very high](https://www.reddit.com/r/dataisbeautiful/comments/b3sirt/i_deployed_over_a_dozen_cyber_honeypots_all_over/?st=JTJ0SC0Q&sh=76e05240) in as a username challenged in hacking attempts. So, for production sites it is highly recommended to use a custom username harder to guess)* + +Switch to `[frappe-user]` (using `su [frappe-user]`) and start the setup + + wget https://raw.githubusercontent.com/frappe/bench/master/playbooks/install.py + + +#### 1. Run the install script + + sudo python3 install.py + +*Note: `user` flag to create a user and install using that user (By default, the script will create a user with the username `frappe` if the --user flag is not used)* + +For production or development, append teh `--production` or `--develop` flag to the command respectively. + + sudo python3 install.py --production --user [frappe-user] + +or + + sudo python3 install.py --develop + sudo python3 install.py --develop --user [frappe-user] + + sudo python3 install.py --production --user [frappe-user] --container + +*Note: `container` flag to install inside a container (this will prevent the `/proc/sys/vm/swappiness: Read-only` file system error)* + + + python3 install.py --production --version 11 --user [frappe-user] + +use --version flag to install specific version + + python3 install.py --production --version 11 --python python2.7 --user [frappe-user] + +use --python flag to specify virtual environments python version, by default script setup python3 + +--- + +## How do I start ERPNext + +1. For development: Go to your bench folder (`frappe-bench` by default) and start the bench with `bench start` +2. For production: Your process will be setup and managed by `nginx` and `supervisor`. Checkout [Setup Production](https://frappe.io/docs/user/en/bench/guides/setup-production.html) for more information. + +--- + +## An error occured mid installation? + +TLDR; Save the logs! + +1. The easy install script starts multiple processes to install prerequisites, system dependencies, requirements, sets up locales, configuration files, etc. + +2. The script pipes all these process outputs and saves it under `/tmp/log/{easy-install-filename}.log` as prompted by the script in the beginning of the script or/and if something went wrong again. + +3. Retain this log file and share it in case you need help with proceeding with the install. Since, the file's saved under `/tmp` it'll be cleared by the system after a reboot. Be careful to save it elsewhere if needed! + +3. A lot of things can go wrong in setting up the environment due to prior settings, company protocols or even breaking changes in system packages and their dependencies. + +4. Sharing your logfile in any issues opened related to this can help us find solutions to it faster and make the sript better! \ No newline at end of file diff --git a/docs/installation.md b/docs/installation.md new file mode 100644 index 00000000..a7e0f06a --- /dev/null +++ b/docs/installation.md @@ -0,0 +1,38 @@ +### Requirements + +You will need a computer/server. Options include: + +- A Normal Computer/VPS/Baremetal Server: This is strongly recommended. Frappe/ERPNext installs properly and works well on these +- A Raspberry Pi, SAN Appliance, Network Router, Gaming Console, etc.: Although you may be able to install Frappe/ERPNext on specialized hardware, it is unlikely to work well and will be difficult for us to support. Strongly consider using a normal computer/VPS/baremetal server instead. **We do not support specialized hardware**. +- A Toaster, Car, Firearm, Thermostat, etc.: Yes, many modern devices now have embedded computing capability. We live in interesting times. However, you should not install Frappe/ERPNext on these devices. Instead, install it on a normal computer/VPS/baremetal server. **We do not support installing on noncomputing devices**. + +To install the Frappe/ERPNext server software, you will need an operating system on your normal computer which is not Windows. Note that the command line interface does work on Windows, and you can use Frappe/ERPNext from any operating system with a web browser. However, the server software does not run on Windows. It does run on other operating systems, so choose one of these instead: + +- Linux: Ubuntu, Debian, CentOS are the preferred distros and are tested. [Arch Linux](https://github.com/frappe/bench/wiki/Install-ERPNext-on-ArchLinux) can also be used +- Mac OS X + +### Manual Install + +To manually install frappe/erpnext, you can follow this [this wiki](https://github.com/frappe/frappe/wiki/The-Hitchhiker%27s-Guide-to-Installing-Frappe-on-Linux) for Linux and [this wiki](https://github.com/frappe/frappe/wiki/The-Hitchhiker's-Guide-to-Installing-Frappe-on-Mac-OS-X) for MacOS. It gives an excellent explanation about the stack. You can also follow the steps mentioned below: + +#### 1. Install Prerequisites +
+• Python 3.6+
+• Node.js 12
+• Redis 5					(caching and realtime updates)
+• MariaDB 10.3 / Postgres 9.5			(to run database driven apps)
+• yarn 1.12+					(js dependency manager)
+• pip 15+					(py dependency manager)
+• cron 						(scheduled jobs)
+• wkhtmltopdf (version 0.12.5 with patched qt) 	(for pdf generation)
+• Nginx 					(for production)
+
+ +#### 2. Install Bench + +Install bench as a *non root* user, + + git clone https://github.com/frappe/bench ~/.bench + pip3 install --user -e ~/.bench + +Note: Please do not remove the bench directory the above commands will create diff --git a/docs/releasing_frappe_erpext.md b/docs/releasing_frappe_apps.md similarity index 100% rename from docs/releasing_frappe_erpext.md rename to docs/releasing_frappe_apps.md From 90c6dacb43b0441c7909b3dea6338ad5d441319f Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Thu, 16 Jan 2020 18:48:04 +0530 Subject: [PATCH 43/66] chore: updated category on discuss forum --- README.md | 4 ++-- docs/easy_install.md | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 06837544..fb8bef97 100755 --- a/README.md +++ b/README.md @@ -69,7 +69,7 @@ To do this install, you must have basic information on how Linux works and shoul As bench is a python application, its installation really depends on `python` + `pip` + `git`. The Frappe Framework, however has various other system dependencies like `nodejs`, `yarn`, `redis` and a database system like `mariadb` or `postgres`. Go through the [installation requirements](https://github.com/frappe/bench/blob/master/docs/installation.md) for an updated list. -If you have questions, please ask them on the [forum](https://discuss.erpnext.com/c/bench). +If you have questions, please ask them on the [forum](https://discuss.erpnext.com/c/bench) under the "Install / Update" category. --- @@ -83,7 +83,7 @@ If you have questions, please ask them on the [forum](https://discuss.erpnext.co - MariaDB (root) password may be `password` on a fresh server - You can then login as **Administrator** with the Administrator password - The log file is saved under `/tmp/logs/install_bench.log` in case you run into any issues during the install. -- If you find any problems, post them on the forum: [https://discuss.erpnext.com](https://discuss.erpnext.com) +- If you find any problems, post them on the forum: [https://discuss.erpnext.com](https://discuss.erpnext.com/c/bench) with the `installation_problem` under "Install / Update" category. wget https://raw.githubusercontent.com/frappe/bench/master/playbooks/install.py python3 install.py --production diff --git a/docs/easy_install.md b/docs/easy_install.md index f83a8c4d..958bd908 100644 --- a/docs/easy_install.md +++ b/docs/easy_install.md @@ -5,7 +5,7 @@ - MariaDB (root) password may be `password` on a fresh server - You can then login as **Administrator** with the Administrator password - The log file is saved under `/tmp/logs/install_bench.log` in case you run into any issues during the install. -- If you find any problems, post them on the forum: [https://discuss.erpnext.com](https://discuss.erpnext.com/tags/installation_problem) with the +- If you find any problems, post them on the forum: [https://discuss.erpnext.com](https://discuss.erpnext.com/tags/installation_problem) under the "Install / Update" category. --- From f75f9ff70614f21dc24336045cba2987ef3a27ac Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Fri, 17 Jan 2020 11:34:17 +0530 Subject: [PATCH 44/66] chore: add license to readme --- README.md | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index fb8bef97..d8cf468a 100755 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ bench is a command-line utility that helps you to install apps, manage multiple sites and update Frappe / ERPNext apps on */nix (macOS, Ubuntu, Debian, CentOS, etc) for development and production. -_Note:_ If you are looking easier ways to get started and evaluate ERPNext, [download the Virtual Machine](https://erpnext.com/download) or take [a free trial on erpnext.com](https://erpnext.com/pricing). +> **Note**: If you are looking for easier ways to get started and evaluate ERPNext, [download the Virtual Machine](https://erpnext.com/download) or take [a free trial on erpnext.com](https://erpnext.com/pricing). --- @@ -19,7 +19,7 @@ _Note:_ If you are looking easier ways to get started and evaluate ERPNext, [dow - [Release Bench](#release-bench) - [Guides](#guides) - [Resources](#resources) - + - [License](#license) --- # bench CLI @@ -133,3 +133,9 @@ Releases can be made for [Frappe](https://github.com/frappe/frappe) apps using b - [Background Services](https://frappe.io/docs/user/en/bench/resources/background-services.html) - [Bench Commands Cheat Sheet](https://frappe.io/docs/user/en/bench/resources/bench-commands-cheatsheet.html) - [Bench Procfile](https://frappe.io/docs/user/en/bench/resources/bench-procfile.html) + +--- + +# License + +bench is licensed under [GNU GPLv3](LICENSE) From a2ea3d90c55236c3dcac2264ed9f5e79c517b035 Mon Sep 17 00:00:00 2001 From: Raffael Meyer Date: Fri, 17 Jan 2020 20:01:38 +0100 Subject: [PATCH 45/66] fix: bump MariaDB version Update MariaDB version from `10.2` to `10.3` to match requirement in README. By the way, is there any reason for the empty string being the default password and version? ```py def install_maridb(mysql_root_password='', version=''): # ... ``` --- bench/commands/install.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bench/commands/install.py b/bench/commands/install.py index e17ad08a..e1fa75f1 100644 --- a/bench/commands/install.py +++ b/bench/commands/install.py @@ -14,7 +14,7 @@ def install_prerequisites(): @click.command('mariadb') @click.option('--mysql_root_password') -@click.option('--version', default="10.2") +@click.option('--version', default="10.3") def install_maridb(mysql_root_password='', version=''): if mysql_root_password: extra_vars.update({ From 5762724cc70639aead1ef4606b42e07fcd5ecc8d Mon Sep 17 00:00:00 2001 From: ckosiegbu Date: Fri, 17 Jan 2020 03:33:12 +0100 Subject: [PATCH 46/66] fix: validate branch in bench update bench update assumes that erpnext is an installed app in the bench. This is a wrong assumption. This work around checks for the intersection of installed apps with the list ['frappe', 'erpnext'] and only attempts to get the current branch of apps that occur in the intersection. This way, if erpnext is not installed (very likely for those using frappe only to build their own apps) there wont be an error when get_current_branch is called for erpnext. The need to perform the kind of check being performed in validate_branch is a strong case for considering apps being able to add "bench update" hooks in their hooks.py See this topic https://discuss.erpnext.com/t/bench-update-error-no-such-file-or-directory-apps-erpnext/56932?u=chude_osiegbu --- bench/app.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/bench/app.py b/bench/app.py index 5ff210b9..b1fddfce 100755 --- a/bench/app.py +++ b/bench/app.py @@ -423,11 +423,14 @@ def get_apps_json(path): return json.load(f) def validate_branch(): - for app in ['frappe', 'erpnext']: + installed_apps = get_apps() + check_apps = ['frappe', 'erpnext'] #this is a dirty work-around. bench update should not assume that erpnext is installed + intersection_apps = list(set(installed_apps) & set(check_apps)) + for app in intersection_apps: branch = get_current_branch(app) if branch == "master": print(''' master branch is renamed to version-11 and develop to version-12. Please switch to new branches to get future updates. To switch to version 11, run the following commands: bench switch-to-branch version-11''') - sys.exit(1) \ No newline at end of file + sys.exit(1) From b85651653b8511fe8f683fb8f101a1be0178b27d Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Fri, 17 Jan 2020 19:42:05 +0530 Subject: [PATCH 47/66] fix: validate branch error message + cleanup --- bench/app.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/bench/app.py b/bench/app.py index b1fddfce..422502ff 100755 --- a/bench/app.py +++ b/bench/app.py @@ -423,14 +423,22 @@ def get_apps_json(path): return json.load(f) def validate_branch(): - installed_apps = get_apps() - check_apps = ['frappe', 'erpnext'] #this is a dirty work-around. bench update should not assume that erpnext is installed - intersection_apps = list(set(installed_apps) & set(check_apps)) + installed_apps = set(get_apps()) + check_apps = set(['frappe', 'erpnext']) + intersection_apps = installed_apps and check_apps + for app in intersection_apps: branch = get_current_branch(app) if branch == "master": - print(''' master branch is renamed to version-11 and develop to version-12. Please switch to new branches to get future updates. + print("""'master' branch is renamed to 'version-11' since 'version-12' release. +As of January 2020, the following branches are +version Frappe ERPNext +11 version-11 version-11 +12 version-12 version-12 +13 develop develop + +Please switch to new branches to get future updates. +To switch to your required branch, run the following commands: bench switch-to-branch [branch-name]""") -To switch to version 11, run the following commands: bench switch-to-branch version-11''') sys.exit(1) From 03c4d9610a7fc8902094cb742596e606f00cddc2 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Tue, 21 Jan 2020 14:27:00 +0530 Subject: [PATCH 48/66] chore: fixed validate branch on update --- bench/app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bench/app.py b/bench/app.py index 32b006f2..ea5d2a22 100755 --- a/bench/app.py +++ b/bench/app.py @@ -425,7 +425,7 @@ def get_apps_json(path): def validate_branch(): installed_apps = set(get_apps()) check_apps = set(['frappe', 'erpnext']) - intersection_apps = installed_apps and check_apps + intersection_apps = installed_apps.intersection(check_apps) for app in intersection_apps: branch = get_current_branch(app) From 0daa5c45ed28bae490ba2dc1e68413ded18f3b95 Mon Sep 17 00:00:00 2001 From: Chinmay Pai Date: Tue, 21 Jan 2020 15:29:17 +0530 Subject: [PATCH 49/66] docs: add db_host and db_port options for new-site (#871) (#906) * Update installation docs to show new options * docs: fix indentation and make the command optional Co-authored-by: Oshosanya Michael --- README.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index d8cf468a..1791b50f 100755 --- a/README.md +++ b/README.md @@ -44,7 +44,11 @@ Bench is a command line tool that helps you install, setup, manage multiple site * Install apps on a particular site - bench --site [site-name] install-app [app-name] + bench new-site site1.local + + **Optional**: If the database for the site does not reside on localhost or listens on a custom port, you can use the flags `--db-host` to set a custom host and/or `--db-port` to set a custom port. + + bench new-site site1.local --db-host 192.168.1.1 --db-port 3307 * Start bench (only for development) From 7498c8e497fa3357b4fc8267cba774525f307946 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Tue, 21 Jan 2020 17:49:18 +0530 Subject: [PATCH 50/66] fix: pip pinned to 19.3.1 after v20 breaking reference: https://github.com/pypa/pip/issues/7217 --- .circleci/config.yml | 4 ++-- .travis.yml | 2 +- bench/utils.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index c067e4ef..6973b24a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -7,7 +7,7 @@ jobs: - checkout - run: name: Setup - command: | + command: | sudo pip install --ignore-installed setuptools sudo pip install urllib3 pyOpenSSL ndg-httpsclient pyasn1 sudo cp -r ~/.ssh/* /root/.ssh @@ -24,7 +24,7 @@ jobs: name: Setup Tests command: | cd ~ - sudo pip install --upgrade pip + sudo pip install --upgrade pip==19.3.1 sudo pip install -e ~/.bench - run: diff --git a/.travis.yml b/.travis.yml index 9df3b6d4..0a28ec6b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -20,7 +20,7 @@ install: script: - cd ~ - - sudo pip install --upgrade pip + - sudo pip install --upgrade pip==19.3.1 - sudo pip install -e ~/.bench # - sudo python -m unittest bench.tests.test_setup_production.TestSetupProduction.test_setup_production_v6 - sudo python -m unittest -v bench.tests.test_setup_production diff --git a/bench/utils.py b/bench/utils.py index 3e291c5c..a81c52e4 100755 --- a/bench/utils.py +++ b/bench/utils.py @@ -203,7 +203,7 @@ def setup_env(bench_path='.', python = 'python3'): pip = os.path.join('env', 'bin', 'pip') exec_cmd('virtualenv -q {} -p {}'.format('env', python), cwd=bench_path) - exec_cmd('{} -q install -U pip wheel six'.format(pip), cwd=bench_path) + exec_cmd('{} -q install -U pip==19.3.1 wheel six'.format(pip), cwd=bench_path) exec_cmd('{} -q install -e git+https://github.com/frappe/python-pdfkit.git#egg=pdfkit'.format(pip), cwd=bench_path) def setup_socketio(bench_path='.'): From 654f1e9c7b72d09d5ab400e2061578a2332b337a Mon Sep 17 00:00:00 2001 From: Chinmay Pai Date: Wed, 22 Jan 2020 00:31:37 +0530 Subject: [PATCH 51/66] chore: fix install-app in readme (#908) Signed-off-by: Chinmay D. Pai --- README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 1791b50f..394448de 100755 --- a/README.md +++ b/README.md @@ -38,18 +38,18 @@ Bench is a command line tool that helps you install, setup, manage multiple site bench new-site [site-name] + **Optional**: If the database for the site does not reside on localhost or listens on a custom port, you can use the flags `--db-host` to set a custom host and/or `--db-port` to set a custom port. + + bench new-site [site-name] --db-host [custom-db-host-ip] --db-port [custom-db-port] + * Add apps to bench bench get-app [app-name] [app-link] * Install apps on a particular site - bench new-site site1.local + bench --site [site-name] install-app [app-name] - **Optional**: If the database for the site does not reside on localhost or listens on a custom port, you can use the flags `--db-host` to set a custom host and/or `--db-port` to set a custom port. - - bench new-site site1.local --db-host 192.168.1.1 --db-port 3307 - * Start bench (only for development) bench start From 3b06683ebf41d4bcf9c48eb342ff7a125fa69a2e Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Wed, 22 Jan 2020 12:04:27 +0530 Subject: [PATCH 52/66] fix: checks for bench requirements to be installed without the --user flag via pip references: https://stackoverflow.com/questions/1871549/determine-if-python-is-running-inside-virtualenv --- bench/utils.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/bench/utils.py b/bench/utils.py index 3e291c5c..37b2bf40 100755 --- a/bench/utils.py +++ b/bench/utils.py @@ -537,10 +537,15 @@ def update_npm_packages(bench_path='.'): exec_cmd('npm install', cwd=bench_path) +def in_virtual_env(): + if sys.version_info.major == 2: + return hasattr(sys, 'real_prefix') + if sys.version_info.major == 3: + return sys.base_prefix != sys.prefix + def install_requirements(pip, req_file, user=False): if os.path.exists(req_file): - # sys.real_prefix exists only in a virtualenv - if hasattr(sys, 'real_prefix'): + if in_virtual_env(): user = False user_flag = "--user" if user else "" From b9bb9440d9cb335a0aa2ec7f807add634d7ff29f Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Wed, 22 Jan 2020 12:54:58 +0530 Subject: [PATCH 53/66] fix: update packages even if pip not in path --- bench/utils.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/bench/utils.py b/bench/utils.py index 37b2bf40..09b34f86 100755 --- a/bench/utils.py +++ b/bench/utils.py @@ -457,8 +457,7 @@ def set_default_site(site, bench_path='.'): def update_bench_requirements(): bench_req_file = os.path.join(os.path.dirname(bench.__path__[0]), 'requirements.txt') - user_pip = which("pip" if PY2 else "pip3") - install_requirements(user_pip, bench_req_file, user=True) + install_requirements(bench_req_file, user=True) def update_env_pip(bench_path): env_pip = os.path.join(bench_path, 'env', 'bin', 'pip') @@ -543,13 +542,19 @@ def in_virtual_env(): if sys.version_info.major == 3: return sys.base_prefix != sys.prefix -def install_requirements(pip, req_file, user=False): +def install_requirements(req_file, user=False): if os.path.exists(req_file): + if user: + python = sys.executable + else: + python = os.path.join("env", "bin", "python") + if in_virtual_env(): user = False user_flag = "--user" if user else "" - exec_cmd("{pip} install {user_flag} -q -U -r {req_file}".format(pip=pip, user_flag=user_flag, req_file=req_file)) + + exec_cmd("{python} -m pip install {user_flag} -q -U -r {req_file}".format(python=python, user_flag=user_flag, req_file=req_file)) def backup_site(site, bench_path='.'): bench.set_frappe_version(bench_path=bench_path) From 63128a21bf723a2b42336b0094fe341bd757aa28 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Wed, 22 Jan 2020 15:19:38 +0530 Subject: [PATCH 54/66] fix: check if virtual env (using pip refs) --- bench/utils.py | 72 +++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 65 insertions(+), 7 deletions(-) diff --git a/bench/utils.py b/bench/utils.py index 09b34f86..06a10571 100755 --- a/bench/utils.py +++ b/bench/utils.py @@ -1,4 +1,4 @@ -import os, sys, shutil, subprocess, logging, itertools, requests, json, platform, select, pwd, grp, multiprocessing, hashlib, glob, errno +import os, sys, shutil, subprocess, logging, itertools, requests, json, platform, select, pwd, grp, multiprocessing, hashlib, glob, re, site, errno from distutils.spawn import find_executable import bench import semantic_version @@ -536,12 +536,6 @@ def update_npm_packages(bench_path='.'): exec_cmd('npm install', cwd=bench_path) -def in_virtual_env(): - if sys.version_info.major == 2: - return hasattr(sys, 'real_prefix') - if sys.version_info.major == 3: - return sys.base_prefix != sys.prefix - def install_requirements(req_file, user=False): if os.path.exists(req_file): if user: @@ -907,3 +901,67 @@ def find_benches(directory=None): benches.extend(find_benches(sub)) return benches + +def in_virtual_env(): + # type: () -> bool + """Returns a boolean, whether running in venv with no system site-packages. + pip really does the best job at this: virtualenv_no_global at https://raw.githubusercontent.com/pypa/pip/master/src/pip/_internal/utils/virtualenv.py + """ + + def running_under_venv(): + # handles PEP 405 compliant virtual environments. + return sys.prefix != getattr(sys, "base_prefix", sys.prefix) + + def running_under_regular_virtualenv(): + # pypa/virtualenv case + return hasattr(sys, 'real_prefix') + + def _no_global_under_venv(): + # type: () -> bool + """Check `{sys.prefix}/pyvenv.cfg` for system site-packages inclusion + PEP 405 specifies that when system site-packages are not supposed to be + visible from a virtual environment, `pyvenv.cfg` must contain the following + line: + include-system-site-packages = false + Additionally, log a warning if accessing the file fails. + """ + def _get_pyvenv_cfg_lines(): + pyvenv_cfg_file = os.path.join(sys.prefix, 'pyvenv.cfg') + try: + with open(pyvenv_cfg_file) as f: + return f.read().splitlines() # avoids trailing newlines + except IOError: + return None + + _INCLUDE_SYSTEM_SITE_PACKAGES_REGEX = re.compile( + r"include-system-site-packages\s*=\s*(?Ptrue|false)" + ) + cfg_lines = _get_pyvenv_cfg_lines() + if cfg_lines is None: + # We're not in a "sane" venv, so assume there is no system + # site-packages access (since that's PEP 405's default state). + return True + + for line in cfg_lines: + match = _INCLUDE_SYSTEM_SITE_PACKAGES_REGEX.match(line) + if match is not None and match.group('value') == 'false': + return True + return False + + def _no_global_under_regular_virtualenv(): + # type: () -> bool + """Check if "no-global-site-packages.txt" exists beside site.py + This mirrors logic in pypa/virtualenv for determining whether system + site-packages are visible in the virtual environment. + """ + site_mod_dir = os.path.dirname(os.path.abspath(site.__file__)) + no_global_site_packages_file = os.path.join(site_mod_dir, 'no-global-site-packages.txt') + return os.path.exists(no_global_site_packages_file) + + if running_under_regular_virtualenv(): + return _no_global_under_regular_virtualenv() + + if running_under_venv(): + return _no_global_under_venv() + + return False From 3ea19e95336e46410e8919f3eb8b67fa8e7ad153 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Wed, 22 Jan 2020 15:26:58 +0530 Subject: [PATCH 55/66] chore: bench.utils format, imports sort, spaces --- bench/utils.py | 34 ++++++++++++++++++++++++++-------- 1 file changed, 26 insertions(+), 8 deletions(-) diff --git a/bench/utils.py b/bench/utils.py index 06a10571..67217a19 100755 --- a/bench/utils.py +++ b/bench/utils.py @@ -1,9 +1,12 @@ -import os, sys, shutil, subprocess, logging, itertools, requests, json, platform, select, pwd, grp, multiprocessing, hashlib, glob, re, site, errno +import errno, glob, grp, itertools, json, logging, multiprocessing, os, platform, pwd, re, select, shutil, site, subprocess, sys from distutils.spawn import find_executable -import bench + +import requests import semantic_version +from six import iteritems + +import bench from bench import env -from six import iteritems, PY2 class PatchError(Exception): @@ -718,11 +721,13 @@ def post_upgrade(from_ver, to_ver, bench_path='.'): if from_ver <= 5 and to_ver == 6: setup_socketio(bench_path=bench_path) - print("As you have setup your bench for production, you will have to reload configuration for nginx and supervisor") - print("To complete the migration, please run the following commands") - print() - print("sudo service nginx restart") - print("sudo supervisorctl reload") + message = """ +As you have setup your bench for production, you will have to reload configuration for nginx and supervisor. To complete the migration, please run the following commands +sudo service nginx restart +sudo supervisorctl reload + """.strip() + print(message) + def update_translations_p(args): try: @@ -730,6 +735,7 @@ def update_translations_p(args): except requests.exceptions.HTTPError: print('Download failed for', args[0], args[1]) + def download_translations_p(): pool = multiprocessing.Pool(4) @@ -739,18 +745,21 @@ def download_translations_p(): pool.map(update_translations_p, args) + def download_translations(): langs = get_langs() apps = ('frappe', 'erpnext') for app, lang in itertools.product(apps, langs): update_translations(app, lang) + def get_langs(): lang_file = 'apps/frappe/frappe/geo/languages.json' with open(lang_file) as f: langs = json.loads(f.read()) return [d['code'] for d in langs] + def update_translations(app, lang): translations_dir = os.path.join('apps', app, app, 'translations') csv_file = os.path.join(translations_dir, lang + '.csv') @@ -795,15 +804,18 @@ def log_line(data, stream): return sys.stderr.write(data) return sys.stdout.write(data) + def get_output(*cmd): s = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE) out = s.stdout.read() s.stdout.close() return out + def before_update(bench_path, requirements): validate_pillow_dependencies(bench_path, requirements) + def validate_pillow_dependencies(bench_path, requirements): if not requirements: return @@ -831,9 +843,11 @@ def validate_pillow_dependencies(bench_path, requirements): raise + def get_bench_name(bench_path): return os.path.basename(os.path.abspath(bench_path)) + def setup_fonts(): fonts_path = os.path.join('/tmp', 'fonts') @@ -848,6 +862,7 @@ def setup_fonts(): shutil.rmtree(fonts_path) exec_cmd("fc-cache -fv") + def set_git_remote_url(git_url, bench_path='.'): "Set app remote git url" app = git_url.rsplit('/', 1)[1].rsplit('.', 1)[0] @@ -860,6 +875,7 @@ def set_git_remote_url(git_url, bench_path='.'): if os.path.exists(os.path.join(app_dir, '.git')): exec_cmd("git remote set-url upstream {}".format(git_url), cwd=app_dir) + def run_playbook(playbook_name, extra_vars=None, tag=None): if not find_executable('ansible'): print("Ansible is needed to run this command, please install it using 'pip install ansible'") @@ -874,6 +890,7 @@ def run_playbook(playbook_name, extra_vars=None, tag=None): subprocess.check_call(args, cwd=os.path.join(os.path.dirname(bench.__path__[0]), 'playbooks')) + def find_benches(directory=None): if not directory: directory = os.path.expanduser("~") @@ -902,6 +919,7 @@ def find_benches(directory=None): return benches + def in_virtual_env(): # type: () -> bool """Returns a boolean, whether running in venv with no system site-packages. From 4474f463dcdf93321c7a188a405db0edb0d7e50f Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Wed, 22 Jan 2020 18:05:02 +0530 Subject: [PATCH 56/66] fix: chown frappe_user home directory --- playbooks/roles/bench/tasks/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playbooks/roles/bench/tasks/main.yml b/playbooks/roles/bench/tasks/main.yml index a227e77d..e01d18b7 100644 --- a/playbooks/roles/bench/tasks/main.yml +++ b/playbooks/roles/bench/tasks/main.yml @@ -33,7 +33,7 @@ - name: Fix permissions become_user: root - command: chown {{ frappe_user }} -R ~ + command: chown {{ frappe_user }} -R /home/{{ frappe_user }} - name: python3 bench init for develop command: bench init {{ bench_path }} --frappe-path {{ frappe_repo_url }} --frappe-branch {{ frappe_branch }} --python {{ python }} From 40f7e09c0f25e8dc05188d272d4c01c58de7229a Mon Sep 17 00:00:00 2001 From: Bernhard Sirlinger Date: Thu, 23 Jan 2020 08:03:14 +0100 Subject: [PATCH 57/66] =?UTF-8?q?fix:=20issue=20when=20ignore=5Fexist=20is?= =?UTF-8?q?=20set=20and=20the=20path=20already=20exists=20=F0=9F=90=9B=20(?= =?UTF-8?q?#910)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit See https://github.com/frappe/bench/commit/7fb1484b7f5dae561a098ab2a01c7f209664f626#diff-237d6b07e1835fbd8e51e251f66b2ec0 where the logic was changes to exclude handling this edge case --- bench/utils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bench/utils.py b/bench/utils.py index a81c52e4..3b5b2bb6 100755 --- a/bench/utils.py +++ b/bench/utils.py @@ -77,7 +77,8 @@ def init(path, apps_path=None, no_procfile=False, no_backups=False, no_auto_upda if os.path.exists(path) and not ignore_exist: log('Path {path} already exists!'.format(path=path)) sys.exit(0) - else: + elif not os.path.exists(path): + # only create dir if it does not exist os.makedirs(path) for dirname in folders_in_bench: From 2257f86feefa3eefef71932449ee28f2e087d688 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Sat, 1 Feb 2020 18:58:37 +0530 Subject: [PATCH 58/66] feat: execute bench/frappe commands from anywhere inside a bench directory --- bench/cli.py | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/bench/cli.py b/bench/cli.py index 7eef2288..0cab4c9b 100755 --- a/bench/cli.py +++ b/bench/cli.py @@ -1,10 +1,11 @@ import click import os, sys, logging, json, pwd, subprocess -from bench.utils import is_root, PatchError, drop_privileges, get_env_cmd, get_cmd_output, get_frappe, log +from bench.utils import is_root, PatchError, drop_privileges, get_env_cmd, get_cmd_output, get_frappe, log, is_bench_directory from bench.app import get_apps from bench.config.common_site_config import get_config from bench.commands import bench_command + logger = logging.getLogger('bench') from_command_line = False @@ -12,6 +13,7 @@ def cli(): global from_command_line from_command_line = True + change_working_directory() check_uid() change_dir() change_uid() @@ -114,3 +116,24 @@ def get_frappe_help(bench_path='.'): return "Framework commands:\n" + out.split('Commands:')[1] except subprocess.CalledProcessError: return "" + +def find_parent_bench(path): + """Checks if parent directories are benches""" + path = os.path.abspath(path) + is_bench = is_bench_directory(directory=path) + home_path = os.path.expanduser("~") + root_path = os.path.abspath(os.sep) + + if path not in {home_path, root_path}: + if is_bench: + return path + + dir_list = os.path.split(path) + parent_dir = dir_list[0] if type(dir_list) == tuple else dir_list + return find_parent_bench(parent_dir) + + +def change_working_directory(): + """Allows bench commands to be run from anywhere inside a bench directory""" + bench_path = find_parent_bench(".") or os.path.abspath(".") + os.chdir(bench_path) From 92cd942fcf32f78e2cd0e8a905700226a04ce332 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Sun, 2 Feb 2020 00:19:58 +0530 Subject: [PATCH 59/66] refactor: `change_working_directory` optimizations --- bench/cli.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/bench/cli.py b/bench/cli.py index 0cab4c9b..733a0dd6 100755 --- a/bench/cli.py +++ b/bench/cli.py @@ -119,21 +119,21 @@ def get_frappe_help(bench_path='.'): def find_parent_bench(path): """Checks if parent directories are benches""" - path = os.path.abspath(path) - is_bench = is_bench_directory(directory=path) + if is_bench_directory(directory=path): + return path + home_path = os.path.expanduser("~") root_path = os.path.abspath(os.sep) if path not in {home_path, root_path}: - if is_bench: - return path - - dir_list = os.path.split(path) - parent_dir = dir_list[0] if type(dir_list) == tuple else dir_list + # NOTE: the os.path.split assumes that given path is absolute + parent_dir = os.path.split(path)[0] return find_parent_bench(parent_dir) - def change_working_directory(): """Allows bench commands to be run from anywhere inside a bench directory""" - bench_path = find_parent_bench(".") or os.path.abspath(".") - os.chdir(bench_path) + cur_dir = os.path.abspath(".") + bench_path = find_parent_bench(cur_dir) + + if bench_path: + os.chdir(bench_path) From 0c29bba6599533a7191b0b97ef0929d25c045bfa Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Thu, 6 Feb 2020 01:32:50 +0530 Subject: [PATCH 60/66] refactor: bench migrate-env command * moved from __init__ to utils and updated sequence of installing applications in env * chanaged import namespaces --- bench/commands/__init__.py | 124 +++---------------------------------- bench/commands/utils.py | 9 +++ bench/utils.py | 67 ++++++++++++++++++++ 3 files changed, 83 insertions(+), 117 deletions(-) diff --git a/bench/commands/__init__.py b/bench/commands/__init__.py index 4572a42b..3c1c45ec 100755 --- a/bench/commands/__init__.py +++ b/bench/commands/__init__.py @@ -1,15 +1,5 @@ import click -import os, shutil -import os.path as osp -import logging - -from datetime import datetime - -from bench.utils import which, exec_cmd - -log = logging.getLogger(__name__) -log.setLevel(logging.DEBUG) def print_bench_version(ctx, param, value): """Prints current bench version""" @@ -31,11 +21,13 @@ def bench_command(bench_path='.'): setup_logging(bench_path=bench_path) -from bench.commands.make import init, get_app, new_app, remove_app +from bench.commands.make import init, get_app, new_app, remove_app, exclude_app_for_update, include_app_for_update bench_command.add_command(init) bench_command.add_command(get_app) bench_command.add_command(new_app) bench_command.add_command(remove_app) +bench_command.add_command(exclude_app_for_update) +bench_command.add_command(include_app_for_update) from bench.commands.update import update, retry_upgrade, switch_to_branch, switch_to_master, switch_to_develop @@ -45,9 +37,10 @@ bench_command.add_command(switch_to_branch) bench_command.add_command(switch_to_master) bench_command.add_command(switch_to_develop) + from bench.commands.utils import (start, restart, set_nginx_port, set_ssl_certificate, set_ssl_certificate_key, set_url_root, set_mariadb_host, set_default_site, download_translations, shell, backup_site, backup_all_sites, release, renew_lets_encrypt, - disable_production, bench_src, prepare_beta_release, set_redis_cache_host, set_redis_queue_host, set_redis_socketio_host, find_benches) + disable_production, bench_src, prepare_beta_release, set_redis_cache_host, set_redis_queue_host, set_redis_socketio_host, find_benches, migrate_env) bench_command.add_command(start) bench_command.add_command(restart) bench_command.add_command(set_nginx_port) @@ -69,6 +62,8 @@ bench_command.add_command(disable_production) bench_command.add_command(bench_src) bench_command.add_command(prepare_beta_release) bench_command.add_command(find_benches) +bench_command.add_command(migrate_env) + from bench.commands.setup import setup bench_command.add_command(setup) @@ -84,108 +79,3 @@ bench_command.add_command(remote_urls) from bench.commands.install import install bench_command.add_command(install) - -from bench.config.common_site_config import get_config -try: - from urlparse import urlparse -except ImportError: - from urllib.parse import urlparse - -@click.command('migrate-env') -@click.argument('python', type = str) -@click.option('--no-backup', is_flag=True) -def migrate_env(python, no_backup = False): - """ - Migrate Virtual Environment to desired Python Version. - """ - try: - # Clear Cache before Bench Dies. - config = get_config(bench_path = os.getcwd()) - rredis = urlparse(config['redis_cache']) - - redis = '{redis} -p {port}'.format( - redis = which('redis-cli'), - port = rredis.port - ) - - log.debug('Clearing Redis Cache...') - exec_cmd('{redis} FLUSHALL'.format(redis = redis)) - log.debug('Clearing Redis DataBase...') - exec_cmd('{redis} FLUSHDB'.format(redis = redis)) - except Exception: - log.warn('Please ensure Redis Connections are running or Daemonized.') - - try: - # This is with the assumption that a bench is set-up within path. - path = os.getcwd() - - # I know, bad name for a flag. Thanks, Ameya! :| - - if not no_backup: - # Back, the f*ck up. - parch = osp.join(path, 'archived_envs') - if not osp.exists(parch): - os.mkdir(parch) - - # Simply moving. Thanks, Ameya. - # I'm keen to zip. - source = osp.join(path, 'env') - target = parch - - log.debug('Backing up Virtual Environment') - stamp = datetime.now().strftime('%Y%m%d_%H%M%S') - dest = osp.join(path, str(stamp)) - - # WARNING: This is an archive, you might have to use virtualenv --relocate - # That's because virtualenv creates symlinks with shebangs pointing to executables. - # shebangs, shebangs - ricky martin. - - # ...and shutil.copytree is a f*cking mess. - os.rename(source, dest) - shutil.move(dest, target) - - log.debug('Setting up a New Virtual {python} Environment'.format( - python = python - )) - - # Path to Python Executable (Basically $PYTHONPTH) - python = which(python) - - - virtualenv = which('virtualenv') - - nvenv = 'env' - pvenv = osp.join(path, nvenv) - - exec_cmd('{virtualenv} --python {python} {pvenv}'.format( - virtualenv = virtualenv, - python = python, - pvenv = pvenv - ), cwd = path) - - pip = osp.join(pvenv, 'bin', 'pip') - exec_cmd('{pip} install --upgrade pip'.format(pip=pip)) - exec_cmd('{pip} install --upgrade setuptools'.format(pip=pip)) - # TODO: Options - - papps = osp.join(path, 'apps') - apps = ['frappe', 'erpnext'] + [app for app in os.listdir(papps) if app not in ['frappe', 'erpnext']] - - for app in apps: - papp = osp.join(papps, app) - if osp.isdir(papp) and osp.exists(osp.join(papp, 'setup.py')): - exec_cmd('{pip} install -e {app}'.format( - pip = pip, app = papp - )) - - log.debug('Migration Successful to {python}'.format( - python = python - )) - except: - log.debug('Migration Error') - raise - -bench_command.add_command(migrate_env) - -from bench.commands.make import exclude_app_for_update, include_app_for_update -bench_command.add_command(exclude_app_for_update) -bench_command.add_command(include_app_for_update) diff --git a/bench/commands/utils.py b/bench/commands/utils.py index 8cdd302d..7e7d5d24 100644 --- a/bench/commands/utils.py +++ b/bench/commands/utils.py @@ -198,3 +198,12 @@ def find_benches(location): """Finds benches recursively from location""" from bench.utils import find_benches find_benches(directory=location) + + +@click.command('migrate-env') +@click.argument('python', type=str) +@click.option('--no-backup', is_flag=True) +def migrate_env(python, no_backup=False): + """Migrate Virtual Environment to desired Python Version""" + from bench.utils import migrate_env + migrate_env(python=python, no_backup=no_backup) diff --git a/bench/utils.py b/bench/utils.py index 6a19347e..143b6b0b 100755 --- a/bench/utils.py +++ b/bench/utils.py @@ -1,9 +1,11 @@ import errno, glob, grp, itertools, json, logging, multiprocessing, os, platform, pwd, re, select, shutil, site, subprocess, sys +from datetime import datetime from distutils.spawn import find_executable import requests import semantic_version from six import iteritems +from six.moves.urllib.parse import urlparse import bench from bench import env @@ -984,3 +986,68 @@ def in_virtual_env(): return _no_global_under_venv() return False + +def migrate_env(python, no_backup=False): + from bench.config.common_site_config import get_config + from bench.app import get_apps + + log = logging.getLogger(__name__) + log.setLevel(logging.DEBUG) + + nvenv = 'env' + path = os.getcwd() + python = which(python) + virtualenv = which('virtualenv') + pvenv = os.path.join(path, nvenv) + pip = os.path.join(pvenv, 'bin', 'pip') + + # Clear Cache before Bench Dies. + try: + config = get_config(bench_path=os.getcwd()) + rredis = urlparse(config['redis_cache']) + + redis = '{redis} -p {port}'.format(redis=which('redis-cli'), port=rredis.port) + + log.debug('Clearing Redis Cache...') + exec_cmd('{redis} FLUSHALL'.format(redis = redis)) + log.debug('Clearing Redis DataBase...') + exec_cmd('{redis} FLUSHDB'.format(redis = redis)) + except: + log.warn('Please ensure Redis Connections are running or Daemonized.') + + # I know, bad name for a flag. Thanks, Ameya! :| - + if not no_backup: + # Back, the f*ck up. + parch = os.path.join(path, 'archived_envs') + if not os.path.exists(parch): + os.mkdir(parch) + + # Simply moving. Thanks, Ameya. + # I'm keen to zip. + source = os.path.join(path, 'env') + target = parch + + log.debug('Backing up Virtual Environment') + stamp = datetime.now().strftime('%Y%m%d_%H%M%S') + dest = os.path.join(path, str(stamp)) + + # WARNING: This is an archive, you might have to use virtualenv --relocate + # That's because virtualenv creates symlinks with shebangs pointing to executables. + # shebangs, shebangs - ricky martin. + + # ...and shutil.copytree is a f*cking mess. + os.rename(source, dest) + shutil.move(dest, target) + + try: + log.debug('Setting up a New Virtual {} Environment'.format(python)) + + exec_cmd('{virtualenv} --python {python} {pvenv}'.format(virtualenv=virtualenv, python=python, pvenv=pvenv)) + + apps = ' '.join(["-e {}".format(os.path.join("apps", app)) for app in get_apps()]) + exec_cmd('{0} install -q -U {1}'.format(pip, apps)) + + log.debug('Migration Successful to {}'.format(python)) + except: + log.debug('Migration Error') + raise \ No newline at end of file From f0596c5928746d45984ee50dbc527d591803e9b1 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Thu, 6 Feb 2020 15:03:29 +0530 Subject: [PATCH 61/66] feat: bench pip * Directly install/update/remove packages from your env using bench pip commands. Pretty much just like using pip To check help don't add the --help flag, use the -h flag instead, use the pip help command. For example, checking help for pip install can be done by bench pip help install bench pip install -h --- bench/commands/__init__.py | 3 ++- bench/commands/make.py | 11 +++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/bench/commands/__init__.py b/bench/commands/__init__.py index 4572a42b..65757f6a 100755 --- a/bench/commands/__init__.py +++ b/bench/commands/__init__.py @@ -31,11 +31,12 @@ def bench_command(bench_path='.'): setup_logging(bench_path=bench_path) -from bench.commands.make import init, get_app, new_app, remove_app +from bench.commands.make import init, get_app, new_app, remove_app, pip bench_command.add_command(init) bench_command.add_command(get_app) bench_command.add_command(new_app) bench_command.add_command(remove_app) +bench_command.add_command(pip) from bench.commands.update import update, retry_upgrade, switch_to_branch, switch_to_master, switch_to_develop diff --git a/bench/commands/make.py b/bench/commands/make.py index a3bbd2e4..b9b01b3e 100755 --- a/bench/commands/make.py +++ b/bench/commands/make.py @@ -93,3 +93,14 @@ def include_app_for_update(app_name): "Include app from updating" from bench.app import remove_from_excluded_apps_txt remove_from_excluded_apps_txt(app_name) + + +@click.command('pip', context_settings={"ignore_unknown_options": True}, help="For pip help use `bench pip help [COMMAND]` or `bench pip [COMMAND] -h`") +@click.argument('args', nargs=-1) +@click.pass_context +def pip(ctx, args): + "Run pip commands in bench env" + import os + from bench.utils import get_env_cmd + env_pip = get_env_cmd('pip') + os.execv(env_pip, (env_pip,) + args) From e3bfe3a82dc2b50a4ace33839568f6c101043b96 Mon Sep 17 00:00:00 2001 From: Wing Yung Chan Date: Fri, 7 Feb 2020 21:37:13 +0000 Subject: [PATCH 62/66] docs: fixes in easy_install.md (#917) Fixed two minor typos --- docs/easy_install.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/easy_install.md b/docs/easy_install.md index 958bd908..8caa7a30 100644 --- a/docs/easy_install.md +++ b/docs/easy_install.md @@ -1,7 +1,7 @@ # Easy Install Script - This script will install the pre-requisites, install bench and setup an ERPNext site `(site1.local under frappe-bench)` -- Passwords for Frappe Administrator and MariaDB (root) will be asked and saved under `~/passwoords.txt` +- Passwords for Frappe Administrator and MariaDB (root) will be asked and saved under `~/passwords.txt` - MariaDB (root) password may be `password` on a fresh server - You can then login as **Administrator** with the Administrator password - The log file is saved under `/tmp/logs/install_bench.log` in case you run into any issues during the install. @@ -45,7 +45,7 @@ Switch to `[frappe-user]` (using `su [frappe-user]`) and start the setup *Note: `user` flag to create a user and install using that user (By default, the script will create a user with the username `frappe` if the --user flag is not used)* -For production or development, append teh `--production` or `--develop` flag to the command respectively. +For production or development, append the `--production` or `--develop` flag to the command respectively. sudo python3 install.py --production --user [frappe-user] @@ -88,4 +88,4 @@ TLDR; Save the logs! 3. A lot of things can go wrong in setting up the environment due to prior settings, company protocols or even breaking changes in system packages and their dependencies. -4. Sharing your logfile in any issues opened related to this can help us find solutions to it faster and make the sript better! \ No newline at end of file +4. Sharing your logfile in any issues opened related to this can help us find solutions to it faster and make the sript better! From 1259b089c8c58230250971e478e20ddb9adeeb2c Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Sun, 9 Feb 2020 13:14:06 +0530 Subject: [PATCH 63/66] refactor: change "no_backup" variable and cleanup comments --- bench/commands/utils.py | 6 +++--- bench/utils.py | 16 ++++------------ 2 files changed, 7 insertions(+), 15 deletions(-) diff --git a/bench/commands/utils.py b/bench/commands/utils.py index 7e7d5d24..f2bac0af 100644 --- a/bench/commands/utils.py +++ b/bench/commands/utils.py @@ -202,8 +202,8 @@ def find_benches(location): @click.command('migrate-env') @click.argument('python', type=str) -@click.option('--no-backup', is_flag=True) -def migrate_env(python, no_backup=False): +@click.option('--no-backup', 'backup', is_flag=True, default=True) +def migrate_env(python, backup=True): """Migrate Virtual Environment to desired Python Version""" from bench.utils import migrate_env - migrate_env(python=python, no_backup=no_backup) + migrate_env(python=python, backup=backup) diff --git a/bench/utils.py b/bench/utils.py index 143b6b0b..fc7f4a1b 100755 --- a/bench/utils.py +++ b/bench/utils.py @@ -987,7 +987,7 @@ def in_virtual_env(): return False -def migrate_env(python, no_backup=False): +def migrate_env(python, backup=False): from bench.config.common_site_config import get_config from bench.app import get_apps @@ -1015,15 +1015,12 @@ def migrate_env(python, no_backup=False): except: log.warn('Please ensure Redis Connections are running or Daemonized.') - # I know, bad name for a flag. Thanks, Ameya! :| - - if not no_backup: - # Back, the f*ck up. + # Backup venv: restore using `virtualenv --relocatable` if needed + if backup: parch = os.path.join(path, 'archived_envs') if not os.path.exists(parch): os.mkdir(parch) - # Simply moving. Thanks, Ameya. - # I'm keen to zip. source = os.path.join(path, 'env') target = parch @@ -1031,17 +1028,12 @@ def migrate_env(python, no_backup=False): stamp = datetime.now().strftime('%Y%m%d_%H%M%S') dest = os.path.join(path, str(stamp)) - # WARNING: This is an archive, you might have to use virtualenv --relocate - # That's because virtualenv creates symlinks with shebangs pointing to executables. - # shebangs, shebangs - ricky martin. - - # ...and shutil.copytree is a f*cking mess. os.rename(source, dest) shutil.move(dest, target) + # Create virtualenv using specified python try: log.debug('Setting up a New Virtual {} Environment'.format(python)) - exec_cmd('{virtualenv} --python {python} {pvenv}'.format(virtualenv=virtualenv, python=python, pvenv=pvenv)) apps = ' '.join(["-e {}".format(os.path.join("apps", app)) for app in get_apps()]) From 66dd5af0871d0c81faee6ed43a595bb2d3ee9c23 Mon Sep 17 00:00:00 2001 From: gavin Date: Mon, 10 Feb 2020 14:56:55 +0530 Subject: [PATCH 64/66] Revert "fix: pip pinned to 19.3.1 after v20 breaking" --- .circleci/config.yml | 4 ++-- .travis.yml | 2 +- bench/utils.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 6973b24a..c067e4ef 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -7,7 +7,7 @@ jobs: - checkout - run: name: Setup - command: | + command: | sudo pip install --ignore-installed setuptools sudo pip install urllib3 pyOpenSSL ndg-httpsclient pyasn1 sudo cp -r ~/.ssh/* /root/.ssh @@ -24,7 +24,7 @@ jobs: name: Setup Tests command: | cd ~ - sudo pip install --upgrade pip==19.3.1 + sudo pip install --upgrade pip sudo pip install -e ~/.bench - run: diff --git a/.travis.yml b/.travis.yml index 0a28ec6b..9df3b6d4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -20,7 +20,7 @@ install: script: - cd ~ - - sudo pip install --upgrade pip==19.3.1 + - sudo pip install --upgrade pip - sudo pip install -e ~/.bench # - sudo python -m unittest bench.tests.test_setup_production.TestSetupProduction.test_setup_production_v6 - sudo python -m unittest -v bench.tests.test_setup_production diff --git a/bench/utils.py b/bench/utils.py index fc7f4a1b..10a4f801 100755 --- a/bench/utils.py +++ b/bench/utils.py @@ -209,7 +209,7 @@ def setup_env(bench_path='.', python = 'python3'): pip = os.path.join('env', 'bin', 'pip') exec_cmd('virtualenv -q {} -p {}'.format('env', python), cwd=bench_path) - exec_cmd('{} -q install -U pip==19.3.1 wheel six'.format(pip), cwd=bench_path) + exec_cmd('{} -q install -U pip wheel six'.format(pip), cwd=bench_path) exec_cmd('{} -q install -e git+https://github.com/frappe/python-pdfkit.git#egg=pdfkit'.format(pip), cwd=bench_path) def setup_socketio(bench_path='.'): From 7ccc2757929ab1aa4a7ccf8474e4d9541f351774 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Tue, 11 Feb 2020 14:02:47 +0530 Subject: [PATCH 65/66] fix(lets-encrypt): run command non interactively if flag passed --- bench/commands/setup.py | 2 +- bench/config/lets_encrypt.py | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/bench/commands/setup.py b/bench/commands/setup.py index c027c0c4..7f93f4cf 100755 --- a/bench/commands/setup.py +++ b/bench/commands/setup.py @@ -122,7 +122,7 @@ def set_ssh_port(port, force=False): @click.command('lets-encrypt') @click.argument('site') @click.option('--custom-domain') -@click.option('-n', '--non-interactive', default=False, is_flag=True, help="Run certbot non-interactively. Shouldn't be used on 1'st attempt") +@click.option('-n', '--non-interactive', default=False, is_flag=True, help="Run command non-interactively. This flag restarts nginx and runs certbot non interactively. Shouldn't be used on 1'st attempt") def setup_letsencrypt(site, custom_domain, non_interactive): "Setup lets-encrypt for site" from bench.config.lets_encrypt import setup_letsencrypt diff --git a/bench/config/lets_encrypt.py b/bench/config/lets_encrypt.py index 40f3450a..45af7b23 100755 --- a/bench/config/lets_encrypt.py +++ b/bench/config/lets_encrypt.py @@ -29,9 +29,10 @@ def setup_letsencrypt(site, custom_domain, bench_path, interactive): print("No custom domain named {0} set for site".format(custom_domain)) return - click.confirm('Running this will stop the nginx service temporarily causing your sites to go offline\n' - 'Do you want to continue?', - abort=True) + if interactive: + click.confirm('Running this will stop the nginx service temporarily causing your sites to go offline\n' + 'Do you want to continue?', + abort=True) if not get_config(bench_path).get("dns_multitenant"): print("You cannot setup SSL without DNS Multitenancy") From 92d2fbdc1383a524579b5337c0e5233eddf35668 Mon Sep 17 00:00:00 2001 From: gavin Date: Mon, 24 Feb 2020 12:09:24 +0530 Subject: [PATCH 66/66] fix: setup cron job for renewing ssl certificate (#918) * fix: setup cronjob for renewing ssl certificate cronjob set for 1st of every month * fix: cannonical PYPI package name updates refs: - https://pypi.org/project/python-crontab/ - https://github.com/frappe/bench/pull/918#issuecomment-589961301 --- bench/config/lets_encrypt.py | 7 +++---- requirements.txt | 2 +- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/bench/config/lets_encrypt.py b/bench/config/lets_encrypt.py index 45af7b23..4beac157 100755 --- a/bench/config/lets_encrypt.py +++ b/bench/config/lets_encrypt.py @@ -83,11 +83,10 @@ def run_certbot_and_setup_ssl(site, custom_domain, bench_path, interactive=True) def setup_crontab(): job_command = '/opt/certbot-auto renew -a nginx --post-hook "systemctl reload nginx"' - system_crontab = CronTab(tabfile='/etc/crontab', user=True) + system_crontab = CronTab(user='root') if job_command not in str(system_crontab): - job = system_crontab.new(command=job_command, comment="Renew lets-encrypt every month") - job.every().month() - job.enable() + job = system_crontab.new(command=job_command, comment="Renew lets-encrypt every month") + job.day.on(1) system_crontab.write() diff --git a/requirements.txt b/requirements.txt index 5c772902..a9b81fae 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ Click==7.0 GitPython==2.1.11 honcho==1.0.1 Jinja2==2.10.3 -python_crontab==2.4.0 +python-crontab==2.4.0 requests==2.22.0 semantic_version==2.8.2 setuptools==40.8.0