[Major] Migrating setup_frappe script from bash to ansible

playbooks/develop/centos.yml

@@ -9,12 +9,6 @@
 
   tasks:
 
-  # install pre-requisites
-  - name: add epel repo
-    yum: name="https://dl.fedoraproject.org/pub/epel/{{ ansible_lsb.major_release }}/x86_64/e/epel-release-*.rpm" state=present
-    become: yes
-    become_user: root
-
   - name: development tools package
     yum: name="@Development tools" state=present
     become: yes
@@ -23,13 +17,17 @@
   - name: install prequisites
     yum: pkg={{ item }} state=present
     with_items:
+      # Install epel-release
+      - epel-release
+
       # basic installs
       - redis
       - nodejs
       - npm
 
       # for mariadb
-      - software-properties-common
+      - mysql-devel
+      - mysql-libs
 
       # for wkhtmltopdf
       - libXrender
@@ -48,6 +46,8 @@
       - tcl-devel
       - tk-devel
 
+    #   To ensure that ansible_lsb is set
+      - redhat-lsb-core  
     become: yes
     become_user: root
 

playbooks/develop/includes/mariadb_centos.yml

@@ -6,6 +6,7 @@
 
   - name: Install MariaDB
     yum: pkg={{ item }} state=present
+    with_items:
       - MariaDB-server
       - MariaDB-client
     become: yes

playbooks/develop/includes/mariadb_ubuntu.yml

@@ -1,10 +1,17 @@
 ---
-  - name: Add apt key
+  - name: Add apt key for mariadb for Ubuntu < 16.04
     apt_key: keyserver=hkp://keyserver.ubuntu.com:80 id=0xcbcb082a1bb943db state=present
     become: yes
     become_user: root
+    when: ansible_distribution_version | version_compare('16.04', 'lt')
 
-  - name: Add apt repositry
+  - name: Add apt key for mariadb for Ubuntu >= 16.04
+    apt_key: keyserver=hkp://keyserver.ubuntu.com:80 id=0xF1656F24C74CD1D8 state=present
+    become: yes
+    become_user: root
+    when: ansible_distribution_version | version_compare('16.04', 'ge')
+
+  - name: Add apt repository
     apt_repository: repo='deb [arch=amd64,i386] http://nyc2.mirrors.digitalocean.com/mariadb/repo/10.1/ubuntu {{ ansible_distribution_release }} main' state=present
     become: yes
     become_user: root
@@ -18,4 +25,3 @@
       - libmariadbclient-dev
     become: yes
     become_user: root
-

playbooks/develop/includes/setup_bench.yml

@@ -4,10 +4,15 @@
     become: yes
     become_user: root
 
+  - name: Check whether bench exists
+    stat: path="{{ bench_path }}"
+    register: bench_stat
+
   - name: init bench
-    command: bench init {{ bench_path }}
+    command: bench init {{ bench_path }} --frappe-branch {{ branch }}
     args:
       creates: "{{ bench_path }}"
+    when: not bench_stat.stat.exists
 
   # setup common_site_config
   - name: setup config
@@ -43,4 +48,3 @@
     args:
       creates: "{{ bench_path }}/config/redis_socketio.conf"
       chdir: "{{ bench_path }}"
-

playbooks/develop/includes/setup_bench_production.yml

@@ -0,0 +1,60 @@
+---
+  - hosts: localhost
+    vars:
+      bench_path: "/home/{{ ansible_user_id }}/benches/frappe-bench"
+
+    tasks:
+
+    # In case we are re-running the script, we would like to skip the site creation
+    - name: Check whether a site exists
+      stat: path="{{ bench_path }}/sites/{{ site }}"
+      register: site_folder
+
+    - name: Create new site
+      command: bench new-site {{ site }} --admin-password {{ admin_password }} --mariadb-root-password {{ mysql_root_password }}
+      args:
+        chdir: "{{ bench_path }}"
+      when: not site_folder.stat.exists
+
+    - name: Check ERPNext App exists
+      stat: path="{{ bench_path }}/apps/erpnext"
+      register: app
+
+    # In case we are re-running the script, we would like to skip getting ERPNext App
+    - name: Get-app erpnext app
+      command: bench get-app erpnext https://github.com/frappe/erpnext.git --branch {{ branch }}
+      args:
+        chdir: '{{ bench_path }}'
+      when: not app.stat.exists
+
+    - name: Install erpnext app
+      command: bench --site {{ site }} install-app erpnext
+      args:
+        chdir: '{{ bench_path }}'
+
+    - name: Change permissions for frappe home folder
+      file:
+        dest: /home/{{ ansible_user_id }}
+        owner: '{{ ansible_user_id }}'
+        group: '{{ ansible_user_id }}'
+        mode: 0755
+        recurse: yes
+
+    - name: Setup production
+      become: yes
+      become_user: root
+      command: bench setup production {{ ansible_user_id }}
+      args:
+        chdir: '{{ bench_path }}'
+
+    - name: Setup Sudoers
+      become: yes
+      become_user: root
+      command: bench setup sudoers {{ ansible_user_id }}
+      args:
+        chdir: '{{ bench_path }}'
+
+    - name: Restart the bench
+      command: bench restart
+      args:
+        chdir: '{{ bench_path }}'

playbooks/develop/includes/setup_mariadb.yml

@@ -2,35 +2,40 @@
   - name: Install MySQLdb in global env
     pip: name=mysql-python version=1.2.5
     become: yes
-    become_method: sudo
+    become_user: root
 
-  - name: Set root Password
+  - name: Set root Password for Ubuntu
     mysql_user:
-      name=root
+      name: root
-      host={{ item }}
+      host: '{{ item }}'
-      password={{ mysql_root_password }}
+      password: '{{ mysql_root_password }}'
-      state=present
+      state: present
-      login_user=root
+      login_user: root
     with_items:
       - localhost
-    when: mysql_root_password is defined
     become: yes
-    become_method: sudo
+    become_user: root
-
     # when you have already defined mysql root password
     ignore_errors: yes
+    when: mysql_root_password is defined and ansible_distribution == 'Ubuntu'
 
   - name: Add configuration
     template: src={{ mysql_config_template }} dest={{ mysql_conf_dir }}/frappe.cnf owner=root mode=0644
     become: yes
-    become_method: sudo
+    become_user: root
 
   - name: restart mysql linux
     service: name=mysql state=restarted
     become: yes
-    become_method: sudo
+    become_user: root
     when: ansible_os_family == 'RedHat' or ansible_os_family == 'Debian'
 
+  - name: Set root password on CentOS
+    command: mysqladmin -u root password '{{ mysql_root_password }}'
+    become: yes
+    become_user: root
+    when: mysql_root_password is defined and ansible_distribution == 'CentOS'
+
   - name: add launchagents folder mac
     file: path=~/Library/LaunchAgents state=directory
     when: ansible_distribution == 'MacOSX'

playbooks/develop/includes/setup_prod_env.yml

@@ -0,0 +1,105 @@
+---
+- hosts: localhost
+
+  tasks:
+
+  #####################################
+  # Ubuntu Production Environment Setup
+  - name: Install production pre-requisites
+    become: yes
+    become_user: root
+    apt: pkg={{ item }} state=present
+    with_items:
+      - nginx
+      - screen
+      - vim
+      - htop
+      - git
+      - postfix
+      - supervisor
+    when: ansible_distribution == 'Ubuntu'
+
+  #####################################
+  # CentOS Production Environment Setup
+  - name: Install production pre-requisites
+    become: yes
+    become_user: root
+    yum: pkg={{ item }} state=present
+    with_items:
+      - nginx
+      - screen
+      - vim
+      - htop
+      - git
+      - postfix
+      - MySQL-python
+    when: ansible_distribution == 'CentOS'
+
+  - name: Install supervisor using yum for Centos 7
+    yum: pkg=supervisor state=present
+    become: yes
+    become_user: root
+    when: ansible_distribution == 'CentOS' and ansible_lsb.major_release == '7'
+
+  ####################################################
+  # Replace default nginx config with nginx template
+  - name: Rename default nginx.conf to nginx.conf.old
+    command: mv /etc/nginx/nginx.conf /etc/nginx/nginx.conf.old
+    become: yes
+    become_user: root
+
+  - name: Copy the nginx_config template
+    template:
+      src: ../templates/default_nginx.j2
+      dest: /etc/nginx/nginx.conf
+    become: yes
+    become_user: root
+
+  - name: Reload the nginx service
+    service:
+      name: nginx
+      state: reloaded
+    become: yes
+    become_user: root
+
+  ####################################################
+  # Enable nginx, mysql, redis and supevisord services
+  - name: Enable nginx, mysql, and redis
+    service:
+      name: '{{ item }}'
+      enabled: yes
+    with_items:
+      - nginx
+      - mysql
+    become: yes
+    become_user: root
+
+  - name: Enable redis.service on centos
+    service:
+      name: redis
+      enabled: yes
+    become: yes
+    become_user: root
+    when: ansible_distribution == 'CentOS'
+
+  - name: Enable redis-server.service on ubuntu
+    service:
+      name: redis-server
+      enabled: yes
+    become: yes
+    become_user: root
+    when: ansible_distribution == 'Ubuntu'
+
+  - name: Stat supervisor.conf
+    stat:
+      path: /etc/supervisord.conf
+    register: supervisor_conf
+
+  - name: Check whether default supervisor.conf exists
+    service:
+      name: supervisord
+      state: started
+      enabled: yes
+    become: yes
+    become_user: root
+    when: supervisor_conf.stat.exists

playbooks/develop/includes/setup_selinux_policy.yml

@@ -0,0 +1,38 @@
+---
+- hosts: localhost
+  tasks:
+    - name: Install SELinux for CentOS
+      yum: name="{{item}}" state=present
+      with_items:
+        - policycoreutils-python
+        - selinux-policy-devel
+      become: yes
+      become_user: root
+      when: ansible_distribution == 'CentOS'
+
+    - name: Install SELinux for Ubuntu
+      apt: name={{ item }} state=present
+      with_items:
+       - selinux
+       - selinux-policy-dev
+      become: yes
+      become_user: root
+      when: ansible_distribution == 'Ubuntu'
+
+    - name: Check enabled SELinux modules
+     shell: semanage module -l
+     register: enabled_modules
+
+    - name: Copy frappe_selinux policy
+      copy: src=templates/frappe_selinux.te dest=/root/frappe_selinux.te
+      register: dest_frappe_selinux_te
+      become: yes
+      become_user: root
+
+    - name: Compile frappe_selinux policy
+      shell: "make -f /usr/share/selinux/devel/Makefile frappe_selinux.pp && semodule -i frappe_selinux.pp"
+      args:
+      chdir: /root/
+      become: yes
+      become_user: root
+      when: "enabled_modules.stdout.find('frappe_selinux') == -1 or dest_frappe_selinux_te.changed"

playbooks/develop/install.yml

@@ -1,9 +1,5 @@
 ---
 - hosts: localhost
-  vars_prompt:
-    - name: mysql_root_password
-      prompt: "MySQL Root Password"
-      when: ansible_distribution == 'Ubuntu'
 
 - include: macosx.yml
   when: ansible_distribution == 'MacOSX'

playbooks/develop/setup_production.yml

@@ -0,0 +1,14 @@
+---
+  - hosts: localhost
+
+  # Install the common pre-requisites for the setting up bench
+  - include: install.yml
+
+  # Install the production environment
+  - include: includes/setup_prod_env.yml
+
+  # Setup Bench for production environment
+  - include: includes/setup_bench_production.yml
+
+  # Setup SELinux Policy, Optional can be done later
+  # - include: includes/setup_selinux_policy.yml

playbooks/develop/templates/default_nginx.j2

@@ -0,0 +1,42 @@
+# For more information on configuration, see:
+#   * Official English Documentation: http://nginx.org/en/docs/
+#   * Official Russian Documentation: http://nginx.org/ru/docs/
+
+{% if ansible_distribution == 'Ubuntu' %}
+	{% set nginx_user = 'www-data'%}
+{% elif ansible_distribution == 'CentOS' %}
+	{% set nginx_user = 'nginx '%}
+{% else %}
+	{% set nginx_user = 'nobody' %}
+{% endif %}
+
+user {{ nginx_user }};
+worker_processes auto;
+error_log /var/log/nginx/error.log;
+pid /run/nginx.pid;
+
+events {
+    worker_connections {{ max_worker_connections }};
+}
+
+http {
+    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
+                      '$status $body_bytes_sent "$http_referer" '
+                      '"$http_user_agent" "$http_x_forwarded_for"';
+
+    access_log  /var/log/nginx/access.log  main;
+
+    sendfile            on;
+    tcp_nopush          on;
+    tcp_nodelay         on;
+    keepalive_timeout   65;
+    types_hash_max_size 2048;
+
+    include             /etc/nginx/mime.types;
+    default_type        application/octet-stream;
+
+    # Load modular configuration files from the /etc/nginx/conf.d directory.
+    # See http://nginx.org/en/docs/ngx_core_module.html#include
+    # for more information.
+    include /etc/nginx/conf.d/*.conf;
+}

playbooks/develop/templates/frappe_selinux.te

@@ -0,0 +1,32 @@
+module frappe_selinux 1.0;
+
+require {
+	type user_home_dir_t;
+	type httpd_t;
+	type user_home_t;
+	type soundd_port_t;
+	class tcp_socket name_connect;
+	class lnk_file read;
+	class dir { getattr search };
+	class file { read open };
+}
+
+#============= httpd_t ==============
+
+#!!!! This avc is allowed in the current policy
+allow httpd_t soundd_port_t:tcp_socket name_connect;
+
+#!!!! This avc is allowed in the current policy
+allow httpd_t user_home_dir_t:dir search;
+
+#!!!! This avc is allowed in the current policy
+allow httpd_t user_home_t:dir { getattr search };
+
+#!!!! This avc can be allowed using the boolean 'httpd_read_user_content'
+allow httpd_t user_home_t:file open;
+
+#!!!! This avc is allowed in the current policy
+allow httpd_t user_home_t:file read;
+
+#!!!! This avc is allowed in the current policy
+allow httpd_t user_home_t:lnk_file read;

playbooks/develop/ubuntu.yml

@@ -2,7 +2,7 @@
 - hosts: localhost
   vars:
     bench_repo_path: "/usr/local/frappe/bench-repo"
-    bench_path: "/home/{{ ansible_user_id }}/frappe/frappe-bench"
+    bench_path: "/home/{{ ansible_user_id }}/benches/frappe-bench"
     mysql_config_template: "templates/simple_mariadb_config.cnf"
     mysql_conf_dir: /etc/mysql/conf.d/
     wkhtmltopdf_version: 0.12.2.1
@@ -45,11 +45,11 @@
       - libtiff4-dev
       - tcl8.5-dev
       - tk8.5-dev
-    when: ansible_distribution_version < 14.04
+    when: ansible_distribution_version | version_compare('14.04', 'lt')
     become: yes
     become_user: root
 
-  - name: install pillow prerequisites for Ubuntu > 14.04
+  - name: install pillow prerequisites for Ubuntu >= 14.04
     apt: pkg={{ item }} state=present
     with_items:
       - libtiff5-dev

playbooks/install.py

@@ -3,6 +3,7 @@ import os
 import sys
 import subprocess
 import getpass
+import json, multiprocessing
 from distutils.spawn import find_executable
 
 bench_repo = '/usr/local/frappe/bench-repo'
@@ -12,12 +13,14 @@ def install_bench(args):
 	success = run_os_command({
 		'apt-get': [
 			'sudo apt-get update',
-			'sudo apt-get install -y git build-essential python-setuptools python-dev'
+			'sudo apt-get install -y git build-essential python-setuptools python-dev libffi-dev libssl-dev'
 		],
 		'yum': [
 			'sudo yum groupinstall -y "Development tools"',
-			'sudo yum install -y git python-setuptools python-devel'
+			'sudo yum install -y git python-setuptools python-devel openssl-devel libffi-devel'
 		],
+		# epel-release is required to install redis, so installing it before the playbook-run.
+		# redhat-lsb-core is required, so that ansible can set ansible_lsb variable
 	})
 
 	if not find_executable("git"):
@@ -41,6 +44,11 @@ def install_bench(args):
 		'yum': 'sudo python get-pip.py',
 	})
 
+	# In some cases setup_tools are not updated, We will force update it.
+	run_os_command({
+		'pip': 'sudo pip install --upgrade setuptools'
+	})
+
 	success = run_os_command({
 		'pip': 'sudo pip install ansible'
 	})
@@ -54,8 +62,11 @@ def install_bench(args):
 	# clone bench repo
 	clone_bench_repo()
 
+	# args is namespace, but we would like to use it as dict in calling function, so use vars()
 	if args.develop:
-		run_playbook('develop/install.yml', sudo=True)
+		run_playbook('develop/install.yml', sudo=True, extra_args=vars(args))
+	elif args.setup_production:
+		run_playbook('develop/setup_production.yml', sudo=True, extra_args=vars(args))
 
 def install_python27():
 	version = (sys.version_info[0], sys.version_info[1])
@@ -123,10 +134,71 @@ def could_not_install(package):
 def is_sudo_user():
 	return os.geteuid() == 0
 
-def run_playbook(playbook_name, sudo=False):
+def get_passwords(run_travis=False):
-	args = ['ansible-playbook', '-c', 'local',  playbook_name]
+	if not run_travis:
+		mysql_root_password, admin_password = '', ''
+		pass_set = True
+		while pass_set:
+			# mysql root password
+			if not mysql_root_password:
+				mysql_root_password = getpass.unix_getpass(prompt='Please enter mysql root password: ')
+				conf_mysql_passwd = getpass.unix_getpass(prompt='Re-enter mysql root password: ')
+
+				if mysql_root_password != conf_mysql_passwd:
+					mysql_root_password = ''
+					continue
+
+			# admin password
+			if not admin_password:
+				admin_password = getpass.unix_getpass(prompt='Please enter Administrator password: ')
+				conf_admin_passswd = getpass.unix_getpass(prompt='Re-enter Administrator password: ')
+
+				if admin_password != conf_admin_passswd:
+					admin_password = ''
+					continue
+
+			pass_set = False
+	else:
+		mysql_root_password = admin_password = 'travis'
+
+	return {
+		'mysql_root_password': mysql_root_password,
+		'admin_password': admin_password
+	}
+
+def get_extra_vars_json(extra_args, run_travis=False):
+	# We need to pass setup_production as extra_vars to the playbook to execute conditionals in the
+	# playbook. Extra variables can passed as json or key=value pair. Here, we will use JSON.
+	json_path = os.path.join(os.path.abspath(os.path.expanduser('~')), 'extra_vars.json')
+	extra_vars = dict(extra_args.items())
+
+	# Get mysql root password and admin password
+	run_travis = extra_args.get('run_travis')
+	extra_vars.update(get_passwords(run_travis))
+
+	# Decide for branch to be cloned depending upon whether we setting up production
+	branch = 'master' if extra_args.get('setup_production') else 'develop'
+	extra_vars.update(branch=branch)
+
+	# Get max worker_connections in nginx.
+	# https://www.digitalocean.com/community/tutorials/how-to-optimize-nginx-configuration
+	# The amount of clients that can be served can be multiplied by the amount of cores.
+ 	extra_vars.update(max_worker_connections=multiprocessing.cpu_count() * 1024)
+
+	with open(json_path, mode='w') as j:
+		json.dump(extra_vars, j, indent=1, sort_keys=True)
+
+	return ('@' + json_path)
+
+def run_playbook(playbook_name, sudo=False, extra_args=None):
+	extra_vars = get_extra_vars_json(extra_args)
+	args = ['ansible-playbook', '-c', 'local',  playbook_name, '-e', extra_vars]
+
 	if sudo:
-		args.append('-K')
+		args.extend(['--become', '--become-user=frappe'])
+
+	if extra_args.get('run_verbose'):
+		args.append('-vvvv')
 
 	success = subprocess.check_call(args, cwd=os.path.join(bench_repo, 'playbooks'))
 	return success
@@ -135,8 +207,28 @@ def parse_commandline_args():
 	import argparse
 
 	parser = argparse.ArgumentParser(description='Frappe Installer')
-	parser.add_argument('--develop', dest='develop', action='store_true', default=False,
+
-						help='Install developer setup')
+	# Arguments develop and setup-production are mutually exclusive both can't be specified together.
+	# Hence, we need to create a group for discouraging use of both options at the same time.
+	args_group = parser.add_mutually_exclusive_group()
+
+	args_group.add_argument('--develop', dest='develop', action='store_true', default=False,
+		help='Install developer setup')
+
+	args_group.add_argument('--setup-production', dest='setup_production', action='store_true',
+		default=False, help='Setup Production environment for bench')
+
+	parser.add_argument('--site', dest='site', action='store', default='site1.local',
+		help='Specifiy name for your first ERPNext site')
+
+	# Hidden arguments to the argsparse
+	parser.add_argument('--run-verbose', dest='run_verbose', action='store_true', default=False,
+		help=argparse.SUPPRESS)
+
+	# To enable testing of script using Travis, this should skip the prompt
+	parser.add_argument('--run-travis', dest='run_travis', action='store_true', default=False,
+		help=argparse.SUPPRESS)
+
 	args = parser.parse_args()
 
 	return args