[Enhancements]Installing pip and bench

.travis.yml

@@ -5,9 +5,11 @@ python:
 
 install:
   - sudo apt-get purge -y mysql-common
-  - sudo bash $TRAVIS_BUILD_DIR/install_scripts/setup_frappe.sh --skip-install-bench --mysql-root-password travis
+  - sudo python $TRAVIS_BUILD_DIR/installer/install.py --user travis --skip-bench-setup
+  # - sudo bash $TRAVIS_BUILD_DIR/install_scripts/setup_frappe.sh --skip-install-bench --mysql-root-password travis
   - mkdir -p ~/bench-repo
   - cp -r $TRAVIS_BUILD_DIR/* ~/bench-repo/
+  - cd ~ && sudo python bench-repo/installer/install.py --only-dependencies
 
 script:
   - cd ~

bench/production_setup.py

@@ -1,4 +1,4 @@
-from .utils import get_program, exec_cmd, get_cmd_output, fix_prod_setup_perms, get_config, get_bench_name
+from .utils import get_program, exec_cmd, get_cmd_output, fix_prod_setup_perms, get_config
 from .config import generate_nginx_config, generate_supervisor_config
 from jinja2 import Environment, PackageLoader
 import os
@@ -52,32 +52,21 @@ def copy_default_nginx_config():
 def setup_production(user, bench='.'):
 	generate_supervisor_config(bench=bench, user=user)
 	generate_nginx_config(bench=bench)
-	fix_prod_setup_perms(bench, frappe_user=user)
+	fix_prod_setup_perms(frappe_user=user)
 	remove_default_nginx_configs()
 
-	bench_name = get_bench_name(bench)
-	nginx_conf = '/etc/nginx/conf.d/{bench_name}.conf'.format(bench_name=bench_name)
-
 	if is_centos7():
-		supervisor_conf_extn = "ini"
+		supervisor_conf_filename = 'frappe.ini'
 		copy_default_nginx_config()
 	else:
-		supervisor_conf_extn = "conf"
+		supervisor_conf_filename = 'frappe.conf'
 
-	supervisor_conf = os.path.join(get_supervisor_confdir(), '{bench_name}.{extn}'.format(
-		bench_name=bench_name, extn=supervisor_conf_extn))
 
-	links = (
-		(os.path.abspath(os.path.join(bench, 'config', 'nginx.conf')), nginx_conf),
-		(os.path.abspath(os.path.join(bench, 'config', 'supervisor.conf')), supervisor_conf),
-	)
-
-	for src, dest in links:
-		if not os.path.exists(dest):
-			os.symlink(src, dest)
+	
 
+	os.symlink(os.path.abspath(os.path.join(bench, 'config', 'supervisor.conf')), os.path.join(get_supervisor_confdir(), supervisor_conf_filename))
+	os.symlink(os.path.abspath(os.path.join(bench, 'config', 'nginx.conf')), '/etc/nginx/conf.d/frappe.conf')
 	exec_cmd('supervisorctl reload')
 	if os.environ.get('NO_SERVICE_RESTART'):
 		return
-
 	restart_service('nginx')

installer/install.py

@@ -4,22 +4,34 @@ import sys
 import pwd
 import stat
 import subprocess
-from distutils.spawn import find_executable
 import string
+
 from random import choice
+from distutils.spawn import find_executable
+from setuptools.command import easy_install as easy_install
 
 def install_bench(args):
 	# pre-requisites for bench repo cloning
+	install_pip()
+	install_ansible()
 	install_git()
 
 	# which user to use for bench repo cloning
 	user_password = add_user(args.user)
 
-	# clone bench repo!
+	# stop install
+	if args.skip_bench_setup:
+		return
 
-	# install pip
+	# clone bench repo
+	cloned = clone_bench_repo(args.user)
+
+	# install pre-requisites
+	installed = install_prerequisites(args.user)
 
 	# install bench
+	# if cloned:
+	# 	install_bench_cmd(user)
 
 def install_python27():
 	version = (sys.version_info[0], sys.version_info[1])
@@ -91,6 +103,40 @@ def add_user(user):
 
 		return user_password
 
+def install_pip():
+	"""Install pip for the user or upgrade to latest version if already present"""
+	try:
+		import pip
+	except ImportError:
+		easy_install.main(['pip'])
+
+def install_ansible():
+	try:
+		import ansible
+	except ImportError:
+		import pip
+		pip.main(["install", "ansible"])
+
+def clone_bench_repo(user):
+	"""Clones the bench repository in the user folder"""
+	bench_repo = os.path.join(get_user_folder(user), 'bench-repo')
+
+	success = run_os_command(
+		{"git": "git clone https://github.com/frappe/bench {bench_repo}".format(bench_repo=bench_repo)}
+	)
+
+	return success
+
+def install_dependencies():
+	"""Installs the pre-requisites like mariadb, nginx, redis etc. for the user"""
+	playbooks_path = get_playbooks_path()
+
+	for playbook in os.listdir(playbooks_path):
+		if playbook.endswith('.yml'):
+			success = run_playbook(os.path.join(playbooks_path, playbook))
+
+	return success
+
 def run_os_command(command_map):
 	"""command_map is a dictionary of {"executable": command}. For ex. {"apt-get": "sudo apt-get install -y python2.7"} """
 	success = False
@@ -115,13 +161,27 @@ def get_random_string(length=16):
 	"""generate a random string"""
 	return ''.join([choice(string.letters + string.digits) for i in range(length)])
 
+def get_playbooks_path():
+	return os.path.abspath(os.path.join(os.getcwd(), 'bench-repo', 'installer', 'playbooks'))
+
+def run_playbook(playbook_name):
+	success = subprocess.check_call("ansible-playbook -c local {playbook_name}".format(playbook_name=playbook_name).split())
+	return success
+
+def install_bench_cmd(user):
+	"""Installs bench using pip from the bench-repo"""
+	pass
+
 def parse_commandline_args():
 	import argparse
 
 	parser = argparse.ArgumentParser(description='Frappe Installer')
 	parser.add_argument('--user', metavar='USER', dest='user', action='store',
 						help="System user which will be used to start various processes required by Frappe framework. If this user doesn't exist, it will be created.")
-
+	parser.add_argument('--skip-bench-setup', dest='skip_bench_setup', action='store_true', default=False,
+						help="Skip cloning and installation of bench.")
+	parser.add_argument('--only-dependencies', dest='only_dependencies', action='store_true', default=False,
+						help="Only install dependencies via ansible")
 	args = parser.parse_args()
 
 	return args
@@ -135,4 +195,8 @@ if __name__ == "__main__":
 
 	args = parse_commandline_args()
 
-	install_bench(args)
+	if args.only_dependencies:
+		install_dependencies()
+
+	else:
+		install_bench(args)

installer/playbooks/install_prereq.yml

@@ -0,0 +1,31 @@
+---
+  - name: Install pre-requisites to setup bench
+    hosts: localhost
+    become: yes
+    become_user: root
+    vars:
+      - mysql_conf_tpl: ../templates/mariadb_config.cnf
+      - nginx_conf_file: ../templates/nginx.conf
+      - mysql_secure_installation: True
+    roles:
+      - mariadb
+      - nginx
+      - wkhtmltopdf
+    tasks:
+      - name: Install the 'Development tools' package group (Redhat)
+        yum: name="@Development tools" state=present
+        when: ansible_os_family == 'RedHat'
+      - name: Install redis # For Redhat
+        yum: name={{ item }} state=present
+        with_items:
+            - redis
+            - nodejs
+            - npm
+        when: ansible_os_family == 'RedHat'
+      - name: Install redis # For Debian
+        apt: pkg={{ item }} state=present force=yes
+        with_items:
+            - redis-server
+            - nodejs
+            - npm
+        when: ansible_os_family == 'Debian'

installer/playbooks/templates/mariadb_config.cnf

@@ -0,0 +1,60 @@
+[mysqld]
+
+# GENERAL #
+user                           = mysql
+default-storage-engine         = InnoDB
+socket                         = /var/lib/mysql/mysql.sock
+pid-file                       = /var/lib/mysql/mysql.pid
+
+# MyISAM #
+key-buffer-size                = 32M
+myisam-recover                 = FORCE,BACKUP
+
+# SAFETY #
+max-allowed-packet             = 16M
+max-connect-errors             = 1000000
+innodb                         = FORCE
+
+# DATA STORAGE #
+datadir                        = /var/lib/mysql/
+
+# BINARY LOGGING #
+log-bin                        = /var/lib/mysql/mysql-bin
+expire-logs-days               = 14
+sync-binlog                    = 1
+
+# REPLICATION #
+server-id                      = 1
+
+# CACHES AND LIMITS #
+tmp-table-size                 = 32M
+max-heap-table-size            = 32M
+query-cache-type               = 0
+query-cache-size               = 0
+max-connections                = 500
+thread-cache-size              = 50
+open-files-limit               = 65535
+table-definition-cache         = 4096
+table-open-cache               = 10240
+
+# INNODB #
+innodb-flush-method            = O_DIRECT
+innodb-log-files-in-group      = 2
+innodb-log-file-size           = 512M
+innodb-flush-log-at-trx-commit = 1
+innodb-file-per-table          = 1
+innodb-buffer-pool-size        = {{ (ansible_memtotal_mb*0.685)|round|int }}M
+innodb-file-format             = barracuda
+innodb-large-prefix            = 1
+collation-server               = utf8mb4_unicode_ci
+character-set-server           = utf8mb4
+character-set-client-handshake = FALSE
+
+# LOGGING #
+log-error                      = /var/lib/mysql/mysql-error.log
+log-queries-not-using-indexes  = 0
+slow-query-log                 = 1
+slow-query-log-file            = /var/lib/mysql/mysql-slow.log
+
+[mysql]
+default-character-set = utf8mb4

installer/playbooks/templates/nginx.conf

@@ -0,0 +1,59 @@
+user  nginx;
+worker_processes  6;
+worker_rlimit_nofile 65535;
+
+error_log  /var/log/nginx/error.log warn;
+pid        /var/run/nginx.pid;
+
+
+events {
+    worker_connections 2048;
+    multi_accept on;
+}
+
+
+http {
+    include       /etc/nginx/mime.types;
+    default_type  application/octet-stream;
+
+    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
+                      '$status $body_bytes_sent "$http_referer" '
+                      '"$http_user_agent" "$http_x_forwarded_for"';
+
+    access_log  /var/log/nginx/access.log  main;
+
+    sendfile        on;
+    tcp_nopush on;
+    tcp_nodelay on;
+    server_tokens off;
+    #tcp_nopush     on;
+
+    keepalive_timeout  10;
+    keepalive_requests   10;
+
+    gzip  on;
+    gzip_disable "msie6";
+
+    gzip_vary on;
+    gzip_proxied any;
+    gzip_comp_level 6;
+    gzip_buffers 16 8k;
+    gzip_http_version 1.1;
+    gzip_types text/plain text/css application/json application/javascript image/svg+xml text/html "application/json; charset: utf-8" "text/html; charset: utf-8" application/font-woff;
+
+    server_names_hash_max_size 4096;
+    #server_names_hash_bucket_size 64;
+
+    open_file_cache max=65000 inactive=1m;
+    open_file_cache_valid 5s;
+    open_file_cache_min_uses 1;
+    open_file_cache_errors on;
+
+    ssl_protocols  SSLv3 TLSv1;
+    ssl_ciphers     ECDHE-RSA-AES256-SHA384:AES256-SHA256:RC4:HIGH:!MD5:!aNULL:!EDH:!AESGCM;
+    ssl_prefer_server_ciphers   on;
+
+    proxy_cache_path /var/cache/nginx levels=1:2 keys_zone=web-cache:8m max_size=1000m inactive=600m;
+
+    include /etc/nginx/conf.d/*.conf;
+}