Add redis cache

bench/app.py

@@ -105,6 +105,12 @@ def check_version_upgrade(bench='.'):
 	if upstream_version - local_version  > 0:
 		raise MajorVersionUpgradeException("Major Upgrade", upstream_version, local_version)
 
+def get_current_frappe_version(bench='.'):
+	apps_dir = os.path.join(bench, 'apps')
+	frappe_dir = os.path.join(apps_dir, 'frappe')
+
+	return get_major_version(get_current_version(frappe_dir))
+
 def get_current_branch(repo_dir):
 	return get_cmd_output("basename $(git symbolic-ref -q HEAD)", cwd=repo_dir)
 

bench/cli.py

@@ -307,7 +307,7 @@ def setup_supervisor():
 	generate_supervisor_config()
 	
 @click.command('redis-cache')
-def setup_supervisor():
+def setup_redis_cache():
 	"generate config for redis cache"
 	generate_redis_config()
 	
@@ -349,6 +349,7 @@ def setup_config():
 setup.add_command(setup_nginx)
 setup.add_command(setup_sudoers)
 setup.add_command(setup_supervisor)
+setup.add_command(setup_redis_cache)
 setup.add_command(setup_auto_update)
 setup.add_command(setup_dnsmasq)
 setup.add_command(setup_backups)

bench/config.py

@@ -1,12 +1,14 @@
 import os
 import getpass
 import json
+import subprocess
 from jinja2 import Environment, PackageLoader
-from .utils import get_sites, get_config, update_config
+from .utils import get_sites, get_config, update_config, get_redis_version
 
 env = Environment(loader=PackageLoader('bench', 'templates'), trim_blocks=True)
 
 def generate_supervisor_config(bench='.', user=None):
+	from .app import get_current_frappe_version
 	template = env.get_template('supervisor.conf')
 	bench_dir = os.path.abspath(bench)
 	sites_dir = os.path.join(bench_dir, "sites")
@@ -20,6 +22,9 @@ def generate_supervisor_config(bench='.', user=None):
 		"sites_dir": sites_dir,
 		"user": user,
 		"http_timeout": config.get("http_timeout", 120),
+		"redis_server": subprocess.check_output('which redis-server', shell=True).strip(),
+		"redis_config": os.path.join(bench_dir, 'config', 'redis.conf'),
+		"frappe_version": get_current_frappe_version()
 	})
 	with open("config/supervisor.conf", 'w') as f:
 		f.write(config)
@@ -67,3 +72,14 @@ def generate_nginx_config(bench='.'):
 	})
 	with open("config/nginx.conf", 'w') as f:
 		f.write(config)
+
+def generate_redis_config(bench='.'):
+	template = env.get_template('redis.conf')
+	conf = {
+		"maxmemory": get_config().get('cache_maxmemory', '50'),
+		"port": get_config().get('redis_cache_port', '11311'),
+		"redis_version": get_redis_version()
+	}
+	config = template.render(**conf)
+	with open("config/redis.conf", 'w') as f:
+		f.write(config)

bench/templates/redis.conf

@@ -0,0 +1,77 @@
+activerehashing yes
+appendfsync everysec
+appendonly no
+auto-aof-rewrite-min-size 64mb
+auto-aof-rewrite-percentage 100
+daemonize no
+databases 16
+dbfilename dump.rdb
+list-max-ziplist-entries 512
+list-max-ziplist-value 64
+no-appendfsync-on-rewrite no
+pidfile /var/run/redis.pid
+port {{port}}
+rdbcompression yes
+set-max-intset-entries 512
+slave-priority 100
+slave-serve-stale-data yes
+slowlog-log-slower-than 10000
+slowlog-max-len 128
+timeout 0
+zset-max-ziplist-entries 128
+zset-max-ziplist-value 64
+
+maxmemory {{maxmemory}}mb
+maxmemory-policy allkeys-lru
+
+{% if redis_version == "2.4"%}
+hash-max-zipmap-entries 512
+hash-max-zipmap-value 64
+loglevel verbose
+vm-enabled no
+vm-max-memory 0
+vm-max-threads 4
+vm-page-size 32
+vm-pages 134217728
+vm-swap-file /tmp/redis.swap
+{% endif %}
+
+{% if redis_version == "2.6"%}
+aof-rewrite-incremental-fsync yes
+client-output-buffer-limit normal 0 0 0
+client-output-buffer-limit pubsub 32mb 8mb 60
+client-output-buffer-limit slave 256mb 64mb 60
+hash-max-ziplist-entries 512
+hash-max-ziplist-value 64
+hz 10
+loglevel notice
+lua-time-limit 5000
+rdbchecksum yes
+repl-disable-tcp-nodelay no
+slave-read-only yes
+stop-writes-on-bgsave-error yes
+tcp-keepalive 0
+{% endif %}
+
+{% if redis_version == "2.8"%}
+aof-load-truncated yes
+aof-rewrite-incremental-fsync yes
+appendfilename "appendonly.aof"
+client-output-buffer-limit normal 0 0 0
+client-output-buffer-limit pubsub 32mb 8mb 60
+client-output-buffer-limit slave 256mb 64mb 60
+hash-max-ziplist-entries 512
+hash-max-ziplist-value 64
+hll-sparse-max-bytes 3000
+hz 10
+latency-monitor-threshold 0
+logfile ""
+loglevel notice
+lua-time-limit 5000
+notify-keyspace-events ""
+rdbchecksum yes
+slave-read-only yes
+stop-writes-on-bgsave-error yes
+tcp-backlog 511
+tcp-keepalive 0
+{% endif %}

bench/templates/supervisor.conf

@@ -28,5 +28,18 @@ stderr_logfile={{ bench_dir }}/logs/workerbeat.error.log
 user={{ user }}
 directory={{ sites_dir }}
 
+
+{% if frappe_version > 4%}
+[program:redis-cache]
+command={{ redis_server }} {{ redis_config }}
+autostart=true
+autorestart=true
+stopsignal=QUIT
+stdout_logfile={{ bench_dir }}/logs/redis.log
+stderr_logfile={{ bench_dir }}/logs/redis.error.log
+user={{ user }}
+directory={{ sites_dir }}
+{% endif %}
+
 [group:frappe]
 programs=frappe-web,frappe-worker,frappe-workerbeat

bench/utils.py

@@ -1,4 +1,5 @@
 import os
+import re
 import sys
 import subprocess
 import getpass
@@ -29,6 +30,7 @@ def get_frappe(bench='.'):
 def init(path, apps_path=None, no_procfile=False, no_backups=False,
 		no_auto_update=False, frappe_path=None, frappe_branch=None, wheel_cache_dir=None):
 	from .app import get_app, install_apps_from_path
+	from .config import generate_redis_config
 	if os.path.exists(path):
 		print 'Directory {} already exists!'.format(path)
 		sys.exit(1)
@@ -55,6 +57,7 @@ def init(path, apps_path=None, no_procfile=False, no_backups=False,
 		setup_auto_update(bench=path)
 	if apps_path:
 		install_apps_from_path(apps_path, bench=path)
+	generate_redis_config()
 
 def exec_cmd(cmd, cwd='.'):
 	try:
@@ -69,10 +72,20 @@ def setup_env(bench='.'):
 	exec_cmd('./env/bin/pip -q install https://github.com/frappe/MySQLdb1/archive/MySQLdb-1.2.5-patched.tar.gz', cwd=bench)
 
 def setup_procfile(bench='.'):
+	from .app import get_current_frappe_version
+	frappe_version = get_current_frappe_version()
+	procfile_contents = {
+		'web': "./env/bin/frappe --serve --sites_path sites",
+		'worker': "sh -c 'cd sites && exec ../env/bin/python -m frappe.celery_app worker'",
+		'workerbeat': "sh -c 'cd sites && exec ../env/bin/python -m frappe.celery_app beat -s scheduler.schedule'"
+	}
+	if frappe_version > 4:
+		procfile_contents['redis_cache'] = "redis-server config/redis.conf"
+	
+	procfile = '\n'.join(["{0}: {1}".format(k, v) for k, v in procfile_contents.items()])
+
 	with open(os.path.join(bench, 'Procfile'), 'w') as f:
-		f.write("""web: ./env/bin/frappe --serve --sites_path sites
+		f.write(procfile)
-worker: sh -c 'cd sites && exec ../env/bin/python -m frappe.celery_app worker'
-workerbeat: sh -c 'cd sites && exec ../env/bin/python -m frappe.celery_app beat -s scheduler.schedule'""")
 
 def new_site(site, mariadb_root_password=None, admin_password=None, bench='.'):
 	import hashlib
@@ -210,7 +223,7 @@ def check_git_for_shallow_clone():
 
 def get_cmd_output(cmd, cwd='.'):
 	try:
-		return subprocess.check_output(cmd, cwd=cwd, shell=True)
+		return subprocess.check_output(cmd, cwd=cwd, shell=True).strip()
 	except subprocess.CalledProcessError, e:
 		print "Error:", e.output
 		raise
@@ -368,3 +381,12 @@ def fix_file_perms():
 		for _file in os.listdir(bin_dir):
 			if not _file.startswith('activate'):
 				os.chmod(os.path.join(bin_dir, _file), 0755)
+
+def get_redis_version():
+	version_string = subprocess.check_output('redis-server --version', shell=True).strip()
+	if re.search("Redis server version 2.4", version_string):
+		return "2.4"
+	if re.search("Redis server v=2.6", version_string):
+		return "2.6"
+	if re.search("Redis server v=2.8", version_string):
+		return "2.8"