Merge pull request #6 from frappe/develop

Develop

MANIFEST.in

@@ -0,0 +1,3 @@
+include *.md
+recursive-include bench *.conf
+recursive-include bench *.py

bench/app.py

@@ -0,0 +1,44 @@
+import os
+from .utils import exec_cmd, get_frappe
+
+import logging
+
+logger = logging.getLogger(__name__)
+
+def get_apps(bench='.'):
+	try:
+		with open(os.path.join(bench, 'sites', 'apps.txt')) as f:
+			return f.read().strip().split('\n')
+	except IOError:
+		return []
+
+def add_to_appstxt(app, bench='.'):
+	apps = get_apps(bench=bench)
+	if app not in apps:
+		apps.append(app)
+		with open(os.path.join(bench, 'sites', 'apps.txt'), 'w') as f:
+			return f.write('\n'.join(apps))
+
+def get_app(app, git_url, bench='.'):
+	logger.info('getting app {}'.format(app))
+	exec_cmd("git clone {} --origin upstream {}".format(git_url, app), cwd=os.path.join(bench, 'apps'))
+	install_app(app, bench=bench)
+
+def new_app(app, bench='.'):
+	logger.info('creating new app {}'.format(app))
+	exec_cmd("{frappe} --make_app {apps}".format(frappe=get_frappe(bench=bench), apps=os.path.join(bench, 'apps')))
+	install_app(app, bench=bench)
+
+def install_app(app, bench='.'):
+	logger.info('installing {}'.format(app))
+	exec_cmd("{pip} install -e {app}".format(pip=os.path.join(bench, 'env', 'bin', 'pip'), app=os.path.join(bench, 'apps', app)))
+	add_to_appstxt(app, bench=bench)
+
+def pull_all_apps(bench='.'):
+	apps_dir = os.path.join(bench, 'apps')
+	apps = [app for app in os.listdir(apps_dir) if os.path.isdir(os.path.join(apps_dir, app))]
+	for app in apps:
+		app_dir = os.path.join(apps_dir, app)
+		if os.path.exists(os.path.join(app_dir, '.git')):
+			logger.info('pulling {}'.format(app))
+			exec_cmd("git pull --rebase upstream HEAD", cwd=app_dir)

bench/cli.py

@@ -0,0 +1,127 @@
+import click
+from .utils import init as _init
+from .utils import setup_env as _setup_env
+from .utils import new_site as _new_site 
+from .utils import setup_backups as _setup_backups
+from .utils import setup_auto_update as _setup_auto_update
+from .utils import setup_sudoers as _setup_sudoers
+from .utils import build_assets, patch_sites, exec_cmd, update_bench, get_frappe, setup_logging
+from .app import get_app as _get_app
+from .app import new_app as _new_app
+from .app import pull_all_apps
+from .config import generate_config
+import os
+import sys
+import logging
+
+logger = logging.getLogger('bench')
+
+def cli():
+	if sys.argv[1] == "frappe":
+		return frappe()
+	return bench()
+
+def frappe(bench='.'):
+	f = get_frappe(bench=bench)
+	os.chdir(os.path.join(bench, 'sites'))
+	os.execv(f, [f] + sys.argv[2:])
+
+@click.group()
+def bench(bench='.'):
+	# TODO add bench path context
+	setup_logging(bench=bench)
+
+@click.command()
+@click.argument('path')
+def init(path):
+	_init(path)
+	click.echo('Bench {} initialized'.format(path))
+
+@click.command('get-app')
+@click.argument('name')
+@click.argument('git-url')
+def get_app(name, git_url):
+	_get_app(name, git_url)
+	
+@click.command('new-app')
+@click.argument('app-name')
+def new_app(app_name):
+	_new_app(app_name)
+	
+@click.command('new-site')
+@click.argument('site')
+def new_site(site):
+	_new_site(site)
+	
+@click.command('update')
+@click.option('--pull', flag_value=True, type=bool)
+@click.option('--patch',flag_value=True, type=bool)
+@click.option('--build',flag_value=True, type=bool)
+@click.option('--bench',flag_value=True, type=bool)
+def update(pull=False, patch=False, build=False, bench=False):
+	if not (pull or patch or build or bench):
+		pull, patch, build, bench = True, True, True, True
+	if bench:
+		update_bench()
+	if pull:
+		pull_all_apps()
+	if patch:
+		patch_sites()
+	if build:
+		build_assets()
+
+@click.command('restart')
+def restart():
+	exec_cmd("sudo supervisorctl restart frappe:")
+
+## Setup
+@click.group()
+def setup():
+	pass
+	
+@click.command('sudoers')
+def setup_sudoers():
+	_setup_sudoers()
+	
+@click.command('nginx')
+def setup_nginx():
+	generate_config('nginx', 'nginx.conf')
+	
+@click.command('supervisor')
+def setup_supervisor():
+	generate_config('supervisor', 'supervisor.conf')
+
+@click.command('auto-update')
+def setup_auto_update():
+	_setup_auto_update()
+	
+@click.command('backups')
+def setup_backups():
+	_setup_backups()
+
+@click.command('dnsmasq')
+def setup_dnsmasq():
+	pass
+
+@click.command('env')
+def setup_env():
+	_setup_env()
+
+setup.add_command(setup_nginx)
+setup.add_command(setup_sudoers)
+setup.add_command(setup_supervisor)
+setup.add_command(setup_auto_update)
+setup.add_command(setup_dnsmasq)
+setup.add_command(setup_backups)
+setup.add_command(setup_env)
+
+#Bench commands
+
+bench.add_command(init)
+bench.add_command(get_app)
+bench.add_command(new_app)
+bench.add_command(new_site)
+bench.add_command(setup)
+bench.add_command(update)
+bench.add_command(restart)
+

bench/config.py

@@ -0,0 +1,25 @@
+import os
+import getpass
+from jinja2 import Environment, PackageLoader
+from .utils import get_sites
+
+env = Environment(loader=PackageLoader('bench', 'templates'), trim_blocks=True)
+
+def generate_config(application, template_name, bench='.'):
+	template = env.get_template(template_name)
+	bench_dir = os.path.abspath(bench)
+	sites_dir = os.path.join(bench_dir, "sites")
+	sites = get_sites(bench=bench)
+	user = getpass.getuser()
+	with open("sites/currentsite.txt") as f:
+		default_site = f.read().strip()
+
+	config = template.render(**{
+		"bench_dir": bench_dir,
+		"sites_dir": sites_dir,
+		"user": user,
+		"default_site": default_site,
+		"sites": sites
+	})
+	with open("config/{}.conf".format(application), 'w') as f:
+		f.write(config)

bench/templates/nginx.conf

@@ -0,0 +1,69 @@
+
+upstream frappe {
+    server 127.0.0.1:8000 fail_timeout=0;
+}
+
+server {
+    listen 80;
+    client_max_body_size 4G;
+    server_name 
+	{% for sitename in sites %}
+	{{ sitename }}
+	{% endfor %}
+	;
+    keepalive_timeout 5;
+    sendfile on;
+    root {{ sites_dir }};
+
+    location /private/ {
+        internal;
+        try_files /$uri =424;
+    }
+
+    location /assets {
+        try_files $uri =404;
+    }
+
+    location / {
+        try_files /$host/public/$uri @magic;
+    }
+
+    location @magic {
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_set_header X-Use-X-Accel-Redirect True;
+        proxy_read_timeout 120;
+        proxy_redirect off;
+        proxy_pass  http://frappe;
+    }
+}
+
+server {
+    listen 80 default;
+    client_max_body_size 4G;
+    server_name localhost;
+    keepalive_timeout 5;
+    sendfile on;
+    root {{ sites_dir }};
+
+    location /private/ {
+        internal;
+        try_files /$uri =424;
+    }
+
+    location /assets {
+        try_files $uri =404;
+    }
+
+    location / {
+        try_files /{{ default_site }}/public/$uri @magic;
+    }
+
+    location @magic {
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_set_header X-Use-X-Accel-Redirect True;
+        proxy_set_header Host {{ default_site }};
+        proxy_read_timeout 120;
+        proxy_redirect off;
+        proxy_pass  http://frappe;
+    }
+}

bench/templates/supervisor.conf

@@ -0,0 +1,32 @@
+[program:frappe-web]
+environment=SITES_PATH='{{ bench_dir }}/sites'
+command={{ bench_dir }}/env/bin/gunicorn -b 127.0.0.1:8000 -w 2 -t 120 frappe.app:application
+autostart=true
+autorestart=true
+stopsignal=QUIT
+stdout_logfile={{ bench_dir }}/logs/web.log
+stderr_logfile={{ bench_dir }}/logs/web.error.log
+user={{ user }}
+
+[program:frappe-worker]
+command={{ bench_dir }}/env/bin/python -m frappe.celery_app worker
+autostart=true
+autorestart=true
+stopsignal=QUIT
+stdout_logfile={{ bench_dir }}/logs/worker.log
+stderr_logfile={{ bench_dir }}/logs/worker.error.log
+user={{ user }}
+directory={{ sites_dir }}
+
+[program:frappe-workerbeat]
+command={{ bench_dir }}/env/bin/python -m frappe.celery_app beat -s test.schedule
+autostart=true
+autorestart=true
+stopsignal=QUIT
+stdout_logfile={{ bench_dir }}/logs/workerbeat.log
+stderr_logfile={{ bench_dir }}/logs/workerbeat.error.log
+user={{ user }}
+directory={{ sites_dir }}
+
+[group:frappe]
+programs=frappe-web,frappe-worker,frappe-workerbeat

bench/utils.py

@@ -0,0 +1,96 @@
+import os
+import sys
+import subprocess
+import getpass
+import logging
+
+logger = logging.getLogger(__name__)
+
+def get_frappe(bench='.'):
+	frappe = os.path.abspath(os.path.join(bench, 'env', 'bin', 'frappe'))
+	if not os.path.exists(frappe):
+		print 'frappe app is not installed. Run the following command to install frappe'
+		print 'bench get-app frappe https://github.com/frappe/frappe.git'
+	return frappe
+
+def init(path):
+	from .app import get_app
+	if os.path.exists(path):
+		print 'Directory {} already exists!'.format(path)
+		sys.exit(1)
+
+	os.mkdir(path)
+	for dirname in ('apps', 'sites', 'config', 'logs'):
+		os.mkdir(os.path.join(path, dirname))
+
+	setup_logging()
+
+	setup_env(bench=path)
+	get_app('frappe', 'https://github.com/frappe/frappe.git', bench=path)
+	setup_backups(bench=path)
+	setup_auto_update(bench=path)
+
+def exec_cmd(cmd, cwd='.'):
+	try:
+		subprocess.check_call(cmd, cwd=cwd, shell=True)
+	except subprocess.CalledProcessError, e:
+		print "Error:", e.output
+		raise
+
+def setup_env(bench='.'):
+	exec_cmd('virtualenv {} -p {}'.format('env', sys.executable), cwd=bench)
+
+def new_site(site, bench='.'):
+	logger.info('creating new site {}'.format(site))
+	exec_cmd("{frappe} --install {site} {site}".format(frappe=get_frappe(bench=bench), site=site), cwd=os.path.join(bench, 'sites'))
+	if len(get_sites(bench=bench)) == 1:
+		exec_cmd("{frappe} --use {site}".format(frappe=get_frappe(bench=bench), site=site), cwd=os.path.join(bench, 'sites'))
+
+def patch_sites(bench='.'):
+	exec_cmd("{frappe} --latest all".format(frappe=get_frappe(bench=bench)), cwd=os.path.join(bench, 'sites'))
+
+def build_assets(bench='.'):
+	exec_cmd("{frappe} --build".format(frappe=get_frappe(bench=bench)), cwd=os.path.join(bench, 'sites'))
+
+def get_sites(bench='.'):
+	sites_dir = os.path.join(bench, "sites")
+	sites = [site for site in os.listdir(sites_dir) 
+		if os.path.isdir(os.path.join(sites_dir, site)) and site not in ('assets',)]
+	return sites
+
+def get_sites_dir(bench='.'):
+	return os.path.abspath(os.path.join(bench, 'sites'))
+
+def get_bench_dir(bench='.'):
+	return os.path.abspath(bench)
+
+def setup_auto_update(bench='.'):
+	logger.info('setting up auto update')
+	exec_cmd('echo \"`crontab -l`\" | uniq | sed -e \"a0 10 * * * cd {bench_dir} &&  {bench} update\" | grep -v "^$" | uniq | crontab'.format(bench_dir=get_bench_dir(bench=bench),
+	bench=os.path.join(get_bench_dir(bench=bench), 'env', 'bin', 'bench')))
+
+def setup_backups(bench='.'):
+	logger.info('setting up backups')
+	exec_cmd('echo \"`crontab -l`\" | uniq | sed -e \"a0 */6 * * * cd {sites_dir} &&  {frappe} --backup all\" | grep -v "^$" | uniq | crontab'.format(sites_dir=get_sites_dir(bench=bench),
+	frappe=get_frappe(bench=bench)))
+
+def update_bench():
+	logger.info('setting up sudoers')
+	cwd = os.path.dirname(os.path.abspath(__file__))
+	exec_cmd("git pull", cwd=cwd)
+
+def setup_sudoers():
+	with open('/etc/sudoers.d/frappe', 'w') as f:
+		f.write("{user} ALL=(ALL) NOPASSWD: {supervisorctl} restart frappe\:\n".format(
+					user=getpass.getuser()),
+					supervisorctl=subprocess.check_output('which supervisorctl', shell=True).strip())
+
+def setup_logging(bench='.'):
+	if os.path.exists(os.path.join(bench, 'logs')):
+		logger = logging.getLogger('bench')
+		log_file = os.path.join(bench, 'logs', 'bench.log')
+		formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s')
+		hdlr = logging.FileHandler(log_file)
+		hdlr.setFormatter(formatter)
+		logger.addHandler(hdlr)
+		logger.setLevel(logging.DEBUG)

install_scripts/setup_frappe.sh

@@ -0,0 +1,193 @@
+set -e
+
+get_distro() {
+	ARCH=$(uname -m | sed 's/x86_/amd/;s/i[3-6]86/x86/') 
+	if [ -f /etc/redhat-release ]; then
+		OS="centos"
+		OS_VER=`cat /etc/redhat-release | cut -d" " -f3 | cut -d "." -f1`
+
+	elif [ -f /etc/lsb-release ]; then
+		. /etc/lsb-release
+		OS=$DISTRIB_ID
+		OS_VER=$DISTRIB_CODENAME
+
+	elif [ -f /etc/debian_version ]; then
+		. /etc/os-release
+		OS="debian"  # XXX or Ubuntu??
+		OS_VER=$VERSION_ID
+	fi
+
+	export OS=$OS
+	export OS_VER=$OS_VER
+	export ARCH=$ARCH
+	echo DEBUG $OS $OS_VER $ARCH
+}
+
+add_centos_mariadb_repo() {
+	echo "
+[mariadb]
+name = MariaDB
+baseurl = http://yum.mariadb.org/5.5/centos$OS_VER-$ARCH
+gpgkey=https://yum.mariadb.org/RPM-GPG-KEY-MariaDB
+gpgcheck=1
+" > /etc/yum.repos.d/mariadb.repo
+}
+
+add_ubuntu_mariadb_repo() {
+	sudo apt-get update
+	sudo apt-get install -y python-software-properties
+	sudo apt-key adv --recv-keys --keyserver hkp://keyserver.ubuntu.com:80 0xcbcb082a1bb943db
+	sudo add-apt-repository "deb http://ams2.mirrors.digitalocean.com/mariadb/repo/5.5/ubuntu $OS_VER main"
+}
+
+add_debian_mariadb_repo() {
+	if [ $OS_VER == "7" ]; then
+		CODENAME="wheezy"
+	
+	elif [ $OS_VER == "6" ]; then
+		CODENAME="squeeze"
+	else
+		echo Unsupported Debian Version
+		exit 1
+	fi
+
+	sudo apt-get update
+	sudo apt-get install -y python-software-properties
+	sudo apt-key adv --recv-keys --keyserver keyserver.ubuntu.com 0xcbcb082a1bb943db
+	sudo add-apt-repository "deb http://ams2.mirrors.digitalocean.com/mariadb/repo/5.5/debian $CODENAME main"
+}
+
+add_maria_db_repo() {
+	if [ "$OS" == "centos" ]; then
+		echo DEBUG adding centos mariadb repo
+		add_centos_mariadb_repo
+	
+	elif [ "$OS" == "debian" ]; then 
+		echo DEBUG adding debian mariadb repo
+		add_debian_mariadb_repo
+
+	elif [ "$OS" == "Ubuntu" ]; then 
+		echo DEBUG adding debian mariadb repo
+		add_ubuntu_mariadb_repo
+	else
+		echo Unsupported Distribution
+		exit 1
+	fi
+}
+
+install_packages() {
+	if [ $OS == "centos" ]; then
+		sudo yum install wget -y
+		add_ius_repo
+		sudo yum groupinstall -y "Development tools"
+		sudo yum install -y sudo yum install MariaDB-server MariaDB-client MariaDB-compat python-setuptools nginx zlib-devel bzip2-devel openssl-devel memcached postfix python27-devel python27 libxml2 libxml2-devel libxslt libxslt-devel redis MariaDB-devel libXrender libXext python27-setuptools
+		wget http://downloads.sourceforge.net/project/wkhtmltopdf/0.12.1/wkhtmltox-0.12.1_linux-centos6-amd64.rpm
+		sudo rpm -Uvh wkhtmltox-0.12.1_linux-centos6-amd64.rpm
+		easy_install-2.7 -U pip
+	
+	elif [ $OS == "debian" ]; then 
+		sudo apt-get update
+		sudo apt-get install python-dev python-setuptools build-essential python-mysqldb git memcached ntp vim screen htop mariadb-server mariadb-common libmariadbclient-dev  libxslt1.1 libxslt1-dev redis-server libssl-dev libcrypto++-dev postfix nginx supervisor python-pip -y 
+
+	elif [ $OS == "Ubuntu" ]; then 
+		sudo apt-get update
+		sudo apt-get install python-dev python-setuptools build-essential python-mysqldb git memcached ntp vim screen htop mariadb-server mariadb-common libmariadbclient-dev  libxslt1.1 libxslt1-dev redis-server libssl-dev libcrypto++-dev postfix nginx supervisor python-pip -y 
+	else
+		echo Unsupported Distribution
+		exit 1
+	fi
+}
+
+add_user() {
+# Check if script is running as root and is not running as sudo. We want to skip
+# this step if the user is already running this script with sudo as a non root
+# user
+	if [ -z $SUDO_UID ] && [ $EUID -eq 0 ]; then
+		useradd -m -d /home/frappe -s $SHELL frappe
+		chmod o+x /home/frappe
+		chmod o+r /home/frappe
+		export FRAPPE_USER="frappe"
+	else
+		export FRAPPE_USER="$SUDO_USER"
+	fi
+}
+
+configure_mariadb_centos() {
+	# Required only for CentOS, Ubuntu and Debian will show dpkg configure screen to set the password
+	if [ $OS == "centos" ]; then
+		echo Enter mysql root password to set:
+		read -s MSQ_PASS
+		mysqladmin -u root password $MSQ_PASS
+	fi
+}
+
+install_supervisor_centos() {
+	# Required only for CentOS, Ubuntu and Debian have them in repositories
+		easy_install supervisor
+		curl https://raw.githubusercontent.com/pdvyas/supervisor-initscripts/master/redhat-init-jkoppe > /etc/init.d/supervisord
+		curl https://raw.githubusercontent.com/pdvyas/supervisor-initscripts/master/redhat-sysconfig-jkoppe > /etc/sysconfig/supervisord
+		curl https://raw.githubusercontent.com/pdvyas/supervisor-initscripts/master/supervisord.conf > /etc/supervisord.conf
+		mkdir /etc/supervisor.d
+		chmod +x /etc/init.d/supervisord
+		bash -c "service supervisord start || true"
+}
+
+
+start_services_centos() {
+	service mysql start
+	service redis start
+	service postfix start
+	service nginx start
+	service memcached start
+}
+
+configure_services_centos() {
+	chkconfig --add supervisord
+	chkconfig redis on
+	chkconfig mysql on
+	chkconfig nginx on
+	chkconfig supervisord on
+}
+
+add_ius_repo() {
+	if [ $ARCH == "amd64" ]; then
+		T_ARCH="x86_64"
+	else
+		T_ARCH="i386"
+	fi 
+	if [ $OS_VER -eq "6" ]; then
+	wget http://dl.iuscommunity.org/pub/ius/stable/CentOS/$OS_VER/$T_ARCH/epel-release-6-5.noarch.rpm
+	wget http://dl.iuscommunity.org/pub/ius/stable/CentOS/$OS_VER/$T_ARCH/ius-release-1.0-11.ius.centos6.noarch.rpm
+	rpm -Uvh epel-release-6-5.noarch.rpm
+	rpm -Uvh ius-release-1.0-11.ius.centos6.noarch.rpm
+	fi
+}
+
+install_bench() {
+	sudo su $FRAPPE_USER -c "cd /home/$FRAPPE_USER && git clone --branch develop https://github.com/frappe/bench"
+	if hash pip-2.7; then
+		PIP="pip-2.7"
+	elif hash pip2.7; then
+		PIP="pip2.7"
+	elif hash pip2; then
+		PIP="pip2"
+	elif hash pip; then
+		PIP="pip"
+	else
+		echo PIP not installed
+		exit 1
+	fi
+	sudo $PIP install /home/$FRAPPE_USER/bench
+}
+
+get_distro
+add_maria_db_repo
+install_packages
+add_user
+if [ $OS == "centos" ]; then
+	install_supervisor_centos
+	configure_services_centos
+	start_services_centos
+	configure_mariadb_centos
+fi
+install_bench

setup.py

@@ -0,0 +1,20 @@
+from setuptools import setup, find_packages
+
+setup(
+	name='bench',
+	version='0.1',
+	py_modules=find_packages(),
+	include_package_data=True,
+	url='https://github.com/frappe/bench',
+	author='Web Notes Technologies Pvt. Ltd.',
+	author_email='info@frappe.io',
+	install_requires=[
+		'Click',
+		'jinja2',
+		'virtualenv'
+    ],
+	entry_points='''
+[console_scripts]
+bench=bench.cli:cli
+''',
+)