diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 00000000..83434ef8 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,21 @@ +language: python +dist: trusty +sudo: required + +python: + - "2.7" + +install: + - sudo apt-get purge -y mysql-common mysql-server mysql-client + # - sudo python $TRAVIS_BUILD_DIR/installer/install.py --user travis --skip-bench-setup + - sudo bash $TRAVIS_BUILD_DIR/install_scripts/setup_frappe.sh --skip-install-bench --mysql-root-password travis + - mkdir -p ~/bench-repo + - cp -r $TRAVIS_BUILD_DIR/* ~/bench-repo/ + # - cd ~ && sudo python bench-repo/installer/install.py --only-dependencies + +script: + - cd ~ + - sudo pip install --upgrade pip + - sudo pip install -e bench-repo + # - sudo python -m unittest bench.tests.test_setup_production.TestSetupProduction.test_new_site + - sudo python -m unittest -v bench.tests.test_setup_production diff --git a/README.md b/README.md index 76ba3805..54e4d2e8 100644 --- a/README.md +++ b/README.md @@ -5,15 +5,15 @@ The bench allows you to setup Frappe / ERPNext apps on your local Linux (CentOS To do this install, you must have basic information on how Linux works and should be able to use the command-line. If you are looking easier ways to get started and evaluate ERPNext, [download the Virtual Machine or take a free trial on ERPNext.com](https://erpnext.com/pricing). -For questions, please join the [developer forum](https://discuss.frappe.io/). +If you have questions, please ask them on our [forum](https://discuss.erpnext.com/). Installation ============ -Easy way --------- - -Supported for CentOS 6, CentOS 7, Debian 7 and Ubuntu 12.04+ +Easy Production Setup +--------------------- +> For production use and which also installs ERPNext. Supported for CentOS 6, CentOS 7, Debian 7 and Ubuntu 12.04+ +> This is an opinionated setup with logging and SE Linux. So, it is best to setup on a blank server. Open your Terminal and enter: @@ -22,11 +22,40 @@ wget https://raw.githubusercontent.com/frappe/bench/master/install_scripts/setup sudo bash setup_frappe.sh --setup-production ``` -This script should install the pre-requisites, install bench and setup an ERPNext site. This will setup ERPNext with nginx, with Supervisor enabled and checkout the master branch of the ERPNext repo. Passwords for Frappe, Frappe Administrator and MariaDB (root) will be generated. You can then login as Administrator with the Administrator password printed. +- This script will install the pre-requisites, install bench and setup an ERPNext site +- This will setup ERPNext with nginx, with Supervisor enabled and checkout the master branch of the ERPNext repo +- Passwords for Frappe, Frappe Administrator and MariaDB (root) will be generated +- You can then login as **Administrator** with the Administrator password printed If you want to develop ERPNext or any Frappe App, you can omit the "--setup-production" part from the command. This will setup ERPNext as well. Use ```bench start``` to run the server. -Note: If you are using a DigitalOcean droplet or any other cloud provider's vps, make sure it has >= 1gb of ram or has swap setup properly. +> Note: If you are using a DigitalOcean droplet or any other cloud provider's vps, make sure it has >= 1gb of ram or has swap setup properly. + +Development Setup (Beta) +------------------------ + +Tested on Ubuntu 14.04+ and MacOS X. If you find any problems, post them on our forum: [https://discuss.erpnext.com](https://discuss.erpnext.com) + +``` +wget https://raw.githubusercontent.com/frappe/bench/master/playbooks/install.py +python install.py +``` + +This script requires Python2.7+ installed on your machine. You need to run this with a user that is **not** `root`, but can `sudo`. If you don't have such a user, you can search the web for *How to add a new user in { your OS }* and *How to add an existing user to sudoers in { your OS }*. + +This script will: + +- Install pre-requisites like git and ansible +- Shallow clones this bench repository under `/usr/local/frappe/bench-repo` +- Runs the Ansible playbook 'playbooks/develop/install.yml', which: + - Installs + - MariaDB and its config + - Redis + - NodeJS + - WKHTMLtoPDF with patched QT + - Initializes a new Bench at `~/frappe/frappe-bench` with `frappe` framework already installed under `apps`. + +You will have to manually create a new site (`bench new-site`) and get apps that you need (`bench get-app`, `bench install-app`). Manual Install -------------- @@ -36,7 +65,7 @@ Install pre-requisites, * [Python 2.7](https://www.python.org/download/releases/2.7/) * [MariaDB](https://mariadb.org/) * [Redis](http://redis.io/topics/quickstart) -* [wkhtmltopdf](http://wkhtmltopdf.org/downloads.html) (optional, required for pdf generation) +* [WKHTMLtoPDF with patched QT](http://wkhtmltopdf.org/downloads.html) (required for pdf generation) [Installing pre-requisites on OSX](https://github.com/frappe/bench/wiki/Installing-Bench-Pre-requisites-on-MacOSX) @@ -47,6 +76,9 @@ Install bench as a *non root* user, Note: Please do not remove the bench directory the above commands will create +Initialize Bench using: `bench init frappe-bench`. Here you can replace `frappe-bench` with a name of your choice for your bench. + + Installing ERPNext ------------------ @@ -75,9 +107,11 @@ Basic Usage * Add apps The get-app command gets and installs frappe apps. Examples include - [erpnext](https://github.com/frappe/erpnext) and - [shopping-cart](https://github.com/frappe/shopping-cart) - + + - [erpnext](https://github.com/frappe/erpnext) + - [erpnext_shopify](https://github.com/frappe/erpnext_shopify) + - [paypal_integration](https://github.com/frappe/paypal_integration) + bench get-app erpnext https://github.com/frappe/erpnext * Add site @@ -118,12 +152,6 @@ You can now either use `bench start` or setup the bench for production use. Updating ======== -On initializing a new bench, a cronjob is added to automatically update the bench -at 1000hrs (as per the time on your machine). You can disable this by running -`bench config auto_update off` and run `bench config auto_update on` to switch -it on again. To change the time of update, you will have to edit the cronjob -manually using `crontab -e`. - To manually update the bench, run `bench update` to update all the apps, run patches, build JS and CSS files and restart supervisor (if configured to). @@ -152,7 +180,7 @@ External services ----------------- * MariaDB (Datastore for frappe) - * Redis (Broker for frappe background workers) + * Redis (Queue for frappe background workers and caching) * nginx (for production deployment) * supervisor (for production deployment) @@ -183,10 +211,10 @@ Production Deployment ===================== -You can setup the bench for production use by configuring two programs -, Supervisor and nginx. These steps are automated if you pass -`--setup-production` to the easy install script or run `sudo bench -setup production` +You can setup the bench for production use by configuring two programs, Supervisor and nginx. + +> These steps are automated if you pass `--setup-production` to the easy install script +> or run `sudo bench setup production` Supervisor ---------- @@ -202,13 +230,13 @@ eg, ``` bench setup supervisor -sudo ln -s `pwd`/config/supervisor.conf /etc/supervisor/conf.d/frappe.conf +sudo ln -s `pwd`/config/supervisor.conf /etc/supervisor/conf.d/frappe-bench.conf ``` Note: For CentOS 7, the extension should be `ini`, thus the command becomes ``` bench setup supervisor -sudo ln -s `pwd`/config/supervisor.conf /etc/supervisor/conf.d/frappe.ini #for CentOS 7 only +sudo ln -s `pwd`/config/supervisor.conf /etc/supervisor/conf.d/frappe-bench.ini #for CentOS 7 only ``` The bench will also need to restart the processes managed by supervisor when you @@ -228,7 +256,7 @@ eg, ``` bench setup nginx -sudo ln -s `pwd`/config/nginx.conf /etc/nginx/conf.d/frappe.conf +sudo ln -s `pwd`/config/nginx.conf /etc/nginx/conf.d/frappe-bench.conf ``` Note: When you restart nginx after the configuration change, it might fail if diff --git a/bench/__init__.py b/bench/__init__.py index e69de29b..2e33efe6 100644 --- a/bench/__init__.py +++ b/bench/__init__.py @@ -0,0 +1,5 @@ +from jinja2 import Environment, PackageLoader + +__version__ = "2.0.0" + +env = Environment(loader=PackageLoader('bench.config'), trim_blocks=True) diff --git a/bench/app.py b/bench/app.py index 9032f540..550e8a3a 100644 --- a/bench/app.py +++ b/bench/app.py @@ -1,5 +1,7 @@ import os -from .utils import exec_cmd, get_frappe, check_git_for_shallow_clone, get_config, build_assets, restart_supervisor_processes, get_cmd_output, run_frappe_cmd +from .utils import (exec_cmd, get_frappe, check_git_for_shallow_clone, build_assets, + restart_supervisor_processes, get_cmd_output, run_frappe_cmd) +from .config.common_site_config import get_config import logging import requests @@ -42,7 +44,7 @@ def write_appstxt(apps, bench='.'): def get_app(app, git_url, branch=None, bench='.', build_asset_files=True, verbose=False): logger.info('getting app {}'.format(app)) - shallow_clone = '--depth 1' if check_git_for_shallow_clone() and get_config().get('shallow_clone') else '' + shallow_clone = '--depth 1' if check_git_for_shallow_clone() else '' branch = '--branch {branch}'.format(branch=branch) if branch else '' exec_cmd("git clone {git_url} {branch} {shallow_clone} --origin upstream {app}".format( git_url=git_url, @@ -54,7 +56,7 @@ def get_app(app, git_url, branch=None, bench='.', build_asset_files=True, verbos install_app(app, bench=bench, verbose=verbose) if build_asset_files: build_assets(bench=bench) - conf = get_config() + conf = get_config(bench=bench) if conf.get('restart_supervisor_on_update'): restart_supervisor_processes(bench=bench) @@ -81,7 +83,7 @@ def install_app(app, bench='.', verbose=False): add_to_appstxt(app, bench=bench) def pull_all_apps(bench='.'): - rebase = '--rebase' if get_config().get('rebase_on_pull') else '' + rebase = '--rebase' if get_config(bench).get('rebase_on_pull') else '' for app in get_apps(bench=bench): app_dir = get_repo_dir(app, bench=bench) diff --git a/bench/cli.py b/bench/cli.py index 8a8de86b..3a53cd40 100644 --- a/bench/cli.py +++ b/bench/cli.py @@ -1,40 +1,13 @@ import click -from .utils import init as _init -from .utils import setup_env as _setup_env -from .utils import new_site as _new_site -from .utils import setup_backups as _setup_backups -from .utils import setup_auto_update as _setup_auto_update -from .utils import setup_sudoers as _setup_sudoers -from .utils import start as _start -from .utils import setup_procfile as _setup_procfile -from .utils import set_nginx_port as _set_nginx_port -from .utils import set_url_root as _set_url_root -from .utils import set_default_site as _set_default_site -from .utils import (build_assets, patch_sites, exec_cmd, update_bench, get_env_cmd, get_frappe, setup_logging, - get_config, update_config, restart_supervisor_processes, put_config, default_config, update_requirements, - backup_all_sites, backup_site, get_sites, prime_wheel_cache, is_root, set_mariadb_host, drop_privileges, - fix_file_perms, fix_prod_setup_perms, set_ssl_certificate, set_ssl_certificate_key, get_cmd_output, post_upgrade, - pre_upgrade, validate_upgrade, PatchError, download_translations_p, setup_socketio, before_update) -from .app import get_app as _get_app -from .app import new_app as _new_app -from .app import pull_all_apps, get_apps, get_current_frappe_version, is_version_upgrade, switch_to_v4, switch_to_v5, switch_to_master, switch_to_develop -from .config import generate_nginx_config, generate_supervisor_config, generate_redis_cache_config, generate_redis_async_broker_config -from .production_setup import setup_production as _setup_production -from .migrate_to_v5 import migrate_to_v5 -import os -import sys -import logging -import copy -import json -import pwd -import grp -import subprocess +import os, sys, logging, json, pwd, subprocess +from bench.utils import is_root, PatchError, drop_privileges, get_env_cmd, get_cmd_output, get_frappe +from bench.app import get_apps +from bench.config.common_site_config import get_config +from bench.commands import bench_command logger = logging.getLogger('bench') from_command_line = False -global FRAPPE_VERSION - def cli(): global from_command_line from_command_line = True @@ -42,45 +15,42 @@ def cli(): check_uid() change_dir() change_uid() + if len(sys.argv) > 2 and sys.argv[1] == "frappe": return old_frappe_cli() + elif len(sys.argv) > 1 and sys.argv[1] in get_frappe_commands(): return frappe_cmd() + elif len(sys.argv) > 1 and sys.argv[1] in ("--site", "--verbose", "--force", "--profile"): return frappe_cmd() + elif len(sys.argv) > 1 and sys.argv[1]=="--help": - print click.Context(bench).get_help() + print click.Context(bench_command).get_help() print print get_frappe_help() return + elif len(sys.argv) > 1 and sys.argv[1] in get_apps(): return app_cmd() + else: try: - bench() + # NOTE: this is the main bench command + bench_command() except PatchError: sys.exit(1) -def cmd_requires_root(): - if len(sys.argv) > 2 and sys.argv[2] in ('production', 'sudoers'): - return True - if len(sys.argv) > 2 and sys.argv[1] in ('patch',): - return True - def check_uid(): if cmd_requires_root() and not is_root(): print 'superuser privileges required for this command' sys.exit(1) -def change_uid(): - if is_root() and not cmd_requires_root(): - frappe_user = get_config().get('frappe_user') - if frappe_user: - drop_privileges(uid_name=frappe_user, gid_name=frappe_user) - os.environ['HOME'] = pwd.getpwnam(frappe_user).pw_dir - else: - print 'You should not run this command as root' - sys.exit(1) +def cmd_requires_root(): + if len(sys.argv) > 2 and sys.argv[2] in ('production', 'sudoers'): + return True + if len(sys.argv) > 2 and sys.argv[1] in ('patch',): + return True def change_dir(): if os.path.exists('config.json') or "init" in sys.argv: @@ -92,6 +62,16 @@ def change_dir(): if os.path.exists(dir_path): os.chdir(dir_path) +def change_uid(): + if is_root() and not cmd_requires_root(): + frappe_user = get_config(".").get('frappe_user') + if frappe_user: + drop_privileges(uid_name=frappe_user, gid_name=frappe_user) + os.environ['HOME'] = pwd.getpwnam(frappe_user).pw_dir + else: + print 'You should not run this command as root' + sys.exit(1) + def old_frappe_cli(bench='.'): f = get_frappe(bench=bench) os.chdir(os.path.join(bench, 'sites')) @@ -127,505 +107,3 @@ def get_frappe_help(bench='.'): return "Framework commands:\n" + out.split('Commands:')[1] except subprocess.CalledProcessError: return "" - -@click.command() -def shell(bench='.'): - if not os.environ.get('SHELL'): - print "Cannot get shell" - sys.exit(1) - if not os.path.exists('sites'): - print "sites dir doesn't exist" - sys.exit(1) - env = copy.copy(os.environ) - env['PS1'] = '(' + os.path.basename(os.path.dirname(os.path.abspath(__file__))) + ')' + env.get('PS1', '') - env['PATH'] = os.path.dirname(os.path.abspath(os.path.join('env','bin')) + ':' + env['PATH']) - os.chdir('sites') - os.execve(env['SHELL'], [env['SHELL']], env) - -@click.group() -def bench(bench='.'): - "Bench manager for Frappe" - # TODO add bench path context - global FRAPPE_VERSION - FRAPPE_VERSION = get_current_frappe_version() - setup_logging(bench=bench) - -@click.command() -@click.argument('path') -@click.option('--apps_path', default=None, help="path to json files with apps to install after init") -@click.option('--frappe-path', default=None, help="path to frappe repo") -@click.option('--frappe-branch', default=None, help="path to frappe repo") -@click.option('--no-procfile', is_flag=True, help="Pull changes in all the apps in bench") -@click.option('--no-backups',is_flag=True, help="Run migrations for all sites in the bench") -@click.option('--no-auto-update',is_flag=True, help="Build JS and CSS artifacts for the bench") -@click.option('--verbose',is_flag=True, help="Verbose output during install") -def init(path, apps_path, frappe_path, frappe_branch, no_procfile, no_backups, - no_auto_update, verbose): - "Create a new bench" - _init(path, apps_path=apps_path, no_procfile=no_procfile, no_backups=no_backups, - no_auto_update=no_auto_update, frappe_path=frappe_path, frappe_branch=frappe_branch, verbose=verbose) - click.echo('Bench {} initialized'.format(path)) - -@click.command('get-app') -@click.argument('name') -@click.argument('git-url') -@click.option('--branch', default=None, help="branch to checkout") -def get_app(name, git_url, branch): - "clone an app from the internet and set it up in your bench" - _get_app(name, git_url, branch=branch) - -@click.command('new-app') -@click.argument('app-name') -def new_app(app_name): - "start a new app" - _new_app(app_name) - -@click.command('new-site') -@click.option('--mariadb-root-password', help="MariaDB root password") -@click.option('--admin-password', help="admin password to set for site") -@click.argument('site') -def new_site(site, mariadb_root_password=None, admin_password=None): - "Create a new site in the bench" - _new_site(site, mariadb_root_password=mariadb_root_password, admin_password=admin_password) - -#TODO: Not DRY -@click.command('update') -@click.option('--pull', is_flag=True, help="Pull changes in all the apps in bench") -@click.option('--patch',is_flag=True, help="Run migrations for all sites in the bench") -@click.option('--build',is_flag=True, help="Build JS and CSS artifacts for the bench") -@click.option('--bench',is_flag=True, help="Update bench") -@click.option('--requirements',is_flag=True, help="Update requirements") -@click.option('--restart-supervisor',is_flag=True, help="restart supervisor processes after update") -@click.option('--auto',is_flag=True) -@click.option('--upgrade',is_flag=True) -@click.option('--no-backup',is_flag=True) -@click.option('--force',is_flag=True) -def _update(pull=False, patch=False, build=False, bench=False, auto=False, restart_supervisor=False, requirements=False, no_backup=False, upgrade=False, force=False): - "Update bench" - - if not (pull or patch or build or bench or requirements): - pull, patch, build, bench, requirements = True, True, True, True, True - - conf = get_config() - - version_upgrade = is_version_upgrade() - - if version_upgrade[0] and not upgrade: - print - print - print "This update will cause a major version change in Frappe/ERPNext from {0} to {1}.".format(*version_upgrade[1:]) - print "This would take significant time to migrate and might break custom apps. Please run `bench update --upgrade` to confirm." - print - print "You can stay on the latest stable release by running `bench switch-to-master` or pin your bench to {0} by running `bench switch-to-v{0}`".format(version_upgrade[1]) - sys.exit(1) - - if conf.get('release_bench'): - print 'Release bench, cannot update' - sys.exit(1) - - if auto: - sys.exit(1) - - if bench and conf.get('update_bench_on_update'): - update_bench() - restart_update({ - 'pull': pull, - 'patch': patch, - 'build': build, - 'requirements': requirements, - 'no-backup': no_backup, - 'restart-supervisor': restart_supervisor, - 'upgrade': upgrade - }) - - update(pull, patch, build, bench, auto, restart_supervisor, requirements, no_backup, upgrade, force=force) - -def update(pull=False, patch=False, build=False, bench=False, auto=False, restart_supervisor=False, requirements=False, no_backup=False, upgrade=False, bench_path='.', force=False): - conf = get_config(bench=bench_path) - version_upgrade = is_version_upgrade(bench=bench_path) - - if version_upgrade[0] and not upgrade: - raise Exception("Major Version Upgrade") - - if upgrade and (version_upgrade[0] or (not version_upgrade[0] and force)): - validate_upgrade(version_upgrade[1], version_upgrade[2], bench=bench_path) - - before_update(bench=bench_path, requirements=requirements) - - if pull: - pull_all_apps(bench=bench_path) - - if requirements: - update_requirements(bench=bench_path) - - if upgrade and (version_upgrade[0] or (not version_upgrade[0] and force)): - pre_upgrade(version_upgrade[1], version_upgrade[2], bench=bench_path) - import utils, app - reload(utils) - reload(app) - - if patch: - if not no_backup: - backup_all_sites(bench=bench_path) - patch_sites(bench=bench_path) - if build: - build_assets(bench=bench_path) - if upgrade and (version_upgrade[0] or (not version_upgrade[0] and force)): - post_upgrade(version_upgrade[1], version_upgrade[2], bench=bench_path) - if restart_supervisor or conf.get('restart_supervisor_on_update'): - restart_supervisor_processes(bench=bench_path) - - print "_"*80 - print "Bench: Open source installer + admin for Frappe and ERPNext (https://erpnext.com)" - print - -@click.command('retry-upgrade') -@click.option('--version', default=5) -def retry_upgrade(version): - pull_all_apps() - patch_sites() - build_assets() - post_upgrade(version-1, version) - -def restart_update(kwargs): - args = ['--'+k for k, v in kwargs.items() if v] - os.execv(sys.argv[0], sys.argv[:2] + args) - -@click.command('restart') -def restart(): - "Restart supervisor processes" - restart_supervisor_processes() - -@click.command('start') -@click.option('--no-dev', is_flag=True) -def start(no_dev=False): - "Start Frappe development processes" - _start(no_dev=no_dev) - -@click.command('migrate-3to4') -@click.argument('path') -def migrate_3to4(path): - "Migrate from ERPNext v3.x" - exec_cmd("{python} {migrate_3to4} {site}".format( - python=os.path.join('env', 'bin', 'python'), - migrate_3to4=os.path.join(os.path.dirname(__file__), 'migrate3to4.py'), - site=path)) - -@click.command('switch-to-master') -@click.option('--upgrade',is_flag=True) -def _switch_to_master(upgrade=False): - "Switch frappe and erpnext to master branch" - switch_to_master(upgrade=upgrade) - print - print 'Switched to master' - print 'Please run `bench update --patch` to be safe from any differences in database schema' - -@click.command('switch-to-develop') -@click.option('--upgrade',is_flag=True) -def _switch_to_develop(upgrade=False): - "Switch frappe and erpnext to develop branch" - switch_to_develop(upgrade=upgrade) - print - print 'Switched to develop' - print 'Please run `bench update --patch` to be safe from any differences in database schema' - -@click.command('switch-to-v4') -@click.option('--upgrade',is_flag=True) -def _switch_to_v4(upgrade=False): - "Switch frappe and erpnext to v4 branch" - switch_to_v4(upgrade=upgrade) - print - print 'Switched to v4' - print 'Please run `bench update --patch` to be safe from any differences in database schema' - -@click.command('switch-to-v5') -@click.option('--upgrade',is_flag=True) -def _switch_to_v5(upgrade=False): - "Switch frappe and erpnext to v4 branch" - switch_to_v5(upgrade=upgrade) - print - print 'Switched to v5' - print 'Please run `bench update --patch` to be safe from any differences in database schema' - -@click.command('set-nginx-port') -@click.argument('site') -@click.argument('port', type=int) -def set_nginx_port(site, port): - "Set nginx port for site" - _set_nginx_port(site, port) - -@click.command('set-ssl-certificate') -@click.argument('site') -@click.argument('ssl-certificate-path') -def _set_ssl_certificate(site, ssl_certificate_path): - "Set ssl certificate path for site" - set_ssl_certificate(site, ssl_certificate_path) - -@click.command('set-ssl-key') -@click.argument('site') -@click.argument('ssl-certificate-key-path') -def _set_ssl_certificate_key(site, ssl_certificate_key_path): - "Set ssl certificate private key path for site" - set_ssl_certificate_key(site, ssl_certificate_key_path) - -@click.command('set-url-root') -@click.argument('site') -@click.argument('url-root') -def set_url_root(site, url_root): - "Set url root for site" - _set_url_root(site, url_root) - -@click.command('set-mariadb-host') -@click.argument('host') -def _set_mariadb_host(host): - "Set MariaDB host for bench" - set_mariadb_host(host) - -@click.command('set-default-site') -@click.argument('site') -def set_default_site(site): - "Set default site for bench" - _set_default_site(site) - -@click.command('backup') -@click.argument('site') -def _backup_site(site): - "backup site" - if not site in get_sites(bench='.'): - print 'site not found' - sys.exit(1) - backup_site(site, bench='.') - -@click.command('backup-all-sites') -def _backup_all_sites(): - "backup all sites" - backup_all_sites(bench='.') - -@click.command('prime-wheel-cache') -def _prime_wheel_cache(): - "Update wheel cache" - prime_wheel_cache(bench='.') - -@click.command('release') -@click.argument('app', type=click.Choice(['frappe', 'erpnext', 'erpnext_shopify', 'paypal_integration'])) -@click.argument('bump-type', type=click.Choice(['major', 'minor', 'patch'])) -@click.option('--develop', default='develop') -@click.option('--master', default='master') -def _release(app, bump_type, develop, master): - "Release app (internal to the Frappe team)" - from .release import release - repo = os.path.join('apps', app) - release(repo, bump_type, develop, master) - -## Setup -@click.group() -def setup(): - "Setup bench" - pass - -@click.command('sudoers') -@click.argument('user') -def setup_sudoers(user): - "Add commands to sudoers list for execution without password" - _setup_sudoers(user) - -@click.command('nginx') -def setup_nginx(): - "generate config for nginx" - generate_nginx_config() - -@click.command('supervisor') -def setup_supervisor(): - "generate config for supervisor" - generate_supervisor_config() - -@click.command('redis-cache') -def setup_redis_cache(): - "generate config for redis cache" - generate_redis_cache_config() - -@click.command('redis-async-broker') -def setup_redis_async_broker(): - "generate config for redis async broker" - generate_redis_async_broker_config() - -@click.command('production') -@click.argument('user') -def setup_production(user): - "setup bench for production" - _setup_production(user=user) - -@click.command('auto-update') -def setup_auto_update(): - "Add cronjob for bench auto update" - _setup_auto_update() - -@click.command('backups') -def setup_backups(): - "Add cronjob for bench backups" - _setup_backups() - -@click.command('dnsmasq') -def setup_dnsmasq(): - pass - -@click.command('env') -def setup_env(): - "Setup virtualenv for bench" - _setup_env() - -@click.command('procfile') -@click.option('--with-watch', is_flag=True) -@click.option('--with-celery-broker', is_flag=True) -def setup_procfile(with_celery_broker, with_watch): - "Setup Procfile for bench start" - _setup_procfile(with_celery_broker, with_watch) - -@click.command('socketio') -def _setup_socketio(): - "Setup node deps for socketio server" - setup_socketio() - -@click.command('config') -def setup_config(): - "overwrite or make config.json" - put_config(default_config) - -setup.add_command(setup_nginx) -setup.add_command(setup_sudoers) -setup.add_command(setup_supervisor) -setup.add_command(setup_redis_cache) -setup.add_command(setup_redis_async_broker) -setup.add_command(setup_auto_update) -setup.add_command(setup_dnsmasq) -setup.add_command(setup_backups) -setup.add_command(setup_env) -setup.add_command(setup_procfile) -setup.add_command(_setup_socketio) -setup.add_command(setup_config) -setup.add_command(setup_production) - -## Config -## Not DRY -@click.group() -def config(): - "change bench configuration" - pass - -@click.command('auto_update') -@click.argument('state', type=click.Choice(['on', 'off'])) -def config_auto_update(state): - "Enable/Disable auto update for bench" - state = True if state == 'on' else False - update_config({'auto_update': state}) - -@click.command('restart_supervisor_on_update') -@click.argument('state', type=click.Choice(['on', 'off'])) -def config_restart_supervisor_on_update(state): - "Enable/Disable auto restart of supervisor processes" - state = True if state == 'on' else False - update_config({'restart_supervisor_on_update': state}) - -@click.command('update_bench_on_update') -@click.argument('state', type=click.Choice(['on', 'off'])) -def config_update_bench_on_update(state): - "Enable/Disable bench updates on running bench update" - state = True if state == 'on' else False - update_config({'update_bench_on_update': state}) - -@click.command('dns_multitenant') -@click.argument('state', type=click.Choice(['on', 'off'])) -def config_dns_multitenant(state): - "Enable/Disable bench updates on running bench update" - state = True if state == 'on' else False - update_config({'dns_multitenant': state}) - -@click.command('serve_default_site') -@click.argument('state', type=click.Choice(['on', 'off'])) -def config_serve_default_site(state): - "Configure nginx to serve the default site on port 80" - state = True if state == 'on' else False - update_config({'serve_default_site': state}) - -@click.command('rebase_on_pull') -@click.argument('state', type=click.Choice(['on', 'off'])) -def config_rebase_on_pull(state): - "Rebase repositories on pulling" - state = True if state == 'on' else False - update_config({'rebase_on_pull': state}) - -@click.command('http_timeout') -@click.argument('seconds', type=int) -def config_http_timeout(seconds): - "set http timeout" - update_config({'http_timeout': seconds}) - -config.add_command(config_auto_update) -config.add_command(config_update_bench_on_update) -config.add_command(config_restart_supervisor_on_update) -config.add_command(config_dns_multitenant) -config.add_command(config_serve_default_site) -config.add_command(config_http_timeout) - - -@click.group() -def patch(): - pass - -@click.command('fix-prod-perms') -def _fix_prod_perms(): - "Fix permissions if supervisor processes were run as root" - if os.path.exists("config/supervisor.conf"): - exec_cmd("supervisorctl stop frappe:") - - fix_prod_setup_perms() - - if os.path.exists("config/supervisor.conf"): - exec_cmd("{bench} setup supervisor".format(bench=sys.argv[0])) - exec_cmd("supervisorctl reload") - - -@click.command('fix-file-perms') -def _fix_file_perms(): - "Fix file permissions" - fix_file_perms() - -patch.add_command(_fix_file_perms) -patch.add_command(_fix_prod_perms) - - -@click.command('download-translations') -def _download_translations(): - "Download latest translations" - download_translations_p() - -#Bench commands - -bench.add_command(init) -bench.add_command(get_app) -bench.add_command(new_app) -bench.add_command(new_site) -bench.add_command(setup) -bench.add_command(_update) -bench.add_command(restart) -bench.add_command(config) -bench.add_command(start) -bench.add_command(set_nginx_port) -bench.add_command(_set_ssl_certificate) -bench.add_command(_set_ssl_certificate_key) -bench.add_command(_set_mariadb_host) -bench.add_command(set_default_site) -bench.add_command(migrate_3to4) -bench.add_command(_switch_to_master) -bench.add_command(_switch_to_develop) -bench.add_command(_switch_to_v4) -bench.add_command(_switch_to_v5) -bench.add_command(shell) -bench.add_command(_backup_all_sites) -bench.add_command(_backup_site) -bench.add_command(_prime_wheel_cache) -bench.add_command(_release) -bench.add_command(patch) -bench.add_command(set_url_root) -bench.add_command(retry_upgrade) -bench.add_command(_download_translations) diff --git a/bench/commands/__init__.py b/bench/commands/__init__.py new file mode 100644 index 00000000..d66c05c6 --- /dev/null +++ b/bench/commands/__init__.py @@ -0,0 +1,53 @@ +import click +global FRAPPE_VERSION + +@click.group() +def bench_command(bench='.'): + "Bench manager for Frappe" + from bench.app import get_current_frappe_version + from bench.utils import setup_logging + + # TODO add bench path context + global FRAPPE_VERSION + FRAPPE_VERSION = get_current_frappe_version() + setup_logging(bench=bench) + + +from bench.commands.make import init, get_app, new_app, new_site +bench_command.add_command(init) +bench_command.add_command(get_app) +bench_command.add_command(new_app) +bench_command.add_command(new_site) + + +from bench.commands.update import update, retry_upgrade, switch_to_master, switch_to_develop, switch_to_v4, switch_to_v5 +bench_command.add_command(update) +bench_command.add_command(retry_upgrade) +bench_command.add_command(switch_to_master) +bench_command.add_command(switch_to_develop) +bench_command.add_command(switch_to_v4) +bench_command.add_command(switch_to_v5) + + +from bench.commands.utils import (start, restart, set_nginx_port, set_ssl_certificate, set_ssl_certificate_key, set_url_root, + set_mariadb_host, set_default_site, download_translations, shell, backup_site, backup_all_sites, release) +bench_command.add_command(start) +bench_command.add_command(restart) +bench_command.add_command(set_nginx_port) +bench_command.add_command(set_ssl_certificate) +bench_command.add_command(set_ssl_certificate_key) +bench_command.add_command(set_url_root) +bench_command.add_command(set_mariadb_host) +bench_command.add_command(set_default_site) +bench_command.add_command(download_translations) +bench_command.add_command(shell) +bench_command.add_command(backup_site) +bench_command.add_command(backup_all_sites) +bench_command.add_command(release) + + +from bench.commands.setup import setup +bench_command.add_command(setup) + +from bench.commands.config import config +bench_command.add_command(config) diff --git a/bench/commands/config.py b/bench/commands/config.py new file mode 100644 index 00000000..b50a1f71 --- /dev/null +++ b/bench/commands/config.py @@ -0,0 +1,72 @@ +import click +from bench.config.common_site_config import update_config + +## Config +## Not DRY +@click.group() +def config(): + "change bench configuration" + pass + + +@click.command('auto_update') +@click.argument('state', type=click.Choice(['on', 'off'])) +def config_auto_update(state): + "Enable/Disable auto update for bench" + state = True if state == 'on' else False + update_config({'auto_update': state}) + + +@click.command('restart_supervisor_on_update') +@click.argument('state', type=click.Choice(['on', 'off'])) +def config_restart_supervisor_on_update(state): + "Enable/Disable auto restart of supervisor processes" + state = True if state == 'on' else False + update_config({'restart_supervisor_on_update': state}) + + +@click.command('update_bench_on_update') +@click.argument('state', type=click.Choice(['on', 'off'])) +def config_update_bench_on_update(state): + "Enable/Disable bench updates on running bench update" + state = True if state == 'on' else False + update_config({'update_bench_on_update': state}) + + +@click.command('dns_multitenant') +@click.argument('state', type=click.Choice(['on', 'off'])) +def config_dns_multitenant(state): + "Enable/Disable bench updates on running bench update" + state = True if state == 'on' else False + update_config({'dns_multitenant': state}) + + +@click.command('serve_default_site') +@click.argument('state', type=click.Choice(['on', 'off'])) +def config_serve_default_site(state): + "Configure nginx to serve the default site on port 80" + state = True if state == 'on' else False + update_config({'serve_default_site': state}) + + +@click.command('rebase_on_pull') +@click.argument('state', type=click.Choice(['on', 'off'])) +def config_rebase_on_pull(state): + "Rebase repositories on pulling" + state = True if state == 'on' else False + update_config({'rebase_on_pull': state}) + + +@click.command('http_timeout') +@click.argument('seconds', type=int) +def config_http_timeout(seconds): + "set http timeout" + update_config({'http_timeout': seconds}) + + +config.add_command(config_auto_update) +config.add_command(config_update_bench_on_update) +config.add_command(config_restart_supervisor_on_update) +config.add_command(config_dns_multitenant) +config.add_command(config_serve_default_site) +config.add_command(config_http_timeout) diff --git a/bench/commands/make.py b/bench/commands/make.py new file mode 100644 index 00000000..a7c9c712 --- /dev/null +++ b/bench/commands/make.py @@ -0,0 +1,48 @@ +import click + +@click.command() +@click.argument('path') +@click.option('--apps_path', default=None, help="path to json files with apps to install after init") +@click.option('--frappe-path', default=None, help="path to frappe repo") +@click.option('--frappe-branch', default=None, help="path to frappe repo") +@click.option('--no-procfile', is_flag=True, help="Pull changes in all the apps in bench") +@click.option('--no-backups',is_flag=True, help="Run migrations for all sites in the bench") +@click.option('--no-auto-update',is_flag=True, help="Build JS and CSS artifacts for the bench") +@click.option('--verbose',is_flag=True, help="Verbose output during install") +def init(path, apps_path, frappe_path, frappe_branch, no_procfile, no_backups, + no_auto_update, verbose): + "Create a new bench" + from bench.utils import init + init(path, apps_path=apps_path, no_procfile=no_procfile, no_backups=no_backups, + no_auto_update=no_auto_update, frappe_path=frappe_path, frappe_branch=frappe_branch, verbose=verbose) + click.echo('Bench {} initialized'.format(path)) + + +@click.command('get-app') +@click.argument('name') +@click.argument('git-url') +@click.option('--branch', default=None, help="branch to checkout") +def get_app(name, git_url, branch): + "clone an app from the internet and set it up in your bench" + from bench.app import get_app + get_app(name, git_url, branch=branch) + + +@click.command('new-app') +@click.argument('app-name') +def new_app(app_name): + "start a new app" + from bench.app import new_app + new_app(app_name) + + +@click.command('new-site') +@click.option('--mariadb-root-password', help="MariaDB root password") +@click.option('--admin-password', help="admin password to set for site") +@click.argument('site') +def new_site(site, mariadb_root_password=None, admin_password=None): + "Create a new site in the bench" + from bench.utils import new_site + new_site(site, mariadb_root_password=mariadb_root_password, admin_password=admin_password) + + diff --git a/bench/commands/setup.py b/bench/commands/setup.py new file mode 100644 index 00000000..c62876a7 --- /dev/null +++ b/bench/commands/setup.py @@ -0,0 +1,97 @@ +import click + +@click.group() +def setup(): + "Setup bench" + pass + + +@click.command('sudoers') +@click.argument('user') +def setup_sudoers(user): + "Add commands to sudoers list for execution without password" + from bench.utils import setup_sudoers + setup_sudoers(user) + + +@click.command('nginx') +def setup_nginx(): + "generate config for nginx" + from bench.config.nginx import make_nginx_conf + make_nginx_conf(bench_path=".") + + +@click.command('supervisor') +def setup_supervisor(): + "generate config for supervisor" + from bench.config.supervisor import generate_supervisor_config + generate_supervisor_config(bench_path=".") + + +@click.command('redis') +def setup_redis(): + "generate config for redis cache" + from bench.config.redis import generate_config + generate_config('.') + + +@click.command('production') +@click.argument('user') +def setup_production(user): + "setup bench for production" + from bench.config.production_setup import setup_production + setup_production(user=user) + + +@click.command('auto-update') +def setup_auto_update(): + "Add cronjob for bench auto update" + from bench.utils import setup_auto_update + setup_auto_update() + + +@click.command('backups') +def setup_backups(): + "Add cronjob for bench backups" + from bench.utils import setup_backups + setup_backups() + +@click.command('env') +def setup_env(): + "Setup virtualenv for bench" + from bench.utils import setup_env + setup_env() + + +@click.command('procfile') +def setup_procfile(): + "Setup Procfile for bench start" + from bench.config.procfile import setup_procfile + setup_procfile('.') + + +@click.command('socketio') +def setup_socketio(): + "Setup node deps for socketio server" + from bench.utils import setup_socketio + setup_socketio() + + +@click.command('config') +def setup_config(): + "overwrite or make config.json" + from bench.config.common_site_config import make_config + make_config('.') + + +setup.add_command(setup_sudoers) +setup.add_command(setup_nginx) +setup.add_command(setup_supervisor) +setup.add_command(setup_redis) +setup.add_command(setup_production) +setup.add_command(setup_auto_update) +setup.add_command(setup_backups) +setup.add_command(setup_env) +setup.add_command(setup_procfile) +setup.add_command(setup_socketio) +setup.add_command(setup_config) diff --git a/bench/commands/update.py b/bench/commands/update.py new file mode 100644 index 00000000..1b66236e --- /dev/null +++ b/bench/commands/update.py @@ -0,0 +1,158 @@ +import click +import sys, os +from bench.config.common_site_config import get_config, deprecate_old_config +from bench.app import pull_all_apps, is_version_upgrade +from bench.utils import (update_bench, validate_upgrade, pre_upgrade, post_upgrade, before_update, + update_requirements, backup_all_sites, patch_sites, build_assets, restart_supervisor_processes) + +#TODO: Not DRY +@click.command('update') +@click.option('--pull', is_flag=True, help="Pull changes in all the apps in bench") +@click.option('--patch',is_flag=True, help="Run migrations for all sites in the bench") +@click.option('--build',is_flag=True, help="Build JS and CSS artifacts for the bench") +@click.option('--bench',is_flag=True, help="Update bench") +@click.option('--requirements',is_flag=True, help="Update requirements") +@click.option('--restart-supervisor',is_flag=True, help="restart supervisor processes after update") +@click.option('--auto',is_flag=True) +@click.option('--upgrade',is_flag=True) +@click.option('--no-backup',is_flag=True) +@click.option('--force',is_flag=True) +def update(pull=False, patch=False, build=False, bench=False, auto=False, restart_supervisor=False, requirements=False, no_backup=False, upgrade=False, force=False): + "Update bench" + + if not (pull or patch or build or bench or requirements): + pull, patch, build, bench, requirements = True, True, True, True, True + + deprecate_old_config(".") + + conf = get_config(".") + + version_upgrade = is_version_upgrade() + + if version_upgrade[0] and not upgrade: + print + print + print "This update will cause a major version change in Frappe/ERPNext from {0} to {1}.".format(*version_upgrade[1:]) + print "This would take significant time to migrate and might break custom apps. Please run `bench update --upgrade` to confirm." + print + print "You can stay on the latest stable release by running `bench switch-to-master` or pin your bench to {0} by running `bench switch-to-v{0}`".format(version_upgrade[1]) + sys.exit(1) + + if conf.get('release_bench'): + print 'Release bench, cannot update' + sys.exit(1) + + if auto: + sys.exit(1) + + if bench and conf.get('update_bench_on_update'): + update_bench() + restart_update({ + 'pull': pull, + 'patch': patch, + 'build': build, + 'requirements': requirements, + 'no-backup': no_backup, + 'restart-supervisor': restart_supervisor, + 'upgrade': upgrade + }) + + _update(pull, patch, build, bench, auto, restart_supervisor, requirements, no_backup, upgrade, force=force) + + +def _update(pull=False, patch=False, build=False, bench=False, auto=False, restart_supervisor=False, requirements=False, no_backup=False, upgrade=False, bench_path='.', force=False): + conf = get_config(bench=bench_path) + version_upgrade = is_version_upgrade(bench=bench_path) + + if version_upgrade[0] and not upgrade: + raise Exception("Major Version Upgrade") + + if upgrade and (version_upgrade[0] or (not version_upgrade[0] and force)): + validate_upgrade(version_upgrade[1], version_upgrade[2], bench=bench_path) + + before_update(bench=bench_path, requirements=requirements) + + if pull: + pull_all_apps(bench=bench_path) + + if requirements: + update_requirements(bench=bench_path) + + if upgrade and (version_upgrade[0] or (not version_upgrade[0] and force)): + pre_upgrade(version_upgrade[1], version_upgrade[2], bench=bench_path) + import utils, app + reload(utils) + reload(app) + + if patch: + if not no_backup: + backup_all_sites(bench=bench_path) + patch_sites(bench=bench_path) + if build: + build_assets(bench=bench_path) + if upgrade and (version_upgrade[0] or (not version_upgrade[0] and force)): + post_upgrade(version_upgrade[1], version_upgrade[2], bench=bench_path) + if restart_supervisor or conf.get('restart_supervisor_on_update'): + restart_supervisor_processes(bench=bench_path) + + print "_"*80 + print "Bench: Open source installer + admin for Frappe and ERPNext (https://erpnext.com)" + print + + +@click.command('retry-upgrade') +@click.option('--version', default=5) +def retry_upgrade(version): + pull_all_apps() + patch_sites() + build_assets() + post_upgrade(version-1, version) + + +def restart_update(kwargs): + args = ['--'+k for k, v in kwargs.items() if v] + os.execv(sys.argv[0], sys.argv[:2] + args) + + +@click.command('switch-to-master') +@click.option('--upgrade',is_flag=True) +def switch_to_master(upgrade=False): + "Switch frappe and erpnext to master branch" + from bench.app import switch_to_master + switch_to_master(upgrade=upgrade) + print + print 'Switched to master' + print 'Please run `bench update --patch` to be safe from any differences in database schema' + + +@click.command('switch-to-develop') +@click.option('--upgrade',is_flag=True) +def switch_to_develop(upgrade=False): + "Switch frappe and erpnext to develop branch" + from bench.app import switch_to_develop + switch_to_develop(upgrade=upgrade) + print + print 'Switched to develop' + print 'Please run `bench update --patch` to be safe from any differences in database schema' + + +@click.command('switch-to-v4') +@click.option('--upgrade',is_flag=True) +def switch_to_v4(upgrade=False): + "Switch frappe and erpnext to v4 branch" + from bench.app import switch_to_v4 + switch_to_v4(upgrade=upgrade) + print + print 'Switched to v4' + print 'Please run `bench update --patch` to be safe from any differences in database schema' + + +@click.command('switch-to-v5') +@click.option('--upgrade',is_flag=True) +def switch_to_v5(upgrade=False): + "Switch frappe and erpnext to v4 branch" + from bench.app import switch_to_v5 + switch_to_v5(upgrade=upgrade) + print + print 'Switched to v5' + print 'Please run `bench update --patch` to be safe from any differences in database schema' diff --git a/bench/commands/utils.py b/bench/commands/utils.py new file mode 100644 index 00000000..c2ddee22 --- /dev/null +++ b/bench/commands/utils.py @@ -0,0 +1,122 @@ +import click +import sys, os, copy + + +@click.command('start') +@click.option('--no-dev', is_flag=True) +def start(no_dev=False): + "Start Frappe development processes" + from bench.utils import start + start(no_dev=no_dev) + + +@click.command('restart') +def restart(): + "Restart supervisor processes" + from bench.utils import restart_supervisor_processes + restart_supervisor_processes() + + +@click.command('set-nginx-port') +@click.argument('site') +@click.argument('port', type=int) +def set_nginx_port(site, port): + "Set nginx port for site" + from bench.utils import set_nginx_port + set_nginx_port(site, port) + + +@click.command('set-ssl-certificate') +@click.argument('site') +@click.argument('ssl-certificate-path') +def set_ssl_certificate(site, ssl_certificate_path): + "Set ssl certificate path for site" + from bench.utils import set_ssl_certificate + set_ssl_certificate(site, ssl_certificate_path) + + +@click.command('set-ssl-key') +@click.argument('site') +@click.argument('ssl-certificate-key-path') +def set_ssl_certificate_key(site, ssl_certificate_key_path): + "Set ssl certificate private key path for site" + from bench.utils import set_ssl_certificate_key + set_ssl_certificate_key(site, ssl_certificate_key_path) + + +@click.command('set-url-root') +@click.argument('site') +@click.argument('url-root') +def set_url_root(site, url_root): + "Set url root for site" + from bench.utils import set_url_root + set_url_root(site, url_root) + + +@click.command('set-mariadb-host') +@click.argument('host') +def set_mariadb_host(host): + "Set MariaDB host for bench" + from bench.utils import set_mariadb_host + set_mariadb_host(host) + + +@click.command('set-default-site') +@click.argument('site') +def set_default_site(site): + "Set default site for bench" + from bench.utils import set_default_site + set_default_site(site) + + +@click.command('download-translations') +def download_translations(): + "Download latest translations" + from bench.utils import download_translations_p + download_translations_p() + + +@click.command() +def shell(bench='.'): + if not os.environ.get('SHELL'): + print "Cannot get shell" + sys.exit(1) + if not os.path.exists('sites'): + print "sites dir doesn't exist" + sys.exit(1) + env = copy.copy(os.environ) + env['PS1'] = '(' + os.path.basename(os.path.dirname(os.path.abspath(__file__))) + ')' + env.get('PS1', '') + env['PATH'] = os.path.dirname(os.path.abspath(os.path.join('env','bin')) + ':' + env['PATH']) + os.chdir('sites') + os.execve(env['SHELL'], [env['SHELL']], env) + + +@click.command('backup') +@click.argument('site') +def backup_site(site): + "backup site" + from bench.utils import get_sites, backup_site + if not site in get_sites(bench='.'): + print 'site not found' + sys.exit(1) + backup_site(site, bench='.') + + +@click.command('backup-all-sites') +def backup_all_sites(): + "backup all sites" + from bench.utils import backup_all_sites + backup_all_sites(bench='.') + + +@click.command('release') +@click.argument('app', type=click.Choice(['frappe', 'erpnext', 'erpnext_shopify', 'paypal_integration'])) +@click.argument('bump-type', type=click.Choice(['major', 'minor', 'patch'])) +@click.option('--develop', default='develop') +@click.option('--master', default='master') +def release(app, bump_type, develop, master): + "Release app (internal to the Frappe team)" + from .release import release + repo = os.path.join('apps', app) + release(repo, bump_type, develop, master) + diff --git a/bench/config.py b/bench/config.py deleted file mode 100644 index 1177aa76..00000000 --- a/bench/config.py +++ /dev/null @@ -1,108 +0,0 @@ -import os -import getpass -import json -import subprocess -import shutil -from distutils.spawn import find_executable -from jinja2 import Environment, PackageLoader -from .utils import get_sites, get_config, update_config, get_redis_version - -env = Environment(loader=PackageLoader('bench', 'templates'), trim_blocks=True) - -def write_config_file(bench, file_name, config): - config_path = os.path.join(bench, 'config') - file_path = os.path.join(config_path, file_name) - number = (len([path for path in os.listdir(config_path) if path.startswith(file_name)]) -1 ) or '' - if number: - number = '.' + str(number) - if os.path.exists(file_path): - shutil.move(file_path, file_path + '.save' + number) - - with open(file_path, 'wb') as f: - f.write(config) - -def generate_supervisor_config(bench='.', user=None): - from .app import get_current_frappe_version - template = env.get_template('supervisor.conf') - bench_dir = os.path.abspath(bench) - sites_dir = os.path.join(bench_dir, "sites") - sites = get_sites(bench=bench) - if not user: - user = getpass.getuser() - config = get_config() - - config = template.render(**{ - "bench_dir": bench_dir, - "sites_dir": sites_dir, - "user": user, - "http_timeout": config.get("http_timeout", 120), - "redis_server": find_executable('redis-server'), - "node": find_executable('node') or find_executable('nodejs'), - "redis_cache_config": os.path.join(bench_dir, 'config', 'redis_cache.conf'), - "redis_async_broker_config": os.path.join(bench_dir, 'config', 'redis_async_broker.conf'), - "frappe_version": get_current_frappe_version() - }) - write_config_file(bench, 'supervisor.conf', config) - update_config({'restart_supervisor_on_update': True}) - -def get_site_config(site, bench='.'): - with open(os.path.join(bench, 'sites', site, 'site_config.json')) as f: - return json.load(f) - -def get_sites_with_config(bench='.'): - sites = get_sites(bench=bench) - ret = [] - for site in sites: - site_config = get_site_config(site, bench=bench) - ret.append({ - "name": site, - "port": site_config.get('nginx_port'), - "ssl_certificate": site_config.get('ssl_certificate'), - "ssl_certificate_key": site_config.get('ssl_certificate_key') - }) - return ret - -def generate_nginx_config(bench='.'): - template = env.get_template('nginx.conf') - bench_dir = os.path.abspath(bench) - sites_dir = os.path.join(bench_dir, "sites") - sites = get_sites_with_config(bench=bench) - user = getpass.getuser() - - if get_config().get('serve_default_site'): - try: - with open("sites/currentsite.txt") as f: - default_site = {'name': f.read().strip()} - except IOError: - default_site = None - else: - default_site = None - - config = template.render(**{ - "sites_dir": sites_dir, - "http_timeout": get_config().get("http_timeout", 120), - "default_site": default_site, - "dns_multitenant": get_config().get('dns_multitenant'), - "sites": sites - }) - write_config_file(bench, 'nginx.conf', config) - -def generate_redis_cache_config(bench='.'): - template = env.get_template('redis_cache.conf') - conf = { - "maxmemory": get_config().get('cache_maxmemory', '50'), - "port": get_config().get('redis_cache_port', '11311'), - "redis_version": get_redis_version() - } - config = template.render(**conf) - write_config_file(bench, 'redis_cache.conf', config) - - -def generate_redis_async_broker_config(bench='.'): - template = env.get_template('redis_async_broker.conf') - conf = { - "port": get_config().get('redis_async_broker_port', '12311'), - "redis_version": get_redis_version() - } - config = template.render(**conf) - write_config_file(bench, 'redis_async_broker.conf', config) diff --git a/bench/config/__init__.py b/bench/config/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/bench/config/common_site_config.py b/bench/config/common_site_config.py new file mode 100644 index 00000000..5e2b14b2 --- /dev/null +++ b/bench/config/common_site_config.py @@ -0,0 +1,145 @@ +import os +import multiprocessing +import getpass +import json +import urlparse + +default_config = { + 'restart_supervisor_on_update': False, + 'auto_update': False, + 'serve_default_site': True, + 'rebase_on_pull': False, + 'update_bench_on_update': True, + 'frappe_user': getpass.getuser(), + 'shallow_clone': True, +} + +def make_config(bench_path): + make_pid_folder(bench_path) + bench_config = get_config(bench_path) + bench_config.update(default_config) + bench_config.update(get_gunicorn_workers()) + update_config_for_frappe(bench_config, bench_path) + + put_config(bench_config, bench_path) + +def get_config(bench): + return get_common_site_config(bench) + +def get_common_site_config(bench_path): + config_path = get_config_path(bench_path) + if not os.path.exists(config_path): + return {} + with open(config_path, 'r') as f: + return json.load(f) + +def put_config(config, bench='.'): + config_path = get_config_path(bench) + with open(config_path, 'w') as f: + return json.dump(config, f, indent=1, sort_keys=True) + +def update_config(new_config, bench='.'): + config = get_config(bench=bench) + config.update(new_config) + put_config(config, bench=bench) + +def get_config_path(bench): + return os.path.join(bench, 'sites', 'common_site_config.json') + +def get_gunicorn_workers(): + '''This function will return the maximum workers that can be started depending upon + number of cpu's present on the machine''' + return { + "gunicorn_workers": multiprocessing.cpu_count() + } + +def update_config_for_frappe(config, bench_path): + ports = make_ports(bench_path) + + for key in ('redis_cache', 'redis_queue', 'redis_socketio'): + if key not in config: + config[key] = "redis://localhost:{0}".format(ports[key]) + + for key in ('webserver_port', 'socketio_port'): + if key not in config: + config[key] = ports[key] + + + # TODO Optionally we need to add the host or domain name in case dns_multitenant is false + +def make_ports(bench_path): + benches_path = os.path.dirname(os.path.abspath(bench_path)) + + default_ports = { + "webserver_port": 8000, + "socketio_port": 9000, + "redis_queue": 11000, + "redis_socketio": 12000, + "redis_cache": 13000 + } + + # collect all existing ports + existing_ports = {} + for folder in os.listdir(benches_path): + bench = os.path.join(benches_path, folder) + if os.path.isdir(bench): + bench_config = get_config(bench) + for key in default_ports.keys(): + value = bench_config.get(key) + + # extract port from redis url + if value and (key in ('redis_cache', 'redis_queue', 'redis_socketio')): + value = urlparse.urlparse(value).port + + if value: + existing_ports.setdefault(key, []).append(value) + + # new port value = max of existing port value + 1 + ports = {} + for key, value in default_ports.items(): + existing_value = existing_ports.get(key, []) + if existing_value: + value = max(existing_value) + 1 + + ports[key] = value + + return ports + +def make_pid_folder(bench_path): + pids_path = os.path.join(bench_path, 'config', 'pids') + if not os.path.exists(pids_path): + os.makedirs(pids_path) + +def deprecate_old_config(bench_path): + # deprecate bench config + bench_config_path = os.path.join(bench_path, 'config.json') + if os.path.exists(bench_config_path): + with open(bench_config_path, "r") as f: + bench_config = json.loads(f.read()) + + common_site_config = get_common_site_config(bench_path) + common_site_config.update(bench_config) + put_config(common_site_config, bench_path) + + # remove bench/config.json + os.remove(bench_config_path) + + # change keys + config = get_config(bench_path) + changed = False + for from_key, to_key, default in ( + ("celery_broker", "redis_queue", "redis://localhost:6379"), + ("async_redis_server", "redis_socketio", "redis://localhost:12311"), + ("cache_redis_server", "redis_cache", "redis://localhost:11311") + ): + if from_key in config: + config[to_key] = config[from_key] + del config[from_key] + changed = True + + elif to_key not in config: + config[to_key] = default + changed = True + + if changed: + put_config(config, bench_path) diff --git a/bench/config/nginx.py b/bench/config/nginx.py new file mode 100644 index 00000000..550d6e59 --- /dev/null +++ b/bench/config/nginx.py @@ -0,0 +1,74 @@ +import os +import json +from bench.utils import get_sites, get_bench_name + +def make_nginx_conf(bench_path): + from bench import env + from bench.config.common_site_config import get_config + + template = env.get_template('nginx.conf') + bench_path = os.path.abspath(bench_path) + sites_path = os.path.join(bench_path, "sites") + + config = get_config(bench_path) + sites = prepare_sites(config, bench_path) + + nginx_conf = template.render(**{ + "sites_path": sites_path, + "http_timeout": config.get("http_timeout"), + "sites": sites, + "webserver_port": config.get('webserver_port'), + "socketio_port": config.get('socketio_port'), + "bench_name": get_bench_name(bench_path) + }) + + with open(os.path.join(bench_path, "config", "nginx.conf"), "w") as f: + f.write(nginx_conf) + +def prepare_sites(config, bench_path): + sites = { + "that_use_dns": [], + "that_use_ssl": [], + "that_use_port": [] + } + ports_in_use = {} + dns_multitenant = config.get('dns_multitenant') + + for site in get_sites_with_config(bench_path=bench_path): + if dns_multitenant: + # assumes site's folder name is same as the domain name + + if site.get("ssl_certificate") and site.get("ssl_certificate_key"): + sites["that_use_ssl"].append(site) + + else: + sites["that_use_dns"].append(site["name"]) + + else: + if not site.get("port"): + site["port"] = 80 + + if site["port"] in ports_in_use: + raise Exception("Port {0} is being used by another site {1}".format(site["port"], ports_in_use[site["port"]])) + + ports_in_use[site["port"]] = site["name"] + sites["that_use_port"].append(site) + + return sites + +def get_sites_with_config(bench_path): + sites = get_sites(bench=bench_path) + ret = [] + for site in sites: + site_config = get_site_config(site, bench_path=bench_path) + ret.append({ + "name": site, + "port": site_config.get('nginx_port'), + "ssl_certificate": site_config.get('ssl_certificate'), + "ssl_certificate_key": site_config.get('ssl_certificate_key') + }) + return ret + +def get_site_config(site, bench_path='.'): + with open(os.path.join(bench_path, 'sites', site, 'site_config.json')) as f: + return json.load(f) diff --git a/bench/config/procfile.py b/bench/config/procfile.py new file mode 100644 index 00000000..ec5c0141 --- /dev/null +++ b/bench/config/procfile.py @@ -0,0 +1,9 @@ +import bench, os +from bench.utils import find_executable + +def setup_procfile(bench_path): + procfile = bench.env.get_template('Procfile').render(node=find_executable("node") \ + or find_executable("nodejs")) + + with open(os.path.join(bench_path, 'Procfile'), 'w') as f: + f.write(procfile) diff --git a/bench/production_setup.py b/bench/config/production_setup.py similarity index 65% rename from bench/production_setup.py rename to bench/config/production_setup.py index f5df633f..f29fe69f 100644 --- a/bench/production_setup.py +++ b/bench/config/production_setup.py @@ -1,8 +1,31 @@ -from .utils import get_program, exec_cmd, get_cmd_output, fix_prod_setup_perms, get_config -from .config import generate_nginx_config, generate_supervisor_config -from jinja2 import Environment, PackageLoader +from bench.utils import get_program, exec_cmd, get_cmd_output, fix_prod_setup_perms, get_bench_name +from bench.config.supervisor import generate_supervisor_config +from bench.config.nginx import make_nginx_conf import os -import shutil + +def setup_production(user, bench='.'): + generate_supervisor_config(bench_path=bench, user=user) + make_nginx_conf(bench_path=bench) + fix_prod_setup_perms(bench, frappe_user=user) + remove_default_nginx_configs() + + bench_name = get_bench_name(bench) + nginx_conf = '/etc/nginx/conf.d/{bench_name}.conf'.format(bench_name=bench_name) + + supervisor_conf_extn = "ini" if is_centos7() else "conf" + supervisor_conf = os.path.join(get_supervisor_confdir(), '{bench_name}.{extn}'.format( + bench_name=bench_name, extn=supervisor_conf_extn)) + + + os.symlink(os.path.abspath(os.path.join(bench, 'config', 'supervisor.conf')), supervisor_conf) + os.symlink(os.path.abspath(os.path.join(bench, 'config', 'nginx.conf')), nginx_conf) + + exec_cmd('supervisorctl reload') + if os.environ.get('NO_SERVICE_RESTART'): + return + + restart_service('nginx') + def restart_service(service): if os.path.basename(get_program(['systemctl']) or '') == 'systemctl' and is_running_systemd(): @@ -45,33 +68,3 @@ def is_running_systemd(): elif comm == "systemd": return True return False - -def copy_default_nginx_config(): - shutil.copy(os.path.join(os.path.dirname(__file__), 'templates', 'nginx_default.conf'), '/etc/nginx/nginx.conf') - -def setup_production(user, bench='.'): - generate_supervisor_config(bench=bench, user=user) - generate_nginx_config(bench=bench) - fix_prod_setup_perms(frappe_user=user) - remove_default_nginx_configs() - - if is_centos7(): - supervisor_conf_filename = 'frappe.ini' - copy_default_nginx_config() - else: - supervisor_conf_filename = 'frappe.conf' - - - links = ( - (os.path.abspath(os.path.join(bench, 'config', 'nginx.conf')), '/etc/nginx/conf.d/frappe.conf'), - (os.path.abspath(os.path.join(bench, 'config', 'supervisor.conf')), os.path.join(get_supervisor_confdir(), supervisor_conf_filename)), - ) - - for src, dest in links: - if not os.path.exists(dest): - os.symlink(src, dest) - - exec_cmd('supervisorctl reload') - if os.environ.get('NO_SERVICE_RESTART'): - return - restart_service('nginx') diff --git a/bench/config/redis.py b/bench/config/redis.py new file mode 100644 index 00000000..8f21bd23 --- /dev/null +++ b/bench/config/redis.py @@ -0,0 +1,62 @@ +from .common_site_config import get_config +import re, os, subprocess, urlparse, semantic_version +import bench + +def generate_config(bench_path): + config = get_config(bench_path) + + ports = {} + for key in ('redis_cache', 'redis_queue', 'redis_socketio'): + ports[key] = urlparse.urlparse(config[key]).port + + write_redis_config( + template_name='redis_queue.conf', + context={ + "port": ports['redis_queue'], + "bench_path": os.path.abspath(bench_path), + }, + bench_path=bench_path + ) + + write_redis_config( + template_name='redis_socketio.conf', + context={ + "port": ports['redis_socketio'], + }, + bench_path=bench_path + ) + + write_redis_config( + template_name='redis_cache.conf', + context={ + "maxmemory": config.get('cache_maxmemory', '50'), + "port": ports['redis_cache'], + "redis_version": get_redis_version(), + }, + bench_path=bench_path + ) + + # make pids folder + pid_path = os.path.join(bench_path, "config", "pids") + if not os.path.exists(pid_path): + os.makedirs(pid_path) + +def write_redis_config(template_name, context, bench_path): + template = bench.env.get_template(template_name) + + if "pid_path" not in context: + context["pid_path"] = os.path.abspath(os.path.join(bench_path, "config", "pids")) + + with open(os.path.join(bench_path, 'config', template_name), 'w') as f: + f.write(template.render(**context)) + +def get_redis_version(): + version_string = subprocess.check_output('redis-server --version', shell=True).strip() + + # extract version number from string + version = re.findall("\d+\.\d+", version_string) + if not version: + return None + + version = semantic_version.Version(version[0], partial=True) + return float('{major}.{minor}'.format(major=version.major, minor=version.minor)) diff --git a/bench/config/supervisor.py b/bench/config/supervisor.py new file mode 100644 index 00000000..a7564e24 --- /dev/null +++ b/bench/config/supervisor.py @@ -0,0 +1,36 @@ +import os, getpass, bench + +def generate_supervisor_config(bench_path, user=None): + from bench.app import get_current_frappe_version + from bench.utils import get_bench_name, find_executable + from bench.config.common_site_config import get_config, update_config, get_gunicorn_workers + + template = bench.env.get_template('supervisor.conf') + if not user: + user = getpass.getuser() + + config = get_config(bench=bench_path) + + bench_dir = os.path.abspath(bench_path) + + config = template.render(**{ + "bench_dir": bench_dir, + "sites_dir": os.path.join(bench_dir, 'sites'), + "user": user, + "http_timeout": config.get("http_timeout", 120), + "redis_server": find_executable('redis-server'), + "node": find_executable('node') or find_executable('nodejs'), + "redis_cache_config": os.path.join(bench_dir, 'config', 'redis_cache.conf'), + "redis_socketio_config": os.path.join(bench_dir, 'config', 'redis_socketio.conf'), + "redis_queue_config": os.path.join(bench_dir, 'config', 'redis_queue.conf'), + "frappe_version": get_current_frappe_version(), + "webserver_port": config.get('webserver_port', 8000), + "gunicorn_workers": config.get('gunicorn_workers', get_gunicorn_workers()["gunicorn_workers"]), + "bench_name": get_bench_name(bench_path) + }) + + with open(os.path.join(bench_path, 'config', 'supervisor.conf'), 'w') as f: + f.write(config) + + update_config({'restart_supervisor_on_update': True}, bench=bench_path) + diff --git a/bench/config/templates/Procfile b/bench/config/templates/Procfile new file mode 100644 index 00000000..e420e0dd --- /dev/null +++ b/bench/config/templates/Procfile @@ -0,0 +1,10 @@ +redis_cache: redis-server config/redis_cache.conf +redis_socketio: redis-server config/redis_socketio.conf +redis_queue: redis-server config/redis_queue.conf +web: bench serve +socketio: {{ node }} apps/frappe/socketio.js +workerbeat: sh -c 'cd sites && exec ../env/bin/python -m frappe.celery_app beat -s scheduler.schedule' +worker: sh -c 'cd sites && exec ../env/bin/python -m frappe.celery_app worker -n jobs@%h -Ofair --soft-time-limit 360 --time-limit 390' +longjob_worker: sh -c 'cd sites && exec ../env/bin/python -m frappe.celery_app worker -n longjobs@%h -Ofair --soft-time-limit 1500 --time-limit 1530' +async_worker: sh -c 'cd sites && exec ../env/bin/python -m frappe.celery_app worker -n async@%h -Ofair --soft-time-limit 1500 --time-limit 1530' +watch: bench watch diff --git a/bench/config/templates/nginx.conf b/bench/config/templates/nginx.conf new file mode 100644 index 00000000..fc7883b7 --- /dev/null +++ b/bench/config/templates/nginx.conf @@ -0,0 +1,106 @@ +{% macro server_block(bench_name, port, server_names, sites_path, ssl_certificate, ssl_certificate_key) %} + +{%- set site_name = server_names[0] if (server_names|length)==1 else "$host" -%} + +server { + listen {{ port }}; + server_name + {% for name in server_names -%} + {{ name }} + {% endfor -%} + ; + + client_max_body_size 4G; + keepalive_timeout 5; + sendfile on; + root {{ sites_path }}; + + {% if ssl_certificate and ssl_certificate_key %} + ssl on; + ssl_certificate {{ ssl_certificate }}; + ssl_certificate_key {{ ssl_certificate_key }}; + ssl_session_timeout 5m; + ssl_protocols TLSv1 TLSv1.1 TLSv1.2; + ssl_ciphers "EECDH+ECDSA+AESGCM EECDH+aRSA+AESGCM EECDH+ECDSA+SHA384 EECDH+ECDSA+SHA256 EECDH+aRSA+SHA384 EECDH+aRSA+SHA256 EECDH+aRSA+RC4 EECDH EDH+aRSA RC4 !aNULL !eNULL !LOW !3DES !MD5 !EXP !PSK !SRP !DSS"; + ssl_prefer_server_ciphers on; + {% endif %} + + location /assets { + try_files $uri =404; + } + + location ~ ^/protected/(.*) { + internal; + try_files /{{ site_name }}/$1 =404; + } + + location /socket.io { + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + proxy_set_header X-Frappe-Site-Name {{ site_name }}; + proxy_set_header Origin $scheme://$http_host; + proxy_set_header Host $host; + + proxy_pass http://{{ bench_name }}-socketio-server; + } + + location / { + try_files /{{ site_name }}/public/$uri @webserver; + } + + location @webserver { + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Frappe-Site-Name {{ site_name }}; + proxy_set_header Host $host; + proxy_set_header X-Use-X-Accel-Redirect True; + proxy_read_timeout {{ http_timeout or 120 }}; + proxy_redirect off; + + proxy_pass http://{{ bench_name }}-frappe; + } +} + +{% if ssl_certificate and ssl_certificate_key %} + # http to https redirect for {{ server_names[0] }} + server { + listen 80; + server_name {{ server_names[0] }}; + return 301 https://$host$request_uri?$query_string; + } + +{% endif %} + +{# keep the empty line above for a pleasant rendering #} +{% endmacro %} + +upstream {{ bench_name }}-frappe { + server 127.0.0.1:{{ webserver_port or 8000 }} fail_timeout=0; +} + +upstream {{ bench_name}}-socketio-server { + server 127.0.0.1:{{ socketio_port or 3000 }} fail_timeout=0; +} + +{% if sites.that_use_dns -%} + + {{ server_block(bench_name, 80, sites.that_use_dns, sites_path) }} + +{%- endif %} + +{%- if sites.that_use_ssl -%} + {% for site in sites.that_use_ssl -%} + + {{ server_block(bench_name, 443, [site.name], sites_path, site.ssl_certificate, site.ssl_certificate_key) }} + + {% endfor %} +{%- endif %} + +{% if sites.that_use_port -%} + {%- for site in sites.that_use_port -%} + + {{ server_block(bench_name, site.port, [site.name], sites_path) }} + + {%- endfor %} +{% endif %} diff --git a/bench/templates/nginx_default.conf b/bench/config/templates/nginx_default.conf similarity index 96% rename from bench/templates/nginx_default.conf rename to bench/config/templates/nginx_default.conf index a7604841..3c714ea9 100644 --- a/bench/templates/nginx_default.conf +++ b/bench/config/templates/nginx_default.conf @@ -33,6 +33,8 @@ http { #keepalive_timeout 0; keepalive_timeout 65; + server_names_hash_bucket_size 64; + #gzip on; index index.html index.htm; @@ -41,4 +43,4 @@ http { # See http://nginx.org/en/docs/ngx_core_module.html#include # for more information. include /etc/nginx/conf.d/*.conf; -} \ No newline at end of file +} diff --git a/bench/config/templates/redis_cache.conf b/bench/config/templates/redis_cache.conf new file mode 100644 index 00000000..99f41815 --- /dev/null +++ b/bench/config/templates/redis_cache.conf @@ -0,0 +1,10 @@ +dbfilename redis_cache.rdb +dir {{ pid_path }} +pidfile {{ pid_path }}/redis_cache.pid +port {{ port }} +maxmemory {{ maxmemory }}mb +maxmemory-policy allkeys-lru +appendonly no +{% if redis_version and redis_version >= 2.2 %} +save "" +{% endif %} diff --git a/bench/config/templates/redis_queue.conf b/bench/config/templates/redis_queue.conf new file mode 100644 index 00000000..9b4fff63 --- /dev/null +++ b/bench/config/templates/redis_queue.conf @@ -0,0 +1,4 @@ +dbfilename redis_queue.rdb +dir {{ pid_path }} +pidfile {{ pid_path }}/redis_queue.pid +port {{ port }} diff --git a/bench/config/templates/redis_socketio.conf b/bench/config/templates/redis_socketio.conf new file mode 100644 index 00000000..25f655cc --- /dev/null +++ b/bench/config/templates/redis_socketio.conf @@ -0,0 +1,4 @@ +dbfilename redis_socketio.rdb +dir {{ pid_path }} +pidfile {{ pid_path }}/redis_socketio.pid +port {{ port }} diff --git a/bench/templates/supervisor.conf b/bench/config/templates/supervisor.conf similarity index 55% rename from bench/templates/supervisor.conf rename to bench/config/templates/supervisor.conf index 0704b970..f97441da 100644 --- a/bench/templates/supervisor.conf +++ b/bench/config/templates/supervisor.conf @@ -2,8 +2,8 @@ ; priority=1 --> Lower priorities indicate programs that start first and shut down last ; killasgroup=true --> send kill signal to child processes too -[program:frappe-web] -command={{ bench_dir }}/env/bin/gunicorn -b 127.0.0.1:8000 -w 2 -t {{http_timeout}} frappe.app:application +[program:{{ bench_name }}-frappe-web] +command={{ bench_dir }}/env/bin/gunicorn -b 127.0.0.1:{{ webserver_port }} -w {{ gunicorn_workers }} -t {{ http_timeout }} frappe.app:application --preload priority=4 autostart=true autorestart=true @@ -12,8 +12,8 @@ stderr_logfile={{ bench_dir }}/logs/web.error.log user={{ user }} directory={{ sites_dir }} -[program:frappe-worker] -command={{ bench_dir }}/env/bin/python -m frappe.celery_app worker -n jobs@%%h --soft-time-limit 360 --time-limit 390 --loglevel INFO -Ofair +[program:{{ bench_name }}-frappe-worker] +command={{ bench_dir }}/env/bin/python -m frappe.celery_app worker -n jobs@%%h -Ofair --soft-time-limit 360 --time-limit 390 --loglevel INFO priority=4 autostart=true autorestart=true @@ -24,8 +24,8 @@ stopwaitsecs=400 directory={{ sites_dir }} killasgroup=true -[program:frappe-longjob-worker] -command={{ bench_dir }}/env/bin/python -m frappe.celery_app worker -n longjobs@%%h --soft-time-limit 1500 --time-limit 1530 --loglevel INFO +[program:{{ bench_name }}-frappe-longjob-worker] +command={{ bench_dir }}/env/bin/python -m frappe.celery_app worker -n longjobs@%%h -Ofair --soft-time-limit 1500 --time-limit 1530 --loglevel INFO priority=2 autostart=true autorestart=true @@ -36,8 +36,8 @@ stopwaitsecs=1540 directory={{ sites_dir }} killasgroup=true -[program:frappe-async-worker] -command={{ bench_dir }}/env/bin/python -m frappe.celery_app worker -n async@%%h --soft-time-limit 1500 --time-limit 1530 --loglevel INFO +[program:{{ bench_name }}-frappe-async-worker] +command={{ bench_dir }}/env/bin/python -m frappe.celery_app worker -n async@%%h -Ofair --soft-time-limit 1500 --time-limit 1530 --loglevel INFO priority=2 autostart=true autorestart=true @@ -48,7 +48,7 @@ stopwaitsecs=1540 directory={{ sites_dir }} killasgroup=true -[program:frappe-workerbeat] +[program:{{ bench_name }}-frappe-workerbeat] command={{ bench_dir }}/env/bin/python -m frappe.celery_app beat -s beat.schedule priority=3 autostart=true @@ -58,9 +58,7 @@ stderr_logfile={{ bench_dir }}/logs/workerbeat.error.log user={{ user }} directory={{ sites_dir }} - -{% if frappe_version > 4%} -[program:redis-cache] +[program:{{ bench_name }}-redis-cache] command={{ redis_server }} {{ redis_cache_config }} priority=1 autostart=true @@ -69,21 +67,30 @@ stdout_logfile={{ bench_dir }}/logs/redis-cache.log stderr_logfile={{ bench_dir }}/logs/redis-cache.error.log user={{ user }} directory={{ sites_dir }} -{% endif %} -{% if frappe_version > 5%} -[program:redis-async-broker] -command={{ redis_server }} {{ redis_async_broker_config }} +[program:{{ bench_name }}-redis-queue] +command={{ redis_server }} {{ redis_queue_config }} priority=1 autostart=true autorestart=true -stdout_logfile={{ bench_dir }}/logs/redis-async-broker.log -stderr_logfile={{ bench_dir }}/logs/redis-async-broker.error.log +stdout_logfile={{ bench_dir }}/logs/redis-queue.log +stderr_logfile={{ bench_dir }}/logs/redis-queue.error.log +user={{ user }} +directory={{ sites_dir }} + +{% if frappe_version > 5 %} +[program:{{ bench_name }}-redis-socketio] +command={{ redis_server }} {{ redis_socketio_config }} +priority=1 +autostart=true +autorestart=true +stdout_logfile={{ bench_dir }}/logs/redis-socketio.log +stderr_logfile={{ bench_dir }}/logs/redis-socketio.error.log user={{ user }} directory={{ sites_dir }} {% if node %} -[program:node-socketio] +[program:{{ bench_name }}-node-socketio] command={{ node }} {{ bench_dir }}/apps/frappe/socketio.js priority=4 autostart=true @@ -91,10 +98,13 @@ autorestart=true stdout_logfile={{ bench_dir }}/logs/node-socketio.log stderr_logfile={{ bench_dir }}/logs/node-socketio.error.log user={{ user }} -directory={{ sites_dir }} +directory={{ bench_dir }} {% endif %} {% endif %} -[group:frappe] -programs=frappe-web,frappe-worker,frappe-workerbeat +[group:{{ bench_name }}-processes] +programs={{ bench_name }}-frappe-web,{{ bench_name }}-frappe-worker,{{ bench_name }}-frappe-longjob-worker,{{ bench_name }}-frappe-async-worker,{{ bench_name }}-frappe-workerbeat {%- if node -%} ,{{ bench_name }}-node-socketio {%- endif%} + +[group:{{ bench_name }}-redis] +programs={{ bench_name }}-redis-cache,{{ bench_name }}-redis-queue {%- if frappe_version > 5 -%} ,{{ bench_name }}-redis-socketio {%- endif %} diff --git a/bench/migrate3to4.py b/bench/migrate3to4.py deleted file mode 100644 index bbec1a50..00000000 --- a/bench/migrate3to4.py +++ /dev/null @@ -1,106 +0,0 @@ -from frappe.installer import add_to_installed_apps -from frappe.cli import latest, backup -from frappe.modules.patch_handler import executed -from frappe.installer import make_site_dirs -import frappe -import argparse -import os -import imp -import json -import shutil -import subprocess - -sites_path = os.environ.get('SITES_PATH', 'sites') -last_3_patch = 'patches.1401.fix_pos_outstanding' - - -def get_frappe(bench='.'): - frappe = os.path.abspath(os.path.join(bench, 'env', 'bin', 'frappe')) - if not os.path.exists(frappe): - print 'frappe app is not installed. Run the following command to install frappe' - print 'bench get-app frappe https://github.com/frappe/frappe.git' - return frappe - -def get_sites(bench='.'): - sites_dir = os.path.join(bench, "sites") - sites = [site for site in os.listdir(sites_dir) - if os.path.isdir(os.path.join(sites_dir, site)) and site not in ('assets',)] - return sites - -def exec_cmd(cmd, cwd='.'): - try: - subprocess.check_call(cmd, cwd=cwd, shell=True) - except subprocess.CalledProcessError, e: - print "Error:", e.output - raise - -def main(path): - site = copy_site(path) - migrate(site) - -def copy_site(path): - if not os.path.exists(path): - raise Exception("Source site does not exist") - site = os.path.basename(path) - site_path = os.path.join(sites_path, site) - confpy_path = os.path.join(path, 'conf.py') - confjson_path = os.path.join(path, 'site_config.json') - if os.path.exists(site_path): - raise Exception("Site already exists") - - os.mkdir(site_path) - print os.path.join(path, 'public') - if os.path.exists(os.path.join(path, 'public')): - exec_cmd("cp -r {src} {dest}".format( - src=os.path.join(path, 'public'), - dest=os.path.join(site_path, 'public'))) - - if os.path.exists(confpy_path): - with open(os.path.join(site_path, 'site_config.json'), 'w') as f: - f.write(module_to_json(confpy_path, indent=1)) - if os.path.exists(confjson_path): - shutil.copy(confjson_path, os.path.join(site_path, 'site_config.json')) - if len(get_sites()) == 1: - exec_cmd("{frappe} --use {site}".format(frappe=get_frappe(), site=site), cwd='sites') - return site - -def validate(site): - frappe.init(site=site, sites_path=sites_path) - make_site_dirs() - backup() - frappe.init(site=site, sites_path=sites_path) - frappe.connect() - if not executed(last_3_patch): - raise Exception, "site not ready to migrate to version 4" - frappe.destroy() - - -def migrate(site): - validate(site) - os.chdir(sites_path) - frappe.init(site=site) - frappe.connect() - add_to_installed_apps('frappe', rebuild_website=False) - add_to_installed_apps('erpnext', rebuild_website=False) - add_to_installed_apps('shopping_cart', rebuild_website=False) - latest() - -def module_to_json(module_path, indent=None, keys=None): - module = imp.load_source('tempmod', module_path) - json_keys = [x for x in dir(module) if not x.startswith('_')] - - if keys: - json_keys = [x for x in json_keys if x in keys] - if 'unicode_literals' in json_keys: - json_keys.remove('unicode_literals') - if 'backup_path' in json_keys: - json_keys.remove('backup_path') - - module = {x:getattr(module, x) for x in json_keys} - return json.dumps(module, indent=indent) - -if __name__ == '__main__': - parser = argparse.ArgumentParser() - parser.add_argument('site') - args = parser.parse_args() - main(args.site) diff --git a/bench/migrate_to_v5.py b/bench/migrate_to_v5.py deleted file mode 100644 index ddc867a7..00000000 --- a/bench/migrate_to_v5.py +++ /dev/null @@ -1,47 +0,0 @@ -from .utils import exec_cmd, get_frappe, run_frappe_cmd -from .release import get_current_version -from .app import remove_from_appstxt -import os -import shutil -import sys - -repos = ('frappe', 'erpnext') - -def migrate_to_v5(bench='.'): - validate_v4(bench=bench) - for repo in repos: - checkout_v5(repo, bench=bench) - remove_shopping_cart(bench=bench) - exec_cmd("{bench} update".format(bench=sys.argv[0])) - -def remove_shopping_cart(bench='.'): - archived_apps_dir = os.path.join(bench, 'archived_apps') - shopping_cart_dir = os.path.join(bench, 'apps', 'shopping_cart') - - if not os.path.exists(shopping_cart_dir): - return - - run_frappe_cmd('--site', 'all', 'remove-from-installed-apps', 'shopping_cart', bench=bench) - remove_from_appstxt('shopping_cart', bench=bench) - exec_cmd("{pip} --no-input uninstall -y shopping_cart".format(pip=os.path.join(bench, 'env', 'bin', 'pip'))) - - if not os.path.exists(archived_apps_dir): - os.mkdir(archived_apps_dir) - shutil.move(shopping_cart_dir, archived_apps_dir) - -def validate_v4(bench='.'): - for repo in repos: - path = os.path.join(bench, 'apps', repo) - if os.path.exists(path): - current_version = get_current_version(path) - if not current_version.startswith('4'): - raise Exception("{} is not on v4.x.x".format(repo)) - -def checkout_v5(repo, bench='.'): - cwd = os.path.join(bench, 'apps', repo) - unshallow = "--unshallow" if os.path.exists(os.path.join(cwd, ".git", "shallow")) else "" - if os.path.exists(cwd): - exec_cmd("git config --add remote.upstream.fetch '+refs/heads/*:refs/remotes/upstream/*'", cwd=cwd) - exec_cmd("git fetch upstream {unshallow}".format(unshallow=unshallow), cwd=cwd) - exec_cmd("git checkout v5.0", cwd=cwd) - exec_cmd("git clean -df", cwd=cwd) diff --git a/bench/release.py b/bench/release.py index 87d44951..384c0e1e 100755 --- a/bench/release.py +++ b/bench/release.py @@ -12,7 +12,8 @@ import re from requests.auth import HTTPBasicAuth import requests.exceptions from time import sleep -from .utils import get_config +from .config.common_site_config import get_config + github_username = None github_password = None @@ -173,7 +174,7 @@ def bump(repo, bump_type, develop='develop', master='master', remote='upstream') print 'Released {tag} for {repo}'.format(tag=tag_name, repo=repo) def release(repo, bump_type, develop, master): - if not get_config().get('release_bench'): + if not get_config(".").get('release_bench'): print 'bench not configured to release' sys.exit(1) global github_username, github_password diff --git a/bench/templates/cached_requirements.txt b/bench/templates/cached_requirements.txt deleted file mode 100644 index dbbb6b05..00000000 --- a/bench/templates/cached_requirements.txt +++ /dev/null @@ -1,26 +0,0 @@ -chardet -cssmin -dropbox -gunicorn -httplib2 -jinja2 -markdown2 -markupsafe -mysql-python -pygeoip -python-dateutil -python-memcached -pytz==2013d -six -slugify -termcolor -werkzeug -semantic_version -rauth>=0.6.2 -requests==1.2.3 -celery -redis -selenium -unidecode -babel -pdfkit diff --git a/bench/templates/nginx.conf b/bench/templates/nginx.conf deleted file mode 100644 index 8899211b..00000000 --- a/bench/templates/nginx.conf +++ /dev/null @@ -1,108 +0,0 @@ - -server_names_hash_bucket_size 64; - -upstream frappe { - server 127.0.0.1:8000 fail_timeout=0; -} - -upstream socketio-server { - server 127.0.0.1:3000 fail_timeout=0; -} - -{% macro location_block(site, port=80, default=False, server_name=None, sites=None, dns_multitenant=False) -%} - keepalive_timeout 5; - sendfile on; - root {{ sites_dir }}; - - location /assets { - try_files $uri =404; - } - - location ~ ^/protected/(.*) { - internal; - try_files /{{ "$host" if dns_multitenant else site.name }}/$1 =404; - } - - location /socket.io { - proxy_pass http://socketio-server; - proxy_http_version 1.1; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection "upgrade"; - {% if not dns_multitenant %} - proxy_set_header X-Frappe-Site-Name {{ site.name }}; - {% endif %} - proxy_set_header Origin $scheme://$http_host; - proxy_set_header Host $host; - } - - location / { - try_files /{{ "$host" if dns_multitenant else site.name }}/public/$uri @magic; - } - - location @magic { - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; - {% if not dns_multitenant %} - proxy_set_header X-Frappe-Site-Name {{ site.name }}; - {% endif %} - proxy_set_header Host $host; - proxy_set_header X-Use-X-Accel-Redirect True; - proxy_read_timeout {{http_timeout}}; - proxy_redirect off; - proxy_pass http://frappe; - } -{%- endmacro %} - -{% macro server_name_block(site, default=False, server_name=None, sites=None, dns_multitenant=False) -%} - client_max_body_size 4G; - {% if dns_multitenant and sites %} - server_name {% for site in sites %} {{ site.name }} {% endfor %}; - {% else %} - server_name {{ site.name if not server_name else server_name }}; - {% endif %} -{%- endmacro %} - -{% macro server_block_http(site, port=80, default=False, server_name=None, sites=None, dns_multitenant=False) -%} - server { - listen {{ site.port if not default and site.port else port }} {% if default %} default {% endif %}; - {{ server_name_block(site, default=default, server_name=server_name, sites=sites, dns_multitenant=dns_multitenant) }} - {{ location_block(site, port=port, default=default, server_name=server_name, sites=sites, dns_multitenant=dns_multitenant) }} - } -{%- endmacro %} - -{% macro server_block_https(site, port=443, default=False, server_name=None, sites=None, dns_multitenant=False) -%} - server { - listen {{ site.ssl_port if not default and site.ssl_port else port }} {% if default %} default {% endif %}; - {{ server_name_block(site, default=default, server_name=server_name, sites=sites, dns_multitenant=dns_multitenant) }} - - ssl on; - ssl_certificate {{ site.ssl_certificate }}; - ssl_certificate_key {{ site.ssl_certificate_key }}; - ssl_session_timeout 5m; - ssl_protocols TLSv1 TLSv1.1 TLSv1.2; - ssl_ciphers "EECDH+ECDSA+AESGCM EECDH+aRSA+AESGCM EECDH+ECDSA+SHA384 EECDH+ECDSA+SHA256 EECDH+aRSA+SHA384 EECDH+aRSA+SHA256 EECDH+aRSA+RC4 EECDH EDH+aRSA RC4 !aNULL !eNULL !LOW !3DES !MD5 !EXP !PSK !SRP !DSS"; - ssl_prefer_server_ciphers on; - - {{ location_block(site, port=port, default=default, server_name=server_name, sites=sites, dns_multitenant=dns_multitenant) }} - } -{%- endmacro %} - -{% for site in sites %} - -{% if site.port %} -{{ server_block_http(site) }} -{% endif %} - -{% if site.ssl_certificate_key and site.ssl_certificate %} -{{ server_block_https(site) }} -{% endif %} - -{% endfor %} - -{% if default_site %} -{{ server_block_http(default_site, default=True, server_name="frappe_default_site") }} -{% endif %} - -{% if dns_multitenant and sites %} -{{ server_block_http(None, default=False, sites=sites, dns_multitenant=True) }} -{% endif %} diff --git a/bench/templates/redis_async_broker.conf b/bench/templates/redis_async_broker.conf deleted file mode 100644 index 7940f17e..00000000 --- a/bench/templates/redis_async_broker.conf +++ /dev/null @@ -1,3 +0,0 @@ -dbfilename redis_async_broker.rdb -pidfile redis_async_broker.pid -port {{port}} \ No newline at end of file diff --git a/bench/templates/redis_cache.conf b/bench/templates/redis_cache.conf deleted file mode 100644 index dfb35872..00000000 --- a/bench/templates/redis_cache.conf +++ /dev/null @@ -1,7 +0,0 @@ -dbfilename redis_cache_dump.rdb -pidfile redis_cache.pid -port {{port}} -maxmemory {{maxmemory}}mb -maxmemory-policy allkeys-lru -save "" -appendonly no \ No newline at end of file diff --git a/bench/tests/__init__.py b/bench/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/bench/tests/test_init.py b/bench/tests/test_init.py new file mode 100644 index 00000000..6f752550 --- /dev/null +++ b/bench/tests/test_init.py @@ -0,0 +1,167 @@ +from __future__ import unicode_literals +import unittest +import json, os, shutil, subprocess +import bench +import bench.utils +import bench.app +import bench.config.common_site_config +import bench.cli + +bench.cli.from_command_line = True + +class TestBenchInit(unittest.TestCase): + def setUp(self): + self.benches_path = "." + self.benches = [] + + def tearDown(self): + for bench_name in self.benches: + bench_path = os.path.join(self.benches_path, bench_name) + if os.path.exists(bench_path): + shutil.rmtree(bench_path, ignore_errors=True) + + def test_init(self, bench_name="test-bench"): + self.init_bench(bench_name) + + self.assert_folders(bench_name) + + self.assert_virtual_env(bench_name) + + self.assert_common_site_config(bench_name, bench.config.common_site_config.default_config) + + self.assert_config(bench_name) + + self.assert_socketio(bench_name) + + def test_multiple_benches(self): + # 1st bench + self.test_init("test-bench-1") + + self.assert_common_site_config("test-bench-1", { + "webserver_port": 8000, + "socketio_port": 9000, + "redis_queue": "redis://localhost:11000", + "redis_socketio": "redis://localhost:12000", + "redis_cache": "redis://localhost:13000" + }) + + # 2nd bench + self.test_init("test-bench-2") + + self.assert_common_site_config("test-bench-2", { + "webserver_port": 8001, + "socketio_port": 9001, + "redis_queue": "redis://localhost:11001", + "redis_socketio": "redis://localhost:12001", + "redis_cache": "redis://localhost:13001" + }) + + def test_new_site(self): + self.new_site("test-site-1.dev") + + def new_site(self, site_name): + self.test_init() + + new_site_cmd = ["bench", "new-site", site_name, "--admin-password", "admin"] + + # set in travis + if os.environ.get("TRAVIS"): + new_site_cmd.extend(["--mariadb-root-password", "travis"]) + + subprocess.check_output(new_site_cmd, cwd=os.path.join(self.benches_path, "test-bench")) + + site_path = os.path.join(self.benches_path, "test-bench", "sites", site_name) + + self.assertTrue(os.path.exists(site_path)) + self.assertTrue(os.path.exists(os.path.join(site_path, "private", "backups"))) + self.assertTrue(os.path.exists(os.path.join(site_path, "private", "files"))) + self.assertTrue(os.path.exists(os.path.join(site_path, "public", "files"))) + + site_config_path = os.path.join(site_path, "site_config.json") + self.assertTrue(os.path.exists(site_config_path)) + with open(site_config_path, "r") as f: + site_config = json.loads(f.read()) + + for key in ("db_name", "db_password"): + self.assertTrue(key in site_config) + self.assertTrue(site_config[key]) + + def test_install_app(self): + site_name = "test-site-2.dev" + + self.new_site(site_name) + + bench_path = os.path.join(self.benches_path, "test-bench") + + # get app + bench.app.get_app("erpnext", "https://github.com/frappe/erpnext", "develop", bench=bench_path) + + self.assertTrue(os.path.exists(os.path.join(bench_path, "apps", "erpnext"))) + + # install app + bench.app.install_app("erpnext", bench=bench_path) + + # install it to site + subprocess.check_output(["bench", "--site", site_name, "install-app", "erpnext"], cwd=bench_path) + + out = subprocess.check_output(["bench", "--site", site_name, "list-apps"], cwd=bench_path) + self.assertTrue("erpnext" in out) + + def init_bench(self, bench_name): + self.benches.append(bench_name) + bench.utils.init(bench_name) + + def assert_folders(self, bench_name): + for folder in bench.utils.folders_in_bench: + self.assert_exists(bench_name, folder) + + self.assert_exists(bench_name, "sites", "assets") + self.assert_exists(bench_name, "apps", "frappe") + self.assert_exists(bench_name, "apps", "frappe", "setup.py") + + def assert_virtual_env(self, bench_name): + bench_path = os.path.abspath(bench_name) + python = os.path.join(bench_path, "env", "bin", "python") + python_path = bench.utils.get_cmd_output('{python} -c "import os; print os.path.dirname(os.__file__)"'.format(python=python)) + + # part of bench's virtualenv + self.assertTrue(python_path.startswith(bench_path)) + self.assert_exists(python_path) + self.assert_exists(python_path, "site-packages") + self.assert_exists(python_path, "site-packages", "IPython") + self.assert_exists(python_path, "site-packages", "pip") + + site_packages = os.listdir(os.path.join(python_path, "site-packages")) + self.assertTrue(any(package.startswith("MySQL_python-1.2.5") for package in site_packages)) + + def assert_config(self, bench_name): + for config, search_key in ( + ("redis_queue.conf", "redis_queue.rdb"), + ("redis_socketio.conf", "redis_socketio.rdb"), + ("redis_cache.conf", "redis_cache.rdb")): + + self.assert_exists(bench_name, "config", config) + + with open(os.path.join(bench_name, "config", config), "r") as f: + f = f.read().decode("utf-8") + self.assertTrue(search_key in f) + + def assert_socketio(self, bench_name): + self.assert_exists(bench_name, "node_modules") + self.assert_exists(bench_name, "node_modules", "socket.io") + + def assert_common_site_config(self, bench_name, expected_config): + common_site_config_path = os.path.join(bench_name, 'sites', 'common_site_config.json') + self.assertTrue(os.path.exists(common_site_config_path)) + + config = self.load_json(common_site_config_path) + + for key, value in expected_config.items(): + self.assertEquals(config.get(key), value) + + def assert_exists(self, *args): + self.assertTrue(os.path.exists(os.path.join(*args))) + + def load_json(self, path): + with open(path, "r") as f: + return json.loads(f.read().decode("utf-8")) diff --git a/bench/tests/test_setup_production.py b/bench/tests/test_setup_production.py new file mode 100644 index 00000000..a280d932 --- /dev/null +++ b/bench/tests/test_setup_production.py @@ -0,0 +1,106 @@ +from __future__ import unicode_literals +from bench.tests import test_init +from bench.config.production_setup import setup_production, get_supervisor_confdir +import bench.utils +import os +import getpass +import re +import unittest +import time + +class TestSetupProduction(test_init.TestBenchInit): + # setUp, tearDown and other tests are defiend in TestBenchInit + + def test_setup_production(self): + self.test_multiple_benches() + + user = getpass.getuser() + + for bench_name in ("test-bench-1", "test-bench-2"): + bench_path = os.path.join(os.path.abspath(self.benches_path), bench_name) + setup_production(user, bench_path) + self.assert_nginx_config(bench_name) + self.assert_supervisor_config(bench_name) + + # test after start of both benches + for bench_name in ("test-bench-1", "test-bench-2"): + self.assert_supervisor_process(bench_name) + + self.assert_nginx_process() + + def assert_nginx_config(self, bench_name): + conf_src = os.path.join(os.path.abspath(self.benches_path), bench_name, 'config', 'nginx.conf') + conf_dest = "/etc/nginx/conf.d/{bench_name}.conf".format(bench_name=bench_name) + + self.assertTrue(os.path.exists(conf_src)) + self.assertTrue(os.path.exists(conf_dest)) + + # symlink matches + self.assertEquals(os.path.realpath(conf_dest), conf_src) + + # file content + with open(conf_src, "r") as f: + f = f.read().decode("utf-8") + + for key in ( + "upstream {bench_name}-frappe", + "upstream {bench_name}-socketio-server" + ): + self.assertTrue(key.format(bench_name=bench_name) in f) + + def assert_supervisor_config(self, bench_name): + conf_src = os.path.join(os.path.abspath(self.benches_path), bench_name, 'config', 'supervisor.conf') + + supervisor_conf_dir = get_supervisor_confdir() + conf_dest = "{supervisor_conf_dir}/{bench_name}.conf".format(supervisor_conf_dir=supervisor_conf_dir, bench_name=bench_name) + + self.assertTrue(os.path.exists(conf_src)) + self.assertTrue(os.path.exists(conf_dest)) + + # symlink matches + self.assertEquals(os.path.realpath(conf_dest), conf_src) + + # file content + with open(conf_src, "r") as f: + f = f.read().decode("utf-8") + + for key in ( + "program:{bench_name}-frappe-web", + "program:{bench_name}-frappe-worker", + "program:{bench_name}-frappe-longjob-worker", + "program:{bench_name}-frappe-async-worker", + "program:{bench_name}-frappe-workerbeat", + "program:{bench_name}-redis-cache", + "program:{bench_name}-redis-queue", + "program:{bench_name}-redis-socketio", + "program:{bench_name}-node-socketio", + "group:{bench_name}-processes", + "group:{bench_name}-redis" + ): + self.assertTrue(key.format(bench_name=bench_name) in f) + + def assert_supervisor_process(self, bench_name): + out = bench.utils.get_cmd_output("sudo supervisorctl status") + + if "STARTING" in out: + time.sleep(10) + out = bench.utils.get_cmd_output("sudo supervisorctl status") + + for key in ( + "{bench_name}-processes:{bench_name}-frappe-web[\s]+RUNNING", + "{bench_name}-processes:{bench_name}-frappe-worker[\s]+RUNNING", + "{bench_name}-processes:{bench_name}-frappe-longjob-worker[\s]+RUNNING", + "{bench_name}-processes:{bench_name}-frappe-async-worker[\s]+RUNNING", + "{bench_name}-processes:{bench_name}-frappe-workerbeat[\s]+RUNNING", + "{bench_name}-processes:{bench_name}-node-socketio[\s]+RUNNING", + "{bench_name}-redis:{bench_name}-redis-cache[\s]+RUNNING", + "{bench_name}-redis:{bench_name}-redis-queue[\s]+RUNNING", + "{bench_name}-redis:{bench_name}-redis-socketio[\s]+RUNNING", + ): + self.assertTrue(re.search(key.format(bench_name=bench_name), out)) + + def assert_nginx_process(self): + out = bench.utils.get_cmd_output("sudo nginx -t 2>&1") + self.assertTrue("nginx: configuration file /etc/nginx/nginx.conf test is successful" in out) + + diff --git a/bench/utils.py b/bench/utils.py index 6a8eaf77..4f5d8712 100644 --- a/bench/utils.py +++ b/bench/utils.py @@ -1,8 +1,6 @@ import os -import re import sys import subprocess -import getpass import logging import itertools import requests @@ -23,15 +21,7 @@ class CommandFailedError(Exception): logger = logging.getLogger(__name__) -default_config = { - 'restart_supervisor_on_update': False, - 'auto_update': False, - 'serve_default_site': True, - 'rebase_on_pull': False, - 'update_bench_on_update': True, - 'frappe_user': getpass.getuser(), - 'shallow_clone': True -} +folders_in_bench = ('apps', 'sites', 'config', 'logs', 'config/pids') def get_frappe(bench='.'): frappe = get_env_cmd('frappe', bench=bench) @@ -47,7 +37,9 @@ def init(path, apps_path=None, no_procfile=False, no_backups=False, no_auto_update=False, frappe_path=None, frappe_branch=None, wheel_cache_dir=None, verbose=False): from .app import get_app, install_apps_from_path - from .config import generate_redis_cache_config, generate_redis_async_broker_config + from .config.common_site_config import make_config + from .config import redis + from .config.procfile import setup_procfile global FRAPPE_VERSION if os.path.exists(path): @@ -55,35 +47,37 @@ def init(path, apps_path=None, no_procfile=False, no_backups=False, raise Exception("Site directory already exists") # sys.exit(1) - os.mkdir(path) - for dirname in ('apps', 'sites', 'config', 'logs'): + os.makedirs(path) + for dirname in folders_in_bench: os.mkdir(os.path.join(path, dirname)) setup_logging() setup_env(bench=path) - put_config(default_config, bench=path) - # if wheel_cache_dir: - # update_config({"wheel_cache_dir":wheel_cache_dir}, bench=path) - # prime_wheel_cache(bench=path) + + make_config(path) if not frappe_path: frappe_path = 'https://github.com/frappe/frappe.git' get_app('frappe', frappe_path, branch=frappe_branch, bench=path, build_asset_files=False, verbose=verbose) + + if apps_path: + install_apps_from_path(apps_path, bench=path) + + FRAPPE_VERSION = get_current_frappe_version(bench=path) + if FRAPPE_VERSION > 5: + setup_socketio(bench=path) + + build_assets(bench=path) + redis.generate_config(path) + if not no_procfile: - setup_procfile(bench=path) + setup_procfile(path) if not no_backups: setup_backups(bench=path) if not no_auto_update: setup_auto_update(bench=path) - if apps_path: - install_apps_from_path(apps_path, bench=path) - FRAPPE_VERSION = get_current_frappe_version(bench=path) - if FRAPPE_VERSION > 5: - setup_socketio(bench=path) - build_assets(bench=path) - generate_redis_cache_config(bench=path) - generate_redis_async_broker_config(bench=path) + def exec_cmd(cmd, cwd='.'): from .cli import from_command_line @@ -108,38 +102,12 @@ def setup_env(bench='.'): exec_cmd('virtualenv -q {} -p {}'.format('env', sys.executable), cwd=bench) exec_cmd('./env/bin/pip -q install --upgrade pip', cwd=bench) exec_cmd('./env/bin/pip -q install wheel', cwd=bench) - exec_cmd('./env/bin/pip -q install https://github.com/frappe/MySQLdb1/archive/MySQLdb-1.2.5-patched.tar.gz', cwd=bench) + # exec_cmd('./env/bin/pip -q install https://github.com/frappe/MySQLdb1/archive/MySQLdb-1.2.5-patched.tar.gz', cwd=bench) exec_cmd('./env/bin/pip -q install -e git+https://github.com/frappe/python-pdfkit.git#egg=pdfkit', cwd=bench) def setup_socketio(bench='.'): exec_cmd("npm install socket.io redis express superagent cookie", cwd=bench) -def setup_procfile(with_celery_broker=False, with_watch=False, bench='.'): - from .app import get_current_frappe_version - frappe_version = get_current_frappe_version() - procfile_contents = { - 'web': "./env/bin/frappe --serve --sites_path sites", - 'worker': "sh -c 'cd sites && exec ../env/bin/python -m frappe.celery_app worker'", - 'longjob_worker': "sh -c 'cd sites && exec ../env/bin/python -m frappe.celery_app -n longjobs@%h worker'", - 'async_worker': "sh -c 'cd sites && exec ../env/bin/python -m frappe.celery_app -n async@%h worker'", - 'workerbeat': "sh -c 'cd sites && exec ../env/bin/python -m frappe.celery_app beat -s scheduler.schedule'" - } - if frappe_version > 4: - procfile_contents['redis_cache'] = "redis-server config/redis_cache.conf" - procfile_contents['redis_async_broker'] = "redis-server config/redis_async_broker.conf" - procfile_contents['web'] = "bench serve" - if with_celery_broker: - procfile_contents['redis_celery'] = "redis-server" - if with_watch: - procfile_contents['watch'] = "bench watch" - if frappe_version > 5: - procfile_contents['socketio'] = "{0} apps/frappe/socketio.js".format(find_executable("node") or find_executable("nodejs")) - - procfile = '\n'.join(["{0}: {1}".format(k, v) for k, v in procfile_contents.items()]) - - with open(os.path.join(bench, 'Procfile'), 'w') as f: - f.write(procfile) - def new_site(site, mariadb_root_password=None, admin_password=None, bench='.'): import hashlib logger.info('creating new site {}'.format(site)) @@ -217,8 +185,11 @@ def read_crontab(): return out def update_bench(): - logger.info('setting up sudoers') - cwd = os.path.dirname(os.path.abspath(__file__)) + logger.info('updating bench') + + # bench-repo folder + cwd = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + exec_cmd("git pull", cwd=cwd) def setup_sudoers(user): @@ -239,22 +210,6 @@ def setup_logging(bench='.'): logger.addHandler(hdlr) logger.setLevel(logging.DEBUG) -def get_config(bench='.'): - config_path = os.path.join(bench, 'config.json') - if not os.path.exists(config_path): - return {} - with open(config_path) as f: - return json.load(f) - -def put_config(config, bench='.'): - with open(os.path.join(bench, 'config.json'), 'w') as f: - return json.dump(config, f, indent=1) - -def update_config(new_config, bench='.'): - config = get_config(bench=bench) - config.update(new_config) - put_config(config, bench=bench) - def get_program(programs): program = None for p in programs: @@ -283,12 +238,15 @@ def check_cmd(cmd, cwd='.'): return False def get_git_version(): - version = get_cmd_output("git --version") - return version.strip().split()[-1] + '''returns git version from `git --version` + extracts version number from string `get version 1.9.1` etc''' + version = get_cmd_output("git --version").strip().split()[2] + version = '.'.join(version.split('.')[0:2]) + return float(version) def check_git_for_shallow_clone(): git_version = get_git_version() - if git_version.startswith('1.9') or git_version.startswith('2'): + if git_version > 1.9: return True return False @@ -301,8 +259,11 @@ def get_cmd_output(cmd, cwd='.'): raise def restart_supervisor_processes(bench='.'): + from .config.common_site_config import get_config conf = get_config(bench=bench) - cmd = conf.get('supervisor_restart_cmd', 'sudo supervisorctl restart frappe:') + bench_name = get_bench_name(bench) + cmd = conf.get('supervisor_restart_cmd', + 'sudo supervisorctl restart {bench_name}-processes:'.format(bench_name=bench_name)) exec_cmd(cmd, cwd=bench) def get_site_config(site, bench='.'): @@ -332,12 +293,12 @@ def set_ssl_certificate_key(site, ssl_certificate_key, bench='.', gen_config=Tru set_site_config_nginx_property(site, {"ssl_certificate_key": ssl_certificate_key}, bench=bench, gen_config=gen_config) def set_site_config_nginx_property(site, config, bench='.', gen_config=True): - from .config import generate_nginx_config + from .config.nginx import make_nginx_conf if site not in get_sites(bench=bench): raise Exception("No such site") update_site_config(site, config, bench=bench) if gen_config: - generate_nginx_config(bench=bench) + make_nginx_conf(bench_path=bench) def set_url_root(site, url_root, bench='.'): update_site_config(site, {"host_name": url_root}, bench=bench) @@ -371,18 +332,6 @@ def backup_all_sites(bench='.'): for site in get_sites(bench=bench): backup_site(site, bench=bench) -def prime_wheel_cache(bench='.'): - conf = get_config(bench=bench) - wheel_cache_dir = conf.get('wheel_cache_dir') - if not wheel_cache_dir: - raise Exception("Wheel cache dir not configured") - requirements = os.path.join(os.path.dirname(__file__), 'templates', 'cached_requirements.txt') - cmd = "{pip} wheel --find-links {wheelhouse} --wheel-dir {wheelhouse} -r {requirements}".format( - pip=os.path.join(bench, 'env', 'bin', 'pip'), - wheelhouse=wheel_cache_dir, - requirements=requirements) - exec_cmd(cmd) - def is_root(): if os.getuid() == 0: return True @@ -395,11 +344,16 @@ def update_common_site_config(ddict, bench='.'): update_json_file(os.path.join(bench, 'sites', 'common_site_config.json'), ddict) def update_json_file(filename, ddict): - with open(filename, 'r') as f: - content = json.load(f) + if os.path.exists(filename): + with open(filename, 'r') as f: + content = json.load(f) + + else: + content = {} + content.update(ddict) with open(filename, 'w') as f: - content = json.dump(content, f, indent=1) + content = json.dump(content, f, indent=1, sort_keys=True) def drop_privileges(uid_name='nobody', gid_name='nogroup'): # from http://stackoverflow.com/a/2699996 @@ -421,7 +375,8 @@ def drop_privileges(uid_name='nobody', gid_name='nogroup'): # Ensure a very conservative umask os.umask(022) -def fix_prod_setup_perms(frappe_user=None): +def fix_prod_setup_perms(bench='.', frappe_user=None): + from .config.common_site_config import get_config files = [ "logs/web.error.log", "logs/web.log", @@ -434,7 +389,7 @@ def fix_prod_setup_perms(frappe_user=None): ] if not frappe_user: - frappe_user = get_config().get('frappe_user') + frappe_user = get_config(bench).get('frappe_user') if not frappe_user: print "frappe user not set" @@ -458,15 +413,6 @@ def fix_file_perms(): if not _file.startswith('activate'): os.chmod(os.path.join(bin_dir, _file), 0755) -def get_redis_version(): - version_string = subprocess.check_output('redis-server --version', shell=True).strip() - if re.search("Redis server version 2.4", version_string): - return "2.4" - if re.search("Redis server v=2.6", version_string): - return "2.6" - if re.search("Redis server v=2.8", version_string): - return "2.8" - def get_current_frappe_version(bench='.'): from .app import get_current_frappe_version as fv return fv(bench=bench) @@ -520,21 +466,23 @@ def pre_upgrade(from_ver, to_ver, bench='.'): exec_cmd("{pip} install --upgrade -e {app}".format(pip=pip, app=cwd)) def post_upgrade(from_ver, to_ver, bench='.'): - from .config import generate_nginx_config, generate_supervisor_config, generate_redis_cache_config, generate_redis_async_broker_config + from .config.common_site_config import get_config + from .config import redis + from .config.supervisor import generate_supervisor_config + from .config.nginx import make_nginx_conf conf = get_config(bench=bench) print "-"*80 print "Your bench was upgraded to version {0}".format(to_ver) if conf.get('restart_supervisor_on_update'): - generate_redis_cache_config(bench=bench) - generate_supervisor_config(bench=bench) - generate_nginx_config(bench=bench) + redis.generate_config(bench_path=bench) + generate_supervisor_config(bench_path=bench) + make_nginx_conf(bench_path=bench) if from_ver == 4 and to_ver == 5: setup_backups(bench=bench) if from_ver <= 5 and to_ver == 6: - generate_redis_async_broker_config(bench=bench) setup_socketio(bench=bench) print "As you have setup your bench for production, you will have to reload configuration for nginx and supervisor" @@ -543,10 +491,6 @@ def post_upgrade(from_ver, to_ver, bench='.'): print "sudo service nginx restart" print "sudo supervisorctl reload" - if to_ver >= 5: - # For dev server. Always set this up incase someone wants to start a dev server. - setup_procfile(bench=bench) - def update_translations_p(args): try: update_translations(*args) @@ -568,7 +512,6 @@ def download_translations(): for app, lang in itertools.product(apps, langs): update_translations(app, lang) - def get_langs(): lang_file = 'apps/frappe/frappe/data/languages.txt' with open(lang_file) as f: @@ -577,7 +520,6 @@ def get_langs(): langs.remove('en') return langs - def update_translations(app, lang): translations_dir = os.path.join('apps', app, app, 'translations') csv_file = os.path.join(translations_dir, lang + '.csv') @@ -658,3 +600,6 @@ def validate_pillow_dependencies(bench, requirements): print "sudo apt-get install -y libtiff5-dev libjpeg8-dev zlib1g-dev libfreetype6-dev liblcms2-dev libwebp-dev tcl8.6-dev tk8.6-dev python-tk" raise + +def get_bench_name(bench_path): + return os.path.basename(os.path.abspath(bench_path)) diff --git a/install_scripts/setup_frappe.sh b/install_scripts/setup_frappe.sh index 8d331195..0e81b118 100755 --- a/install_scripts/setup_frappe.sh +++ b/install_scripts/setup_frappe.sh @@ -16,7 +16,7 @@ get_passwd() { } set_opts () { - OPTS=`getopt -o v --long verbose,mysql-root-password:,frappe-user:,bench-branch:,setup-production,skip-setup-bench,help -n 'parse-options' -- "$@"` + OPTS=`getopt -o v --long verbose,mysql-root-password:,frappe-user:,bench-branch:,setup-production,skip-install-bench,skip-setup-bench,help -n 'parse-options' -- "$@"` if [ $? != 0 ] ; then echo "Failed parsing options." >&2 ; exit 1 ; fi @@ -27,6 +27,7 @@ set_opts () { FRAPPE_USER=false BENCH_BRANCH="master" SETUP_PROD=false + INSTALL_BENCH=true SETUP_BENCH=true if [ -f ~/frappe_passwords.sh ]; then @@ -50,6 +51,7 @@ set_opts () { --setup-production ) SETUP_PROD=true; shift;; --bench-branch ) BENCH_BRANCH="$2"; shift;; --skip-setup-bench ) SETUP_BENCH=false; shift;; + --skip-install-bench ) INSTALL_BENCH=false; shift;; -- ) shift; break ;; * ) break ;; esac @@ -202,14 +204,12 @@ install_packages() { elif [ $OS == "debian" ] || [ $OS == "Ubuntu" ]; then export DEBIAN_FRONTEND=noninteractive setup_debconf - if [ $OS == "debian" ]; then - run_cmd bash -c "curl -sL https://deb.nodesource.com/setup_0.12 | bash -" - fi + run_cmd bash -c "curl -sL https://deb.nodesource.com/setup_0.12 | sudo bash -" run_cmd sudo apt-get update run_cmd sudo apt-get install -y python-dev python-setuptools build-essential python-mysqldb git \ ntp vim screen htop mariadb-server mariadb-common libmariadbclient-dev \ libxslt1.1 libxslt1-dev redis-server libssl-dev libcrypto++-dev postfix nginx \ - supervisor python-pip fontconfig libxrender1 libxext6 xfonts-75dpi xfonts-base nodejs npm + supervisor python-pip fontconfig libxrender1 libxext6 xfonts-75dpi xfonts-base nodejs if [ $OS_VER == "precise" ]; then run_cmd sudo apt-get install -y libtiff4-dev libjpeg8-dev zlib1g-dev libfreetype6-dev liblcms2-dev libwebp-dev tcl8.5-dev tk8.5-dev python-tk @@ -454,9 +454,12 @@ main() { configure_mariadb echo "Adding frappe user" add_user - install_bench - if $SETUP_BENCH; then - setup_bench + + if $INSTALL_BENCH; then + install_bench + if $SETUP_BENCH; then + setup_bench + fi fi echo diff --git a/installer/playbooks/install_dependencies.yml b/installer/playbooks/install_dependencies.yml new file mode 100644 index 00000000..990478bf --- /dev/null +++ b/installer/playbooks/install_dependencies.yml @@ -0,0 +1,99 @@ +--- + - name: Install dependencies + hosts: localhost + become: yes + become_user: root + vars: + - mysql_conf_tpl: ../templates/mariadb_config.cnf + - nginx_conf_file: ../templates/nginx.conf + - mysql_secure_installation: True + roles: + - locale + - swap + - mariadb + - { role: epel, when: "ansible_os_family == 'RedHat'" } + - nginx + - logwatch + - fail2ban + - bash_screen_wall + - frappe_selinux + - dns_caching + - wkhtmltopdf + - ntpd + tasks: + - name: Set hostname + hostname: name='{{ hostname }}' + - name: Install the 'Development tools' package group (Redhat) + yum: name="@Development tools" state=present + when: ansible_os_family == 'RedHat' + - name: Install packages + yum: name={{ item }} state=present + with_items: + - bzip2-devel + - cronie + - freetype-devel + - git + - lcms2-devel + - libjpeg-devel + - libtiff-devel + - libwebp-devel + - libXext + - libXrender + - libzip-devel + - nodejs + - npm + - openssl-devel + - postfix + - python-devel + - python-pip + - redis + - screen + - sudo + - supervisor + - tcl-devel + - tk-devel + - vim + - which + - xorg-x11-fonts-75dpi + - xorg-x11-fonts-Type1 + - zlib-devel + when: ansible_os_family == 'RedHat' + - name: Install packages + apt: pkg={{ item }} state=present force=yes + with_items: + - build-essential + - fontconfig + - git + - htop + - libcrypto++-dev + + - libfreetype6-dev + - libjpeg8-dev + - liblcms2-dev + - libssl-dev + - libtiff5-dev + - libwebp-dev + - libxext6 + - libxrender1 + - libxslt1-dev + - libxslt1.1 + - tcl8.6-dev + - tk8.6-dev + - zlib1g-dev + - libopenjpeg-dev + + - nodejs + - npm + - ntp + - postfix + - python-dev + - python-pip + - python-tk + - redis-server + - screen + - supervisor + - vim + - xfonts-75dpi + - xfonts-base + + when: ansible_os_family == 'Debian' diff --git a/installer/playbooks/roles/bash_screen_wall/files/screen_wall.sh b/installer/playbooks/roles/bash_screen_wall/files/screen_wall.sh new file mode 100644 index 00000000..dec411e2 --- /dev/null +++ b/installer/playbooks/roles/bash_screen_wall/files/screen_wall.sh @@ -0,0 +1,8 @@ +if [ $TERM != 'screen' ] +then + PS1='HEY! USE SCREEN '$PS1 +fi + +sw() { + screen -x $1 || screen -S $1 +} diff --git a/installer/playbooks/roles/bash_screen_wall/tasks/main.yml b/installer/playbooks/roles/bash_screen_wall/tasks/main.yml new file mode 100644 index 00000000..338b6fbc --- /dev/null +++ b/installer/playbooks/roles/bash_screen_wall/tasks/main.yml @@ -0,0 +1,3 @@ +--- +- name: Setup bash screen wall + copy: src=screen_wall.sh dest=/etc/profile.d/screen_wall.sh \ No newline at end of file diff --git a/installer/playbooks/roles/dns_caching/handlers/main.yml b/installer/playbooks/roles/dns_caching/handlers/main.yml new file mode 100644 index 00000000..8197a1c5 --- /dev/null +++ b/installer/playbooks/roles/dns_caching/handlers/main.yml @@ -0,0 +1,3 @@ +--- +- name: restart network manager + service: name=NetworkManager state=restarted \ No newline at end of file diff --git a/installer/playbooks/roles/dns_caching/tasks/main.yml b/installer/playbooks/roles/dns_caching/tasks/main.yml new file mode 100644 index 00000000..26afaf9a --- /dev/null +++ b/installer/playbooks/roles/dns_caching/tasks/main.yml @@ -0,0 +1,7 @@ +- name: add dnsmasq to network config + lineinfile: > + dest=/etc/NetworkManager/NetworkManager.conf + regexp="dns=" + line="dns=dnsmasq" + state=present + notify: restart network manager \ No newline at end of file diff --git a/installer/playbooks/roles/epel/README.md b/installer/playbooks/roles/epel/README.md new file mode 100644 index 00000000..059424bf --- /dev/null +++ b/installer/playbooks/roles/epel/README.md @@ -0,0 +1,42 @@ +# Ansible Role: EPEL Repository + +Installs the EPEL repository (Extra Packages for Enterprise Linux) for RHEL/CentOS. + +## Requirements + +This role only is needed/runs on RHEL and its derivatives. + +## Role Variables + +Available variables are listed below, along with default values (see `defaults/main.yml`): + + epel_release: + "4": 10 + "5": 4 + "6": 8 + "7": 5 + +A mapping from RHEL major version to current EPEL release version. + + epel_repo_url: "http://download.fedoraproject.org/pub/epel/{{ ansible_distribution_major_version }}/{{ ansible_userspace_architecture }}{{ '/' if ansible_distribution_major_version < '7' else '/e/' }}epel-release-{{ ansible_distribution_major_version }}-{{ epel_release[ansible_distribution_major_version] }}.noarch.rpm" + epel_repo_gpg_key_url: "/etc/pki/rpm-gpg/RPM-GPG-KEY-EPEL-{{ ansible_distribution_major_version }}" + +The EPEL repo URL and GPG key URL. Generally, these should not be changed, but if this role is out of date, or if you need a very specific version, these can both be overridden. + +## Dependencies + +None. + +## Example Playbook + + - hosts: servers + roles: + - { role: geerlingguy.repo-epel } + +## License + +MIT / BSD + +## Author Information + +This role was created in 2014 by [Jeff Geerling](http://jeffgeerling.com/), author of [Ansible for DevOps](http://ansiblefordevops.com/). diff --git a/installer/playbooks/roles/epel/defaults/main.yml b/installer/playbooks/roles/epel/defaults/main.yml new file mode 100644 index 00000000..b0c89be3 --- /dev/null +++ b/installer/playbooks/roles/epel/defaults/main.yml @@ -0,0 +1,9 @@ +--- +epel_release: + "4": 10 + "5": 4 + "6": 8 + "7": 5 + +epel_repo_url: "http://download.fedoraproject.org/pub/epel/{{ ansible_distribution_major_version }}/{{ ansible_userspace_architecture }}{{ '/' if ansible_distribution_major_version < '7' else '/e/' }}epel-release-{{ ansible_distribution_major_version }}-{{ epel_release[ansible_distribution_major_version] }}.noarch.rpm" +epel_repo_gpg_key_url: "/etc/pki/rpm-gpg/RPM-GPG-KEY-EPEL-{{ ansible_distribution_major_version }}" diff --git a/installer/playbooks/roles/epel/meta/main.yml b/installer/playbooks/roles/epel/meta/main.yml new file mode 100644 index 00000000..3731f6fa --- /dev/null +++ b/installer/playbooks/roles/epel/meta/main.yml @@ -0,0 +1,18 @@ +--- +dependencies: [] + +galaxy_info: + author: geerlingguy + description: EPEL repository for RHEL/CentOS. + company: "Midwestern Mac, LLC" + license: "license (BSD, MIT)" + min_ansible_version: 1.4 + platforms: + - name: EL + versions: + - 4 + - 5 + - 6 + - 7 + categories: + - packaging diff --git a/installer/playbooks/roles/epel/tasks/main.yml b/installer/playbooks/roles/epel/tasks/main.yml new file mode 100644 index 00000000..8ac24a69 --- /dev/null +++ b/installer/playbooks/roles/epel/tasks/main.yml @@ -0,0 +1,10 @@ +--- +- name: Install EPEL repo. + yum: + name: "{{ epel_repo_url }}" + state: present + +- name: Import EPEL GPG key. + rpm_key: + key: "{{ epel_repo_gpg_key_url }}" + state: present diff --git a/installer/playbooks/roles/fail2ban/defaults/main.yml b/installer/playbooks/roles/fail2ban/defaults/main.yml new file mode 100644 index 00000000..3feadba7 --- /dev/null +++ b/installer/playbooks/roles/fail2ban/defaults/main.yml @@ -0,0 +1,2 @@ +--- +fail2ban_nginx_access_log: /var/log/nginx/access.log \ No newline at end of file diff --git a/installer/playbooks/roles/fail2ban/handlers/main.yml b/installer/playbooks/roles/fail2ban/handlers/main.yml new file mode 100644 index 00000000..d675d4d5 --- /dev/null +++ b/installer/playbooks/roles/fail2ban/handlers/main.yml @@ -0,0 +1,3 @@ +--- +- name: restart fail2ban + service: name=fail2ban state=restarted \ No newline at end of file diff --git a/installer/playbooks/roles/fail2ban/tasks/main.yml b/installer/playbooks/roles/fail2ban/tasks/main.yml new file mode 100644 index 00000000..a76bda54 --- /dev/null +++ b/installer/playbooks/roles/fail2ban/tasks/main.yml @@ -0,0 +1,21 @@ +--- +- name: Install fail2ban + yum: name=fail2ban state=present + +- name: Enable fail2ban + service: name=fail2ban enabled=yes + +- name: Create jail.d + file: path=/etc/fail2ban/jail.d state=directory + +- name: Setup filters + template: src="{{item}}-filter.conf.j2" dest="/etc/fail2ban/filter.d/{{item}}.conf" + with_items: + - nginx-proxy + notify: restart fail2ban + +- name: setup jails + template: src="{{item}}-jail.conf.j2" dest="/etc/fail2ban/jail.d/{{item}}.conf" + with_items: + - nginx-proxy + notify: restart fail2ban diff --git a/installer/playbooks/roles/fail2ban/templates/nginx-proxy-filter.conf.j2 b/installer/playbooks/roles/fail2ban/templates/nginx-proxy-filter.conf.j2 new file mode 100644 index 00000000..27f74cd5 --- /dev/null +++ b/installer/playbooks/roles/fail2ban/templates/nginx-proxy-filter.conf.j2 @@ -0,0 +1,10 @@ +# Block IPs trying to use server as proxy. +[Definition] +failregex = .*\" 400 + .*"[A-Z]* /(cms|muieblackcat|db|cpcommerce|cgi-bin|wp-login|joomla|awstatstotals|wp-content|wp-includes|pma|phpmyadmin|myadmin|mysql|mysqladmin|sqladmin|mypma|admin|xampp|mysqldb|pmadb|phpmyadmin1|phpmyadmin2).*" 4[\d][\d] + .*".*supports_implicit_sdk_logging.*" 4[\d][\d] + .*".*activities?advertiser_tracking_enabled.*" 4[\d][\d] + .*".*/picture?type=normal.*" 4[\d][\d] + .*".*/announce.php?info_hash=.*" 4[\d][\d] + +ignoreregex = \ No newline at end of file diff --git a/installer/playbooks/roles/fail2ban/templates/nginx-proxy-jail.conf.j2 b/installer/playbooks/roles/fail2ban/templates/nginx-proxy-jail.conf.j2 new file mode 100644 index 00000000..23a1dfc7 --- /dev/null +++ b/installer/playbooks/roles/fail2ban/templates/nginx-proxy-jail.conf.j2 @@ -0,0 +1,8 @@ +## block hosts trying to abuse our server as a forward proxy +[nginx-proxy] +enabled = true +filter = nginx-proxy +logpath = {{ fail2ban_nginx_access_log }} +action = iptables-multiport[name=NoNginxProxy, port="http,https"] +maxretry = 2 +bantime = 86400 \ No newline at end of file diff --git a/installer/playbooks/roles/frappe_selinux/files/frappe_selinux.te b/installer/playbooks/roles/frappe_selinux/files/frappe_selinux.te new file mode 100644 index 00000000..b8cd1f0f --- /dev/null +++ b/installer/playbooks/roles/frappe_selinux/files/frappe_selinux.te @@ -0,0 +1,32 @@ +module frappe_selinux 1.0; + +require { + type user_home_dir_t; + type httpd_t; + type user_home_t; + type soundd_port_t; + class tcp_socket name_connect; + class lnk_file read; + class dir { getattr search }; + class file { read open }; +} + +#============= httpd_t ============== + +#!!!! This avc is allowed in the current policy +allow httpd_t soundd_port_t:tcp_socket name_connect; + +#!!!! This avc is allowed in the current policy +allow httpd_t user_home_dir_t:dir search; + +#!!!! This avc is allowed in the current policy +allow httpd_t user_home_t:dir { getattr search }; + +#!!!! This avc can be allowed using the boolean 'httpd_read_user_content' +allow httpd_t user_home_t:file open; + +#!!!! This avc is allowed in the current policy +allow httpd_t user_home_t:file read; + +#!!!! This avc is allowed in the current policy +allow httpd_t user_home_t:lnk_file read; \ No newline at end of file diff --git a/installer/playbooks/roles/frappe_selinux/tasks/main.yml b/installer/playbooks/roles/frappe_selinux/tasks/main.yml new file mode 100644 index 00000000..67ac1dd1 --- /dev/null +++ b/installer/playbooks/roles/frappe_selinux/tasks/main.yml @@ -0,0 +1,21 @@ +--- +- name: Install deps + yum: name="{{item}}" state=present + with_items: + - policycoreutils-python + - selinux-policy-devel + +- name: Check enabled SELinux modules + shell: semanage module -l + register: enabled_modules + +- name: Copy frappe_selinux policy + copy: src=frappe_selinux.te dest=/root/frappe_selinux.te + register: dest_frappe_selinux_te + +- name: Compile frappe_selinux policy + shell: "make -f /usr/share/selinux/devel/Makefile frappe_selinux.pp && semodule -i frappe_selinux.pp" + args: + chdir: /root/ + when: "enabled_modules.stdout.find('frappe_selinux') == -1 or dest_frappe_selinux_te.changed" + diff --git a/installer/playbooks/roles/locale/defaults/main.yml b/installer/playbooks/roles/locale/defaults/main.yml new file mode 100644 index 00000000..3b713b45 --- /dev/null +++ b/installer/playbooks/roles/locale/defaults/main.yml @@ -0,0 +1,2 @@ +locale_keymap: us +locale_lang: en_US.utf8 \ No newline at end of file diff --git a/installer/playbooks/roles/locale/tasks/main.yml b/installer/playbooks/roles/locale/tasks/main.yml new file mode 100644 index 00000000..3c211eb5 --- /dev/null +++ b/installer/playbooks/roles/locale/tasks/main.yml @@ -0,0 +1,12 @@ +--- +- name: Check current locale + shell: localectl + register: locale_test + +- name: Set Locale + command: "localectl set-locale LANG={{ locale_lang }}" + when: locale_test.stdout.find('LANG={{ locale_lang }}') == -1 + +- name: Set keymap + command: "localectl set-keymap {{ locale_keymap }}" + when: "locale_test.stdout.find('Keymap: {{locale_keymap}}') == -1" \ No newline at end of file diff --git a/installer/playbooks/roles/logwatch/defaults/main.yml b/installer/playbooks/roles/logwatch/defaults/main.yml new file mode 100644 index 00000000..7c82c654 --- /dev/null +++ b/installer/playbooks/roles/logwatch/defaults/main.yml @@ -0,0 +1,3 @@ +--- +logwatch_emails: "{{ admin_emails }}" +logwatch_detail: High diff --git a/installer/playbooks/roles/logwatch/tasks/main.yml b/installer/playbooks/roles/logwatch/tasks/main.yml new file mode 100644 index 00000000..6d129c69 --- /dev/null +++ b/installer/playbooks/roles/logwatch/tasks/main.yml @@ -0,0 +1,6 @@ +--- +- name: Install logwatch + yum: name=logwatch state=present + +- name: Copy logwatch config + template: src=logwatch.conf.j2 dest=/etc/logwatch/conf/logwatch.conf backup=yes \ No newline at end of file diff --git a/installer/playbooks/roles/logwatch/templates/logwatch.conf.j2 b/installer/playbooks/roles/logwatch/templates/logwatch.conf.j2 new file mode 100644 index 00000000..a5c45cf8 --- /dev/null +++ b/installer/playbooks/roles/logwatch/templates/logwatch.conf.j2 @@ -0,0 +1,2 @@ +MailTo = {{ logwatch_emails }} +Detail = {{ logwatch_detail }} \ No newline at end of file diff --git a/installer/playbooks/roles/mariadb/README.md b/installer/playbooks/roles/mariadb/README.md new file mode 100644 index 00000000..bc872db5 --- /dev/null +++ b/installer/playbooks/roles/mariadb/README.md @@ -0,0 +1,64 @@ +# Ansible Role: MariaDB + +Installs MariaDB + +## Supported platforms + +``` +CentOS 6 & 7 +Ubuntu 14.04 +``` + +## Post install + +Run `mysql_secure_installation` + +## Requirements + +None + +## Role Variables + +MariaDB version: + +``` +mariadb_version: 10.0 +``` + +Configuration template: + +``` +mysql_conf_tpl: change_me +``` + +Configuration filename: + +``` +mysql_conf_file: settings.cnf +``` + +### Experimental unattended mysql_secure_installation + +``` +ansible-playbook release.yml --extra-vars "mysql_secure_installation=true mysql_root_password=your_very_secret_password" +``` + +## Dependencies + +None + +## Example Playbook + +``` +- hosts: servers + roles: + - { role: pcextreme.mariadb } +``` + +## License + +MIT / BSD + +## Author Information + +Created by [Attila van der Velde](https://github.com/vdvm) diff --git a/installer/playbooks/roles/mariadb/defaults/main.yml b/installer/playbooks/roles/mariadb/defaults/main.yml new file mode 100644 index 00000000..c091ab71 --- /dev/null +++ b/installer/playbooks/roles/mariadb/defaults/main.yml @@ -0,0 +1,8 @@ +--- +mariadb_version: 10.0 + +mysql_conf_tpl: change_me +mysql_conf_file: settings.cnf + +mysql_secure_installation: false +mysql_root_password: frappe diff --git a/installer/playbooks/roles/mariadb/handlers/main.yml b/installer/playbooks/roles/mariadb/handlers/main.yml new file mode 100644 index 00000000..3755d8ce --- /dev/null +++ b/installer/playbooks/roles/mariadb/handlers/main.yml @@ -0,0 +1,3 @@ +--- +- name: restart mysql + service: name=mysql state=restarted diff --git a/installer/playbooks/roles/mariadb/meta/main.yml b/installer/playbooks/roles/mariadb/meta/main.yml new file mode 100644 index 00000000..b2beef2c --- /dev/null +++ b/installer/playbooks/roles/mariadb/meta/main.yml @@ -0,0 +1,19 @@ +--- +galaxy_info: + author: "Attila van der Velde" + description: "Installs MariaDB" + company: "PCextreme B.V." + license: "license (MIT, BSD)" + min_ansible_version: 1.8 + platforms: + - name: EL + versions: + - 6 + - 7 + - name: Ubuntu + versions: + - trusty + categories: + - database:sql + +dependencies: [] diff --git a/vm/ansible/roles/mariadb/tasks/centos.yml~ b/installer/playbooks/roles/mariadb/tasks/centos.yml similarity index 93% rename from vm/ansible/roles/mariadb/tasks/centos.yml~ rename to installer/playbooks/roles/mariadb/tasks/centos.yml index 00e4b590..84b49c55 100644 --- a/vm/ansible/roles/mariadb/tasks/centos.yml~ +++ b/installer/playbooks/roles/mariadb/tasks/centos.yml @@ -8,3 +8,5 @@ - MariaDB-server - MariaDB-client - MySQL-python + - MariaDB-devel + diff --git a/installer/playbooks/roles/mariadb/tasks/main.yml b/installer/playbooks/roles/mariadb/tasks/main.yml new file mode 100644 index 00000000..2c185324 --- /dev/null +++ b/installer/playbooks/roles/mariadb/tasks/main.yml @@ -0,0 +1,18 @@ +--- +- include: centos.yml + when: ansible_distribution == 'CentOS' and ansible_distribution_major_version|int >= 6 + +- include: ubuntu.yml + when: ansible_distribution == 'Ubuntu' and ansible_distribution_version == '14.04' + +- name: Add configuration + template: src={{ mysql_conf_tpl }} dest={{ mysql_conf_dir[ansible_distribution] }}/{{ mysql_conf_file }} owner=root group=root mode=0644 + when: mysql_conf_tpl != 'change_me' + notify: restart mysql + +- name: Start and enable service + service: name=mysql state=started enabled=yes + +- include: mysql_secure_installation.yml +- debug: var=mysql_secure_installation + when: mysql_secure_installation and mysql_root_password is defined diff --git a/installer/playbooks/roles/mariadb/tasks/mysql_secure_installation.yml b/installer/playbooks/roles/mariadb/tasks/mysql_secure_installation.yml new file mode 100644 index 00000000..6d1d9996 --- /dev/null +++ b/installer/playbooks/roles/mariadb/tasks/mysql_secure_installation.yml @@ -0,0 +1,59 @@ +--- +# Set root password +# UPDATE mysql.user SET Password=PASSWORD('mysecret') WHERE User='root'; +# FLUSH PRIVILEGES; + + +- name: Set root Password + mysql_user: name=root host={{ item }} password={{ mysql_root_password }} state=present + with_items: + - localhost + +- name: Add .my.cnf + command: '/usr/bin/whoami' + register: current_user + template: src=my.cnf.j2 dest=/home/{{ current_user }}/.my.cnf owner=root group=root mode=0600 + +- name: Set root Password + mysql_user: name=root host={{ item }} password={{ mysql_root_password }} state=present + with_items: + - 127.0.0.1 + - ::1 + +- name: Reload privilege tables + command: 'mysql -ne "{{ item }}"' + with_items: + - FLUSH PRIVILEGES + changed_when: False + +- name: Reload privilege tables + command: 'mysql -ne "{{ item }}"' + with_items: + - FLUSH PRIVILEGES + changed_when: False + +- name: Remove anonymous users + command: 'mysql -ne "{{ item }}"' + with_items: + - DELETE FROM mysql.user WHERE User='' + changed_when: False + +- name: Disallow root login remotely + command: 'mysql -ne "{{ item }}"' + with_items: + - DELETE FROM mysql.user WHERE User='root' AND Host NOT IN ('localhost', '127.0.0.1', '::1') + changed_when: False + +- name: Remove test database and access to it + command: 'mysql -ne "{{ item }}"' + with_items: + - DROP DATABASE if exists test + - DELETE FROM mysql.db WHERE Db='test' OR Db='test\\_%' + changed_when: False + ignore_errors: True + +- name: Reload privilege tables + command: 'mysql -ne "{{ item }}"' + with_items: + - FLUSH PRIVILEGES + changed_when: False diff --git a/installer/playbooks/roles/mariadb/tasks/ubuntu.yml b/installer/playbooks/roles/mariadb/tasks/ubuntu.yml new file mode 100644 index 00000000..ea5c1031 --- /dev/null +++ b/installer/playbooks/roles/mariadb/tasks/ubuntu.yml @@ -0,0 +1,23 @@ +--- +- name: Add repo file + template: src=mariadb_ubuntu.list.j2 dest=/etc/apt/sources.list.d/mariadb.list owner=root group=root mode=0644 + register: mariadb_list + +- name: Add repo key + apt_key: id=1BB943DB url=http://keyserver.ubuntu.com/pks/lookup?op=get&search=0xCBCB082A1BB943DB state=present + register: mariadb_key + +- name: Update apt cache + apt: update_cache=yes + when: mariadb_list.changed == True or mariadb_key.changed == True + +- name: Unattended package installation + shell: export DEBIAN_FRONTEND=noninteractive + changed_when: false + +- name: Install MariaDB + apt: pkg={{ item }} state=present + with_items: + - mariadb-server + - mariadb-client + - python-mysqldb diff --git a/installer/playbooks/roles/mariadb/templates/mariadb_centos.repo.j2 b/installer/playbooks/roles/mariadb/templates/mariadb_centos.repo.j2 new file mode 100644 index 00000000..64738cc1 --- /dev/null +++ b/installer/playbooks/roles/mariadb/templates/mariadb_centos.repo.j2 @@ -0,0 +1,7 @@ +# MariaDB CentOS {{ ansible_distribution_major_version|int }} repository list +# http://mariadb.org/mariadb/repositories/ +[mariadb] +name = MariaDB +baseurl = http://yum.mariadb.org/{{ mariadb_version }}/centos{{ ansible_distribution_major_version|int }}-amd64 +gpgkey=https://yum.mariadb.org/RPM-GPG-KEY-MariaDB +gpgcheck=1 diff --git a/installer/playbooks/roles/mariadb/templates/mariadb_ubuntu.list.j2 b/installer/playbooks/roles/mariadb/templates/mariadb_ubuntu.list.j2 new file mode 100644 index 00000000..981b4d6b --- /dev/null +++ b/installer/playbooks/roles/mariadb/templates/mariadb_ubuntu.list.j2 @@ -0,0 +1,4 @@ +# MariaDB Ubuntu {{ ansible_distribution_release | title }} repository list +# http://mariadb.org/mariadb/repositories/ +deb http://ams2.mirrors.digitalocean.com/mariadb/repo/{{ mariadb_version }}/ubuntu {{ ansible_distribution_release | lower }} main +deb-src http://ams2.mirrors.digitalocean.com/mariadb/repo/{{ mariadb_version }}/ubuntu {{ ansible_distribution_release | lower }} main diff --git a/installer/playbooks/roles/mariadb/templates/my.cnf.j2 b/installer/playbooks/roles/mariadb/templates/my.cnf.j2 new file mode 100644 index 00000000..b63b4e63 --- /dev/null +++ b/installer/playbooks/roles/mariadb/templates/my.cnf.j2 @@ -0,0 +1,3 @@ +[client] +user=root +password={{ mysql_root_password }} diff --git a/installer/playbooks/roles/mariadb/vars/main.yml b/installer/playbooks/roles/mariadb/vars/main.yml new file mode 100644 index 00000000..4248ca90 --- /dev/null +++ b/installer/playbooks/roles/mariadb/vars/main.yml @@ -0,0 +1,4 @@ +--- +mysql_conf_dir: + "CentOS": /etc/my.cnf.d + "Ubuntu": /etc/mysql/conf.d diff --git a/installer/playbooks/roles/nginx/.travis.yml b/installer/playbooks/roles/nginx/.travis.yml new file mode 100644 index 00000000..24648b13 --- /dev/null +++ b/installer/playbooks/roles/nginx/.travis.yml @@ -0,0 +1,35 @@ +--- +language: python +python: "2.7" + +env: + - SITE=test.yml + +before_install: + - sudo apt-get update -qq + - sudo apt-get install -y curl + +install: + # Install Ansible. + - pip install ansible + + # Add ansible.cfg to pick up roles path. + - "{ echo '[defaults]'; echo 'roles_path = ../'; } >> ansible.cfg" + +script: + # Check the role/playbook's syntax. + - "ansible-playbook -i tests/inventory tests/$SITE --syntax-check" + + # Run the role/playbook with ansible-playbook. + - "ansible-playbook -i tests/inventory tests/$SITE --connection=local --sudo" + + # Run the role/playbook again, checking to make sure it's idempotent. + - > + ansible-playbook -i tests/inventory tests/$SITE --connection=local --sudo + | grep -q 'changed=0.*failed=0' + && (echo 'Idempotence test: pass' && exit 0) + || (echo 'Idempotence test: fail' && exit 1) + + # TODO - get the test working. Probably need to add a virtual host. + # Request a page via Nginx, to make sure Nginx is running and responds. + # - "curl http://localhost/" diff --git a/installer/playbooks/roles/nginx/README.md b/installer/playbooks/roles/nginx/README.md new file mode 100644 index 00000000..00bfb8a2 --- /dev/null +++ b/installer/playbooks/roles/nginx/README.md @@ -0,0 +1,82 @@ +# Ansible Role: Nginx + +[![Build Status](https://travis-ci.org/geerlingguy/ansible-role-nginx.svg?branch=master)](https://travis-ci.org/geerlingguy/ansible-role-nginx) + +Installs Nginx on RedHat/CentOS or Debian/Ubuntu linux servers. + +This role installs and configures the latest version of Nginx from the Nginx yum repository (on RedHat-based systems) or via apt (on Debian-based systems). You will likely need to do extra setup work after this role has installed Nginx, like adding your own [virtualhost].conf file inside `/etc/nginx/conf.d/`, describing the location and options to use for your particular website. + +## Requirements + +None. + +## Role Variables + +Available variables are listed below, along with default values (see `defaults/main.yml`): + + nginx_vhosts: [] + +A list of vhost definitions (server blocks) for Nginx virtual hosts. If left empty, you will need to supply your own virtual host configuration. See the commented example in `defaults/main.yml` for available server options. If you have a large number of customizations required for your server definition(s), you're likely better off managing the vhost configuration file yourself, leaving this variable set to `[]`. + + nginx_remove_default_vhost: false + +Whether to remove the 'default' virtualhost configuration supplied by Nginx. Useful if you want the base `/` URL to be directed at one of your own virtual hosts configured in a separate .conf file. + + nginx_upstreams: [] + +If you are configuring Nginx as a load balancer, you can define one or more upstream sets using this variable. In addition to defining at least one upstream, you would need to configure one of your server blocks to proxy requests through the defined upstream (e.g. `proxy_pass http://myapp1;`). See the commented example in `defaults/main.yml` for more information. + + nginx_user: "nginx" + +The user under which Nginx will run. Defaults to `nginx` for RedHat, and `www-data` for Debian. + + nginx_worker_processes: "1" + nginx_worker_connections: "1024" + +`nginx_worker_processes` should be set to the number of cores present on your machine. Connections (find this number with `grep processor /proc/cpuinfo | wc -l`). `nginx_worker_connections` is the number of connections per process. Set this higher to handle more simultaneous connections (and remember that a connection will be used for as long as the keepalive timeout duration for every client!). + + nginx_error_log: "/var/log/nginx/error.log warn" + nginx_access_log: "/var/log/nginx/access.log main buffer=16k" + +Configuration of the default error and access logs. Set to `off` to disable a log entirely. + + nginx_sendfile: "on" + nginx_tcp_nopush: "on" + nginx_tcp_nodelay: "on" + +TCP connection options. See [this blog post](https://t37.net/nginx-optimization-understanding-sendfile-tcp_nodelay-and-tcp_nopush.html) for more information on these directives. + + nginx_keepalive_timeout: "65" + nginx_keepalive_requests: "100" + +Nginx keepalive settings. Timeout should be set higher (10s+) if you have more polling-style traffic (AJAX-powered sites especially), or lower (<10s) if you have a site where most users visit a few pages and don't send any further requests. + + nginx_client_max_body_size: "64m" + +This value determines the largest file upload possible, as uploads are passed through Nginx before hitting a backend like `php-fpm`. If you get an error like `client intended to send too large body`, it means this value is set too low. + + nginx_proxy_cache_path: "" + +Set as the `proxy_cache_path` directive in the `nginx.conf` file. By default, this will not be configured (if left as an empty string), but if you wish to use Nginx as a reverse proxy, you can set this to a valid value (e.g. `"/var/cache/nginx keys_zone=cache:32m"`) to use Nginx's cache (further proxy configuration can be done in individual server configurations). + + nginx_default_release: "" + +(For Debian/Ubuntu only) Allows you to set a different repository for the installation of Nginx. As an example, if you are running Debian's wheezy release, and want to get a newer version of Nginx, you can install the `wheezy-backports` repository and set that value here, and Ansible will use that as the `-t` option while installing Nginx. + +## Dependencies + +None. + +## Example Playbook + + - hosts: server + roles: + - { role: geerlingguy.nginx } + +## License + +MIT / BSD + +## Author Information + +This role was created in 2014 by [Jeff Geerling](http://jeffgeerling.com/), author of [Ansible for DevOps](http://ansiblefordevops.com/). diff --git a/installer/playbooks/roles/nginx/defaults/main.yml b/installer/playbooks/roles/nginx/defaults/main.yml new file mode 100644 index 00000000..8aacad8c --- /dev/null +++ b/installer/playbooks/roles/nginx/defaults/main.yml @@ -0,0 +1,47 @@ +--- +# Used only for Debian/Ubuntu installation, as the -t option for apt. +nginx_default_release: "" + +nginx_worker_processes: "1" +nginx_worker_connections: "1024" + +nginx_error_log: "/var/log/nginx/error.log warn" +nginx_access_log: "/var/log/nginx/access.log main buffer=16k" + +nginx_sendfile: "on" +nginx_tcp_nopush: "on" +nginx_tcp_nodelay: "on" + +nginx_keepalive_timeout: "65" +nginx_keepalive_requests: "100" + +nginx_client_max_body_size: "64m" + +nginx_proxy_cache_path: "" + +nginx_remove_default_vhost: false +nginx_vhosts: [] +# Example vhost below, showing all available options: +# - { +# listen: "80 default_server", # default: "80 default_server" +# server_name: "example.com", # default: N/A +# root: "/var/www/example.com", # default: N/A +# index: "index.html index.htm", # default: "index.html index.htm" +# +# # Properties that are only added if defined: +# error_page: "", +# access_log: "", +# extra_config: "" # Can be used to add extra config blocks (multiline). +# } + +nginx_upstreams: [] +# - { +# name: myapp1, +# strategy: "ip_hash", # "least_conn", etc. +# servers: { +# "srv1.example.com", +# "srv2.example.com weight=3", +# "srv3.example.com" +# } +# } +nginx_conf_file: nginx.conf.j2 \ No newline at end of file diff --git a/installer/playbooks/roles/nginx/handlers/main.yml b/installer/playbooks/roles/nginx/handlers/main.yml new file mode 100644 index 00000000..92971d2c --- /dev/null +++ b/installer/playbooks/roles/nginx/handlers/main.yml @@ -0,0 +1,3 @@ +--- +- name: restart nginx + service: name=nginx state=restarted diff --git a/installer/playbooks/roles/nginx/meta/main.yml b/installer/playbooks/roles/nginx/meta/main.yml new file mode 100644 index 00000000..efbe68f7 --- /dev/null +++ b/installer/playbooks/roles/nginx/meta/main.yml @@ -0,0 +1,23 @@ +--- +dependencies: [] + +galaxy_info: + author: geerlingguy + description: Nginx installation for Linux/UNIX. + company: "Midwestern Mac, LLC" + license: "license (BSD, MIT)" + min_ansible_version: 1.4 + platforms: + - name: EL + versions: + - 6 + - 7 + - name: Debian + versions: + - all + - name: Ubuntu + versions: + - all + categories: + - development + - web diff --git a/installer/playbooks/roles/nginx/tasks/main.yml b/installer/playbooks/roles/nginx/tasks/main.yml new file mode 100644 index 00000000..da4be94d --- /dev/null +++ b/installer/playbooks/roles/nginx/tasks/main.yml @@ -0,0 +1,31 @@ +--- +# Variable setup. +- name: Include OS-specific variables. + include_vars: "{{ ansible_os_family }}.yml" + +- name: Define nginx_user. + set_fact: + nginx_user: "{{ __nginx_user }}" + when: nginx_user is not defined + +# Setup/install tasks. +- include: setup-RedHat.yml + when: ansible_os_family == 'RedHat' + +- include: setup-Debian.yml + when: ansible_os_family == 'Debian' + +# Nginx setup. +- name: Copy nginx configuration in place. + template: + src: "{{ nginx_conf_file }}" + dest: /etc/nginx/nginx.conf + owner: root + group: root + mode: 0644 + notify: restart nginx + +- name: Ensure nginx is started and enabled to start at boot. + service: name=nginx state=started enabled=yes + +- include: vhosts.yml diff --git a/installer/playbooks/roles/nginx/tasks/setup-Debian.yml b/installer/playbooks/roles/nginx/tasks/setup-Debian.yml new file mode 100644 index 00000000..ced11b65 --- /dev/null +++ b/installer/playbooks/roles/nginx/tasks/setup-Debian.yml @@ -0,0 +1,6 @@ +--- +- name: Ensure nginx is installed. + apt: + pkg: nginx + state: installed + default_release: "{{ nginx_default_release }}" diff --git a/installer/playbooks/roles/nginx/tasks/setup-RedHat.yml b/installer/playbooks/roles/nginx/tasks/setup-RedHat.yml new file mode 100644 index 00000000..73f205e5 --- /dev/null +++ b/installer/playbooks/roles/nginx/tasks/setup-RedHat.yml @@ -0,0 +1,11 @@ +--- +- name: Enable nginx repo. + template: + src: nginx.repo.j2 + dest: /etc/yum.repos.d/nginx.repo + owner: root + group: root + mode: 0644 + +- name: Ensure nginx is installed. + yum: pkg=nginx state=installed enablerepo=nginx diff --git a/installer/playbooks/roles/nginx/tasks/vhosts.yml b/installer/playbooks/roles/nginx/tasks/vhosts.yml new file mode 100644 index 00000000..5ee8ec22 --- /dev/null +++ b/installer/playbooks/roles/nginx/tasks/vhosts.yml @@ -0,0 +1,22 @@ +--- +- name: Remove default nginx vhost config file (if configured). + file: + path: "{{ nginx_default_vhost_path }}" + state: absent + when: nginx_remove_default_vhost + notify: restart nginx + +- name: Add managed vhost config file (if any vhosts are configured). + template: + src: vhosts.j2 + dest: "{{ nginx_vhost_path }}/vhosts.conf" + mode: 0644 + when: nginx_vhosts + notify: restart nginx + +- name: Remove managed vhost config file (if no vhosts are configured). + file: + path: "{{ nginx_vhost_path }}/vhosts.conf" + state: absent + when: not nginx_vhosts + notify: restart nginx diff --git a/installer/playbooks/roles/nginx/templates/nginx.conf.j2 b/installer/playbooks/roles/nginx/templates/nginx.conf.j2 new file mode 100644 index 00000000..a43202ce --- /dev/null +++ b/installer/playbooks/roles/nginx/templates/nginx.conf.j2 @@ -0,0 +1,51 @@ +user {{ nginx_user }}; + +error_log {{ nginx_error_log }}; +pid /var/run/nginx.pid; + +worker_processes {{ nginx_worker_processes }}; + +events { + worker_connections {{ nginx_worker_connections }}; +} + +http { + include /etc/nginx/mime.types; + default_type application/octet-stream; + + server_names_hash_bucket_size 64; + + client_max_body_size {{ nginx_client_max_body_size }}; + + log_format main '$remote_addr - $remote_user [$time_local] "$request" ' + '$status $body_bytes_sent "$http_referer" ' + '"$http_user_agent" "$http_x_forwarded_for"'; + + access_log {{ nginx_access_log }}; + + sendfile {{ nginx_sendfile }}; + tcp_nopush {{ nginx_tcp_nopush }}; + tcp_nodelay {{ nginx_tcp_nodelay }}; + + keepalive_timeout {{ nginx_keepalive_timeout }}; + keepalive_requests {{ nginx_keepalive_requests }}; + + #gzip on; + +{% if nginx_proxy_cache_path %} + proxy_cache_path {{ nginx_proxy_cache_path }}; +{% endif %} + +{% for upstream in nginx_upstreams %} + upstream {{ upstream.name }} { +{% if upstream.strategy is defined %} + {{ upstream.strategy }}; +{% endif %} +{% for server in upstream.servers %} + server {{ server }}; +{% endfor %} + } +{% endfor %} + + include {{ nginx_vhost_path }}/*; +} diff --git a/installer/playbooks/roles/nginx/templates/nginx.repo.j2 b/installer/playbooks/roles/nginx/templates/nginx.repo.j2 new file mode 100644 index 00000000..9a853b70 --- /dev/null +++ b/installer/playbooks/roles/nginx/templates/nginx.repo.j2 @@ -0,0 +1,5 @@ +[nginx] +name=nginx repo +baseurl=http://nginx.org/packages/centos/{{ ansible_distribution_major_version }}/$basearch/ +gpgcheck=0 +enabled=1 diff --git a/installer/playbooks/roles/nginx/templates/vhosts.j2 b/installer/playbooks/roles/nginx/templates/vhosts.j2 new file mode 100644 index 00000000..09bda352 --- /dev/null +++ b/installer/playbooks/roles/nginx/templates/vhosts.j2 @@ -0,0 +1,24 @@ +{% for vhost in nginx_vhosts %} +server { + listen {{ vhost.listen | default('80 default_server') }}; + server_name {{ vhost.server_name }}; + + root {{ vhost.root }}; + index {{ vhost.index | default('index.html index.htm') }}; + + {% if vhost.error_page is defined %} + error_page {{ vhost.error_page }}; + {% endif %} + {% if vhost.access_log is defined %} + access_log {{ vhost.access_log }}; + {% endif %} + + {% if vhost.return is defined %} + return {{ vhost.return }}; + {% endif %} + + {% if vhost.extra_parameters is defined %} + {{ vhost.extra_parameters }}; + {% endif %} +} +{% endfor %} diff --git a/installer/playbooks/roles/nginx/tests/inventory b/installer/playbooks/roles/nginx/tests/inventory new file mode 100644 index 00000000..2fbb50c4 --- /dev/null +++ b/installer/playbooks/roles/nginx/tests/inventory @@ -0,0 +1 @@ +localhost diff --git a/installer/playbooks/roles/nginx/tests/test.yml b/installer/playbooks/roles/nginx/tests/test.yml new file mode 100644 index 00000000..42bba2c0 --- /dev/null +++ b/installer/playbooks/roles/nginx/tests/test.yml @@ -0,0 +1,5 @@ +--- +- hosts: localhost + remote_user: root + roles: + - ansible-role-nginx diff --git a/installer/playbooks/roles/nginx/vars/Debian.yml b/installer/playbooks/roles/nginx/vars/Debian.yml new file mode 100644 index 00000000..b78b7c55 --- /dev/null +++ b/installer/playbooks/roles/nginx/vars/Debian.yml @@ -0,0 +1,4 @@ +--- +nginx_vhost_path: /etc/nginx/sites-enabled +nginx_default_vhost_path: /etc/nginx/sites-enabled/default +__nginx_user: "www-data" diff --git a/installer/playbooks/roles/nginx/vars/RedHat.yml b/installer/playbooks/roles/nginx/vars/RedHat.yml new file mode 100644 index 00000000..24123048 --- /dev/null +++ b/installer/playbooks/roles/nginx/vars/RedHat.yml @@ -0,0 +1,4 @@ +--- +nginx_vhost_path: /etc/nginx/conf.d +nginx_default_vhost_path: /etc/nginx/conf.d/default.conf +__nginx_user: "nginx" diff --git a/installer/playbooks/roles/ntpd/tasks/main.yml b/installer/playbooks/roles/ntpd/tasks/main.yml new file mode 100644 index 00000000..19881f82 --- /dev/null +++ b/installer/playbooks/roles/ntpd/tasks/main.yml @@ -0,0 +1,9 @@ +--- +- name: Install ntpd + yum: name="{{item}}" state=installed + with_items: + - ntp + - ntpdate + +- name: enable ntpd + service: name=ntpd enabled=yes state=started \ No newline at end of file diff --git a/installer/playbooks/roles/swap/defaults/main.yml b/installer/playbooks/roles/swap/defaults/main.yml new file mode 100644 index 00000000..3eac0ae0 --- /dev/null +++ b/installer/playbooks/roles/swap/defaults/main.yml @@ -0,0 +1 @@ +swap_size_mb: 1024 \ No newline at end of file diff --git a/installer/playbooks/roles/swap/tasks/main.yml b/installer/playbooks/roles/swap/tasks/main.yml new file mode 100644 index 00000000..1d61d2f4 --- /dev/null +++ b/installer/playbooks/roles/swap/tasks/main.yml @@ -0,0 +1,18 @@ +- name: Create swap space + command: dd if=/dev/zero of=/extraswap bs=1M count={{swap_size_mb}} + when: ansible_swaptotal_mb < 1 + +- name: Make swap + command: mkswap /extraswap + when: ansible_swaptotal_mb < 1 + +- name: Add to fstab + action: lineinfile dest=/etc/fstab regexp="extraswap" line="/extraswap none swap sw 0 0" state=present + when: ansible_swaptotal_mb < 1 + +- name: Turn swap on + command: swapon -a + when: ansible_swaptotal_mb < 1 + +- name: Set swapiness + shell: echo 1 | tee /proc/sys/vm/swappiness \ No newline at end of file diff --git a/installer/playbooks/roles/wkhtmltopdf/defaults/main.yml b/installer/playbooks/roles/wkhtmltopdf/defaults/main.yml new file mode 100644 index 00000000..76266ab0 --- /dev/null +++ b/installer/playbooks/roles/wkhtmltopdf/defaults/main.yml @@ -0,0 +1 @@ +wkhtmltopdf_version: 0.12.2.1 \ No newline at end of file diff --git a/installer/playbooks/roles/wkhtmltopdf/tasks/main.yml b/installer/playbooks/roles/wkhtmltopdf/tasks/main.yml new file mode 100644 index 00000000..7f246b39 --- /dev/null +++ b/installer/playbooks/roles/wkhtmltopdf/tasks/main.yml @@ -0,0 +1,32 @@ +--- +- name: install base fonts + yum: name={{ item }} state=present + with_items: + - libXrender + - libXext + - xorg-x11-fonts-75dpi + - xorg-x11-fonts-Type1 + when: ansible_os_family == 'RedHat' + +- name: Install wkhtmltopdf rpm + yum: name=http://download.gna.org/wkhtmltopdf/0.12/{{ wkhtmltopdf_version }}/wkhtmltox-{{ wkhtmltopdf_version }}_linux-centos{{ ansible_distribution_major_version }}-{{ "amd64" if ansible_architecture == "x86_64" else "i386"}}.rpm + when: ansible_os_family == 'RedHat' + +- name: install base fonts + apt: name={{ item }} state=present force=yes + with_items: + - libxrender1 + - libxext6 + - xfonts-75dpi + - xfonts-base + when: ansible_os_family == 'Debian' + +- name: Download wkhtmltopdf + get_url: + url=http://download.gna.org/wkhtmltopdf/0.12/{{ wkhtmltopdf_version }}/wkhtmltox-{{ wkhtmltopdf_version }}_linux-{{ ansible_distribution_release }}-{{ "amd64" if ansible_architecture == "x86_64" else "i386"}}.deb + dest="/tmp/" + when: ansible_os_family == 'Debian' + +- name: Install wkhtmltopdf deb + apt: deb=/tmp/wkhtmltox-{{ wkhtmltopdf_version }}_linux-{{ ansible_distribution_release }}-{{ "amd64" if ansible_architecture == "x86_64" else "i386"}}.deb + when: ansible_os_family == 'Debian' diff --git a/installer/playbooks/templates/mariadb_config.cnf b/installer/playbooks/templates/mariadb_config.cnf new file mode 100644 index 00000000..6d4bcf28 --- /dev/null +++ b/installer/playbooks/templates/mariadb_config.cnf @@ -0,0 +1,60 @@ +[mysqld] + +# GENERAL # +user = mysql +default-storage-engine = InnoDB +socket = /var/lib/mysql/mysql.sock +pid-file = /var/lib/mysql/mysql.pid + +# MyISAM # +key-buffer-size = 32M +myisam-recover = FORCE,BACKUP + +# SAFETY # +max-allowed-packet = 16M +max-connect-errors = 1000000 +innodb = FORCE + +# DATA STORAGE # +datadir = /var/lib/mysql/ + +# BINARY LOGGING # +log-bin = /var/lib/mysql/mysql-bin +expire-logs-days = 14 +sync-binlog = 1 + +# REPLICATION # +server-id = 1 + +# CACHES AND LIMITS # +tmp-table-size = 32M +max-heap-table-size = 32M +query-cache-type = 0 +query-cache-size = 0 +max-connections = 500 +thread-cache-size = 50 +open-files-limit = 65535 +table-definition-cache = 4096 +table-open-cache = 10240 + +# INNODB # +innodb-flush-method = O_DIRECT +innodb-log-files-in-group = 2 +innodb-log-file-size = 512M +innodb-flush-log-at-trx-commit = 1 +innodb-file-per-table = 1 +innodb-buffer-pool-size = {{ (ansible_memtotal_mb*0.685)|round|int }}M +innodb-file-format = barracuda +innodb-large-prefix = 1 +collation-server = utf8mb4_unicode_ci +character-set-server = utf8mb4 +character-set-client-handshake = FALSE + +# LOGGING # +log-error = /var/lib/mysql/mysql-error.log +log-queries-not-using-indexes = 0 +slow-query-log = 1 +slow-query-log-file = /var/lib/mysql/mysql-slow.log + +[mysql] +default-character-set = utf8mb4 diff --git a/installer/playbooks/templates/nginx.conf b/installer/playbooks/templates/nginx.conf new file mode 100644 index 00000000..fa487e86 --- /dev/null +++ b/installer/playbooks/templates/nginx.conf @@ -0,0 +1,59 @@ +user nginx; +worker_processes 6; +worker_rlimit_nofile 65535; + +error_log /var/log/nginx/error.log warn; +pid /var/run/nginx.pid; + + +events { + worker_connections 2048; + multi_accept on; +} + + +http { + include /etc/nginx/mime.types; + default_type application/octet-stream; + + log_format main '$remote_addr - $remote_user [$time_local] "$request" ' + '$status $body_bytes_sent "$http_referer" ' + '"$http_user_agent" "$http_x_forwarded_for"'; + + access_log /var/log/nginx/access.log main; + + sendfile on; + tcp_nopush on; + tcp_nodelay on; + server_tokens off; + #tcp_nopush on; + + keepalive_timeout 10; + keepalive_requests 10; + + gzip on; + gzip_disable "msie6"; + + gzip_vary on; + gzip_proxied any; + gzip_comp_level 6; + gzip_buffers 16 8k; + gzip_http_version 1.1; + gzip_types text/plain text/css application/json application/javascript image/svg+xml text/html "application/json; charset: utf-8" "text/html; charset: utf-8" application/font-woff; + + server_names_hash_max_size 4096; + #server_names_hash_bucket_size 64; + + open_file_cache max=65000 inactive=1m; + open_file_cache_valid 5s; + open_file_cache_min_uses 1; + open_file_cache_errors on; + + ssl_protocols SSLv3 TLSv1; + ssl_ciphers ECDHE-RSA-AES256-SHA384:AES256-SHA256:RC4:HIGH:!MD5:!aNULL:!EDH:!AESGCM; + ssl_prefer_server_ciphers on; + + proxy_cache_path /var/cache/nginx levels=1:2 keys_zone=web-cache:8m max_size=1000m inactive=600m; + + include /etc/nginx/conf.d/*.conf; +} \ No newline at end of file diff --git a/playbooks/__init__.py b/playbooks/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/playbooks/develop/centos.yml b/playbooks/develop/centos.yml new file mode 100644 index 00000000..5fc52f19 --- /dev/null +++ b/playbooks/develop/centos.yml @@ -0,0 +1,64 @@ +--- +- hosts: localhost + vars: + bench_repo_path: "/usr/local/frappe/bench-repo" + bench_path: "/home/{{ ansible_user_id }}/frappe/frappe-bench" + mysql_config_template: "templates/simple_mariadb_config.cnf" + mysql_conf_dir: /etc/my.cnf.d/ + wkhtmltopdf_version: 0.12.2.1 + + tasks: + + # install pre-requisites + - name: add epel repo + yum: name="https://dl.fedoraproject.org/pub/epel/{{ ansible_lsb.major_release }}/x86_64/e/epel-release-*.rpm" state=present + become: yes + become_user: root + + - name: development tools package + yum: name="@Development tools" state=present + become: yes + become_user: root + + - name: install prequisites + yum: pkg={{ item }} state=present + with_items: + # basic installs + - redis + - nodejs + - npm + + # for mariadb + - software-properties-common + + # for wkhtmltopdf + - libXrender + - libXext + - xorg-x11-fonts-75dpi + - xorg-x11-fonts-Type1 + + # for Pillow + - libjpeg-devel + - zlib-devel + - libzip-devel + - freetype-devel + - lcms2-devel + - libwebp-devel + - libtiff-devel + - tcl-devel + - tk-devel + + become: yes + become_user: root + + # install MariaDB + - include: includes/mariadb_centos.yml + + # install WKHTMLtoPDF + - include: includes/wkhtmltopdf_centos.yml + + # setup MariaDB + - include: includes/setup_mariadb.yml + + # setup frappe-bench + - include: includes/setup_bench.yml diff --git a/playbooks/develop/includes/mariadb_centos.yml b/playbooks/develop/includes/mariadb_centos.yml new file mode 100644 index 00000000..766fd276 --- /dev/null +++ b/playbooks/develop/includes/mariadb_centos.yml @@ -0,0 +1,12 @@ +--- + - name: Add repository + template: src=templates/mariadb_centos.repo dest=/etc/yum.repos.d/MariaDB.repo owner=root mode=0644 + become: yes + become_user: root + + - name: Install MariaDB + yum: pkg={{ item }} state=present + - MariaDB-server + - MariaDB-client + become: yes + become_user: root diff --git a/playbooks/develop/includes/mariadb_ubuntu.yml b/playbooks/develop/includes/mariadb_ubuntu.yml new file mode 100644 index 00000000..735df132 --- /dev/null +++ b/playbooks/develop/includes/mariadb_ubuntu.yml @@ -0,0 +1,21 @@ +--- + - name: Add apt key + apt_key: keyserver=hkp://keyserver.ubuntu.com:80 id=0xcbcb082a1bb943db state=present + become: yes + become_user: root + + - name: Add apt repositry + apt_repository: repo='deb [arch=amd64,i386] http://nyc2.mirrors.digitalocean.com/mariadb/repo/10.1/ubuntu {{ ansible_distribution_release }} main' state=present + become: yes + become_user: root + + - name: apt-get install + apt: pkg={{ item }} update_cache=yes state=present + with_items: + - mariadb-server + - mariadb-client + - mariadb-common + - libmariadbclient-dev + become: yes + become_user: root + diff --git a/playbooks/develop/includes/setup_bench.yml b/playbooks/develop/includes/setup_bench.yml new file mode 100644 index 00000000..5cc0a4f8 --- /dev/null +++ b/playbooks/develop/includes/setup_bench.yml @@ -0,0 +1,46 @@ +--- + - name: install bench + pip: name={{ bench_repo_path }} extra_args='-e' + become: yes + become_user: root + + - name: init bench + command: bench init {{ bench_path }} + args: + creates: "{{ bench_path }}" + + # setup common_site_config + - name: setup config + command: bench setup config + args: + creates: "{{ bench_path }}/sites/common_site_config.json" + chdir: "{{ bench_path }}" + + - name: install frappe app + command: bench get-app frappe https://github.com/frappe/frappe + args: + creates: "{{ bench_path }}/apps/frappe" + chdir: "{{ bench_path }}" + + # setup procfile + - name: setup procfile + command: bench setup socketio + args: + creates: "{{ bench_path }}/node_modules" + chdir: "{{ bench_path }}" + + # setup procfile + - name: setup procfile + command: bench setup procfile + args: + creates: "{{ bench_path }}/Procfile" + chdir: "{{ bench_path }}" + + + # setup config for redis/socketio + - name: setup redis + command: bench setup redis + args: + creates: "{{ bench_path }}/config/redis_socketio.conf" + chdir: "{{ bench_path }}" + diff --git a/playbooks/develop/includes/setup_mariadb.yml b/playbooks/develop/includes/setup_mariadb.yml new file mode 100644 index 00000000..6e00be92 --- /dev/null +++ b/playbooks/develop/includes/setup_mariadb.yml @@ -0,0 +1,33 @@ +--- + - name: Install MySQLdb in global env + pip: name=mysql-python version=1.2.5 + become: yes + become_method: sudo + + - name: Set root Password + mysql_user: + name=root + host={{ item }} + password={{ mysql_root_password }} + state=present + login_user=root + with_items: + - localhost + when: mysql_root_password is defined + become: yes + become_method: sudo + + # when you have already defined mysql root password + ignore_errors: yes + + - name: Add configuration + template: src={{ mysql_config_template }} dest={{ mysql_conf_dir }}/frappe.cnf owner=root mode=0644 + notify: + - restart mysql + become: yes + become_method: sudo + + - name: restart mysql + service: name=mysql state=restarted + become: yes + become_method: sudo diff --git a/playbooks/develop/includes/wkhtmltopdf_centos.yml b/playbooks/develop/includes/wkhtmltopdf_centos.yml new file mode 100644 index 00000000..175801a2 --- /dev/null +++ b/playbooks/develop/includes/wkhtmltopdf_centos.yml @@ -0,0 +1,12 @@ +--- + - name: Download wkhtmltopdf + get_url: + url=http://download.gna.org/wkhtmltopdf/0.12/{{ wkhtmltopdf_version }}/wkhtmltox-{{ wkhtmltopdf_version }}_linux-centos{{ ansible_lsb.major_release }}-{{ "amd64" if ansible_architecture == "x86_64" else "i386"}}.rpm + dest="/tmp/" + become: yes + become_user: root + + - name: Install wkhtmltopdf deb + yum: name=/tmp/wkhtmltox-{{ wkhtmltopdf_version }}_linux-centos{{ ansible_lsb.major_release }}-{{ "amd64" if ansible_architecture == "x86_64" else "i386"}}.rpm state=present + become: yes + become_user: root diff --git a/playbooks/develop/includes/wkhtmltopdf_ubuntu_debian.yml b/playbooks/develop/includes/wkhtmltopdf_ubuntu_debian.yml new file mode 100644 index 00000000..2c7a15eb --- /dev/null +++ b/playbooks/develop/includes/wkhtmltopdf_ubuntu_debian.yml @@ -0,0 +1,28 @@ +--- + - name: Download wkhtmltopdf + get_url: + url=http://download.gna.org/wkhtmltopdf/0.12/{{ wkhtmltopdf_version }}/wkhtmltox-{{ wkhtmltopdf_version }}_linux-{{ ansible_distribution_release }}-{{ "amd64" if ansible_architecture == "x86_64" else "i386"}}.deb + dest="/tmp/" + become: yes + become_user: root + when: ansible_distribution_release in ("jessie", "precise", "trusty", "wheezy") + + - name: Install wkhtmltopdf deb + apt: deb=/tmp/wkhtmltox-{{ wkhtmltopdf_version }}_linux-{{ ansible_distribution_release }}-{{ "amd64" if ansible_architecture == "x86_64" else "i386"}}.deb + become: yes + become_user: root + when: ansible_distribution_release in ("jessie", "precise", "trusty", "wheezy") + + - name: Download wkhtmltopdf for Ubuntu Wily + get_url: + url=http://download.gna.org/wkhtmltopdf/0.12/{{ wkhtmltopdf_version }}/wkhtmltox-{{ wkhtmltopdf_version }}_linux-trusty-{{ "amd64" if ansible_architecture == "x86_64" else "i386"}}.deb + dest="/tmp/" + become: yes + become_user: root + when: ansible_distribution_release=="wily" + + - name: Install wkhtmltopdf deb for Ubuntu Wily + apt: deb=/tmp/wkhtmltox-{{ wkhtmltopdf_version }}_linux-trusty-{{ "amd64" if ansible_architecture == "x86_64" else "i386"}}.deb + become: yes + become_user: root + when: ansible_distribution_release=="wily" diff --git a/playbooks/develop/install.yml b/playbooks/develop/install.yml new file mode 100644 index 00000000..dfec2ceb --- /dev/null +++ b/playbooks/develop/install.yml @@ -0,0 +1,13 @@ +--- +- hosts: localhost + vars_prompt: + - name: mysql_root_password + prompt: "MySQL Root Password" + when: ansible_distribution == 'Ubuntu' + +- include: macosx.yml + when: ansible_distribution == 'MacOSX' +- include: ubuntu.yml + when: ansible_distribution == 'Ubuntu' +- include: centos.yml + when: ansible_distribution == 'CentOS' diff --git a/playbooks/develop/macosx.yml b/playbooks/develop/macosx.yml new file mode 100644 index 00000000..fbc0418b --- /dev/null +++ b/playbooks/develop/macosx.yml @@ -0,0 +1,31 @@ +--- +- hosts: localhost + vars: + bench_repo_path: "/usr/local/frappe/bench-repo" + bench_path: "/Users/{{ ansible_user_id }}/frappe/frappe-bench" + mysql_config_template: "templates/simple_mariadb_config.cnf" + mysql_conf_dir: /usr/local/etc/my.cnf.d + + tasks: + + # install pre-requisites + - name: install prequisites + homebrew: name={{ item }} state=present + with_items: + - cmake + - redis + - mariadb + - nodejs + + # install wkhtmltopdf + - name: cask installs + homebrew_cask: name={{ item }} state=present + with_items: + - wkhtmltopdf + + # setup MariaDB + - include: includes/setup_mariadb.yml + + # setup frappe-bench + - include: includes/setup_bench.yml + diff --git a/playbooks/develop/templates/mariadb_centos.repo b/playbooks/develop/templates/mariadb_centos.repo new file mode 100644 index 00000000..b0d7c819 --- /dev/null +++ b/playbooks/develop/templates/mariadb_centos.repo @@ -0,0 +1,7 @@ +# MariaDB 10.1 CentOS repository list - created 2016-03-18 09:56 UTC +# http://mariadb.org/mariadb/repositories/ +[mariadb] +name = MariaDB +baseurl = http://yum.mariadb.org/10.1/centos{{ ansible_lsb.major_release }}-{{ "amd64" if ansible_architecture == "x86_64" else "x86"}} +gpgkey=https://yum.mariadb.org/RPM-GPG-KEY-MariaDB +gpgcheck=1 diff --git a/playbooks/develop/templates/simple_mariadb_config.cnf b/playbooks/develop/templates/simple_mariadb_config.cnf new file mode 100644 index 00000000..d2122d44 --- /dev/null +++ b/playbooks/develop/templates/simple_mariadb_config.cnf @@ -0,0 +1,10 @@ +[mysqld] +innodb-file-format=barracuda +innodb-file-per-table=1 +innodb-large-prefix=1 +character-set-client-handshake = FALSE +character-set-server = utf8mb4 +collation-server = utf8mb4_unicode_ci + +[mysql] +default-character-set = utf8mb4 diff --git a/playbooks/develop/ubuntu.yml b/playbooks/develop/ubuntu.yml new file mode 100644 index 00000000..6896bbcf --- /dev/null +++ b/playbooks/develop/ubuntu.yml @@ -0,0 +1,72 @@ +--- +- hosts: localhost + vars: + bench_repo_path: "/usr/local/frappe/bench-repo" + bench_path: "/home/{{ ansible_user_id }}/frappe/frappe-bench" + mysql_config_template: "templates/simple_mariadb_config.cnf" + mysql_conf_dir: /etc/mysql/conf.d/ + wkhtmltopdf_version: 0.12.2.1 + + tasks: + + # install pre-requisites + - name: install prequisites + apt: pkg={{ item }} state=present + with_items: + # basic installs + - build-essential + - redis-server + - nodejs + - npm + + # for mariadb + - software-properties-common + + # for wkhtmltopdf + - libxrender1 + - libxext6 + - xfonts-75dpi + - xfonts-base + + # for Pillow + - libjpeg8-dev + - zlib1g-dev + - libfreetype6-dev + - liblcms2-dev + - libwebp-dev + - python-tk + + become: yes + become_user: root + + - name: install pillow prerequisites for Ubuntu < 14.04 + apt: pkg={{ item }} state=present + with_items: + - libtiff4-dev + - tcl8.5-dev + - tk8.5-dev + when: ansible_distribution_version < 14.04 + become: yes + become_user: root + + - name: install pillow prerequisites for Ubuntu > 14.04 + apt: pkg={{ item }} state=present + with_items: + - libtiff5-dev + - tcl8.6-dev + - tk8.6-dev + when: ansible_distribution_version >= 14.04 + become: yes + become_user: root + + # install MariaDB + - include: includes/mariadb_ubuntu.yml + + # install WKHTMLtoPDF + - include: includes/wkhtmltopdf_ubuntu_debian.yml + + # setup MariaDB + - include: includes/setup_mariadb.yml + + # setup frappe-bench + - include: includes/setup_bench.yml diff --git a/playbooks/install.py b/playbooks/install.py new file mode 100644 index 00000000..87654707 --- /dev/null +++ b/playbooks/install.py @@ -0,0 +1,153 @@ +# wget setup_frappe.py | python +import os +import sys +import subprocess +import getpass +from distutils.spawn import find_executable + +bench_repo = '/usr/local/frappe/bench-repo' + +def install_bench(args): + # pre-requisites for bench repo cloning + success = run_os_command({ + 'apt-get': [ + 'sudo apt-get update', + 'sudo apt-get install -y git build-essential python-setuptools python-dev' + ], + 'yum': [ + 'sudo yum groupinstall -y "Development tools"', + 'sudo yum install -y git python-setuptools python-devel' + ], + }) + + if not find_executable("git"): + success = run_os_command({ + 'brew': 'brew install git' + }) + + if not success: + print 'Could not install pre-requisites. Please check for errors or install them manually.' + return + + # secure pip installation + if not os.path.exists("get-pip.py"): + run_os_command({ + 'apt-get': 'wget https://bootstrap.pypa.io/get-pip.py', + 'yum': 'wget https://bootstrap.pypa.io/get-pip.py' + }) + + run_os_command({ + 'apt-get': 'sudo python get-pip.py', + 'yum': 'sudo python get-pip.py', + }) + + success = run_os_command({ + 'pip': 'sudo pip install ansible' + }) + + if not success: + could_not_install('Ansible') + + if is_sudo_user(): + raise Exception('Please run this script as a non-root user with sudo privileges, but without using sudo') + + # clone bench repo + clone_bench_repo() + + if args.develop: + run_playbook('develop/install.yml', sudo=True) + +def install_python27(): + version = (sys.version_info[0], sys.version_info[1]) + + if version == (2, 7): + return + + print 'Installing Python 2.7' + + # install python 2.7 + success = run_os_command({ + 'apt-get': 'sudo apt-get install -y python2.7', + 'yum': 'sudo yum install -y python27', + 'brew': 'brew install python' + }) + + if not success: + could_not_install('Python 2.7') + + # replace current python with python2.7 + os.execvp('python2.7', ([] if is_sudo_user() else ['sudo']) + ['python2.7', __file__] + sys.argv[1:]) + +def clone_bench_repo(): + '''Clones the bench repository in the user folder''' + + if os.path.exists(bench_repo): + return 0 + + run_os_command({ + 'brew': 'mkdir -p /usr/local/frappe', + 'apt-get': 'sudo mkdir -p /usr/local/frappe', + 'yum': 'sudo mkdir -p /usr/local/frappe', + }) + + # change user + run_os_command({ + 'ls': 'sudo chown -R {user}:{user} /usr/local/frappe'.format(user=getpass.getuser()), + }) + + success = run_os_command( + {'git': 'git clone https://github.com/frappe/bench {bench_repo} --depth 1 --branch new-install'.format(bench_repo=bench_repo)} + ) + + return success + +def run_os_command(command_map): + '''command_map is a dictionary of {'executable': command}. For ex. {'apt-get': 'sudo apt-get install -y python2.7'} ''' + success = True + for executable, commands in command_map.items(): + if find_executable(executable): + if isinstance(commands, basestring): + commands = [commands] + + for command in commands: + returncode = subprocess.check_call(command, shell=True) + success = success and ( returncode == 0 ) + + break + + return success + +def could_not_install(package): + raise Exception('Could not install {0}. Please install it manually.'.format(package)) + +def is_sudo_user(): + return os.geteuid() == 0 + +def run_playbook(playbook_name, sudo=False): + args = ['ansible-playbook', '-c', 'local', playbook_name] + if sudo: + args.append('-K') + + success = subprocess.check_call(args, cwd=os.path.join(bench_repo, 'playbooks')) + return success + +def parse_commandline_args(): + import argparse + + parser = argparse.ArgumentParser(description='Frappe Installer') + parser.add_argument('--develop', dest='develop', action='store_true', default=False, + help='Install developer setup') + args = parser.parse_args() + + return args + +if __name__ == '__main__': + try: + import argparse + except ImportError: + # install python2.7 + install_python27() + + args = parse_commandline_args() + + install_bench(args) diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..2629b148 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,7 @@ +Click +jinja2 +virtualenv +requests +honcho +semantic_version +GitPython==0.3.2.rc1 diff --git a/setup.py b/setup.py index 91b33dc8..5decf181 100644 --- a/setup.py +++ b/setup.py @@ -1,22 +1,28 @@ from setuptools import setup, find_packages +from pip.req import parse_requirements +import re +import ast + +# get version from __version__ variable in bench/__init__.py +_version_re = re.compile(r'__version__\s+=\s+(.*)') + +with open('bench/__init__.py', 'rb') as f: + version = str(ast.literal_eval(_version_re.search( + f.read().decode('utf-8')).group(1))) + +requirements = parse_requirements("requirements.txt", session="") setup( name='bench', description='Metadata driven, full-stack web framework', author='Frappe Technologies', author_email='info@frappe.io', + version=version, packages=find_packages(), zip_safe=False, include_package_data=True, - install_requires=[ - 'Click', - 'jinja2', - 'virtualenv', - 'requests', - 'honcho', - 'semantic_version', - 'GitPython==0.3.2.RC1' - ], + install_requires=[str(ir.req) for ir in requirements], + dependency_links=[str(ir._link) for ir in requirements if ir._link], entry_points=''' [console_scripts] bench=bench.cli:cli