diff --git a/.gitignore b/.gitignore index 3abf0da4..35826d3c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,8 +1,139 @@ -*.pyc -*.py~ -*.swp +# MAC OS .DS_Store -build/ -dist/ -*.egg-info/ + +# Vim Gitignore +## Swap +[._]*.s[a-v][a-z] +[._]*.sw[a-p] +[._]s[a-v][a-z] +[._]sw[a-p] + +## Session +Session.vim + +## Temporary +.netrwhist +*~ + +## Auto-generated tag files tags + +# Python Gitignore +## Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +## C extensions +*.so + +## Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +## PyInstaller +## Usually these files are written by a python script from a template +## before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +## Installer logs +pip-log.txt +pip-delete-this-directory.txt + +## Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +.hypothesis/ +.pytest_cache/ + +## Translations +*.mo +*.pot + +## Django stuff: +*.log +.static_storage/ +.media/ +local_settings.py + +## Flask stuff: +instance/ +.webassets-cache + +## Scrapy stuff: +.scrapy + +## Sphinx documentation +docs/_build/ + +## PyBuilder +target/ + +## Jupyter Notebook +.ipynb_checkpoints + +## pyenv +.python-version + +## celery beat schedule file +celerybeat-schedule + +## SageMath parsed files +*.sage.py + +## Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +## Spyder project settings +.spyderproject +.spyproject + +## Rope project settings +.ropeproject + +## mkdocs documentation +/site + +## mypy +.mypy_cache/ + +# Packer Gitignore +## Cache objects +packer_cache/ +*.checksum + +## For built virtualmachines +*.ova +*.iso + +## For built boxes +*.box + diff --git a/.travis.yml b/.travis.yml index 9c42c989..b0399dfd 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,6 +7,7 @@ python: - "2.7" install: + - sudo rm /etc/apt/sources.list.d/mongodb*.list - sudo rm /etc/apt/sources.list.d/docker.list - sudo pip install urllib3 pyOpenSSL ndg-httpsclient pyasn1 - sudo apt-get purge -y mysql-common mysql-server mysql-client diff --git a/MANIFEST.in b/MANIFEST.in index 5e899217..898efefa 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -3,3 +3,5 @@ include *.py recursive-include bench *.conf recursive-include bench *.py recursive-include bench *.txt +recursive-include bench *.json +recursive-include bench/templates * diff --git a/README.md b/README.md index 487c3946..9cfff9e4 100755 --- a/README.md +++ b/README.md @@ -1,8 +1,12 @@ -# Bench + +
+ +

Frappe Bench

+
[![Build Status](https://travis-ci.org/frappe/bench.svg?branch=master)](https://travis-ci.org/frappe/bench) -The bench is a command-line utility that helps you to install apps, manage multiple sites and update Frappe / ERPNext apps on */nix (CentOS 6, Debian 7, Ubuntu, etc) for development and production. Bench will also create nginx and supervisor config files, setup backups and much more. +The bench is a command-line utility that helps you to install apps, manage multiple sites and update Frappe / ERPNext apps on */nix (CentOS, Debian, Ubuntu, etc) for development and production. Bench will also create nginx and supervisor config files, setup backups and much more. If you are using on a VPS make sure it has >= 1Gb of RAM or has swap setup properly. @@ -12,26 +16,40 @@ If you have questions, please ask them on the [forum](https://discuss.erpnext.co ## Installation -## Manual Install +### Installation Requirements -To manually install frappe/erpnext here are the steps +You will need a computer/server. Options include: + +- A Normal Computer/VPS/Baremetal Server: This is strongly recommended. Frappe/ERPNext installs properly and works well on these +- A Raspberry Pi, SAN Appliance, Network Router, Gaming Console, etc.: Although you may be able to install Frappe/ERPNext on specialized hardware, it is unlikely to work well and will be difficult for us to support. Strongly consider using a normal computer/VPS/baremetal server instead. **We do not support specialized hardware**. +- A Toaster, Car, Firearm, Thermostat, etc.: Yes, many modern devices now have embedded computing capability. We live in interesting times. However, you should not install Frappe/ERPNext on these devices. Instead, install it on a normal computer/VPS/baremetal server. **We do not support installing on noncomputing devices**. + +To install the Frappe/ERPNext server software, you will need an operating system on your normal computer which is not Windows. Note that the command line interface does work on Windows, and you can use Frappe/ERPNext from any operating system with a web browser. However, the server software does not run on Windows. It does run on other operating systems, so choose one of these instead: + +- Linux: Debian, Ubuntu, CentOS are the preferred distros and are well tested. [Arch Linux](https://github.com/frappe/bench/wiki/Install-ERPNext-on-ArchLinux) can also be used +- Mac OS X + +### Manual Install + +To manually install frappe/erpnext, you can follow this [this wiki](https://github.com/frappe/frappe/wiki/The-Hitchhiker's-Guide-to-Installing-Frapp%C3%A9-on-Linux-OS) for Linux and [this wiki](https://github.com/frappe/frappe/wiki/The-Hitchhiker's-Guide-to-Installing-Frapp%C3%A9-on-Mac-OS-X) for MacOS. It gives an excellent explanation about the stack. You can also follow the steps mentioned below: #### 1. Install Pre-requisites -- Python 2.7 +- Python 2.7 [Python3.5+ also supported, but not recommended for production] - MariaDB 10+ - Nginx (for production) - Nodejs +- yarn - Redis - cron (crontab is required) -- wkhtmltopdf with patched Qt (for pdf generation) +- wkhtmltopdf with patched Qt (version 0.12.3) (for pdf generation) #### 2. Install Bench Install bench as a *non root* user, git clone https://github.com/frappe/bench bench-repo - sudo pip install -e bench-repo + pip install --user -e bench-repo Note: Please do not remove the bench directory the above commands will create @@ -80,11 +98,12 @@ Note: Please do not remove the bench directory the above commands will create ## Easy Install - This is an opinionated setup so it is best to setup on a blank server. -- Works on Ubuntu 14.04 to 16.04, CentOS 7+, Debian 7 to 8 and MacOS X. +- Works on Ubuntu 16.04, CentOS 7+, Debian 8+ - You may have to install Python 2.7 (eg on Ubuntu 16.04+) by running `apt-get install python-minimal` - You may also have to install build-essential and python-setuptools by running `apt-get install build-essential python-setuptools` - This script will install the pre-requisites, install bench and setup an ERPNext site - Passwords for Frappe Administrator and MariaDB (root) will be asked +- MariaDB (root) password may be `password` on a fresh server - You can then login as **Administrator** with the Administrator password - If you find any problems, post them on the forum: [https://discuss.erpnext.com](https://discuss.erpnext.com) @@ -96,16 +115,6 @@ For Linux: wget https://raw.githubusercontent.com/frappe/bench/master/playbooks/install.py -For Mac OSX: - -Install X Code (from App store) and HomeBrew (http://brew.sh/) first - - brew install python - brew install git - -Download the Script - - curl "https://raw.githubusercontent.com/frappe/bench/master/playbooks/install.py" -o install.py #### 2. Run the install script @@ -126,12 +135,12 @@ For production: - Install all the pre-requisites - Install the command line `bench` - Create a new bench (a folder that will contain your entire frappe/erpnext setup) -- Create a new site on the bench +- Create a new ERPNext site on the bench #### How do I start ERPNext 1. For development: Go to your bench folder (`frappe-bench` by default) and start the bench with `bench start` -2. For production: Your process will be setup and managed by `nginx` and `supervisor`. [Setup Production](https://frappe.github.io/frappe/user/en/bench/guides/setup-production.html) +2. For production: Your process will be setup and managed by `nginx` and `supervisor`. [Setup Production](https://frappe.io/docs/user/en/bench/guides/setup-production.html) --- @@ -169,19 +178,19 @@ You can also run the parts of the bench selectively. Guides ======= -- [Configuring HTTPS](https://frappe.github.io/frappe/user/en/bench/guides/configuring-https.html) -- [Using Let's Encrypt to setup HTTPS](https://frappe.github.io/frappe/user/en/bench/guides/lets-encrypt-ssl-setup.html) -- [Diagnosing the Scheduler](https://frappe.github.io/frappe/user/en/bench/guides/diagnosing-the-scheduler.html) -- [Change Hostname](https://frappe.github.io/frappe/user/en/bench/guides/adding-custom-domains) -- [Manual Setup](https://frappe.github.io/frappe/user/en/bench/guides/manual-setup.html) -- [Setup Production](https://frappe.github.io/frappe/user/en/bench/guides/setup-production.html) -- [Setup Multitenancy](https://frappe.github.io/frappe/user/en/bench/guides/setup-multitenancy.html) +- [Configuring HTTPS](https://frappe.io/docs/user/en/bench/guides/configuring-https.html) +- [Using Let's Encrypt to setup HTTPS](https://frappe.io/docs/user/en/bench/guides/lets-encrypt-ssl-setup.html) +- [Diagnosing the Scheduler](https://frappe.io/docs/user/en/bench/guides/diagnosing-the-scheduler.html) +- [Change Hostname](https://frappe.io/docs/user/en/bench/guides/adding-custom-domains) +- [Manual Setup](https://frappe.io/docs/user/en/bench/guides/manual-setup.html) +- [Setup Production](https://frappe.io/docs/user/en/bench/guides/setup-production.html) +- [Setup Multitenancy](https://frappe.io/docs/user/en/bench/guides/setup-multitenancy.html) - [Stopping Production](https://github.com/frappe/bench/wiki/Stopping-Production-and-starting-Development) Resources ======= -- [Background Services](https://frappe.github.io/frappe/user/en/bench/resources/background-services.html) -- [Bench Commands Cheat Sheet](https://frappe.github.io/frappe/user/en/bench/resources/bench-commands-cheatsheet.html) -- [Bench Procfile](https://frappe.github.io/frappe/user/en/bench/resources/bench-procfile.html) +- [Background Services](https://frappe.io/docs/user/en/bench/resources/background-services.html) +- [Bench Commands Cheat Sheet](https://frappe.io/docs/user/en/bench/resources/bench-commands-cheatsheet.html) +- [Bench Procfile](https://frappe.io/docs/user/en/bench/resources/bench-procfile.html) diff --git a/bench/__init__.py b/bench/__init__.py index b5c69ef0..e120c37b 100644 --- a/bench/__init__.py +++ b/bench/__init__.py @@ -10,4 +10,4 @@ def set_frappe_version(bench_path='.'): from .app import get_current_frappe_version global FRAPPE_VERSION if not FRAPPE_VERSION: - FRAPPE_VERSION = get_current_frappe_version(bench_path=bench_path) + FRAPPE_VERSION = get_current_frappe_version(bench_path=bench_path) \ No newline at end of file diff --git a/bench/app.py b/bench/app.py index 84103b7b..24f0d214 100755 --- a/bench/app.py +++ b/bench/app.py @@ -48,10 +48,40 @@ def write_appstxt(apps, bench_path='.'): with open(os.path.join(bench_path, 'sites', 'apps.txt'), 'w') as f: return f.write('\n'.join(apps)) +def check_url(url, raise_err = True): + try: + from urlparse import urlparse + except ImportError: + from urllib.parse import urlparse + + parsed = urlparse(url) + if not parsed.scheme: + if raise_err: + raise TypeError('{url} Not a valid URL'.format(url = url)) + else: + return False + + return True + def get_app(git_url, branch=None, bench_path='.', build_asset_files=True, verbose=False): - #less verbose app install - if '/' not in git_url: - git_url = 'https://github.com/frappe/' + git_url + # from bench.utils import check_url + try: + from urlparse import urljoin + except ImportError: + from urllib.parse import urljoin + + if not check_url(git_url, raise_err = False): + orgs = ['frappe', 'erpnext'] + for org in orgs: + url = 'https://api.github.com/repos/{org}/{app}'.format(org = org, app = git_url) + res = requests.get(url) + if res.ok: + data = res.json() + if 'name' in data: + if git_url == data['name']: + git_url = 'https://github.com/{org}/{app}'.format(org = org, app = git_url) + break + #Gets repo name from URL repo_name = git_url.rsplit('/', 1)[1].rsplit('.', 1)[0] logger.info('getting app {}'.format(repo_name)) @@ -142,6 +172,7 @@ def pull_all_apps(bench_path='.', reset=False): app_dir = get_repo_dir(app, bench_path=bench_path) if os.path.exists(os.path.join(app_dir, '.git')): out = subprocess.check_output(["git", "status"], cwd=app_dir) + out = out.decode('utf-8') if not re.search(r'nothing to commit, working (directory|tree) clean', out): print(''' @@ -184,7 +215,7 @@ def is_version_upgrade(app='frappe', bench_path='.', branch=None): local_version = get_major_version(get_current_version(app, bench_path=bench_path)) upstream_version = get_major_version(upstream_version) - + if upstream_version - local_version > 0: return (True, local_version, upstream_version) @@ -204,6 +235,7 @@ def get_remote(app, bench_path='.'): repo_dir = get_repo_dir(app, bench_path=bench_path) contents = subprocess.check_output(['git', 'remote', '-v'], cwd=repo_dir, stderr=subprocess.STDOUT) + contents = contents.decode('utf-8') if re.findall('upstream[\s]+', contents): remote = 'upstream' else: @@ -232,14 +264,20 @@ def get_current_version(app, bench_path='.'): with open(os.path.join(repo_dir, 'setup.py')) as f: return get_version_from_string(f.read(), field='version') +def get_develop_version(app, bench_path='.'): + repo_dir = get_repo_dir(app, bench_path=bench_path) + with open(os.path.join(repo_dir, os.path.basename(repo_dir), 'hooks.py')) as f: + return get_version_from_string(f.read(), field='develop_version') + def get_upstream_version(app, branch=None, bench_path='.'): repo_dir = get_repo_dir(app, bench_path=bench_path) if not branch: branch = get_current_branch(app, bench_path=bench_path) try: contents = subprocess.check_output(['git', 'show', 'upstream/{branch}:{app}/__init__.py'.format(branch=branch, app=app)], cwd=repo_dir, stderr=subprocess.STDOUT) + contents = contents.decode('utf-8') except subprocess.CalledProcessError as e: - if "Invalid object" in e.output: + if b"Invalid object" in e.output: return None else: raise @@ -253,7 +291,7 @@ def get_repo_dir(app, bench_path='.'): return os.path.join(bench_path, 'apps', app) def switch_branch(branch, apps=None, bench_path='.', upgrade=False, check_upgrade=True): - from .utils import update_requirements, update_npm_packages, backup_all_sites, patch_sites, build_assets, pre_upgrade, post_upgrade + from .utils import update_requirements, update_node_packages, backup_all_sites, patch_sites, build_assets, pre_upgrade, post_upgrade from . import utils apps_dir = os.path.join(bench_path, 'apps') version_upgrade = (False,) @@ -293,7 +331,7 @@ def switch_branch(branch, apps=None, bench_path='.', upgrade=False, check_upgrad if version_upgrade[0] and upgrade: update_requirements() - update_npm_packages() + update_node_packages() pre_upgrade(version_upgrade[1], version_upgrade[2]) reload(utils) backup_all_sites() @@ -304,10 +342,10 @@ def switch_branch(branch, apps=None, bench_path='.', upgrade=False, check_upgrad def switch_to_branch(branch=None, apps=None, bench_path='.', upgrade=False): switch_branch(branch, apps=apps, bench_path=bench_path, upgrade=upgrade) -def switch_to_master(apps=None, bench_path='.', upgrade=False): +def switch_to_master(apps=None, bench_path='.', upgrade=True): switch_branch('master', apps=apps, bench_path=bench_path, upgrade=upgrade) -def switch_to_develop(apps=None, bench_path='.', upgrade=False): +def switch_to_develop(apps=None, bench_path='.', upgrade=True): switch_branch('develop', apps=apps, bench_path=bench_path, upgrade=upgrade) def get_version_from_string(contents, field='__version__'): diff --git a/bench/cli.py b/bench/cli.py index 4d6bad36..e6ad9984 100755 --- a/bench/cli.py +++ b/bench/cli.py @@ -47,10 +47,12 @@ def check_uid(): sys.exit(1) def cmd_requires_root(): - if len(sys.argv) > 2 and sys.argv[2] in ('production', 'sudoers', 'lets-encrypt', 'fonts', 'reload-nginx', 'firewall', 'ssh-port'): - return True - if len(sys.argv) >= 2 and sys.argv[1] in ('patch', 'renew-lets-encrypt', 'disable-production'): - return True + if len(sys.argv) > 2 and sys.argv[2] in ('production', 'sudoers', 'lets-encrypt', 'fonts', + 'print', 'firewall', 'ssh-port', 'role', 'fail2ban'): + return True + if len(sys.argv) >= 2 and sys.argv[1] in ('patch', 'renew-lets-encrypt', 'disable-production', + 'install'): + return True def change_dir(): if os.path.exists('config.json') or "init" in sys.argv: @@ -93,7 +95,9 @@ def get_frappe_commands(bench_path='.'): if not os.path.exists(sites_path): return [] try: - return json.loads(get_cmd_output("{python} -m frappe.utils.bench_helper get-frappe-commands".format(python=python), cwd=sites_path)) + output = get_cmd_output("{python} -m frappe.utils.bench_helper get-frappe-commands".format(python=python), cwd=sites_path) + # output = output.decode('utf-8') + return json.loads(output) except subprocess.CalledProcessError: return [] diff --git a/bench/commands/__init__.py b/bench/commands/__init__.py index 86caf0bf..fa4f48ec 100755 --- a/bench/commands/__init__.py +++ b/bench/commands/__init__.py @@ -1,5 +1,16 @@ import click +import os, shutil +import os.path as osp +import logging + +from datetime import datetime + +from bench.utils import which, exec_cmd + +log = logging.getLogger(__name__) +log.setLevel(logging.DEBUG) + def print_bench_version(ctx, param, value): """Prints current bench version""" if not value or ctx.resilient_parsing: @@ -14,7 +25,6 @@ def print_bench_version(ctx, param, value): def bench_command(bench_path='.'): """Bench manager for Frappe""" import bench - from bench.app import get_current_frappe_version from bench.utils import setup_logging bench.set_frappe_version(bench_path=bench_path) @@ -67,3 +77,107 @@ from bench.commands.git import remote_set_url, remote_reset_url, remote_urls bench_command.add_command(remote_set_url) bench_command.add_command(remote_reset_url) bench_command.add_command(remote_urls) + +from bench.commands.install import install +bench_command.add_command(install) + +from bench.config.common_site_config import get_config +try: + from urlparse import urlparse +except ImportError: + from urllib.parse import urlparse + +@click.command('migrate-env') +@click.argument('python', type = str) +@click.option('--no-backup', default = False, help = 'Do not backup the existing Virtual Environment') +def migrate_env(python, no_backup = False): + """ + Migrate Virtual Environment to desired Python Version. + """ + try: + # Clear Cache before Bench Dies. + config = get_config(bench_path = os.getcwd()) + rredis = urlparse(config['redis_cache']) + + redis = '{redis} -p {port}'.format( + redis = which('redis-cli'), + port = rredis.port + ) + + log.debug('Clearing Redis Cache...') + exec_cmd('{redis} FLUSHALL'.format(redis = redis)) + log.debug('Clearing Redis DataBase...') + exec_cmd('{redis} FLUSHDB'.format(redis = redis)) + except Exception: + log.warn('Please ensure Redis Connections are running or Daemonized.') + + try: + # This is with the assumption that a bench is set-up within path. + path = os.getcwd() + + # I know, bad name for a flag. Thanks, Ameya! :| - + if not no_backup: + # Back, the f*ck up. + parch = osp.join(path, 'archived_envs') + if not osp.exists(parch): + os.mkdir(parch) + + # Simply moving. Thanks, Ameya. + # I'm keen to zip. + source = osp.join(path, 'env') + target = parch + + log.debug('Backing up Virtual Environment') + stamp = datetime.now().strftime('%Y%m%d_%H%M%S') + dest = osp.join(path, str(stamp)) + + # WARNING: This is an archive, you might have to use virtualenv --relocate + # That's because virtualenv creates symlinks with shebangs pointing to executables. + # shebangs, shebangs - ricky martin. + + # ...and shutil.copytree is a f*cking mess. + os.rename(source, dest) + shutil.move(dest, target) + + log.debug('Setting up a New Virtual {python} Environment'.format( + python = python + )) + + # Path to Python Executable (Basically $PYTHONPTH) + python = which(python) + + + virtualenv = which('virtualenv') + + nvenv = 'env' + pvenv = osp.join(path, nvenv) + + exec_cmd('{virtualenv} --python {python} {pvenv}'.format( + virtualenv = virtualenv, + python = python, + pvenv = pvenv + ), cwd = path) + + pip = osp.join(pvenv, 'bin', 'pip') + exec_cmd('{pip} install --upgrade pip'.format(pip=pip)) + exec_cmd('{pip} install --upgrade setuptools'.format(pip=pip)) + # TODO: Options + + papps = osp.join(path, 'apps') + apps = ['frappe', 'erpnext'] + [app for app in os.listdir(papps) if app not in ['frappe', 'erpnext']] + + for app in apps: + papp = osp.join(papps, app) + if osp.isdir(papp) and osp.exists(osp.join(papp, 'setup.py')): + exec_cmd('{pip} install -e {app}'.format( + pip = pip, app = papp + )) + + log.debug('Migration Successful to {python}'.format( + python = python + )) + except: + log.debug('Migration Error') + raise + +bench_command.add_command(migrate_env) \ No newline at end of file diff --git a/bench/commands/install.py b/bench/commands/install.py new file mode 100644 index 00000000..5a271bf4 --- /dev/null +++ b/bench/commands/install.py @@ -0,0 +1,74 @@ +import os, sys, json, click +from bench.utils import run_playbook, setup_sudoers, is_root + +extra_vars = {"production": True} + +@click.group() +def install(): + "Install system dependancies" + pass + +@click.command('prerequisites') +def install_prerequisites(): + run_playbook('site.yml', tag='common, redis') + +@click.command('mariadb') +@click.option('--mysql_root_password') +def install_maridb(mysql_root_password=''): + if mysql_root_password: + extra_vars.update({"mysql_root_password": mysql_root_password}) + + run_playbook('site.yml', extra_vars=extra_vars, tag='mariadb') + +@click.command('wkhtmltopdf') +def install_wkhtmltopdf(): + run_playbook('site.yml', extra_vars=extra_vars, tag='wkhtmltopdf') + +@click.command('nodejs') +def install_nodejs(): + run_playbook('site.yml', extra_vars=extra_vars, tag='nodejs') + +@click.command('psutil') +def install_psutil(): + run_playbook('site.yml', extra_vars=extra_vars, tag='psutil') + +@click.command('supervisor') +@click.option('--user') +def install_supervisor(user=None): + run_playbook('site.yml', extra_vars=extra_vars, tag='supervisor') + if user: + setup_sudoers(user) + +@click.command('nginx') +@click.option('--user') +def install_nginx(user=None): + run_playbook('site.yml', extra_vars=extra_vars, tag='nginx') + if user: + setup_sudoers(user) + +@click.command('virtualbox') +def install_virtualbox(): + run_playbook('vm_build.yml', tag='virtualbox') + +@click.command('packer') +def install_packer(): + run_playbook('vm_build.yml', tag='packer') + +@click.command('fail2ban') +@click.option('--maxretry', default=6, help="Number of matches (i.e. value of the counter) which triggers ban action on the IP.") +@click.option('--bantime', default=600, help="The counter is set to zero if no match is found within 'findtime' seconds.") +@click.option('--findtime', default=600, help='Duration (in seconds) for IP to be banned for. Negative number for "permanent" ban.') +def install_failtoban(**kwargs): + extra_vars.update(kwargs) + run_playbook('site.yml', extra_vars=extra_vars, tag='fail2ban') + +install.add_command(install_prerequisites) +install.add_command(install_maridb) +install.add_command(install_wkhtmltopdf) +install.add_command(install_nodejs) +install.add_command(install_psutil) +install.add_command(install_supervisor) +install.add_command(install_nginx) +install.add_command(install_failtoban) +install.add_command(install_virtualbox) +install.add_command(install_packer) diff --git a/bench/commands/make.py b/bench/commands/make.py index 56d51c6e..f838e293 100755 --- a/bench/commands/make.py +++ b/bench/commands/make.py @@ -2,6 +2,8 @@ import click @click.command() @click.argument('path') +@click.option('--python', type = str, default = 'python', help = 'Path to Python Executable.') +@click.option('--ignore-exist', is_flag = True, default = False, help = "Ignore if Bench instance exists.") @click.option('--apps_path', default=None, help="path to json files with apps to install after init") @click.option('--frappe-path', default=None, help="path to frappe repo") @click.option('--frappe-branch', default=None, help="path to frappe repo") @@ -9,19 +11,23 @@ import click @click.option('--no-procfile', is_flag=True, help="Pull changes in all the apps in bench") @click.option('--no-backups',is_flag=True, help="Run migrations for all sites in the bench") @click.option('--no-auto-update',is_flag=True, help="Build JS and CSS artifacts for the bench") -@click.option('--verbose',is_flag=True, help="Verbose output during install") -@click.option('--skip-bench-mkdir', is_flag=True, help="Skip mkdir frappe-bench") @click.option('--skip-redis-config-generation', is_flag=True, help="Skip redis config generation if already specifying the common-site-config file") +@click.option('--verbose',is_flag=True, help="Verbose output during install") def init(path, apps_path, frappe_path, frappe_branch, no_procfile, no_backups, - no_auto_update, clone_from, verbose, skip_bench_mkdir, skip_redis_config_generation): - "Create a new bench" + no_auto_update, clone_from, verbose, skip_redis_config_generation, + ignore_exist = False, + python = 'python'): # Let's change we're ready. - + ''' + Create a New Bench Instance. + ''' from bench.utils import init init(path, apps_path=apps_path, no_procfile=no_procfile, no_backups=no_backups, no_auto_update=no_auto_update, frappe_path=frappe_path, frappe_branch=frappe_branch, - verbose=verbose, clone_from=clone_from, skip_bench_mkdir=skip_bench_mkdir, skip_redis_config_generation=skip_redis_config_generation) + verbose=verbose, clone_from=clone_from, skip_redis_config_generation=skip_redis_config_generation, + ignore_exist = ignore_exist, + python = python) click.echo('Bench {} initialized'.format(path)) - @click.command('get-app') @click.argument('name', nargs=-1) # Dummy argument for backward compatibility @click.argument('git-url') diff --git a/bench/commands/setup.py b/bench/commands/setup.py index 1d83c750..379a8b8c 100755 --- a/bench/commands/setup.py +++ b/bench/commands/setup.py @@ -1,3 +1,4 @@ +from bench.utils import exec_cmd import click, sys, json @click.group() @@ -5,7 +6,6 @@ def setup(): "Setup bench" pass - @click.command('sudoers') @click.argument('user') def setup_sudoers(user): @@ -13,7 +13,6 @@ def setup_sudoers(user): from bench.utils import setup_sudoers setup_sudoers(user) - @click.command('nginx') @click.option('--yes', help='Yes to regeneration of nginx config file', default=False, is_flag=True) def setup_nginx(yes=False): @@ -47,13 +46,19 @@ def setup_fonts(): from bench.utils import setup_fonts setup_fonts() - @click.command('production') @click.option('--yes', is_flag=True, default=False, help='Dont prompt for confirmation') @click.argument('user') +@click.option('--yes', help='Yes to regeneration config', is_flag=True, default=False) def setup_production(user, yes=False): "setup bench for production" from bench.config.production_setup import setup_production + from bench.utils import run_playbook + # Install prereqs for production + exec_cmd("sudo pip install ansible") + exec_cmd("bench setup role fail2ban") + exec_cmd("bench setup role nginx") + exec_cmd("bench setup role supervisor") setup_production(user=user, yes=yes) @@ -71,30 +76,42 @@ def setup_backups(): setup_backups() @click.command('env') -def setup_env(): +@click.option('--python', type = str, default = 'python', help = 'Path to Python Executable.') +def setup_env(python='python'): "Setup virtualenv for bench" from bench.utils import setup_env - setup_env() + setup_env(python=python) @click.command('firewall') -def setup_firewall(): +@click.option('--ssh_port') +@click.option('--force') +def setup_firewall(ssh_port=None, force=False): "Setup firewall" from bench.utils import run_playbook - click.confirm('Setting up the firewall will block all ports except 80, 443 and 22\n' - 'Do you want to continue?', - abort=True) - run_playbook('production/setup_firewall.yml') + + if not force: + click.confirm('Setting up the firewall will block all ports except 80, 443 and 22\n' + 'Do you want to continue?', + abort=True) + + if not ssh_port: + ssh_port = 22 + + run_playbook('roles/bench/tasks/setup_firewall.yml', {"ssh_port": ssh_port}) @click.command('ssh-port') @click.argument('port') -def set_ssh_port(port): +@click.option('--force') +def set_ssh_port(port, force=False): "Set SSH Port" from bench.utils import run_playbook - click.confirm('This will change your SSH Port to {}\n' - 'Do you want to continue?'.format(port), - abort=True) - run_playbook('production/change_ssh_port.yml', {"ssh_port": port}) + if not force: + click.confirm('This will change your SSH Port to {}\n' + 'Do you want to continue?'.format(port), + abort=True) + + run_playbook('roles/bench/tasks/change_ssh_port.yml', {"ssh_port": port}) @click.command('lets-encrypt') @click.argument('site') @@ -118,12 +135,24 @@ def setup_socketio(): from bench.utils import setup_socketio setup_socketio() -@click.command('requirements') -def setup_requirements(): +@click.command('requirements', help="Update Python and Node packages") +@click.option('--node', help="Update only Node packages", default=False, is_flag=True) +@click.option('--python', help="Update only Python packages", default=False, is_flag=True) +def setup_requirements(node=False, python=False): "Setup python and node requirements" - from bench.utils import update_requirements, update_npm_packages + + if not node: + setup_python_requirements() + if not python: + setup_node_requirements() + +def setup_python_requirements(): + from bench.utils import update_requirements update_requirements() - update_npm_packages() + +def setup_node_requirements(): + from bench.utils import update_node_packages + update_node_packages() @click.command('config') def setup_config(): @@ -181,6 +210,30 @@ def sync_domains(domain=None, site=None): # if changed, success, else failure sys.exit(0 if changed else 1) +@click.command('role') +@click.argument('role') +@click.option('--admin_emails', default='') +@click.option('--mysql_root_password') +def setup_roles(role, **kwargs): + "Install dependancies via roles" + from bench.utils import run_playbook + + extra_vars = {"production": True} + extra_vars.update(kwargs) + + if role: + run_playbook('site.yml', extra_vars=extra_vars, tag=role) + else: + run_playbook('site.yml', extra_vars=extra_vars) + +@click.command('fail2ban') +@click.option('--maxretry', default=6, help="Number of matches (i.e. value of the counter) which triggers ban action on the IP. Default is 6 seconds" ) +@click.option('--bantime', default=600, help="The counter is set to zero if no match is found within 'findtime' seconds. Default is 600 seconds") +@click.option('--findtime', default=600, help='Duration (in seconds) for IP to be banned for. Negative number for "permanent" ban. Default is 600 seconds') +def setup_nginx_proxy_jail(**kwargs): + from bench.utils import run_playbook + run_playbook('roles/fail2ban/tasks/configure_nginx_jail.yml', extra_vars=kwargs) + setup.add_command(setup_sudoers) setup.add_command(setup_nginx) setup.add_command(reload_nginx) @@ -201,3 +254,5 @@ setup.add_command(remove_domain) setup.add_command(sync_domains) setup.add_command(setup_firewall) setup.add_command(set_ssh_port) +setup.add_command(setup_roles) +setup.add_command(setup_nginx_proxy_jail) \ No newline at end of file diff --git a/bench/commands/update.py b/bench/commands/update.py index d38e82e0..5de03cbe 100755 --- a/bench/commands/update.py +++ b/bench/commands/update.py @@ -3,7 +3,7 @@ import sys, os from bench.config.common_site_config import get_config from bench.app import pull_all_apps, is_version_upgrade from bench.utils import (update_bench, validate_upgrade, pre_upgrade, post_upgrade, before_update, - update_requirements, update_npm_packages, backup_all_sites, patch_sites, build_assets, restart_supervisor_processes) + update_requirements, update_node_packages, backup_all_sites, patch_sites, build_assets, restart_supervisor_processes) from bench import patches #TODO: Not DRY @@ -15,11 +15,10 @@ from bench import patches @click.option('--requirements',is_flag=True, help="Update requirements") @click.option('--restart-supervisor',is_flag=True, help="restart supervisor processes after update") @click.option('--auto',is_flag=True) -@click.option('--upgrade',is_flag=True, help="Required for major version updates") @click.option('--no-backup',is_flag=True) @click.option('--force',is_flag=True) @click.option('--reset', is_flag=True, help="Hard resets git branch's to their new states overriding any changes and overriding rebase on pull") -def update(pull=False, patch=False, build=False, bench=False, auto=False, restart_supervisor=False, requirements=False, no_backup=False, upgrade=False, force=False, reset=False): +def update(pull=False, patch=False, build=False, bench=False, auto=False, restart_supervisor=False, requirements=False, no_backup=False, force=False, reset=False): "Update bench" if not (pull or patch or build or bench or requirements): @@ -40,7 +39,6 @@ def update(pull=False, patch=False, build=False, bench=False, auto=False, restar 'requirements': requirements, 'no-backup': no_backup, 'restart-supervisor': restart_supervisor, - 'upgrade': upgrade, 'reset':reset }) @@ -50,27 +48,21 @@ def update(pull=False, patch=False, build=False, bench=False, auto=False, restar version_upgrade = is_version_upgrade() - if version_upgrade[0] and not upgrade: + if version_upgrade[0]: print() print() print("This update will cause a major version change in Frappe/ERPNext from {0} to {1}.".format(*version_upgrade[1:])) - print("This would take significant time to migrate and might break custom apps. Please run `bench update --upgrade` to confirm.") - print() - print("You can stay on the latest stable release by running `bench switch-to-master` or pin your bench to {0} by running `bench switch-to-v{0}`".format(version_upgrade[1])) - sys.exit(1) - - _update(pull, patch, build, bench, auto, restart_supervisor, requirements, no_backup, upgrade, force=force, reset=reset) + print("This would take significant time to migrate and might break custom apps.") + click.confirm('Do you want to continue?', abort=True) + _update(pull, patch, build, bench, auto, restart_supervisor, requirements, no_backup, force=force, reset=reset) def _update(pull=False, patch=False, build=False, update_bench=False, auto=False, restart_supervisor=False, - requirements=False, no_backup=False, upgrade=False, bench_path='.', force=False, reset=False): + requirements=False, no_backup=False, bench_path='.', force=False, reset=False): conf = get_config(bench_path=bench_path) version_upgrade = is_version_upgrade(bench_path=bench_path) - if version_upgrade[0] and not upgrade: - raise Exception("Major Version Upgrade") - - if upgrade and (version_upgrade[0] or (not version_upgrade[0] and force)): + if version_upgrade[0] or (not version_upgrade[0] and force): validate_upgrade(version_upgrade[1], version_upgrade[2], bench_path=bench_path) before_update(bench_path=bench_path, requirements=requirements) @@ -80,9 +72,9 @@ def _update(pull=False, patch=False, build=False, update_bench=False, auto=False if requirements: update_requirements(bench_path=bench_path) - update_npm_packages(bench_path=bench_path) + update_node_packages(bench_path=bench_path) - if upgrade and (version_upgrade[0] or (not version_upgrade[0] and force)): + if version_upgrade[0] or (not version_upgrade[0] and force): pre_upgrade(version_upgrade[1], version_upgrade[2], bench_path=bench_path) import bench.utils, bench.app print('Reloading bench...') @@ -98,7 +90,7 @@ def _update(pull=False, patch=False, build=False, update_bench=False, auto=False patch_sites(bench_path=bench_path) if build: build_assets(bench_path=bench_path) - if upgrade and (version_upgrade[0] or (not version_upgrade[0] and force)): + if version_upgrade[0] or (not version_upgrade[0] and force): post_upgrade(version_upgrade[1], version_upgrade[2], bench_path=bench_path) if restart_supervisor or conf.get('restart_supervisor_on_update'): restart_supervisor_processes(bench_path=bench_path) @@ -108,7 +100,6 @@ def _update(pull=False, patch=False, build=False, update_bench=False, auto=False print("Open source depends on your contributions, so please contribute bug reports, patches, fixes or cash and be a part of the community") print() - @click.command('retry-upgrade') @click.option('--version', default=5) def retry_upgrade(version): @@ -117,12 +108,10 @@ def retry_upgrade(version): build_assets() post_upgrade(version-1, version) - def restart_update(kwargs): args = ['--'+k for k, v in list(kwargs.items()) if v] os.execv(sys.argv[0], sys.argv[:2] + args) - @click.command('switch-to-branch') @click.argument('branch') @click.argument('apps', nargs=-1) @@ -134,26 +123,20 @@ def switch_to_branch(branch, apps, upgrade=False): print('Switched to ' + branch) print('Please run `bench update --patch` to be safe from any differences in database schema') - @click.command('switch-to-master') -@click.option('--upgrade',is_flag=True) -def switch_to_master(upgrade=False): +def switch_to_master(): "Switch frappe and erpnext to master branch" from bench.app import switch_to_master - switch_to_master(upgrade=upgrade, apps=['frappe', 'erpnext']) + switch_to_master(apps=['frappe', 'erpnext']) print() print('Switched to master') print('Please run `bench update --patch` to be safe from any differences in database schema') - @click.command('switch-to-develop') -@click.option('--upgrade',is_flag=True) def switch_to_develop(upgrade=False): "Switch frappe and erpnext to develop branch" from bench.app import switch_to_develop - switch_to_develop(upgrade=upgrade, apps=['frappe', 'erpnext']) + switch_to_develop(apps=['frappe', 'erpnext']) print() print('Switched to develop') print('Please run `bench update --patch` to be safe from any differences in database schema') - - diff --git a/bench/config/procfile.py b/bench/config/procfile.py index 3e4183e9..3dd7cece 100755 --- a/bench/config/procfile.py +++ b/bench/config/procfile.py @@ -13,7 +13,8 @@ def setup_procfile(bench_path, yes=False): procfile = bench.env.get_template('Procfile').render( node=find_executable("node") or find_executable("nodejs"), use_rq=use_rq(bench_path), - webserver_port=config.get('webserver_port')) + webserver_port=config.get('webserver_port'), + CI=os.environ.get('CI')) with open(procfile_path, 'w') as f: f.write(procfile) diff --git a/bench/config/redis.py b/bench/config/redis.py index 43423c8e..fe102135 100644 --- a/bench/config/redis.py +++ b/bench/config/redis.py @@ -56,8 +56,8 @@ def write_redis_config(template_name, context, bench_path): f.write(template.render(**context)) def get_redis_version(): - version_string = subprocess.check_output('redis-server --version', shell=True).strip() - + version_string = subprocess.check_output('redis-server --version', shell=True) + version_string = version_string.decode('utf-8').strip() # extract version number from string version = re.findall("\d+\.\d+", version_string) if not version: diff --git a/bench/config/templates/Procfile b/bench/config/templates/Procfile index 9a1e3ce2..b9c81118 100644 --- a/bench/config/templates/Procfile +++ b/bench/config/templates/Procfile @@ -4,12 +4,14 @@ redis_queue: redis-server config/redis_queue.conf web: bench serve {% if webserver_port -%} --port {{ webserver_port }} {%- endif %} socketio: {{ node }} apps/frappe/socketio.js +{% if not CI %} watch: bench watch +{% endif %} {% if use_rq -%} schedule: bench schedule -worker_short: bench worker --queue short -worker_long: bench worker --queue long -worker_default: bench worker --queue default +worker_short: bench worker --queue short --quiet +worker_long: bench worker --queue long --quiet +worker_default: bench worker --queue default --quiet {% else %} workerbeat: sh -c 'cd sites && exec ../env/bin/python -m frappe.celery_app beat -s scheduler.schedule' worker: sh -c 'cd sites && exec ../env/bin/python -m frappe.celery_app worker -n jobs@%h -Ofair --soft-time-limit 360 --time-limit 390' diff --git a/bench/config/templates/letsencrypt.cfg b/bench/config/templates/letsencrypt.cfg index ba57815a..cde2520d 100755 --- a/bench/config/templates/letsencrypt.cfg +++ b/bench/config/templates/letsencrypt.cfg @@ -17,4 +17,3 @@ text = True # Uncomment to use the standalone authenticator on port 443 authenticator = standalone -standalone-supported-challenges = tls-sni-01 diff --git a/bench/package.json b/bench/package.json deleted file mode 100644 index 31121283..00000000 --- a/bench/package.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "name": "frappe", - "description": "Default package.json for frappe apps", - "dependencies": { - "babel-core": "^6.24.1", - "babel-preset-babili": "0.0.12", - "babel-preset-es2015": "^6.24.1", - "babel-preset-es2016": "^6.24.1", - "babel-preset-es2017": "^6.24.1", - "chokidar": "^1.7.0", - "cookie": "^0.3.1", - "express": "^4.15.3", - "less": "^2.7.2", - "redis": "^2.7.1", - "socket.io": "^2.0.1", - "superagent": "^3.5.2" - } -} diff --git a/bench/patches/patches.txt b/bench/patches/patches.txt index e979b0d6..754068c5 100644 --- a/bench/patches/patches.txt +++ b/bench/patches/patches.txt @@ -3,4 +3,4 @@ bench.patches.v3.celery_to_rq bench.patches.v3.redis_bind_ip bench.patches.v4.update_node bench.patches.v4.update_socketio - +bench.patches.v4.install_yarn #2 diff --git a/bench/patches/v4/install_yarn.py b/bench/patches/v4/install_yarn.py new file mode 100644 index 00000000..e29868e9 --- /dev/null +++ b/bench/patches/v4/install_yarn.py @@ -0,0 +1,5 @@ +import os +from bench.utils import exec_cmd + +def execute(bench_path): + exec_cmd('npm install yarn', os.path.join(bench_path, 'apps/frappe')) diff --git a/bench/release.py b/bench/release.py index c58a446a..ab35a9fd 100755 --- a/bench/release.py +++ b/bench/release.py @@ -24,6 +24,7 @@ github_password = None def release(bench_path, app, bump_type, from_branch='develop', to_branch='master', remote='upstream', owner='frappe', repo_name=None): + confirm_testing() config = get_config(bench_path) if not config.get('release_bench'): @@ -53,10 +54,20 @@ def validate(bench_path, config): r = requests.get('https://api.github.com/user', auth=HTTPBasicAuth(github_username, github_password)) r.raise_for_status() +def confirm_testing(): + print('') + print('================ CAUTION ==================') + print('Never miss this, even if it is a really small release!!') + print('Manual Testing Checklisk: https://github.com/frappe/bench/wiki/Testing-Checklist') + print('') + print('') + click.confirm('Is manual testing done?', abort = True) + def bump(bench_path, app, bump_type, from_branch, to_branch, remote, owner, repo_name=None): assert bump_type in ['minor', 'major', 'patch', 'stable', 'prerelease'] repo_path = os.path.join(bench_path, 'apps', app) + push_branch_for_old_major_version(bench_path, bump_type, app, repo_path, from_branch, to_branch, remote, owner) update_branches_and_check_for_changelog(repo_path, from_branch, to_branch, remote=remote) message = get_release_message(repo_path, from_branch=from_branch, to_branch=to_branch, remote=remote) @@ -299,3 +310,24 @@ def create_github_release(repo_path, tag_name, message, remote='upstream', owner raise return r +def push_branch_for_old_major_version(bench_path, bump_type, app, repo_path, from_branch, to_branch, remote, owner): + if bump_type != 'major': + return + + current_version = get_current_version(repo_path) + old_major_version_branch = "v{major}.x.x".format(major=current_version.split('.')[0]) + + click.confirm('Do you want to push {branch}?'.format(branch=old_major_version_branch), abort=True) + + update_branch(repo_path, to_branch, remote=remote) + + g = git.Repo(repo_path).git + g.checkout(b=old_major_version_branch) + + args = [ + '{old_major_version_branch}:{old_major_version_branch}'.format(old_major_version_branch=old_major_version_branch), + ] + + print("Pushing {old_major_version_branch} ".format(old_major_version_branch=old_major_version_branch)) + print(g.push(remote, *args)) + diff --git a/bench/tests/test_init.py b/bench/tests/test_init.py index b4c16b47..c4509163 100755 --- a/bench/tests/test_init.py +++ b/bench/tests/test_init.py @@ -212,7 +212,9 @@ class TestBenchInit(unittest.TestCase): self.assert_exists(python_path, "site-packages", "pip") site_packages = os.listdir(os.path.join(python_path, "site-packages")) - self.assertTrue(any(package.startswith("mysqlclient-1.3.10") for package in site_packages)) + # removing test case temporarily + # as develop and master branch havin differnt version of mysqlclient + #self.assertTrue(any(package.startswith("mysqlclient-1.3.12") for package in site_packages)) def assert_config(self, bench_name): for config, search_key in ( diff --git a/bench/utils.py b/bench/utils.py index b0859231..a9f96229 100755 --- a/bench/utils.py +++ b/bench/utils.py @@ -1,6 +1,7 @@ import os, sys, shutil, subprocess, logging, itertools, requests, json, platform, select, pwd, grp, multiprocessing, hashlib from distutils.spawn import find_executable import bench +import semantic_version from bench import env from six import iteritems @@ -15,6 +16,13 @@ logger = logging.getLogger(__name__) folders_in_bench = ('apps', 'sites', 'config', 'logs', 'config/pids') +def safe_decode(string, encoding = 'utf-8'): + try: + string = string.decode(encoding) + except Exception: + pass + return string + def get_frappe(bench_path='.'): frappe = get_env_cmd('frappe', bench_path=bench_path) if not os.path.exists(frappe): @@ -27,31 +35,33 @@ def get_env_cmd(cmd, bench_path='.'): def init(path, apps_path=None, no_procfile=False, no_backups=False, no_auto_update=False, frappe_path=None, frappe_branch=None, wheel_cache_dir=None, - verbose=False, clone_from=None, skip_bench_mkdir=False, skip_redis_config_generation=False): + verbose=False, clone_from=None, skip_redis_config_generation=False, + ignore_exist = False, + python = 'python'): # Let's change when we're ready. - from .app import get_app, install_apps_from_path from .config.common_site_config import make_config from .config import redis from .config.procfile import setup_procfile from bench.patches import set_all_patches_executed - if(skip_bench_mkdir): - pass + import os.path as osp + + if osp.exists(path): + if not ignore_exist: + raise ValueError('Bench Instance {path} already exists.'.format(path = path)) else: - if os.path.exists(path): - print('Directory {} already exists!'.format(path)) - raise Exception("Site directory already exists") os.makedirs(path) for dirname in folders_in_bench: try: os.makedirs(os.path.join(path, dirname)) - except OSError, e: + except OSError as e: if e.errno != os.errno.EEXIST: pass setup_logging() - setup_env(bench_path=path) + setup_env(bench_path=path, python = python) make_config(path) @@ -69,7 +79,7 @@ def init(path, apps_path=None, no_procfile=False, no_backups=False, bench.set_frappe_version(bench_path=path) if bench.FRAPPE_VERSION > 5: - update_npm_packages(bench_path=path) + update_node_packages(bench_path=path) set_all_patches_executed(bench_path=path) build_assets(bench_path=path) @@ -89,8 +99,10 @@ def clone_apps_from(bench_path, clone_from): print('Copying apps from {0}...'.format(clone_from)) subprocess.check_output(['cp', '-R', os.path.join(clone_from, 'apps'), bench_path]) - print('Copying node_modules from {0}...'.format(clone_from)) - subprocess.check_output(['cp', '-R', os.path.join(clone_from, 'node_modules'), bench_path]) + node_modules_path = os.path.join(clone_from, 'node_modules') + if os.path.exists(node_modules_path): + print('Copying node_modules from {0}...'.format(clone_from)) + subprocess.check_output(['cp', '-R', node_modules_path, bench_path]) def setup_app(app): # run git reset --hard in each branch, pull latest updates and install_app @@ -139,8 +151,21 @@ def exec_cmd(cmd, cwd='.'): if return_code > 0: raise CommandFailedError(cmd) -def setup_env(bench_path='.'): - exec_cmd('virtualenv -q {} -p {}'.format('env', sys.executable), cwd=bench_path) +def which(executable, raise_err = False): + from distutils.spawn import find_executable + exec_ = find_executable(executable) + + if not exec_ and raise_err: + raise ValueError('{executable} not found.'.format( + executable = executable + )) + + return exec_ + +def setup_env(bench_path='.', python = 'python'): + python = which(python, raise_err = True) + + exec_cmd('virtualenv -q {} -p {}'.format('env', python), cwd=bench_path) exec_cmd('./env/bin/pip -q install --upgrade pip', cwd=bench_path) exec_cmd('./env/bin/pip -q install wheel', cwd=bench_path) # exec_cmd('./env/bin/pip -q install https://github.com/frappe/MySQLdb1/archive/MySQLdb-1.2.5-patched.tar.gz', cwd=bench_path) @@ -203,13 +228,14 @@ def setup_backups(bench_path='.'): def add_to_crontab(line): current_crontab = read_crontab() + line = str.encode(line) if not line in current_crontab: cmd = ["crontab"] if platform.system() == 'FreeBSD': cmd = ["crontab", "-"] s = subprocess.Popen(cmd, stdin=subprocess.PIPE) s.stdin.write(current_crontab) - s.stdin.write(line + '\n') + s.stdin.write(line + b'\n') s.stdin.close() def read_crontab(): @@ -251,9 +277,10 @@ def setup_sudoers(user): 'nginx': find_executable('nginx'), 'bench': find_executable('bench') }) + frappe_sudoers = safe_decode(frappe_sudoers) with open(sudoers_file, 'w') as f: - f.write(frappe_sudoers.encode('utf-8')) + f.write(frappe_sudoers) os.chmod(sudoers_file, 0o440) @@ -302,7 +329,9 @@ def check_cmd(cmd, cwd='.'): def get_git_version(): '''returns git version from `git --version` extracts version number from string `get version 1.9.1` etc''' - version = get_cmd_output("git --version").strip().split()[2] + version = get_cmd_output("git --version") + version = safe_decode(version) + version = version.strip().split()[2] version = '.'.join(version.split('.')[0:2]) return float(version) @@ -323,12 +352,22 @@ def check_git_for_shallow_clone(): def get_cmd_output(cmd, cwd='.'): try: - return subprocess.check_output(cmd, cwd=cwd, shell=True, stderr=open(os.devnull, 'wb')).strip() + output = subprocess.check_output(cmd, cwd=cwd, shell=True, stderr=open(os.devnull, 'wb')).strip() + output = output.decode('utf-8') + return output except subprocess.CalledProcessError as e: if e.output: print(e.output) raise +def safe_encode(what, encoding = 'utf-8'): + try: + what = what.encode(encoding) + except Exception: + pass + + return what + def restart_supervisor_processes(bench_path='.', web_workers=False): from .config.common_site_config import get_config conf = get_config(bench_path=bench_path) @@ -340,7 +379,8 @@ def restart_supervisor_processes(bench_path='.', web_workers=False): else: supervisor_status = subprocess.check_output(['sudo', 'supervisorctl', 'status'], cwd=bench_path) - + supervisor_status = safe_decode(supervisor_status) + if web_workers and '{bench_name}-web:'.format(bench_name=bench_name) in supervisor_status: group = '{bench_name}-web: '.format(bench_name=bench_name) @@ -380,8 +420,35 @@ def update_requirements(bench_path='.'): req_file = os.path.join(apps_dir, app, 'requirements.txt') install_requirements(pip, req_file) +def update_node_packages(bench_path='.'): + print('Updating node packages...') + from bench.app import get_develop_version + from distutils.version import LooseVersion + v = LooseVersion(get_develop_version('frappe', bench_path = bench_path)) + + + # After rollup was merged, frappe_version = 10.1 + # if develop_verion is 11 and up, only then install yarn + if v < LooseVersion('11.x.x-develop'): + update_npm_packages(bench_path) + else: + update_yarn_packages(bench_path) + +def update_yarn_packages(bench_path='.'): + apps_dir = os.path.join(bench_path, 'apps') + + if not find_executable('yarn'): + print("Please install yarn using below command and try again.") + print("`npm install -g yarn`") + return + + for app in os.listdir(apps_dir): + app_path = os.path.join(apps_dir, app) + if os.path.exists(os.path.join(app_path, 'package.json')): + exec_cmd('yarn install', cwd=app_path) + + def update_npm_packages(bench_path='.'): - print('Updating node libraries...') apps_dir = os.path.join(bench_path, 'apps') package_json = {} @@ -412,6 +479,7 @@ def update_npm_packages(bench_path='.'): exec_cmd('npm install', cwd=bench_path) + def install_requirements(pip, req_file): if os.path.exists(req_file): exec_cmd("{pip} install -q -r {req_file}".format(pip=pip, req_file=req_file)) @@ -732,11 +800,16 @@ def set_git_remote_url(git_url, bench_path='.'): if os.path.exists(os.path.join(app_dir, '.git')): exec_cmd("git remote set-url upstream {}".format(git_url), cwd=app_dir) -def run_playbook(playbook_name, extra_vars=None): +def run_playbook(playbook_name, extra_vars=None, tag=None): if not find_executable('ansible'): print("Ansible is needed to run this command, please install it using 'pip install ansible'") sys.exit(1) args = ['ansible-playbook', '-c', 'local', playbook_name] + if extra_vars: args.extend(['-e', json.dumps(extra_vars)]) + + if tag: + args.extend(['-t', tag]) + subprocess.check_call(args, cwd=os.path.join(os.path.dirname(bench.__path__[0]), 'playbooks')) diff --git a/docs/releasing_frappe_erpext.md b/docs/releasing_frappe_erpext.md index 606c4c4d..52010c0c 100644 --- a/docs/releasing_frappe_erpext.md +++ b/docs/releasing_frappe_erpext.md @@ -36,4 +36,6 @@ Usage: bench release [OPTIONS] APP BUMP_TYPE * --to-branch git master branch, default is master * --remote git remote, default is upstream * --owner git owner, default is frappe - * --repo-name git repo name if different from app name \ No newline at end of file + * --repo-name git repo name if different from app name + +* When updating major version, update `develop_version` in hooks.py, e.g. `9.x.x-develop` diff --git a/install_scripts/setup_frappe.sh b/install_scripts/setup_frappe.sh deleted file mode 100755 index e6c7ac7c..00000000 --- a/install_scripts/setup_frappe.sh +++ /dev/null @@ -1,483 +0,0 @@ -#!/bin/bash - -set -e - - -## Utils - -print_msg() { - echo "Frappe password: $FRAPPE_USER_PASS" - echo "MariaDB root password: $MSQ_PASS" - echo "Administrator password: $ADMIN_PASS" -} - -get_passwd() { - echo `cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w 16 | head -n 1` -} - -set_opts () { - OPTS=`getopt -o v --long verbose,mysql-root-password:,frappe-user:,bench-branch:,setup-production,skip-install-bench,skip-setup-bench,help -n 'parse-options' -- "$@"` - - if [ $? != 0 ] ; then echo "Failed parsing options." >&2 ; exit 1 ; fi - - eval set -- "$OPTS" - - VERBOSE=false - HELP=false - FRAPPE_USER=false - BENCH_BRANCH="master" - SETUP_PROD=false - INSTALL_BENCH=true - SETUP_BENCH=true - - if [ -f ~/frappe_passwords.sh ]; then - source ~/frappe_passwords.sh - else - FRAPPE_USER_PASS=`get_passwd` - MSQ_PASS=`get_passwd` - ADMIN_PASS=`get_passwd` - - echo "FRAPPE_USER_PASS=$FRAPPE_USER_PASS" > ~/frappe_passwords.sh - echo "MSQ_PASS=$MSQ_PASS" >> ~/frappe_passwords.sh - echo "ADMIN_PASS=$ADMIN_PASS" >> ~/frappe_passwords.sh - fi - - while true; do - case "$1" in - -v | --verbose ) VERBOSE=true; shift ;; - -h | --help ) HELP=true; shift ;; - --mysql-root-password ) MSQ_PASS="$2"; shift; shift ;; - --frappe-user ) FRAPPE_USER="$2"; shift; shift ;; - --setup-production ) SETUP_PROD=true; shift;; - --bench-branch ) BENCH_BRANCH="$2"; shift;; - --skip-setup-bench ) SETUP_BENCH=false; shift;; - --skip-install-bench ) INSTALL_BENCH=false; shift;; - -- ) shift; break ;; - * ) break ;; - esac - done -} - -get_distro() { - ARCH=$(uname -m | sed 's/x86_/amd/;s/i[3-6]86/x86/') - - if [ $ARCH == "amd64" ]; then - T_ARCH="x86_64" - WK_ARCH="amd64" - else - T_ARCH="i386" - WK_ARCH="i386" - fi - - if [ -f /etc/redhat-release ]; then - OS="centos" - OS_VER=`cat /etc/redhat-release | sed 's/Linux\ //g' | cut -d" " -f3 | cut -d. -f1` - - elif [ -f /etc/lsb-release ]; then - . /etc/lsb-release - OS=$DISTRIB_ID - OS_VER=$DISTRIB_CODENAME - - elif [ -f /etc/debian_version ]; then - . /etc/os-release - OS="debian" # XXX or Ubuntu?? - OS_VER=$VERSION_ID - fi - - export OS=$OS - export OS_VER=$OS_VER - export ARCH=$ARCH - export T_ARCH=$T_ARCH - export WK_ARCH=$WK_ARCH - echo Installing for $OS $OS_VER $ARCH - echo "In case you encounter an error, you can post on https://discuss.frappe.io" - echo -} - -run_cmd() { - if $VERBOSE; then - "$@" - else - # $@ - "$@" > /tmp/cmdoutput.txt 2>&1 || (cat /tmp/cmdoutput.txt && exit 1) - fi -} - -## add repos - -add_centos6_mariadb_repo() { - echo " -[mariadb] -name = MariaDB -baseurl = http://yum.mariadb.org/10.0/centos$OS_VER-$ARCH -gpgkey=https://yum.mariadb.org/RPM-GPG-KEY-MariaDB -gpgcheck=1 -" > /etc/yum.repos.d/mariadb.repo -} - - -add_ubuntu_mariadb_repo() { - run_cmd sudo apt-get update - run_cmd sudo apt-get install -y software-properties-common python-software-properties - run_cmd sudo apt-key adv --recv-keys --keyserver hkp://keyserver.ubuntu.com:80 0xcbcb082a1bb943db - run_cmd sudo add-apt-repository "deb http://ams2.mirrors.digitalocean.com/mariadb/repo/10.0/ubuntu $OS_VER main" -} - -add_debian_mariadb_repo() { - if [ $OS_VER == "7" ]; then - CODENAME="wheezy" - elif [ $OS_VER == "6" ]; then - CODENAME="squeeze" - elif [ $OS_VER == "8" ]; then - CODENAME="jessie" - else - echo Unsupported Debian Version - exit 1 - fi - run_cmd sudo apt-get update - run_cmd sudo apt-get install -y software-properties-common python-software-properties - run_cmd sudo apt-key adv --recv-keys --keyserver keyserver.ubuntu.com 0xcbcb082a1bb943db - $repo_test=$(apt-cache search --names-only 'mariadb-server') - if [ -z "$repo_test" ]; then - run_cmd sudo add-apt-repository "deb http://ams2.mirrors.digitalocean.com/mariadb/repo/10.0/debian $CODENAME main" - fi -} - -add_ius_repo() { - if [ $OS_VER -eq "6" ]; then - wget http://dl.iuscommunity.org/pub/ius/stable/CentOS/$OS_VER/$T_ARCH/epel-release-6-5.noarch.rpm - wget http://dl.iuscommunity.org/pub/ius/stable/CentOS/$OS_VER/$T_ARCH/ius-release-1.0-14.ius.centos6.noarch.rpm - rpm --quiet -q epel-release || rpm -Uvh epel-release-6-5.noarch.rpm - rpm --quiet -q ius-release || rpm -Uvh ius-release-1.0-14.ius.centos6.noarch.rpm - fi -} - -add_epel_centos7() { - yum install -y epel-release -} - -add_maria_db_repo() { - if [ "$OS" == "Ubuntu" ] && [ $OS_VER == "utopic" ]; then - return - elif [ "$OS" == "centos" ]; then - echo Adding centos mariadb repo - add_centos6_mariadb_repo - - elif [ "$OS" == "debian" ]; then - echo Adding debian mariadb repo - add_debian_mariadb_repo - - elif [ "$OS" == "Ubuntu" ]; then - echo Adding ubuntu mariadb repo - add_ubuntu_mariadb_repo - else - echo Unsupported Distribution - exit 1 - fi -} - -## install - -install_packages() { - if [ $OS == "centos" ]; then - run_cmd sudo yum install wget -y - run_cmd sudo yum groupinstall -y "Development tools" - if [ $OS_VER == "6" ]; then - run_cmd add_ius_repo - run_cmd sudo yum install -y git MariaDB-server MariaDB-client MariaDB-compat python-setuptools nginx \ - zlib-devel bzip2-devel openssl-devel postfix python27-devel python27 \ - libxml2 libxml2-devel libxslt libxslt-devel redis MariaDB-devel libXrender libXext \ - python27-setuptools cronie sudo which xorg-x11-fonts-Type1 xorg-x11-fonts-75dpi nodejs npm \ - libtiff-devel libjpeg-devel libzip-devel freetype-devel lcms2-devel libwebp-devel tcl-devel tk-devel - elif [ $OS_VER == "7" ]; then - run_cmd add_epel_centos7 - run_cmd sudo yum install -y git mariadb-server mariadb-devel python-setuptools nginx \ - zlib-devel bzip2-devel openssl-devel postfix python-devel \ - libxml2 libxml2-devel libxslt libxslt-devel redis libXrender libXext \ - supervisor cronie sudo which xorg-x11-fonts-75dpi xorg-x11-fonts-Type1 nodejs npm \ - libtiff-devel libjpeg-devel libzip-devel freetype-devel lcms2-devel libwebp-devel tcl-devel tk-devel - fi - - echo "Installing wkhtmltopdf" - install_wkhtmltopdf_centos - run_cmd easy_install-2.7 -U pip - - - elif [ $OS == "debian" ] || [ $OS == "Ubuntu" ]; then - export DEBIAN_FRONTEND=noninteractive - setup_debconf - run_cmd bash -c "curl -sL https://deb.nodesource.com/setup_0.12 | sudo bash -" - run_cmd sudo apt-get update - run_cmd sudo apt-get install -y python-dev python-setuptools build-essential python-mysqldb git \ - ntp vim screen htop mariadb-server mariadb-common libmariadbclient-dev \ - libxslt1.1 libxslt1-dev redis-server libssl-dev libcrypto++-dev postfix nginx \ - supervisor python-pip fontconfig libxrender1 libxext6 xfonts-75dpi xfonts-base nodejs - - if [ $OS_VER == "precise" ]; then - run_cmd sudo apt-get install -y libtiff4-dev libjpeg8-dev zlib1g-dev libfreetype6-dev \ - liblcms2-dev libwebp-dev tcl8.5-dev tk8.5-dev python-tk - elif [ $OS_VER == "8" ]; then - run_cmd sudo apt-get install -y libtiff5-dev libjpeg62-turbo-dev zlib1g-dev libfreetype6-dev \ - liblcms2-dev libwebp-dev tcl8.5-dev tk8.5-dev python-tk npm - else - run_cmd sudo apt-get install -y libtiff5-dev libjpeg8-dev zlib1g-dev libfreetype6-dev \ - liblcms2-dev libwebp-dev tcl8.6-dev tk8.6-dev python-tk - fi - - echo "Installing wkhtmltopdf" - install_wkhtmltopdf_deb - - else - echo Unsupported Distribution - exit 1 - fi -} - -install_wkhtmltopdf_centos () { - - if [[ $OS == "centos" && $OS_VER == "7" && $T_ARCH == "i386" ]]; then - echo "Cannot install wkhtmltodpdf. Skipping..." - return 0 - fi - RPM="wkhtmltox-0.12.2.1_linux-$OS$OS_VER-$WK_ARCH.rpm" - run_cmd wget http://download.gna.org/wkhtmltopdf/0.12/0.12.2.1/$RPM - rpm --quiet -q wkhtmltox || run_cmd rpm -Uvh $RPM -} - -install_wkhtmltopdf_deb () { - WK_VER=$OS_VER - - if [[ $OS_VER == "utopic" || $OS_VER == "vivid" || $OS_VER == "wily" ]]; then - echo "Installing wkhtmltox package for trusty (Ubuntu 14.4) even if you are using $OS_VER." - WK_VER="trusty" - fi - if [[ $OS == "debian" && $OS_VER == "7" ]]; then - WK_VER="wheezy" - elif [[ $OS == "debian" && $OS_VER == "8" ]]; then - WK_VER="jessie" - fi - - run_cmd wget http://download.gna.org/wkhtmltopdf/0.12/0.12.2.1/wkhtmltox-0.12.2.1_linux-$WK_VER-$WK_ARCH.deb - run_cmd dpkg -i wkhtmltox-0.12.2.1_linux-$WK_VER-$WK_ARCH.deb -} - - -install_supervisor_centos6() { - run_cmd easy_install supervisor - curl -Ss https://raw.githubusercontent.com/pdvyas/supervisor-initscripts/master/redhat-init-jkoppe > /etc/init.d/supervisord - curl -Ss https://raw.githubusercontent.com/pdvyas/supervisor-initscripts/master/redhat-sysconfig-jkoppe > /etc/sysconfig/supervisord - curl -Ss https://raw.githubusercontent.com/pdvyas/supervisor-initscripts/master/supervisord.conf > /etc/supervisord.conf - run_cmd mkdir /etc/supervisor.d - run_cmd chmod +x /etc/init.d/supervisord - bash -c "service supervisord start || true" -} - -### config - -get_mariadb_password() { - get_password "MariaDB root" MSQ_PASS -} - -get_site_admin_password() { - get_password "Admin password" ADMIN_PASS -} - -get_password() { - if [ -z "$2" ]; then - read -t 1 -n 10000 discard || true - echo - read -p "Enter $1 password to set:" -s TMP_PASS1 - echo - read -p "Re enter $1 password to set:" -s TMP_PASS2 - echo - if [ $TMP_PASS1 == $TMP_PASS2 ]; then - export $2=$TMP_PASS1 - else - echo Passwords do not match - get_password $1 $2 - fi - fi -} - -configure_mariadb_centos() { - # Required only for CentOS, Ubuntu and Debian will show dpkg configure screen to set the password - if [ $OS == "centos" ]; then - mysqladmin -u root password $MSQ_PASS - fi -} - -start_services_centos6() { - run_cmd service nginx start - run_cmd service mysql start - run_cmd service redis start -} - -configure_services_centos6() { - run_cmd chkconfig --add supervisord - run_cmd chkconfig redis on - run_cmd chkconfig mysql on - run_cmd chkconfig nginx on - run_cmd chkconfig supervisord on -} - -configure_services_centos7() { - run_cmd systemctl enable nginx - run_cmd systemctl enable mysql - run_cmd systemctl enable redis - run_cmd systemctl enable supervisord -} - -start_services_centos7() { - run_cmd systemctl start nginx - run_cmd systemctl start mysql - run_cmd systemctl start redis - run_cmd systemctl start supervisord -} - -configure_mariadb() { - config=" -[mysqld] -innodb-file-format=barracuda -innodb-file-per-table=1 -innodb-large-prefix=1 -character-set-client-handshake = FALSE -character-set-server = utf8mb4 -collation-server = utf8mb4_unicode_ci - -[mysql] -default-character-set = utf8mb4 - " - deb_cnf_path="/etc/mysql/conf.d/barracuda.cnf" - centos_cnf_path="/etc/my.cnf.d/barracuda.cnf" - - if [ $OS == "centos" ]; then - - echo "$config" > $centos_cnf_path - if [ $OS_VER == "6" ]; then - run_cmd sudo service mysql restart - elif [ $OS_VER == "7" ]; then - run_cmd sudo systemctl restart mysql - fi - - elif [ $OS == "debian" ] || [ $OS == "Ubuntu" ]; then - echo "$config" > $deb_cnf_path - sudo service mysql restart - fi -} - -setup_debconf() { - debconf-set-selections <<< "postfix postfix/mailname string `hostname`" - debconf-set-selections <<< "postfix postfix/main_mailer_type string 'Internet Site'" - debconf-set-selections <<< "mariadb-server-5.5 mysql-server/root_password password $MSQ_PASS" - debconf-set-selections <<< "mariadb-server-5.5 mysql-server/root_password_again password $MSQ_PASS" -} - -install_bench() { - run_cmd sudo su $FRAPPE_USER -c "cd /home/$FRAPPE_USER && git clone https://github.com/frappe/bench --branch $BENCH_BRANCH bench-repo" - if hash pip-2.7 &> /dev/null; then - PIP="pip-2.7" - elif hash pip2.7 &> /dev/null; then - PIP="pip2.7" - elif hash pip2 &> /dev/null; then - PIP="pip2" - elif hash pip &> /dev/null; then - PIP="pip" - else - echo PIP not installed - exit 1 - fi - - run_cmd sudo $PIP install --upgrade pip - run_cmd sudo $PIP install -e /home/$FRAPPE_USER/bench-repo -} - -setup_bench() { - echo Installing frappe-bench - FRAPPE_BRANCH="develop" - ERPNEXT_APPS_JSON="https://raw.githubusercontent.com/frappe/bench/master/install_scripts/erpnext-apps.json" - if $SETUP_PROD; then - FRAPPE_BRANCH="master" - ERPNEXT_APPS_JSON="https://raw.githubusercontent.com/frappe/bench/master/install_scripts/erpnext-apps-master.json" - fi - - run_cmd sudo su $FRAPPE_USER -c "cd /home/$FRAPPE_USER && bench init frappe-bench --frappe-branch $FRAPPE_BRANCH --apps_path $ERPNEXT_APPS_JSON" - echo Setting up first site - echo /home/$FRAPPE_USER/frappe-bench > /etc/frappe_bench_dir - run_cmd sudo su $FRAPPE_USER -c "cd /home/$FRAPPE_USER/frappe-bench && bench new-site site1.local --mariadb-root-password $MSQ_PASS --admin-password $ADMIN_PASS" - run_cmd sudo su $FRAPPE_USER -c "cd /home/$FRAPPE_USER/frappe-bench && bench install-app erpnext" - run_cmd bash -c "cd /home/$FRAPPE_USER/frappe-bench && bench setup sudoers $FRAPPE_USER" - if $SETUP_PROD; then - run_cmd bash -c "cd /home/$FRAPPE_USER/frappe-bench && bench setup production $FRAPPE_USER" - fi - chown $FRAPPE_USER /home/$FRAPPE_USER/frappe-bench/logs/* -} - -add_user() { -# Check if script is running as root and is not running as sudo. We want to skip -# this step if the user is already running this script with sudo as a non root -# user - if [ "$FRAPPE_USER" == "false" ]; then - if [[ $SUDO_UID -eq 0 ]] && [[ $EUID -eq 0 ]]; then - export FRAPPE_USER="frappe" - else - export FRAPPE_USER="$SUDO_USER" - fi - fi - - USER_EXISTS=`bash -c "id $FRAPPE_USER > /dev/null 2>&1 && echo true || (echo false && exit 0)"` - - if [ $USER_EXISTS == "false" ]; then - useradd -m -d /home/$FRAPPE_USER -s $SHELL $FRAPPE_USER - echo $FRAPPE_USER:$FRAPPE_USER_PASS | chpasswd - chmod o+x /home/$FRAPPE_USER - chmod o+r /home/$FRAPPE_USER - fi -} - -main() { - set_opts $@ - get_distro - add_maria_db_repo - echo Installing packages for $OS\. This might take time... - install_packages - if [ $OS == "centos" ]; then - if [ $OS_VER == "6" ]; then - echo "Installing supervisor" - install_supervisor_centos6 - echo "Configuring CentOS services" - configure_services_centos6 - echo "Starting services" - start_services_centos6 - elif [ $OS_VER == "7" ]; then - echo "Configuring CentOS services" - configure_services_centos7 - echo "Starting services" - start_services_centos7 - fi - configure_mariadb_centos - fi - configure_mariadb - echo "Adding frappe user" - add_user - - if $INSTALL_BENCH; then - install_bench - if $SETUP_BENCH; then - setup_bench - fi - fi - - echo - RUNNING="" - if $SETUP_PROD; then - RUNNING=" and is running on port 80" - fi - echo "Frappe/ERPNext is installed successfully$RUNNING." - print_msg > ~/frappe_passwords.txt - print_msg - echo - echo "The passwords are also stored at ~/frappe_passwords.txt" - echo "You can remove this file after making a note of the passwords." -} - -main $@ diff --git a/patches.txt b/patches.txt new file mode 100644 index 00000000..8f127fe4 --- /dev/null +++ b/patches.txt @@ -0,0 +1 @@ +bench.patches.v3.deprecate_old_config diff --git a/playbooks/README.md b/playbooks/README.md new file mode 100644 index 00000000..4c524ff1 --- /dev/null +++ b/playbooks/README.md @@ -0,0 +1,10 @@ +# Deploying a, developer/production-ready ERPNext website with Ansible + +## Supported Platforms + - Debian 8, 9 + - Ubuntu 14.04, 16.04 + - CentOS 7 + +## Notes for maintainers + - For MariaDB playbooks refer https://github.com/PCextreme/ansible-role-mariadb + - Any changes made in relation to a role should be dont inside the role and not outside it diff --git a/playbooks/__init__.py b/playbooks/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/playbooks/develop/create_user.yml b/playbooks/create_user.yml old mode 100755 new mode 100644 similarity index 85% rename from playbooks/develop/create_user.yml rename to playbooks/create_user.yml index 2980ed4e..acda02e2 --- a/playbooks/develop/create_user.yml +++ b/playbooks/create_user.yml @@ -1,4 +1,5 @@ --- + - hosts: localhost tasks: - name: Create user @@ -30,4 +31,8 @@ owner: '{{ frappe_user }}' group: '{{ frappe_user }}' recurse: yes - when: ansible_distribution == 'Ubuntu' or ansible_distribution == 'CentOS' or ansible_distribution == 'Debian' \ No newline at end of file + when: ansible_distribution == 'Ubuntu' or ansible_distribution == 'CentOS' or ansible_distribution == 'Debian' + + - name: Change default shell to bash + shell: "chsh {{ frappe_user }} -s $(which bash)" +... \ No newline at end of file diff --git a/playbooks/develop/centos.yml b/playbooks/develop/centos.yml deleted file mode 100755 index 8090340d..00000000 --- a/playbooks/develop/centos.yml +++ /dev/null @@ -1,86 +0,0 @@ ---- -- hosts: localhost - vars: - bench_repo_path: "/home/{{ ansible_user_id }}/.bench" - bench_path: "/home/{{ ansible_user_id }}/frappe-bench" - mysql_config_template: "templates/simple_mariadb_config.cnf" - mysql_conf_dir: /etc/my.cnf.d/ - - tasks: - - - name: development tools package - yum: name="@Development tools" state=present - become: yes - become_user: root - - - name: install prequisites - yum: pkg={{ item }} state=present - with_items: - # basic installs - - redis - - # for mariadb - - libselinux-python - - mysql-devel - - mysql-libs - - # for wkhtmltopdf - - libXrender - - libXext - - xorg-x11-fonts-75dpi - - xorg-x11-fonts-Type1 - - # for Pillow - - libjpeg-devel - - zlib-devel - - libzip-devel - - freetype-devel - - lcms2-devel - - libwebp-devel - - libtiff-devel - - tcl-devel - - tk-devel - - # Python LDAP - - openldap-devel - - become: yes - become_user: root - - - name: Import Node source RPM key - rpm_key: - key: https://rpm.nodesource.com/pub/el/NODESOURCE-GPG-SIGNING-KEY-EL - state: present - become: yes - become_user: root - - - name: Add Node Repo - yum: - name: 'https://rpm.nodesource.com/pub_6.x/el/{{ ansible_distribution_major_version }}/{{ ansible_architecture }}/nodesource-release-el{{ ansible_distribution_major_version }}-1.noarch.rpm' - state: present - become: yes - become_user: root - - - name: Install nodejs - yum: - name: nodejs - state: present - become: yes - become_user: root - - # install MariaDB - - include: includes/mariadb_centos.yml - - # install WKHTMLtoPDF - - include: includes/wkhtmltopdf.yml - - # setup MariaDB - - include: includes/setup_mariadb.yml - - # setup frappe-bench - - include: includes/setup_bench.yml - when: not run_travis and not without_bench_setup - - # setup development environment - - include: includes/setup_dev_env.yml - when: not run_travis and not production diff --git a/playbooks/develop/debian.yml b/playbooks/develop/debian.yml deleted file mode 100755 index 767d2b6a..00000000 --- a/playbooks/develop/debian.yml +++ /dev/null @@ -1,126 +0,0 @@ ---- -- hosts: localhost - vars: - bench_repo_path: "/home/{{ ansible_user_id }}/.bench" - bench_path: "/home/{{ ansible_user_id }}/frappe-bench" - mysql_config_template: "templates/simple_mariadb_config.cnf" - mysql_conf_dir: /etc/mysql/conf.d/ - - tasks: - - - name: Add dotdeb apt repository key for Debian < 8 - apt_key: - url: http://www.dotdeb.org/dotdeb.gpg - state: present - become: yes - become_user: root - when: ansible_distribution == 'Debian' and ansible_distribution_version | version_compare('8', 'lt') - - - name: Add dotdeb apt repository for redis-server for Debian < 8 - apt_repository: - repo: 'deb http://packages.dotdeb.org wheezy all' - state: present - become: yes - become_user: root - when: ansible_distribution == 'Debian' and ansible_distribution_version | version_compare('8', 'lt') - - # install pre-requisites - - name: install prequisites - apt: pkg={{ item }} state=present - with_items: - # basic installs - - build-essential - - redis-server - - # for mariadb - - software-properties-common - - # for wkhtmltopdf - - libxrender1 - - libxext6 - - xfonts-75dpi - - xfonts-base - - # for Pillow - - zlib1g-dev - - libfreetype6-dev - - liblcms2-dev - - libwebp-dev - - python-tk - - # Ensure apt-transport-https - - apt-transport-https - - # Python LDAP - - libsasl2-dev - - libldap2-dev - - become: yes - become_user: root - - - name: install pillow prerequisites for Debian < 8 - apt: pkg={{ item }} state=present - with_items: - - libjpeg8-dev - - libtiff4-dev - - tcl8.5-dev - - tk8.5-dev - when: ansible_distribution_version | version_compare('8', 'lt') - become: yes - become_user: root - - - name: install pillow prerequisites for Debian >= 8 - apt: pkg={{ item }} state=present - with_items: - - libjpeg62-turbo-dev - - libtiff5-dev - - tcl8.5-dev - - tk8.5-dev - when: ansible_distribution_version | version_compare('8', 'ge') - become: yes - become_user: root - - - name: Add apt key for node repo - apt_key: - url: https://keyserver.ubuntu.com/pks/lookup?op=get&fingerprint=on&search=0x1655A0AB68576280 - id: "68576280" - state: present - become: yes - become_user: root - - - name: Add repo - apt_repository: - repo: "deb [arch=amd64,i386] https://deb.nodesource.com/node_6.x {{ ansible_distribution_release }} main" - state: present - update_cache: yes - become: yes - become_user: root - - - name: Install nodejs 6.x - apt: - name: nodejs - state: present - update_cache: yes - force: yes - become: yes - become_user: root - - # install MariaDB - - include: includes/mariadb_debian.yml - - # install WKHTMLtoPDF - - include: includes/wkhtmltopdf.yml - - # setup MariaDB - - include: includes/setup_mariadb.yml - - - debug: - var: run_travis - - # setup frappe-bench - - include: includes/setup_bench.yml - when: not run_travis - - # setup development environment - - include: includes/setup_dev_env.yml - when: not production and not run_travis diff --git a/playbooks/develop/includes/mariadb_centos.yml b/playbooks/develop/includes/mariadb_centos.yml deleted file mode 100644 index 37454f1f..00000000 --- a/playbooks/develop/includes/mariadb_centos.yml +++ /dev/null @@ -1,13 +0,0 @@ ---- - - name: Add repository - template: src=templates/mariadb_centos.repo dest=/etc/yum.repos.d/MariaDB.repo owner=root mode=0644 - become: yes - become_user: root - - - name: Install MariaDB - yum: pkg={{ item }} state=present - with_items: - - MariaDB-server - - MariaDB-client - become: yes - become_user: root diff --git a/playbooks/develop/includes/mariadb_debian.yml b/playbooks/develop/includes/mariadb_debian.yml deleted file mode 100644 index 96e480a2..00000000 --- a/playbooks/develop/includes/mariadb_debian.yml +++ /dev/null @@ -1,27 +0,0 @@ ---- - - - name: Add apt key for mariadb for Debian <= 8 - apt_key: keyserver=hkp://keyserver.ubuntu.com:80 id=0xcbcb082a1bb943db state=present - become: yes - become_user: root - when: ansible_distribution_major_version | version_compare('8', 'le') - - - name: Add apt repository - apt_repository: - repo: 'deb [arch=amd64,i386] http://nyc2.mirrors.digitalocean.com/mariadb/repo/10.1/debian {{ ansible_distribution_release }} main' - state: present - become: yes - become_user: root - - - name: Unattended package installation - shell: export DEBIAN_FRONTEND=noninteractive - - - name: apt-get install - apt: pkg={{ item }} update_cache=yes state=present - with_items: - - mariadb-server - - mariadb-client - - mariadb-common - - libmariadbclient-dev - become: yes - become_user: root diff --git a/playbooks/develop/includes/mariadb_ubuntu.yml b/playbooks/develop/includes/mariadb_ubuntu.yml deleted file mode 100644 index 9d64c198..00000000 --- a/playbooks/develop/includes/mariadb_ubuntu.yml +++ /dev/null @@ -1,32 +0,0 @@ ---- - - name: Add apt key for mariadb for Ubuntu < 16.04 - apt_key: keyserver=hkp://keyserver.ubuntu.com:80 id=0xcbcb082a1bb943db state=present - become: yes - become_user: root - when: ansible_distribution_version | version_compare('16.04', 'lt') - - - name: Add apt key for mariadb for Ubuntu >= 16.04 - apt_key: keyserver=hkp://keyserver.ubuntu.com:80 id=0xF1656F24C74CD1D8 state=present - become: yes - become_user: root - when: ansible_distribution_version | version_compare('16.04', 'ge') - - - name: Add apt repository - apt_repository: - repo: 'deb [arch=amd64,i386] http://nyc2.mirrors.digitalocean.com/mariadb/repo/10.1/ubuntu {{ ansible_distribution_release }} main' - state: present - become: yes - become_user: root - - - name: Unattended package installation - shell: export DEBIAN_FRONTEND=noninteractive - - - name: apt-get install - apt: pkg={{ item }} update_cache=yes state=present - with_items: - - mariadb-server - - mariadb-client - - mariadb-common - - libmariadbclient-dev - become: yes - become_user: root diff --git a/playbooks/develop/includes/setup_dev_env.yml b/playbooks/develop/includes/setup_dev_env.yml deleted file mode 100644 index 4e07360e..00000000 --- a/playbooks/develop/includes/setup_dev_env.yml +++ /dev/null @@ -1,16 +0,0 @@ ---- - - # Setup Procfile - - name: setup procfile - command: bench setup procfile - args: - creates: "{{ bench_path }}/Procfile" - chdir: "{{ bench_path }}" - - - # Setup Redis env for RQ - - name: setup redis - command: bench setup redis - args: - creates: "{{ bench_path }}/config/redis_socketio.conf" - chdir: "{{ bench_path }}" diff --git a/playbooks/develop/includes/setup_mariadb.yml b/playbooks/develop/includes/setup_mariadb.yml deleted file mode 100644 index 2f563742..00000000 --- a/playbooks/develop/includes/setup_mariadb.yml +++ /dev/null @@ -1,40 +0,0 @@ ---- - - name: Install MySQLdb in global env - pip: name=mysql-python version=1.2.5 - become: yes - become_user: root - - - name: Add configuration - template: src={{ mysql_config_template }} dest={{ mysql_conf_dir }}/frappe.cnf owner=root mode=0644 - become: yes - become_user: root - - - name: restart mysql linux - service: name=mysql state=restarted - become: yes - become_user: root - when: ansible_os_family == 'RedHat' or ansible_os_family == 'Debian' - - - name: Set root Password - command: mysqladmin -u root password {{ mysql_root_password }} - become: yes - become_user: root - when: mysql_root_password is defined - # incase root password is already set - ignore_errors: yes - - - name: add launchagents folder mac - file: path=~/Library/LaunchAgents state=directory - when: ansible_distribution == 'MacOSX' - - - name: add mysql to mac startup - file: src=/usr/local/opt/mariadb/homebrew.mxcl.mariadb.plist path=~/Library/LaunchAgents/homebrew.mxcl.mariadb.plist state=link force=yes - when: ansible_distribution == 'MacOSX' - - - name: stop mysql mac - command: launchctl unload ~/Library/LaunchAgents/homebrew.mxcl.mariadb.plist - when: ansible_distribution == 'MacOSX' - - - name: start mysql mac - command: launchctl load ~/Library/LaunchAgents/homebrew.mxcl.mariadb.plist - when: ansible_distribution == 'MacOSX' diff --git a/playbooks/develop/includes/setup_selinux_policy.yml b/playbooks/develop/includes/setup_selinux_policy.yml deleted file mode 100644 index 98904e53..00000000 --- a/playbooks/develop/includes/setup_selinux_policy.yml +++ /dev/null @@ -1,38 +0,0 @@ ---- -- hosts: localhost - tasks: - - name: Install SELinux for CentOS - yum: name="{{item}}" state=present - with_items: - - policycoreutils-python - - selinux-policy-devel - become: yes - become_user: root - when: ansible_distribution == 'CentOS' - - - name: Install SELinux for Ubuntu - apt: name={{ item }} state=present - with_items: - - selinux - - selinux-policy-dev - become: yes - become_user: root - when: ansible_distribution == 'Ubuntu' - - - name: Check enabled SELinux modules - shell: semanage module -l - register: enabled_modules - - - name: Copy frappe_selinux policy - copy: src=templates/frappe_selinux.te dest=/root/frappe_selinux.te - register: dest_frappe_selinux_te - become: yes - become_user: root - - - name: Compile frappe_selinux policy - shell: "make -f /usr/share/selinux/devel/Makefile frappe_selinux.pp && semodule -i frappe_selinux.pp" - args: - chdir: /root/ - become: yes - become_user: root - when: "enabled_modules.stdout.find('frappe_selinux') == -1 or dest_frappe_selinux_te.changed" diff --git a/playbooks/develop/includes/wkhtmltopdf.yml b/playbooks/develop/includes/wkhtmltopdf.yml deleted file mode 100644 index ea4c0c28..00000000 --- a/playbooks/develop/includes/wkhtmltopdf.yml +++ /dev/null @@ -1,19 +0,0 @@ ---- - - name: download wkthmltox linux - get_url: url=https://github.com/frappe/wkhtmltopdf/raw/master/wkhtmltox-0.12.3_linux-generic-{{ "amd64" if ansible_architecture == "x86_64" else "i386"}}.tar.xz dest=/tmp/wkhtmltox.tar.xz - - - name: Creates directory - file: path=/tmp/wkhtmltox state=directory - - - name: unarchive wkhtmltopdf - unarchive: src=/tmp/wkhtmltox.tar.xz dest=/tmp - - - name: copy to /usr/local/bin - copy: src="/tmp/wkhtmltox/bin/wkhtmltopdf" dest="/usr/local/bin/wkhtmltopdf" - become: true - become_user: root - - - name: make wkhtmltopdf executable - file: path=/usr/local/bin/wkhtmltopdf mode="o+x" - become: true - become_user: root diff --git a/playbooks/develop/install.yml b/playbooks/develop/install.yml deleted file mode 100644 index be793add..00000000 --- a/playbooks/develop/install.yml +++ /dev/null @@ -1,14 +0,0 @@ ---- -- hosts: localhost - -- include: macosx.yml - when: ansible_distribution == 'MacOSX' - -- include: ubuntu.yml - when: ansible_distribution == 'Ubuntu' - -- include: centos.yml - when: ansible_distribution == 'CentOS' - -- include: debian.yml - when: ansible_distribution == 'Debian' \ No newline at end of file diff --git a/playbooks/develop/macosx.yml b/playbooks/develop/macosx.yml deleted file mode 100644 index 0a846562..00000000 --- a/playbooks/develop/macosx.yml +++ /dev/null @@ -1,34 +0,0 @@ ---- -- hosts: localhost - vars: - bench_repo_path: "/Users/{{ ansible_user_id }}/.bench" - bench_path: "/Users/{{ ansible_user_id }}/frappe-bench" - mysql_config_template: "templates/simple_mariadb_config.cnf" - mysql_conf_dir: /usr/local/etc/my.cnf.d - - tasks: - - # install pre-requisites - - name: install prequisites - homebrew: name={{ item }} state=present - with_items: - - cmake - - redis - - mariadb - - nodejs - - # install wkhtmltopdf - - name: cask installs - homebrew_cask: name={{ item }} state=present - with_items: - - wkhtmltopdf - - # setup MariaDB - - include: includes/setup_mariadb.yml - - # setup frappe-bench - - include: includes/setup_bench.yml - - # setup development environment - - include: includes/setup_dev_env.yml - when: not production diff --git a/playbooks/develop/templates/mariadb_centos.repo b/playbooks/develop/templates/mariadb_centos.repo deleted file mode 100644 index b0d7c819..00000000 --- a/playbooks/develop/templates/mariadb_centos.repo +++ /dev/null @@ -1,7 +0,0 @@ -# MariaDB 10.1 CentOS repository list - created 2016-03-18 09:56 UTC -# http://mariadb.org/mariadb/repositories/ -[mariadb] -name = MariaDB -baseurl = http://yum.mariadb.org/10.1/centos{{ ansible_lsb.major_release }}-{{ "amd64" if ansible_architecture == "x86_64" else "x86"}} -gpgkey=https://yum.mariadb.org/RPM-GPG-KEY-MariaDB -gpgcheck=1 diff --git a/playbooks/develop/ubuntu.yml b/playbooks/develop/ubuntu.yml deleted file mode 100644 index c2d433a4..00000000 --- a/playbooks/develop/ubuntu.yml +++ /dev/null @@ -1,109 +0,0 @@ ---- -- hosts: localhost - vars: - bench_repo_path: "/home/{{ ansible_user_id }}/.bench" - bench_path: "/home/{{ ansible_user_id }}/frappe-bench" - mysql_config_template: "templates/simple_mariadb_config.cnf" - mysql_conf_dir: /etc/mysql/conf.d/ - - tasks: - - # install pre-requisites - - name: install prequisites - apt: pkg={{ item }} state=present - with_items: - # basic installs - - build-essential - - redis-server - - # for mariadb - - software-properties-common - - # for wkhtmltopdf - - libxrender1 - - libxext6 - - xfonts-75dpi - - xfonts-base - - # for Pillow - - libjpeg8-dev - - zlib1g-dev - - libfreetype6-dev - - liblcms2-dev - - libwebp-dev - - python-tk - - # Ensure apt-transport-https - - apt-transport-https - - # Python LDAP - - libsasl2-dev - - libldap2-dev - - become: yes - become_user: root - - - name: install pillow prerequisites for Ubuntu < 14.04 - apt: pkg={{ item }} state=present - with_items: - - libtiff4-dev - - tcl8.5-dev - - tk8.5-dev - when: ansible_distribution_version | version_compare('14.04', 'lt') - become: yes - become_user: root - - - name: install pillow prerequisites for Ubuntu >= 14.04 - apt: pkg={{ item }} state=present - with_items: - - libtiff5-dev - - tcl8.6-dev - - tk8.6-dev - when: ansible_distribution_version | version_compare('14.04', 'ge') - become: yes - become_user: root - - - name: Add apt key for node repo - apt_key: - url: https://keyserver.ubuntu.com/pks/lookup?op=get&fingerprint=on&search=0x1655A0AB68576280 - id: "68576280" - state: present - become: yes - become_user: root - - - name: Add repo - apt_repository: - repo: "deb [arch=amd64,i386] https://deb.nodesource.com/node_6.x {{ ansible_distribution_release }} main" - state: present - register: node_repo - become: yes - become_user: root - - - name: Install nodejs 6.x - apt: - name: nodejs - state: present - update_cache: yes - force: yes - become: yes - become_user: root - - # install MariaDB - - include: includes/mariadb_ubuntu.yml - - # install WKHTMLtoPDF - - include: includes/wkhtmltopdf.yml - - # setup MariaDB - - include: includes/setup_mariadb.yml - - - debug: - var: run_travis - - # setup frappe-bench - - include: includes/setup_bench.yml - when: not without_bench_setup and not run_travis - - # setup development environment - - include: includes/setup_dev_env.yml - when: not production and not run_travis and not without_bench_setup diff --git a/playbooks/install.py b/playbooks/install.py old mode 100755 new mode 100644 index 1c6f7406..fb478ddf --- a/playbooks/install.py +++ b/playbooks/install.py @@ -58,8 +58,6 @@ def install_bench(args): 'pip': 'sudo pip install --upgrade pip setuptools', }) - # Restricting ansible version due to following bug in ansible 2.1 - # https://github.com/ansible/ansible-modules-core/issues/3752 success = run_os_command({ 'pip': "sudo pip install ansible" }) @@ -84,6 +82,14 @@ def install_bench(args): if args.user == 'root': raise Exception('Please run this script as a non-root user with sudo privileges, but without using sudo or pass --user=USER') + # Python executable + if not args.production: + dist_name, dist_version = get_distribution_info() + if dist_name=='centos': + args.python = 'python3.6' + else: + args.python = 'python3' + # create user if not exists extra_vars = vars(args) extra_vars.update(frappe_user=args.user) @@ -95,7 +101,7 @@ def install_bench(args): repo_path = os.path.join(os.path.expanduser('~'), 'bench') extra_vars.update(repo_path=repo_path) - run_playbook('develop/create_user.yml', extra_vars=extra_vars) + run_playbook('create_user.yml', extra_vars=extra_vars) extra_vars.update(get_passwords(args)) if args.production: @@ -103,18 +109,22 @@ def install_bench(args): branch = 'master' if args.production else 'develop' extra_vars.update(branch=branch) + + bench_name = 'frappe-bench' if not args.bench_name else args.bench_name + extra_vars.update(bench_name=bench_name) - if args.develop: - run_playbook('develop/install.yml', sudo=True, extra_vars=extra_vars) + # Will install ERPNext production setup by default + run_playbook('site.yml', sudo=True, extra_vars=extra_vars) - elif args.production: - run_playbook('production/install.yml', sudo=True, extra_vars=extra_vars) + # # Will do changes for production if --production flag is passed + # if args.production: + # run_playbook('production.yml', sudo=True, extra_vars=extra_vars) if os.path.exists(tmp_bench_repo): shutil.rmtree(tmp_bench_repo) def check_distribution_compatibility(): - supported_dists = {'ubuntu': [14, 15, 16], 'debian': [7, 8], + supported_dists = {'ubuntu': [14, 15, 16], 'debian': [8, 9], 'centos': [7], 'macos': [10.9, 10.10, 10.11, 10.12]} dist_name, dist_version = get_distribution_info() @@ -146,7 +156,7 @@ def install_python27(): # install python 2.7 success = run_os_command({ - 'apt-get': 'sudo apt-get install -y python2.7', + 'apt-get': 'sudo apt-get install -y python-dev', 'yum': 'sudo yum install -y python27', 'brew': 'brew install python' }) @@ -261,7 +271,7 @@ def get_passwords(args): mysql_root_password = getpass.unix_getpass(prompt='Please enter mysql root password: ') conf_mysql_passwd = getpass.unix_getpass(prompt='Re-enter mysql root password: ') - if mysql_root_password != conf_mysql_passwd: + if mysql_root_password != conf_mysql_passwd or mysql_root_password == '': mysql_root_password = '' continue @@ -270,7 +280,7 @@ def get_passwords(args): admin_password = getpass.unix_getpass(prompt='Please enter the default Administrator user password: ') conf_admin_passswd = getpass.unix_getpass(prompt='Re-enter Administrator password: ') - if admin_password != conf_admin_passswd: + if admin_password != conf_admin_passswd or admin_password == '': admin_password = '' continue @@ -340,6 +350,9 @@ def parse_commandline_args(): parser.add_argument('--site', dest='site', action='store', default='site1.local', help='Specifiy name for your first ERPNext site') + + parser.add_argument('--without-site', dest='without_site', action='store_true', + default=False) parser.add_argument('--verbose', dest='verbosity', action='store_true', default=False, help='Run the script in verbose mode') @@ -364,6 +377,12 @@ def parse_commandline_args(): # set passwords parser.add_argument('--mysql-root-password', dest='mysql_root_password', help='Set mysql root password') parser.add_argument('--admin-password', dest='admin_password', help='Set admin password') + parser.add_argument('--bench-name', dest='bench_name', help='Create bench with specified name. Default name is frappe-bench') + + # Python interpreter to be used + parser.add_argument('--python', dest='python', default='python', + help=argparse.SUPPRESS + ) args = parser.parse_args() diff --git a/playbooks/macosx.yml b/playbooks/macosx.yml new file mode 100644 index 00000000..2ca0e174 --- /dev/null +++ b/playbooks/macosx.yml @@ -0,0 +1,39 @@ +--- +- hosts: localhost + become: yes + become_user: root + vars: + bench_repo_path: "/Users/{{ ansible_user_id }}/.bench" + bench_path: "/Users/{{ ansible_user_id }}/frappe-bench" + tasks: + # install pre-requisites + - name: install prequisites + homebrew: name={{ item }} state=present + with_items: + - cmake + - redis + - mariadb + - nodejs + + # install wkhtmltopdf + - name: cask installs + homebrew_cask: name={{ item }} state=present + with_items: + - wkhtmltopdf + + - name: configure mariadb + include: roles/mariadb/tasks/main.yml + vars: + mysql_conf_tpl: roles/mariadb/files/mariadb_config.cnf + + - name: Install MySQLdb in global env + pip: name=mysql-python version=1.2.5 + + # setup frappe-bench + - include: includes/setup_bench.yml + + # setup development environment + - include: includes/setup_dev_env.yml + when: not production + +... \ No newline at end of file diff --git a/playbooks/production/includes/setup_bench_production.yml b/playbooks/production/includes/setup_bench_production.yml deleted file mode 100644 index 2acbb71f..00000000 --- a/playbooks/production/includes/setup_bench_production.yml +++ /dev/null @@ -1,63 +0,0 @@ ---- - - hosts: localhost - vars: - bench_path: "/home/{{ ansible_user_id }}/frappe-bench" - - tasks: - - # In case we are re-running the script, we would like to skip the site creation - - name: Check whether a site exists - stat: path="{{ bench_path }}/sites/{{ site }}" - register: site_folder - - - name: Create new site - command: bench new-site {{ site }} --admin-password {{ admin_password }} --mariadb-root-password {{ mysql_root_password }} - args: - chdir: "{{ bench_path }}" - when: not site_folder.stat.exists - - - name: Check ERPNext App exists - stat: path="{{ bench_path }}/apps/erpnext" - register: app - - # In case we are re-running the script, we would like to skip getting ERPNext App - - name: Get-app erpnext app - command: bench get-app erpnext https://github.com/frappe/erpnext.git --branch {{ branch }} - args: - chdir: '{{ bench_path }}' - when: not app.stat.exists - - - name: Install erpnext app - command: bench --site {{ site }} install-app erpnext - args: - chdir: '{{ bench_path }}' - - - name: Change permissions for frappe home folder - become: yes - become_user: root - file: - dest: '{{ ansible_env.HOME }}' - owner: '{{ ansible_user_id }}' - group: '{{ ansible_user_id }}' - mode: 0755 - recurse: yes - state: directory - - - name: Setup production - become: yes - become_user: root - command: bench setup production {{ ansible_user_id }} - args: - chdir: '{{ bench_path }}' - - - name: Setup Sudoers - become: yes - become_user: root - command: bench setup sudoers {{ ansible_user_id }} - args: - chdir: '{{ bench_path }}' - - - name: Restart the bench - command: bench restart - args: - chdir: '{{ bench_path }}' diff --git a/playbooks/production/includes/setup_prod_env.yml b/playbooks/production/includes/setup_prod_env.yml deleted file mode 100755 index ffa00c20..00000000 --- a/playbooks/production/includes/setup_prod_env.yml +++ /dev/null @@ -1,157 +0,0 @@ ---- -- hosts: localhost - - tasks: - - - name: Add nginx apt repository key for Debian < 8 - apt_key: - url: http://nginx.org/keys/nginx_signing.key - state: present - become: yes - become_user: root - when: ansible_distribution == 'Debian' and ansible_distribution_version | version_compare('8', 'lt') - - - name: Add nginx apt repository for Debian < 8 - apt_repository: - repo: 'deb [arch=amd64,i386] http://nginx.org/packages/debian/ {{ ansible_distribution_release }} nginx' - state: present - become: yes - become_user: root - when: ansible_distribution == 'Debian' and ansible_distribution_version | version_compare('8', 'lt') - - ##################################### - # Ubuntu Production Environment Setup - - name: Install production pre-requisites - become: yes - become_user: root - apt: pkg={{ item }} state=present - with_items: - - nginx - - screen - - vim - - htop - - git - - postfix - - supervisor - when: ansible_distribution == 'Ubuntu' or ansible_distribution == 'Debian' - - ##################################### - # CentOS Production Environment Setup - - name: Install production pre-requisites - become: yes - become_user: root - yum: pkg={{ item }} state=present - with_items: - - nginx - - screen - - vim - - htop - - git - - postfix - - MySQL-python - when: ansible_distribution == 'CentOS' - - - name: Install supervisor using yum for Centos 7 - yum: pkg=supervisor state=present - become: yes - become_user: root - when: ansible_distribution == 'CentOS' and ansible_lsb.major_release == '7' - - #################################################### - # Replace default nginx config with nginx template - - name: Rename default nginx.conf to nginx.conf.old - command: mv /etc/nginx/nginx.conf /etc/nginx/nginx.conf.old - become: yes - become_user: root - - - name: Copy the nginx_config template - template: - src: ../templates/default_nginx.j2 - dest: /etc/nginx/nginx.conf - become: yes - become_user: root - - - name: Reload the nginx service - service: - name: nginx - state: restarted - become: yes - become_user: root - - ###################################################### - # Set InnoDB Buffer Pool size to half of total RAM - - name: Set InnoDB buffer pool - lineinfile: > - dest=/etc/my.cnf.d/frappe.cnf - regexp="^\[mysqld\]$" - line="[mysqld]\ninnodb_buffer_pool_size={{ (ansible_memtotal_mb/2)|round|int }}M" - state=present - when: ansible_distribution == 'CentOS' - become: yes - become_user: root - - - name: Set InnoDB buffer pool - lineinfile: > - dest=/etc/mysql/conf.d/frappe.cnf - regexp="^\[mysqld\]$" - line="[mysqld]\ninnodb_buffer_pool_size={{ (ansible_memtotal_mb/2)|round|int }}M" - state=present - when: ansible_distribution == 'Ubuntu' or ansible_distribution == 'Debian' - become: yes - become_user: root - - #################################################### - # Enable nginx, mysql, redis and supevisord services - - name: Enable nginx, mysql, and redis - service: - name: '{{ item }}' - enabled: yes - with_items: - - nginx - - mysql - become: yes - become_user: root - - - name: Enable redis.service on centos - service: - name: redis - enabled: yes - become: yes - become_user: root - when: ansible_distribution == 'CentOS' - - - name: Enable redis-server.service on ubuntu - service: - name: redis-server - enabled: yes - become: yes - become_user: root - when: ansible_distribution == 'Ubuntu' or ansible_distribution == 'Debian' - - - name: Check whether default supervisor.conf exists - service: - name: supervisord - state: started - enabled: yes - become: yes - become_user: root - when: ansible_distribution == 'CentOS' - - - name: Check whether default supervisor.conf exists - service: - name: supervisor - state: started - enabled: yes - become: yes - become_user: root - when: ansible_distribution == 'Ubuntu' or ansible_distribution == 'Debian' - - - name: insert/update inputrc for history - blockinfile: - dest: "/home/{{ ansible_user_id }}/.inputrc" - create: yes - block: | - ## arrow up - "\e[A":history-search-backward - ## arrow down - "\e[B":history-search-forward \ No newline at end of file diff --git a/playbooks/production/install.yml b/playbooks/production/install.yml deleted file mode 100644 index 0a2fd77d..00000000 --- a/playbooks/production/install.yml +++ /dev/null @@ -1,15 +0,0 @@ ---- - - hosts: localhost - - # Install the common pre-requisites for the setting up bench - - include: ../develop/install.yml - - # Install the production environment - - include: includes/setup_prod_env.yml - - # Setup Bench for production environment - - include: includes/setup_bench_production.yml - when: not run_travis - - # Setup SELinux Policy, Optional can be done later - # - include: includes/setup_selinux_policy.yml diff --git a/playbooks/production/templates/default_nginx.j2 b/playbooks/production/templates/default_nginx.j2 deleted file mode 100644 index 4170888d..00000000 --- a/playbooks/production/templates/default_nginx.j2 +++ /dev/null @@ -1,42 +0,0 @@ -# For more information on configuration, see: -# * Official English Documentation: http://nginx.org/en/docs/ -# * Official Russian Documentation: http://nginx.org/ru/docs/ - -{% if ansible_distribution == 'Ubuntu' or ansible_distribution == 'Debian' %} - {% set nginx_user = 'www-data'%} -{% elif ansible_distribution == 'CentOS' %} - {% set nginx_user = 'nginx '%} -{% else %} - {% set nginx_user = 'nobody' %} -{% endif %} - -user {{ nginx_user }}; -worker_processes auto; -error_log /var/log/nginx/error.log; -pid /run/nginx.pid; - -events { - worker_connections {{ max_worker_connections }}; -} - -http { - log_format main '$remote_addr - $remote_user [$time_local] "$request" ' - '$status $body_bytes_sent "$http_referer" ' - '"$http_user_agent" "$http_x_forwarded_for"'; - - access_log /var/log/nginx/access.log main; - - sendfile on; - tcp_nopush on; - tcp_nodelay on; - keepalive_timeout 65; - types_hash_max_size 2048; - - include /etc/nginx/mime.types; - default_type application/octet-stream; - - # Load modular configuration files from the /etc/nginx/conf.d directory. - # See http://nginx.org/en/docs/ngx_core_module.html#include - # for more information. - include /etc/nginx/conf.d/*.conf; -} diff --git a/playbooks/roles/bash_screen_wall/files/screen_wall.sh b/playbooks/roles/bash_screen_wall/files/screen_wall.sh new file mode 100644 index 00000000..dec411e2 --- /dev/null +++ b/playbooks/roles/bash_screen_wall/files/screen_wall.sh @@ -0,0 +1,8 @@ +if [ $TERM != 'screen' ] +then + PS1='HEY! USE SCREEN '$PS1 +fi + +sw() { + screen -x $1 || screen -S $1 +} diff --git a/playbooks/roles/bash_screen_wall/tasks/main.yml b/playbooks/roles/bash_screen_wall/tasks/main.yml new file mode 100644 index 00000000..452b7a49 --- /dev/null +++ b/playbooks/roles/bash_screen_wall/tasks/main.yml @@ -0,0 +1,4 @@ +--- +- name: Setup bash screen wall + copy: src=screen_wall.sh dest=/etc/profile.d/screen_wall.sh +... \ No newline at end of file diff --git a/playbooks/production/change_ssh_port.yml b/playbooks/roles/bench/tasks/change_ssh_port.yml old mode 100755 new mode 100644 similarity index 74% rename from playbooks/production/change_ssh_port.yml rename to playbooks/roles/bench/tasks/change_ssh_port.yml index a0bd21f8..5e850ed5 --- a/playbooks/production/change_ssh_port.yml +++ b/playbooks/roles/bench/tasks/change_ssh_port.yml @@ -1,14 +1,20 @@ -- name: Change ssh port - gather_facts: false - hosts: localhost - user: root - tasks: - - name: change sshd config - lineinfile: > - dest=/etc/ssh/sshd_config - regexp="^Port" - line="Port {{ ssh_port }}" - state=present - - - name: restart ssh - service: name=sshd state=reloaded +--- +- name: Change ssh port + gather_facts: false + hosts: localhost + user: root + tasks: + - name: change sshd config + lineinfile: > + dest=/etc/ssh/sshd_config + regexp="^Port" + line="Port {{ ssh_port }}" + state=present + + - name: restart ssh + service: name=sshd state=reloaded + + - name: Change ansible ssh port to 2332 + set_fact: + ansible_ssh_port: '{{ ssh_port }}' +... \ No newline at end of file diff --git a/playbooks/develop/includes/setup_bench.yml b/playbooks/roles/bench/tasks/main.yml similarity index 52% rename from playbooks/develop/includes/setup_bench.yml rename to playbooks/roles/bench/tasks/main.yml index 7b768c4c..0a963b30 100644 --- a/playbooks/develop/includes/setup_bench.yml +++ b/playbooks/roles/bench/tasks/main.yml @@ -28,11 +28,17 @@ stat: path="{{ bench_path }}" register: bench_stat - - name: init bench + - name: python3 bench init for develop + command: bench init {{ bench_path }} --frappe-branch {{ branch }} --python {{ python }} + args: + creates: "{{ bench_path }}" + when: not bench_stat.stat.exists and not production + + - name: python2 bench init for production command: bench init {{ bench_path }} --frappe-branch {{ branch }} args: creates: "{{ bench_path }}" - when: not bench_stat.stat.exists + when: not bench_stat.stat.exists and production # setup common_site_config - name: setup config @@ -41,8 +47,29 @@ creates: "{{ bench_path }}/sites/common_site_config.json" chdir: "{{ bench_path }}" - - name: install frappe app - command: bench get-app frappe https://github.com/frappe/frappe + - include_tasks: setup_inputrc.yml + + # Setup Procfile + - name: Setup Procfile + command: bench setup procfile args: - creates: "{{ bench_path }}/apps/frappe" + creates: "{{ bench_path }}/Procfile" chdir: "{{ bench_path }}" + + # Setup Redis env for RQ + - name: Setup Redis + command: bench setup redis + args: + creates: "{{ bench_path }}/config/redis_socketio.conf" + chdir: "{{ bench_path }}" + + # Setup an ERPNext site + - include_tasks: setup_erpnext.yml + when: not run_travis + + # Setup Bench for production environment + - include_tasks: setup_bench_production.yml + vars: + bench_path: "/home/{{ frappe_user }}/{{ bench_name }}" + when: not run_travis and production +... diff --git a/playbooks/roles/bench/tasks/setup_bench_production.yml b/playbooks/roles/bench/tasks/setup_bench_production.yml new file mode 100644 index 00000000..1262d542 --- /dev/null +++ b/playbooks/roles/bench/tasks/setup_bench_production.yml @@ -0,0 +1,20 @@ +--- +- name: Setup production + become: yes + become_user: root + command: bench setup production {{ frappe_user }} + args: + chdir: '{{ bench_path }}' + +- name: Setup Sudoers + become: yes + become_user: root + command: bench setup sudoers {{ frappe_user }} + args: + chdir: '{{ bench_path }}' + +- name: Restart the bench + command: bench restart + args: + chdir: '{{ bench_path }}' +... \ No newline at end of file diff --git a/playbooks/roles/bench/tasks/setup_erpnext.yml b/playbooks/roles/bench/tasks/setup_erpnext.yml new file mode 100644 index 00000000..b7f83691 --- /dev/null +++ b/playbooks/roles/bench/tasks/setup_erpnext.yml @@ -0,0 +1,27 @@ +--- + - name: Check if ERPNext App exists + stat: path="{{ bench_path }}/apps/erpnext" + register: app + + - name: Get the ERPNext app + command: bench get-app erpnext https://github.com/frappe/erpnext --branch {{ branch }} + args: + creates: "{{ bench_path }}/apps/erpnext" + chdir: "{{ bench_path }}" + when: not app.stat.exists + + - name: Check whether the site already exists + stat: path="{{ bench_path }}/sites/{{ site }}" + register: site_folder + + - name: Create a new site + command: "bench new-site {{ site }} --admin-password {{ admin_password }} --mariadb-root-password {{ mysql_root_password }}" + args: + chdir: "{{ bench_path }}" + when: not site_folder.stat.exists + + - name: Install ERPNext to default site + command: "bench --site {{ site }} install-app erpnext" + args: + chdir: "{{ bench_path }}" +... \ No newline at end of file diff --git a/playbooks/production/setup_firewall.yml b/playbooks/roles/bench/tasks/setup_firewall.yml old mode 100755 new mode 100644 similarity index 79% rename from playbooks/production/setup_firewall.yml rename to playbooks/roles/bench/tasks/setup_firewall.yml index 00b6b893..f1fc9fa8 --- a/playbooks/production/setup_firewall.yml +++ b/playbooks/roles/bench/tasks/setup_firewall.yml @@ -1,9 +1,14 @@ +--- - name: Setup Firewall user: root hosts: localhost tasks: # For CentOS + - name: Enable SELinux + selinux: policy=targeted state=permissive + when: ansible_distribution == 'CentOS' + - name: Install firewalld yum: name=firewalld state=present when: ansible_distribution == 'CentOS' @@ -17,7 +22,7 @@ with_items: - 80 - 443 - - 22 + - "{{ ssh_port }}" when: ansible_distribution == 'CentOS' - name: Restart Firewall @@ -26,7 +31,10 @@ # For Ubuntu / Debian - name: Install ufw - apt: name=ufw state=present + apt: name={{ item }} state=present force=yes + with_items: + - python-selinux + - ufw when: ansible_distribution == 'Ubuntu' or ansible_distribution == 'Debian' - name: Enable Firewall @@ -38,6 +46,6 @@ with_items: - 80 - 443 - - 22 + - "{{ ssh_port }}" when: ansible_distribution == 'Ubuntu' or ansible_distribution == 'Debian' - +... \ No newline at end of file diff --git a/playbooks/roles/bench/tasks/setup_inputrc.yml b/playbooks/roles/bench/tasks/setup_inputrc.yml new file mode 100644 index 00000000..9c88b933 --- /dev/null +++ b/playbooks/roles/bench/tasks/setup_inputrc.yml @@ -0,0 +1,11 @@ +--- +- name: insert/update inputrc for history + blockinfile: + dest: "/home/{{ frappe_user }}/.inputrc" + create: yes + block: | + ## arrow up + "\e[A":history-search-backward + ## arrow down + "\e[B":history-search-forward +... \ No newline at end of file diff --git a/playbooks/roles/common/tasks/debian.yml b/playbooks/roles/common/tasks/debian.yml new file mode 100644 index 00000000..29372a7c --- /dev/null +++ b/playbooks/roles/common/tasks/debian.yml @@ -0,0 +1,24 @@ +--- + +- name: Setup OpenSSL dependancy + pip: name=pyOpenSSL version=16.2.0 + +- name: install pillow prerequisites for Debian < 8 + apt: pkg={{ item }} state=present + with_items: + - libjpeg8-dev + - libtiff4-dev + - tcl8.5-dev + - tk8.5-dev + when: ansible_distribution_version | version_compare('8', 'lt') + +- name: install pillow prerequisites for Debian >= 8 + apt: pkg={{ item }} state=present + with_items: + - libjpeg62-turbo-dev + - libtiff5-dev + - tcl8.5-dev + - tk8.5-dev + when: ansible_distribution_version | version_compare('8', 'ge') + +... \ No newline at end of file diff --git a/playbooks/roles/common/tasks/debian_family.yml b/playbooks/roles/common/tasks/debian_family.yml new file mode 100644 index 00000000..77671fcd --- /dev/null +++ b/playbooks/roles/common/tasks/debian_family.yml @@ -0,0 +1,42 @@ +--- + +- name: Install prerequisites using apt-get + become: yes + become_user: root + apt: pkg={{ item }} state=present force=yes + with_items: + - dnsmasq + - fontconfig + - git # Version control + - htop # Server stats + - libcrypto++-dev + - libfreetype6-dev + - liblcms2-dev + - libssl-dev + - libwebp-dev + - libxext6 + - libxrender1 + - libxslt1-dev + - libxslt1.1 + - libffi-dev + - ntp # Clock synchronization + - postfix # Mail Server + - python-dev # Installing python developer suite + - python3-dev # For python3 compatibility + - python-tk + - screen # To aid ssh sessions with connectivity problems + - vim # Is that supposed to be a question!? + - xfonts-75dpi + - xfonts-base + - zlib1g-dev + - apt-transport-https + - libsasl2-dev + - libldap2-dev + +- include_tasks: debian.yml + when: ansible_distribution == 'Debian' + +- include_tasks: ubuntu.yml + when: ansible_distribution == 'Ubuntu' + +... \ No newline at end of file diff --git a/playbooks/roles/common/tasks/macos.yml b/playbooks/roles/common/tasks/macos.yml new file mode 100644 index 00000000..a5fbd0bf --- /dev/null +++ b/playbooks/roles/common/tasks/macos.yml @@ -0,0 +1,37 @@ +--- + +- hosts: localhost + become: yes + become_user: root + vars: + bench_repo_path: "/Users/{{ ansible_user_id }}/.bench" + bench_path: "/Users/{{ ansible_user_id }}/frappe-bench" + tasks: + # install pre-requisites + - name: install prequisites + homebrew: name={{ item }} state=present + with_items: + - cmake + - redis + - mariadb + - nodejs + + # install wkhtmltopdf + - name: cask installs + homebrew_cask: name={{ item }} state=present + with_items: + - wkhtmltopdf + + - name: configure mariadb + include_tasks: roles/mariadb/tasks/main.yml + vars: + mysql_conf_tpl: roles/mariadb/files/mariadb_config.cnf + + # setup frappe-bench + - include_tasks: includes/setup_bench.yml + + # setup development environment + - include_tasks: includes/setup_dev_env.yml + when: not production + +... \ No newline at end of file diff --git a/playbooks/roles/common/tasks/main.yml b/playbooks/roles/common/tasks/main.yml new file mode 100644 index 00000000..e5f6ac0d --- /dev/null +++ b/playbooks/roles/common/tasks/main.yml @@ -0,0 +1,9 @@ +--- +# Install's prerequisites, like fonts, image libraries, vim, screen, python-dev + +- include_tasks: debian_family.yml + when: ansible_os_family == 'Debian' + +- include_tasks: redhat_family.yml + when: ansible_os_family == "RedHat" +... \ No newline at end of file diff --git a/playbooks/roles/common/tasks/redhat_family.yml b/playbooks/roles/common/tasks/redhat_family.yml new file mode 100644 index 00000000..8604c0ae --- /dev/null +++ b/playbooks/roles/common/tasks/redhat_family.yml @@ -0,0 +1,50 @@ +--- + +- name: Install IUS repo for python 3.6 + become: yes + become_user: root + yum: + name: https://centos7.iuscommunity.org/ius-release.rpm + state: present + +- name: "Setup prerequisites using yum" + become: yes + become_user: root + yum: name={{ item }} state=present + with_items: + - bzip2-devel + - cronie + - dnsmasq + - freetype-devel + - git + - htop + - lcms2-devel + - libjpeg-devel + - libtiff-devel + - libffi-devel + - libwebp-devel + - libXext + - libXrender + - libzip-devel + - libffi-devel + - ntp + - openssl-devel + - postfix + - python36u + - python-devel + - python-setuptools + - python-pip + - redis + - screen + - sudo + - tcl-devel + - tk-devel + - vim + - which + - xorg-x11-fonts-75dpi + - xorg-x11-fonts-Type1 + - zlib-devel + - openssl-devel + - openldap-devel + +... \ No newline at end of file diff --git a/playbooks/roles/common/tasks/ubuntu.yml b/playbooks/roles/common/tasks/ubuntu.yml new file mode 100644 index 00000000..13fb7450 --- /dev/null +++ b/playbooks/roles/common/tasks/ubuntu.yml @@ -0,0 +1,21 @@ +--- + +- name: install pillow prerequisites for Ubuntu < 14.04 + apt: pkg={{ item }} state=present force=yes + with_items: + - libjpeg8-dev + - libtiff4-dev + - tcl8.5-dev + - tk8.5-dev + when: ansible_distribution_version | version_compare('14.04', 'lt') + +- name: install pillow prerequisites for Ubuntu >= 14.04 + apt: pkg={{ item }} state=present force=yes + with_items: + - libjpeg8-dev + - libtiff5-dev + - tcl8.6-dev + - tk8.6-dev + when: ansible_distribution_version | version_compare('14.04', 'ge') + +... \ No newline at end of file diff --git a/playbooks/roles/dns_caching/handlers/main.yml b/playbooks/roles/dns_caching/handlers/main.yml new file mode 100644 index 00000000..afe7ac3e --- /dev/null +++ b/playbooks/roles/dns_caching/handlers/main.yml @@ -0,0 +1,4 @@ +--- +- name: Restart network manager + service: name=NetworkManager state=restarted +... \ No newline at end of file diff --git a/playbooks/roles/dns_caching/tasks/main.yml b/playbooks/roles/dns_caching/tasks/main.yml new file mode 100644 index 00000000..7c34f0fa --- /dev/null +++ b/playbooks/roles/dns_caching/tasks/main.yml @@ -0,0 +1,20 @@ +--- +- name: Check NetworkManager.conf exists + stat: + path: /etc/NetworkManager/NetworkManager.conf + register: result + +- name: Unmask NetworkManager service + command: systemctl unmask NetworkManager + when: result.stat.exists + +- name: Add dnsmasq to network config + lineinfile: > + dest=/etc/NetworkManager/NetworkManager.conf + regexp="dns=" + line="dns=dnsmasq" + state=present + when: result.stat.exists + notify: + - restart network manager +... diff --git a/playbooks/roles/fail2ban/defaults/main.yml b/playbooks/roles/fail2ban/defaults/main.yml new file mode 100644 index 00000000..5aae2800 --- /dev/null +++ b/playbooks/roles/fail2ban/defaults/main.yml @@ -0,0 +1,5 @@ +--- +fail2ban_nginx_access_log: /var/log/nginx/access.log +maxretry: 6 +bantime: 600 +findtime: 600 \ No newline at end of file diff --git a/playbooks/roles/fail2ban/handlers/main.yml b/playbooks/roles/fail2ban/handlers/main.yml new file mode 100644 index 00000000..d675d4d5 --- /dev/null +++ b/playbooks/roles/fail2ban/handlers/main.yml @@ -0,0 +1,3 @@ +--- +- name: restart fail2ban + service: name=fail2ban state=restarted \ No newline at end of file diff --git a/playbooks/roles/fail2ban/tasks/configure_nginx_jail.yml b/playbooks/roles/fail2ban/tasks/configure_nginx_jail.yml new file mode 100644 index 00000000..4d2ede07 --- /dev/null +++ b/playbooks/roles/fail2ban/tasks/configure_nginx_jail.yml @@ -0,0 +1,11 @@ +- name: Configure fail2ban jail options + hosts: localhost + become: yes + become_user: root + vars_files: + - ../defaults/main.yml + tasks: + - name: Setup jail + template: src="../templates/nginx-proxy-jail.conf.j2" dest="/etc/fail2ban/jail.d/nginx-proxy.conf" + - name: restart service + service: name=fail2ban state=restarted \ No newline at end of file diff --git a/playbooks/roles/fail2ban/tasks/main.yml b/playbooks/roles/fail2ban/tasks/main.yml new file mode 100644 index 00000000..d7866036 --- /dev/null +++ b/playbooks/roles/fail2ban/tasks/main.yml @@ -0,0 +1,28 @@ +--- +- name: Install fail2ban + yum: name=fail2ban state=present + when: ansible_distribution == 'CentOS' + +- name: Install fail2ban + apt: name=fail2ban state=present + when: ansible_distribution == 'Debian' or ansible_distribution == 'Ubuntu' + +- name: Enable fail2ban + service: name=fail2ban enabled=yes + +- name: Create jail.d + file: path=/etc/fail2ban/jail.d state=directory + +- name: Setup filters + template: src="{{item}}-filter.conf.j2" dest="/etc/fail2ban/filter.d/{{item}}.conf" + with_items: + - nginx-proxy + notify: + - restart fail2ban + +- name: setup jails + template: src="{{item}}-jail.conf.j2" dest="/etc/fail2ban/jail.d/{{item}}.conf" + with_items: + - nginx-proxy + notify: + - restart fail2ban diff --git a/playbooks/roles/fail2ban/templates/nginx-proxy-filter.conf.j2 b/playbooks/roles/fail2ban/templates/nginx-proxy-filter.conf.j2 new file mode 100644 index 00000000..27f74cd5 --- /dev/null +++ b/playbooks/roles/fail2ban/templates/nginx-proxy-filter.conf.j2 @@ -0,0 +1,10 @@ +# Block IPs trying to use server as proxy. +[Definition] +failregex = .*\" 400 + .*"[A-Z]* /(cms|muieblackcat|db|cpcommerce|cgi-bin|wp-login|joomla|awstatstotals|wp-content|wp-includes|pma|phpmyadmin|myadmin|mysql|mysqladmin|sqladmin|mypma|admin|xampp|mysqldb|pmadb|phpmyadmin1|phpmyadmin2).*" 4[\d][\d] + .*".*supports_implicit_sdk_logging.*" 4[\d][\d] + .*".*activities?advertiser_tracking_enabled.*" 4[\d][\d] + .*".*/picture?type=normal.*" 4[\d][\d] + .*".*/announce.php?info_hash=.*" 4[\d][\d] + +ignoreregex = \ No newline at end of file diff --git a/playbooks/roles/fail2ban/templates/nginx-proxy-jail.conf.j2 b/playbooks/roles/fail2ban/templates/nginx-proxy-jail.conf.j2 new file mode 100644 index 00000000..c351773b --- /dev/null +++ b/playbooks/roles/fail2ban/templates/nginx-proxy-jail.conf.j2 @@ -0,0 +1,9 @@ +## block hosts trying to abuse our server as a forward proxy +[nginx-proxy] +enabled = true +filter = nginx-proxy +logpath = {{ fail2ban_nginx_access_log }} +action = iptables-multiport[name=NoNginxProxy, port="http,https"] +maxretry = {{ maxretry }} +bantime = {{ bantime }} +findtime = {{ findtime }} \ No newline at end of file diff --git a/playbooks/production/templates/frappe_selinux.te b/playbooks/roles/frappe_selinux/files/frappe_selinux.te similarity index 95% rename from playbooks/production/templates/frappe_selinux.te rename to playbooks/roles/frappe_selinux/files/frappe_selinux.te index 0551ebad..b8cd1f0f 100644 --- a/playbooks/production/templates/frappe_selinux.te +++ b/playbooks/roles/frappe_selinux/files/frappe_selinux.te @@ -29,4 +29,4 @@ allow httpd_t user_home_t:file open; allow httpd_t user_home_t:file read; #!!!! This avc is allowed in the current policy -allow httpd_t user_home_t:lnk_file read; +allow httpd_t user_home_t:lnk_file read; \ No newline at end of file diff --git a/playbooks/roles/frappe_selinux/tasks/main.yml b/playbooks/roles/frappe_selinux/tasks/main.yml new file mode 100644 index 00000000..43602163 --- /dev/null +++ b/playbooks/roles/frappe_selinux/tasks/main.yml @@ -0,0 +1,24 @@ +--- +- name: Install deps + yum: name="{{item}}" state=present + with_items: + - policycoreutils-python + - selinux-policy-devel + when: ansible_distribution == 'CentOS' + +- name: Check enabled SELinux modules + shell: semanage module -l + register: enabled_modules + when: ansible_distribution == 'CentOS' + +- name: Copy frappe_selinux policy + copy: src=frappe_selinux.te dest=/root/frappe_selinux.te + register: dest_frappe_selinux_te + when: ansible_distribution == 'CentOS' + +- name: Compile frappe_selinux policy + shell: "make -f /usr/share/selinux/devel/Makefile frappe_selinux.pp && semodule -i frappe_selinux.pp" + args: + chdir: /root/ + when: "ansible_distribution == 'CentOS' and enabled_modules.stdout.find('frappe_selinux') == -1 or dest_frappe_selinux_te.changed" +... \ No newline at end of file diff --git a/playbooks/roles/locale/defaults/main.yml b/playbooks/roles/locale/defaults/main.yml new file mode 100644 index 00000000..82343e32 --- /dev/null +++ b/playbooks/roles/locale/defaults/main.yml @@ -0,0 +1,4 @@ +--- +locale_keymap: us +locale_lang: en_US.utf8 +... \ No newline at end of file diff --git a/playbooks/roles/locale/tasks/main.yml b/playbooks/roles/locale/tasks/main.yml new file mode 100644 index 00000000..8551f690 --- /dev/null +++ b/playbooks/roles/locale/tasks/main.yml @@ -0,0 +1,21 @@ +--- +- name: Check current locale + shell: localectl + register: locale_test + when: ansible_distribution == 'Centos' or ansible_distribution == 'Ubuntu' + +- name: Set Locale + command: "localectl set-locale LANG={{ locale_lang }}" + when: (ansible_distribution == 'Centos' or ansible_distribution == 'Ubuntu') and locale_test.stdout.find('LANG=locale_lang') == -1 + +- name: Set keymap + command: "localectl set-keymap {{ locale_keymap }}" + when: "(ansible_distribution == 'Centos' or ansible_distribution == 'Ubuntu') and locale_test.stdout.find('Keymap:locale_keymap') == -1" + +- name: Set Locale as en_US + lineinfile: dest=/etc/environment backup=yes line="{{ item }}" + with_items: + - "LC_ALL=en_US.UTF-8" + - "LC_CTYPE=en_US.UTF-8" + - "LANG=en_US.UTF-8" +... \ No newline at end of file diff --git a/playbooks/roles/logwatch/defaults/main.yml b/playbooks/roles/logwatch/defaults/main.yml new file mode 100644 index 00000000..fd3675e0 --- /dev/null +++ b/playbooks/roles/logwatch/defaults/main.yml @@ -0,0 +1,4 @@ +--- +logwatch_emails: "{{ admin_emails }}" +logwatch_detail: High +... \ No newline at end of file diff --git a/playbooks/roles/logwatch/tasks/main.yml b/playbooks/roles/logwatch/tasks/main.yml new file mode 100644 index 00000000..2450ac75 --- /dev/null +++ b/playbooks/roles/logwatch/tasks/main.yml @@ -0,0 +1,13 @@ +--- +- name: Install logwatch + yum: name=logwatch state=present + when: ansible_distribution == 'CentOS' + +- name: Install logwatch on Ubuntu or Debian + apt: name=logwatch state=present + when: ansible_distribution == 'Debian' or ansible_distribution == 'Ubuntu' + +- name: Copy logwatch config + template: src=logwatch.conf.j2 dest=/etc/logwatch/conf/logwatch.conf backup=yes + when: admin_emails is defined +... \ No newline at end of file diff --git a/playbooks/roles/logwatch/templates/logwatch.conf.j2 b/playbooks/roles/logwatch/templates/logwatch.conf.j2 new file mode 100644 index 00000000..a5c45cf8 --- /dev/null +++ b/playbooks/roles/logwatch/templates/logwatch.conf.j2 @@ -0,0 +1,2 @@ +MailTo = {{ logwatch_emails }} +Detail = {{ logwatch_detail }} \ No newline at end of file diff --git a/vm/ansible/roles/mariadb/README.md b/playbooks/roles/mariadb/README.md similarity index 100% rename from vm/ansible/roles/mariadb/README.md rename to playbooks/roles/mariadb/README.md diff --git a/vm/ansible/roles/mariadb/defaults/main.yml b/playbooks/roles/mariadb/defaults/main.yml similarity index 79% rename from vm/ansible/roles/mariadb/defaults/main.yml rename to playbooks/roles/mariadb/defaults/main.yml index a5bb87ff..fce5bc00 100644 --- a/vm/ansible/roles/mariadb/defaults/main.yml +++ b/playbooks/roles/mariadb/defaults/main.yml @@ -1,7 +1,8 @@ --- -mariadb_version: 10.0 +mariadb_version: 10.2 mysql_conf_tpl: change_me mysql_conf_file: settings.cnf mysql_secure_installation: false +... \ No newline at end of file diff --git a/playbooks/develop/templates/simple_mariadb_config.cnf b/playbooks/roles/mariadb/files/debian_mariadb_config.cnf similarity index 79% rename from playbooks/develop/templates/simple_mariadb_config.cnf rename to playbooks/roles/mariadb/files/debian_mariadb_config.cnf index 2e0c0d60..f2ac13f3 100644 --- a/playbooks/develop/templates/simple_mariadb_config.cnf +++ b/playbooks/roles/mariadb/files/debian_mariadb_config.cnf @@ -5,7 +5,10 @@ innodb-large-prefix=1 character-set-client-handshake = FALSE character-set-server = utf8mb4 collation-server = utf8mb4_unicode_ci -max_allowed_packet=64M +max_allowed_packet=256M [mysql] default-character-set = utf8mb4 + +[mysqldump] +max_allowed_packet=256M diff --git a/vm/ansible/templates/mariadb_config.cnf b/playbooks/roles/mariadb/files/mariadb_config.cnf similarity index 91% rename from vm/ansible/templates/mariadb_config.cnf rename to playbooks/roles/mariadb/files/mariadb_config.cnf index 26ed5482..19443454 100644 --- a/vm/ansible/templates/mariadb_config.cnf +++ b/playbooks/roles/mariadb/files/mariadb_config.cnf @@ -11,7 +11,7 @@ key-buffer-size = 32M myisam-recover = FORCE,BACKUP # SAFETY # -max-allowed-packet = 16M +max-allowed-packet = 256M max-connect-errors = 1000000 innodb = FORCE @@ -23,6 +23,9 @@ log-bin = /var/lib/mysql/mysql-bin expire-logs-days = 14 sync-binlog = 1 +# REPLICATION # +server-id = 1 + # CACHES AND LIMITS # tmp-table-size = 32M max-heap-table-size = 32M @@ -46,6 +49,7 @@ innodb-large-prefix = 1 collation-server = utf8mb4_unicode_ci character-set-server = utf8mb4 character-set-client-handshake = FALSE +max_allowed_packet = 256M # LOGGING # log-error = /var/lib/mysql/mysql-error.log @@ -56,3 +60,5 @@ slow-query-log-file = /var/lib/mysql/mysql-slow.log [mysql] default-character-set = utf8mb4 +[mysqldump] +max_allowed_packet=256M \ No newline at end of file diff --git a/vm/ansible/roles/mariadb/handlers/main.yml b/playbooks/roles/mariadb/handlers/main.yml similarity index 95% rename from vm/ansible/roles/mariadb/handlers/main.yml rename to playbooks/roles/mariadb/handlers/main.yml index 3755d8ce..7072dbd9 100644 --- a/vm/ansible/roles/mariadb/handlers/main.yml +++ b/playbooks/roles/mariadb/handlers/main.yml @@ -1,3 +1,4 @@ --- - name: restart mysql service: name=mysql state=restarted +... \ No newline at end of file diff --git a/vm/ansible/roles/mariadb/tasks/centos.yml b/playbooks/roles/mariadb/tasks/centos.yml similarity index 80% rename from vm/ansible/roles/mariadb/tasks/centos.yml rename to playbooks/roles/mariadb/tasks/centos.yml index 84b49c55..2b5ced27 100644 --- a/vm/ansible/roles/mariadb/tasks/centos.yml +++ b/playbooks/roles/mariadb/tasks/centos.yml @@ -6,7 +6,5 @@ yum: name={{ item }} enablerepo=mariadb state=present with_items: - MariaDB-server - - MariaDB-client - - MySQL-python - - MariaDB-devel - + - MySQL-python # required for secure_install +... diff --git a/playbooks/roles/mariadb/tasks/debian.yml b/playbooks/roles/mariadb/tasks/debian.yml new file mode 100644 index 00000000..6cfe4b2e --- /dev/null +++ b/playbooks/roles/mariadb/tasks/debian.yml @@ -0,0 +1,31 @@ +--- +- name: Add apt key for mariadb for Debian <= 8 + apt_key: keyserver=hkp://keyserver.ubuntu.com:80 id=0xcbcb082a1bb943db state=present + when: ansible_distribution_major_version | version_compare('8', 'le') + +- name: Add apt key for mariadb for Debian > 8 + apt_key: keyserver=hkp://keyserver.ubuntu.com:80 id=0xF1656F24C74CD1D8 state=present + when: ansible_distribution_major_version | version_compare('8', 'gt') + +- name: Add apt repository + apt_repository: + repo: 'deb [arch=amd64,i386] http://ams2.mirrors.digitalocean.com/mariadb/repo/{{ mariadb_version }}/debian {{ ansible_distribution_release }} main' + state: present + +- name: Add apt repository + apt_repository: + repo: 'deb-src [arch=amd64,i386] http://ams2.mirrors.digitalocean.com/mariadb/repo/{{ mariadb_version }}/debian {{ ansible_distribution_release }} main' + state: present + +- name: Unattended package installation + shell: export DEBIAN_FRONTEND=noninteractive + +- name: apt-get install + apt: pkg={{ item }} update_cache=yes state=present + with_items: + - mariadb-server + - mariadb-client + - mariadb-common + - libmariadbclient18 + - python-mysqldb +... \ No newline at end of file diff --git a/playbooks/roles/mariadb/tasks/main.yml b/playbooks/roles/mariadb/tasks/main.yml new file mode 100644 index 00000000..8d3e6ea0 --- /dev/null +++ b/playbooks/roles/mariadb/tasks/main.yml @@ -0,0 +1,56 @@ +--- + +- include_tasks: centos.yml + when: ansible_distribution == 'CentOS' + +- include_tasks: ubuntu.yml + when: ansible_distribution == 'Ubuntu' + +- name: Add configuration + template: src={{ mysql_conf_tpl }} dest={{ mysql_conf_dir[ansible_distribution] }}/{{ mysql_conf_file }} owner=root group=root mode=0644 + when: mysql_conf_tpl != 'change_me' and ansible_distribution != 'Debian' + notify: restart mysql + +- include_tasks: debian.yml + when: ansible_distribution == 'Debian' + +- name: Add configuration + template: src={{ mysql_conf_tpl }} dest={{ mysql_conf_dir[ansible_distribution] }}/{{ mysql_conf_file }} owner=root group=root mode=0644 + when: mysql_conf_tpl != 'change_me' and ansible_distribution == 'Debian' + notify: restart mysql + +- name: Add additional conf for MariaDB 10.2 in mariadb.conf.d + blockinfile: + path: /etc/mysql/conf.d/settings.cnf + block: | + # Import all .cnf files from configuration directory + !includedir /etc/mysql/mariadb.conf.d/ + become: yes + become_user: root + when: ansible_distribution == 'Ubuntu' or ansible_distribution == 'Debian' + +- name: Add additional conf for MariaDB 10.2 in mariadb.conf.d + blockinfile: + path: /etc/mysql/mariadb.conf.d/erpnext.cnf + block: | + [mysqld] + pid-file = /var/run/mysqld/mysqld.pid + socket = /var/run/mysqld/mysqld.sock + create: yes + become: yes + become_user: root + when: ansible_distribution == 'Ubuntu' or ansible_distribution == 'Debian' + +- name: Start and enable service + service: name=mysql state=started enabled=yes + +- debug: + msg: "{{ mysql_root_password }}" + +- include_tasks: mysql_secure_installation.yml + when: mysql_root_password is defined + +- debug: var=mysql_secure_installation + when: mysql_secure_installation and mysql_root_password is defined + +... diff --git a/vm/ansible/roles/mariadb/tasks/mysql_secure_installation.yml b/playbooks/roles/mariadb/tasks/mysql_secure_installation.yml similarity index 56% rename from vm/ansible/roles/mariadb/tasks/mysql_secure_installation.yml rename to playbooks/roles/mariadb/tasks/mysql_secure_installation.yml index b92dc143..621ab4b0 100644 --- a/vm/ansible/roles/mariadb/tasks/mysql_secure_installation.yml +++ b/playbooks/roles/mariadb/tasks/mysql_secure_installation.yml @@ -3,44 +3,52 @@ # UPDATE mysql.user SET Password=PASSWORD('mysecret') WHERE User='root'; # FLUSH PRIVILEGES; +- debug: + msg: "{{ mysql_root_password }}" - name: Set root Password mysql_user: name=root host={{ item }} password={{ mysql_root_password }} state=present with_items: - localhost + ignore_errors: yes - name: Add .my.cnf template: src=my.cnf.j2 dest=/root/.my.cnf owner=root group=root mode=0600 +- name: display .my.cnf + command: cat /root/.my.cnf + register: details + +- debug: + msg: "{{ details.stdout_lines }}" + - name: Set root Password mysql_user: name=root host={{ item }} password={{ mysql_root_password }} state=present with_items: - 127.0.0.1 - ::1 + when: run_travis is not defined - name: Reload privilege tables command: 'mysql -ne "{{ item }}"' with_items: - FLUSH PRIVILEGES changed_when: False - -- name: Reload privilege tables - command: 'mysql -ne "{{ item }}"' - with_items: - - FLUSH PRIVILEGES - changed_when: False + when: run_travis is not defined - name: Remove anonymous users command: 'mysql -ne "{{ item }}"' with_items: - DELETE FROM mysql.user WHERE User='' changed_when: False + when: run_travis is not defined - name: Disallow root login remotely command: 'mysql -ne "{{ item }}"' with_items: - DELETE FROM mysql.user WHERE User='root' AND Host NOT IN ('localhost', '127.0.0.1', '::1') changed_when: False + when: run_travis is not defined - name: Remove test database and access to it command: 'mysql -ne "{{ item }}"' @@ -48,10 +56,28 @@ - DROP DATABASE if exists test - DELETE FROM mysql.db WHERE Db='test' OR Db='test\\_%' changed_when: False - ignore_errors: True + when: run_travis is not defined - name: Reload privilege tables command: 'mysql -ne "{{ item }}"' with_items: - FLUSH PRIVILEGES changed_when: False + when: run_travis is not defined + +- name: add launchagents folder mac + file: path=~/Library/LaunchAgents state=directory + when: ansible_distribution == 'MacOSX' + +- name: add mysql to mac startup + file: src=/usr/local/opt/mariadb/homebrew.mxcl.mariadb.plist path=~/Library/LaunchAgents/homebrew.mxcl.mariadb.plist state=link force=yes + when: ansible_distribution == 'MacOSX' + +- name: stop mysql mac + command: launchctl unload ~/Library/LaunchAgents/homebrew.mxcl.mariadb.plist + when: ansible_distribution == 'MacOSX' + +- name: start mysql mac + command: launchctl load ~/Library/LaunchAgents/homebrew.mxcl.mariadb.plist + when: ansible_distribution == 'MacOSX' +... diff --git a/vm/ansible/roles/mariadb/tasks/ubuntu.yml b/playbooks/roles/mariadb/tasks/ubuntu.yml similarity index 61% rename from vm/ansible/roles/mariadb/tasks/ubuntu.yml rename to playbooks/roles/mariadb/tasks/ubuntu.yml index ea5c1031..5abec54b 100644 --- a/vm/ansible/roles/mariadb/tasks/ubuntu.yml +++ b/playbooks/roles/mariadb/tasks/ubuntu.yml @@ -6,6 +6,12 @@ - name: Add repo key apt_key: id=1BB943DB url=http://keyserver.ubuntu.com/pks/lookup?op=get&search=0xCBCB082A1BB943DB state=present register: mariadb_key + when: ansible_distribution_version | version_compare('16.04', 'lt') + +- name: Add apt key for mariadb for Ubuntu >= 16.04 + apt_key: id=C74CD1D8 url=http://keyserver.ubuntu.com/pks/lookup?op=get&search=0xF1656F24C74CD1D8 state=present + register: mariadb_key + when: ansible_distribution_version | version_compare('16.04', 'ge') - name: Update apt cache apt: update_cache=yes @@ -19,5 +25,5 @@ apt: pkg={{ item }} state=present with_items: - mariadb-server - - mariadb-client - - python-mysqldb + - python-mysqldb # required to set the MySQL password using ansible +... diff --git a/vm/ansible/roles/mariadb/templates/mariadb_centos.repo.j2 b/playbooks/roles/mariadb/templates/mariadb_centos.repo.j2 similarity index 70% rename from vm/ansible/roles/mariadb/templates/mariadb_centos.repo.j2 rename to playbooks/roles/mariadb/templates/mariadb_centos.repo.j2 index 64738cc1..0a025305 100644 --- a/vm/ansible/roles/mariadb/templates/mariadb_centos.repo.j2 +++ b/playbooks/roles/mariadb/templates/mariadb_centos.repo.j2 @@ -1,4 +1,4 @@ -# MariaDB CentOS {{ ansible_distribution_major_version|int }} repository list +# MariaDB {{ mariadb_version }} CentOS {{ ansible_distribution_major_version|int }} repository list # http://mariadb.org/mariadb/repositories/ [mariadb] name = MariaDB diff --git a/playbooks/roles/mariadb/templates/mariadb_debian.list.j2 b/playbooks/roles/mariadb/templates/mariadb_debian.list.j2 new file mode 100644 index 00000000..710566f8 --- /dev/null +++ b/playbooks/roles/mariadb/templates/mariadb_debian.list.j2 @@ -0,0 +1,4 @@ +# MariaDB {{ mariadb_version }} Debian {{ ansible_distribution_release | title }} repository list +# http://mariadb.org/mariadb/repositories/ +deb http://ams2.mirrors.digitalocean.com/mariadb/repo/{{ mariadb_version }}/debian {{ ansible_distribution_release | lower }} main +deb-src http://ams2.mirrors.digitalocean.com/mariadb/repo/{{ mariadb_version }}/debian {{ ansible_distribution_release | lower }} main diff --git a/vm/ansible/roles/mariadb/templates/mariadb_ubuntu.list.j2 b/playbooks/roles/mariadb/templates/mariadb_ubuntu.list.j2 similarity index 75% rename from vm/ansible/roles/mariadb/templates/mariadb_ubuntu.list.j2 rename to playbooks/roles/mariadb/templates/mariadb_ubuntu.list.j2 index 981b4d6b..dca4e70b 100644 --- a/vm/ansible/roles/mariadb/templates/mariadb_ubuntu.list.j2 +++ b/playbooks/roles/mariadb/templates/mariadb_ubuntu.list.j2 @@ -1,4 +1,4 @@ -# MariaDB Ubuntu {{ ansible_distribution_release | title }} repository list +# MariaDB {{ mariadb_version }} Ubuntu {{ ansible_distribution_release | title }} repository list # http://mariadb.org/mariadb/repositories/ deb http://ams2.mirrors.digitalocean.com/mariadb/repo/{{ mariadb_version }}/ubuntu {{ ansible_distribution_release | lower }} main deb-src http://ams2.mirrors.digitalocean.com/mariadb/repo/{{ mariadb_version }}/ubuntu {{ ansible_distribution_release | lower }} main diff --git a/vm/ansible/roles/mariadb/templates/my.cnf.j2 b/playbooks/roles/mariadb/templates/my.cnf.j2 similarity index 100% rename from vm/ansible/roles/mariadb/templates/my.cnf.j2 rename to playbooks/roles/mariadb/templates/my.cnf.j2 diff --git a/playbooks/roles/mariadb/vars/main.yml b/playbooks/roles/mariadb/vars/main.yml new file mode 100644 index 00000000..b7bf4bcf --- /dev/null +++ b/playbooks/roles/mariadb/vars/main.yml @@ -0,0 +1,9 @@ +--- +mysql_conf_dir: + "CentOS": /etc/my.cnf.d + "Ubuntu": /etc/mysql/conf.d + "Debian": /etc/mysql/conf.d + "MacOSX": /usr/local/etc/my.cnf.d +mysql_conf_tpl: files/mariadb_config.cnf +mysql_secure_installation: True +... \ No newline at end of file diff --git a/vm/ansible/roles/nginx/README.md b/playbooks/roles/nginx/README.md similarity index 100% rename from vm/ansible/roles/nginx/README.md rename to playbooks/roles/nginx/README.md diff --git a/vm/ansible/roles/nginx/defaults/main.yml b/playbooks/roles/nginx/defaults/main.yml similarity index 95% rename from vm/ansible/roles/nginx/defaults/main.yml rename to playbooks/roles/nginx/defaults/main.yml index 8aacad8c..07eee907 100644 --- a/vm/ansible/roles/nginx/defaults/main.yml +++ b/playbooks/roles/nginx/defaults/main.yml @@ -44,4 +44,5 @@ nginx_upstreams: [] # "srv3.example.com" # } # } -nginx_conf_file: nginx.conf.j2 \ No newline at end of file +nginx_conf_file: nginx.conf.j2 +setup_www_redirect: false \ No newline at end of file diff --git a/vm/ansible/roles/nginx/handlers/main.yml b/playbooks/roles/nginx/handlers/main.yml similarity index 100% rename from vm/ansible/roles/nginx/handlers/main.yml rename to playbooks/roles/nginx/handlers/main.yml diff --git a/vm/ansible/roles/nginx/meta/main.yml b/playbooks/roles/nginx/meta/main.yml similarity index 100% rename from vm/ansible/roles/nginx/meta/main.yml rename to playbooks/roles/nginx/meta/main.yml diff --git a/vm/ansible/roles/nginx/tasks/main.yml b/playbooks/roles/nginx/tasks/main.yml similarity index 51% rename from vm/ansible/roles/nginx/tasks/main.yml rename to playbooks/roles/nginx/tasks/main.yml index da4be94d..8786b345 100644 --- a/vm/ansible/roles/nginx/tasks/main.yml +++ b/playbooks/roles/nginx/tasks/main.yml @@ -9,10 +9,15 @@ when: nginx_user is not defined # Setup/install tasks. -- include: setup-RedHat.yml +- include_tasks: setup-RedHat.yml when: ansible_os_family == 'RedHat' -- include: setup-Debian.yml +- include_tasks: setup-Debian.yml + when: ansible_os_family == 'Debian' + +# Replace default nginx config with nginx template +- name: Rename default nginx.conf to nginx.conf.old + command: mv /etc/nginx/nginx.conf /etc/nginx/nginx.conf.old when: ansible_os_family == 'Debian' # Nginx setup. @@ -25,7 +30,22 @@ mode: 0644 notify: restart nginx +- name: Setup www redirect + template: + src: ../files/www_redirect.conf + dest: /etc/nginx/conf.d/ + owner: root + group: root + mode: 0644 + notify: restart nginx + when: setup_www_redirect + +- name: Enable SELinux + selinux: policy=targeted state=permissive + when: ansible_distribution == 'CentOS' + - name: Ensure nginx is started and enabled to start at boot. service: name=nginx state=started enabled=yes -- include: vhosts.yml +- include_tasks: vhosts.yml +... \ No newline at end of file diff --git a/playbooks/roles/nginx/tasks/setup-Debian.yml b/playbooks/roles/nginx/tasks/setup-Debian.yml new file mode 100644 index 00000000..2f50fb37 --- /dev/null +++ b/playbooks/roles/nginx/tasks/setup-Debian.yml @@ -0,0 +1,18 @@ +--- +- name: Add nginx apt repository key for Debian < 8 + apt_key: + url: http://nginx.org/keys/nginx_signing.key + state: present + when: ansible_distribution == 'Debian' and ansible_distribution_version | version_compare('8', 'lt') + +- name: Add nginx apt repository for Debian < 8 + apt_repository: + repo: 'deb [arch=amd64,i386] http://nginx.org/packages/debian/ {{ ansible_distribution_release }} nginx' + state: present + when: ansible_distribution == 'Debian' and ansible_distribution_version | version_compare('8', 'lt') + +- name: Ensure nginx is installed. + apt: + pkg: nginx + state: installed + default_release: "{{ nginx_default_release }}" diff --git a/vm/ansible/roles/nginx/tasks/setup-RedHat.yml b/playbooks/roles/nginx/tasks/setup-RedHat.yml similarity index 100% rename from vm/ansible/roles/nginx/tasks/setup-RedHat.yml rename to playbooks/roles/nginx/tasks/setup-RedHat.yml diff --git a/vm/ansible/roles/nginx/tasks/vhosts.yml b/playbooks/roles/nginx/tasks/vhosts.yml similarity index 100% rename from vm/ansible/roles/nginx/tasks/vhosts.yml rename to playbooks/roles/nginx/tasks/vhosts.yml diff --git a/playbooks/roles/nginx/templates/nginx.conf.j2 b/playbooks/roles/nginx/templates/nginx.conf.j2 new file mode 100644 index 00000000..573e3ed2 --- /dev/null +++ b/playbooks/roles/nginx/templates/nginx.conf.j2 @@ -0,0 +1,76 @@ +user {{ nginx_user }}; +worker_processes auto; +worker_rlimit_nofile 65535; + +error_log /var/log/nginx/error.log warn; +pid /var/run/nginx.pid; + + +events { + worker_connections {{ nginx_worker_connections or 2048 }}; + multi_accept on; +} + + +http { + include /etc/nginx/mime.types; + default_type application/octet-stream; + + log_format main '$remote_addr - $remote_user [$time_local] "$request" ' + '$status $body_bytes_sent "$http_referer" ' + '"$http_user_agent" "$http_x_forwarded_for"'; + + access_log /var/log/nginx/access.log main; + + sendfile on; + tcp_nopush on; + tcp_nodelay on; + server_tokens off; + + # keepalive_timeout 10; + # keepalive_requests 10; + + gzip on; + gzip_disable "msie6"; + gzip_http_version 1.1; + gzip_comp_level 5; + gzip_min_length 256; + gzip_proxied any; + gzip_vary on; + gzip_types + application/atom+xml + application/javascript + application/json + application/rss+xml + application/vnd.ms-fontobject + application/x-font-ttf + application/font-woff + application/x-web-app-manifest+json + application/xhtml+xml + application/xml + font/opentype + image/svg+xml + image/x-icon + text/css + text/plain + text/x-component + ; + + server_names_hash_max_size 4096; + + open_file_cache max=65000 inactive=1m; + open_file_cache_valid 5s; + open_file_cache_min_uses 1; + open_file_cache_errors on; + + ssl_protocols SSLv3 TLSv1; + ssl_ciphers ECDHE-RSA-AES256-SHA384:AES256-SHA256:RC4:HIGH:!MD5:!aNULL:!EDH:!AESGCM; + ssl_prefer_server_ciphers on; + + client_max_body_size 50m; + large_client_header_buffers 4 32k; + + proxy_cache_path /var/cache/nginx levels=1:2 keys_zone=web-cache:8m max_size=1000m inactive=600m; + + include /etc/nginx/conf.d/*.conf; +} diff --git a/vm/ansible/roles/nginx/templates/nginx.repo.j2 b/playbooks/roles/nginx/templates/nginx.repo.j2 similarity index 100% rename from vm/ansible/roles/nginx/templates/nginx.repo.j2 rename to playbooks/roles/nginx/templates/nginx.repo.j2 diff --git a/vm/ansible/roles/nginx/templates/vhosts.j2 b/playbooks/roles/nginx/templates/vhosts.j2 similarity index 100% rename from vm/ansible/roles/nginx/templates/vhosts.j2 rename to playbooks/roles/nginx/templates/vhosts.j2 diff --git a/vm/ansible/roles/nginx/tests/inventory b/playbooks/roles/nginx/tests/inventory similarity index 100% rename from vm/ansible/roles/nginx/tests/inventory rename to playbooks/roles/nginx/tests/inventory diff --git a/vm/ansible/roles/nginx/tests/test.yml b/playbooks/roles/nginx/tests/test.yml similarity index 100% rename from vm/ansible/roles/nginx/tests/test.yml rename to playbooks/roles/nginx/tests/test.yml diff --git a/vm/ansible/roles/nginx/vars/Debian.yml b/playbooks/roles/nginx/vars/Debian.yml similarity index 100% rename from vm/ansible/roles/nginx/vars/Debian.yml rename to playbooks/roles/nginx/vars/Debian.yml diff --git a/vm/ansible/roles/nginx/vars/RedHat.yml b/playbooks/roles/nginx/vars/RedHat.yml similarity index 100% rename from vm/ansible/roles/nginx/vars/RedHat.yml rename to playbooks/roles/nginx/vars/RedHat.yml diff --git a/playbooks/roles/nodejs/tasks/debian_family.yml b/playbooks/roles/nodejs/tasks/debian_family.yml new file mode 100644 index 00000000..c64bac55 --- /dev/null +++ b/playbooks/roles/nodejs/tasks/debian_family.yml @@ -0,0 +1,20 @@ +--- +- name: Add apt key for node repo + apt_key: + url: https://keyserver.ubuntu.com/pks/lookup?op=get&fingerprint=on&search=0x1655A0AB68576280 + id: "68576280" + state: present + +- name: Add repo + apt_repository: + repo: "deb [arch=amd64,i386] https://deb.nodesource.com/node_8.x {{ ansible_distribution_release }} main" + state: present + register: node_repo + +- name: Install nodejs 8.x + apt: + name: nodejs + state: present + update_cache: yes + force: yes +... \ No newline at end of file diff --git a/playbooks/roles/nodejs/tasks/main.yml b/playbooks/roles/nodejs/tasks/main.yml new file mode 100644 index 00000000..85379727 --- /dev/null +++ b/playbooks/roles/nodejs/tasks/main.yml @@ -0,0 +1,14 @@ +--- +# Install's prerequisites, like fonts, image libraries, vim, screen, python-dev and gcc + +- include_tasks: debian_family.yml + when: ansible_os_family == 'Debian' + +- include_tasks: redhat_family.yml + when: ansible_os_family == "RedHat" + +- name: Install yarn + command: npm install -g yarn + become: yes + become_user: root +... \ No newline at end of file diff --git a/playbooks/roles/nodejs/tasks/redhat_family.yml b/playbooks/roles/nodejs/tasks/redhat_family.yml new file mode 100644 index 00000000..15af3846 --- /dev/null +++ b/playbooks/roles/nodejs/tasks/redhat_family.yml @@ -0,0 +1,18 @@ +--- +- name: Import Node source RPM key + rpm_key: + key: https://rpm.nodesource.com/pub/el/NODESOURCE-GPG-SIGNING-KEY-EL + state: present + when: ansible_os_family == 'RedHat' + +- name: Add Node Repo for RedHat + yum: + name: 'https://rpm.nodesource.com/pub_8.x/el/{{ ansible_distribution_major_version }}/{{ ansible_architecture }}/nodesource-release-el{{ ansible_distribution_major_version }}-1.noarch.rpm' + state: present + update_cache: yes + when: ansible_os_family == 'RedHat' + +- name: Install node v8 + yum: name=nodejs state=present + when: ansible_os_family == 'RedHat' +... \ No newline at end of file diff --git a/playbooks/roles/ntpd/tasks/main.yml b/playbooks/roles/ntpd/tasks/main.yml new file mode 100644 index 00000000..b2cfb190 --- /dev/null +++ b/playbooks/roles/ntpd/tasks/main.yml @@ -0,0 +1,23 @@ +--- +- name: Install ntpd + yum: name="{{item}}" state=installed + with_items: + - ntp + - ntpdate + when: ansible_distribution == 'CentOS' + +- name: Enable ntpd + service: name=ntpd enabled=yes state=started + when: ansible_distribution == 'CentOS' + +- name: Install ntpd + apt: name="{{item}}" state=installed + with_items: + - ntp + - ntpdate + when: ansible_distribution == 'Debian' or ansible_distribution == 'Ubuntu' + +- name: Enable ntpd + service: name=ntp enabled=yes state=started + when: ansible_distribution == 'Debian' or ansible_distribution == 'Ubuntu' +... \ No newline at end of file diff --git a/playbooks/roles/packer/tasks/debian_family.yml b/playbooks/roles/packer/tasks/debian_family.yml new file mode 100644 index 00000000..52d34486 --- /dev/null +++ b/playbooks/roles/packer/tasks/debian_family.yml @@ -0,0 +1,6 @@ +--- +- name: Install unzip + apt: pkg={{ item }} update_cache=yes state=present + with_items: + - unzip +... \ No newline at end of file diff --git a/playbooks/roles/packer/tasks/main.yml b/playbooks/roles/packer/tasks/main.yml new file mode 100644 index 00000000..e31041ca --- /dev/null +++ b/playbooks/roles/packer/tasks/main.yml @@ -0,0 +1,37 @@ +--- +- name: Check if packer already exists + stat: + path: /opt/packer + register: packer + +- name: Check if packer version is 1.2.1 + command: /opt/packer --version + register: packer_version + when: packer.stat.exists + +- include_tasks: debian_family.yml + when: ansible_os_family == 'Debian' and packer.stat.exists == False + +- include_tasks: redhat_family.yml + when: ansible_os_family == "RedHat" and packer.stat.exists == False + +- name: Delete packer if < 1.2.1 + file: + state: absent + path: /opt/packer + when: (packer.stat.exists) and (packer_version | version_compare('1.2.1', '<')) + +- name: Download packer zip file + command: chdir=/opt/ wget https://releases.hashicorp.com/packer/1.2.1/packer_1.2.1_linux_amd64.zip + when: (packer.stat.exists == False) or (packer_version | version_compare('1.2.1', '<')) + +- name: Unzip the packer binary in /opt + command: chdir=/opt/ unzip packer_1.2.1_linux_amd64.zip + when: (packer.stat.exists == False) or (packer_version | version_compare('1.2.1', '<')) + +- name: Remove the downloaded packer zip file + file: + state: absent + path: /opt/packer_1.2.1_linux_amd64.zip + when: (packer.stat.exists == False) or (packer_version | version_compare('1.2.1', '<')) +... \ No newline at end of file diff --git a/playbooks/roles/packer/tasks/redhat_family.yml b/playbooks/roles/packer/tasks/redhat_family.yml new file mode 100644 index 00000000..937adfc0 --- /dev/null +++ b/playbooks/roles/packer/tasks/redhat_family.yml @@ -0,0 +1,7 @@ +--- + +- name: Install unzip + yum: name={{ item }} state=present + with_items: + - unzip +... diff --git a/playbooks/roles/psutil/tasks/main.yml b/playbooks/roles/psutil/tasks/main.yml new file mode 100644 index 00000000..9a38e344 --- /dev/null +++ b/playbooks/roles/psutil/tasks/main.yml @@ -0,0 +1,3 @@ +--- +- name: Install psutil + pip: name=psutil state=latest \ No newline at end of file diff --git a/playbooks/roles/redis/tasks/main.yml b/playbooks/roles/redis/tasks/main.yml new file mode 100644 index 00000000..472c65c4 --- /dev/null +++ b/playbooks/roles/redis/tasks/main.yml @@ -0,0 +1,21 @@ +--- + - name: Install yum packages + yum: name={{ item }} state=present + with_items: + - redis + when: ansible_os_family == 'RedHat' + + # Prerequisite for Debian and Ubuntu + - name: Install apt packages + apt: pkg={{ item }} state=present force=yes + with_items: + - redis-server + when: ansible_distribution == 'Debian' or ansible_distribution == 'Ubuntu' + + # Prerequisite for MACOS + - name: install prequisites for macos + homebrew: name={{ item }} state=present + with_items: + - redis + when: ansible_distribution == 'MacOSX' +... \ No newline at end of file diff --git a/playbooks/roles/supervisor/tasks/main.yml b/playbooks/roles/supervisor/tasks/main.yml new file mode 100644 index 00000000..5926f27d --- /dev/null +++ b/playbooks/roles/supervisor/tasks/main.yml @@ -0,0 +1,8 @@ +--- +- name: Install supervisor on centos + yum: name=supervisor state=present + when: ansible_os_family == 'RedHat' + +- name: Install supervisor on debian + apt: pkg=supervisor state=present force=yes + when: ansible_distribution == 'Debian' or ansible_distribution == 'Ubuntu' diff --git a/playbooks/roles/swap/defaults/main.yml b/playbooks/roles/swap/defaults/main.yml new file mode 100644 index 00000000..cf25d056 --- /dev/null +++ b/playbooks/roles/swap/defaults/main.yml @@ -0,0 +1,3 @@ +--- +swap_size_mb: 1024 +... \ No newline at end of file diff --git a/vm/ansible/roles/swap/tasks/main.yml b/playbooks/roles/swap/tasks/main.yml similarity index 100% rename from vm/ansible/roles/swap/tasks/main.yml rename to playbooks/roles/swap/tasks/main.yml diff --git a/playbooks/roles/virtualbox/defaults/main.yml b/playbooks/roles/virtualbox/defaults/main.yml new file mode 100644 index 00000000..c2c83404 --- /dev/null +++ b/playbooks/roles/virtualbox/defaults/main.yml @@ -0,0 +1,3 @@ +--- +virtualbox_version: 5.2 +... \ No newline at end of file diff --git a/playbooks/roles/virtualbox/files/virtualbox_centos.repo b/playbooks/roles/virtualbox/files/virtualbox_centos.repo new file mode 100644 index 00000000..9ad836ab --- /dev/null +++ b/playbooks/roles/virtualbox/files/virtualbox_centos.repo @@ -0,0 +1,7 @@ +[virtualbox] +name=Oracle Linux / RHEL / CentOS-$releasever / $basearch - VirtualBox +baseurl=http://download.virtualbox.org/virtualbox/rpm/el/$releasever/$basearch +enabled=1 +gpgcheck=1 +repo_gpgcheck=1 +gpgkey=https://www.virtualbox.org/download/oracle_vbox.asc \ No newline at end of file diff --git a/playbooks/roles/virtualbox/tasks/debian_family.yml b/playbooks/roles/virtualbox/tasks/debian_family.yml new file mode 100644 index 00000000..8dbcab79 --- /dev/null +++ b/playbooks/roles/virtualbox/tasks/debian_family.yml @@ -0,0 +1,29 @@ +--- +- name: Install dependencies + apt: pkg={{ item }} state=present + with_items: + - apt-transport-https + - ca-certificates + +- name: Add VirtualBox to sources.list + apt_repository: + repo: deb https://download.virtualbox.org/virtualbox/debian {{ ansible_distribution_release }} contrib + state: present + +- name: Add apt signing key for VirtualBox for Debian >= 8 and Ubuntu >= 16 + apt_key: + url: https://www.virtualbox.org/download/oracle_vbox_2016.asc + state: present + when: (ansible_distribution == "Debian" and ansible_distribution_major_version >= "8") or (ansible_distribution == "Ubuntu" and ansible_distribution_major_version >= "16") + +- name: Add apt signing key for VirtualBox for Debian < 8 and Ubuntu < 16 + apt_key: + url: https://www.virtualbox.org/download/oracle_vbox.asc + state: present + when: (ansible_distribution == "Debian" and ansible_distribution_major_version < "8") or (ansible_distribution == "Ubuntu" and ansible_distribution_major_version < "16") + +- name: Install VirtualBox + apt: pkg={{ item }} update_cache=yes state=present + with_items: + - virtualbox-{{ virtualbox_version }} +... diff --git a/playbooks/roles/virtualbox/tasks/main.yml b/playbooks/roles/virtualbox/tasks/main.yml new file mode 100644 index 00000000..d62fbe76 --- /dev/null +++ b/playbooks/roles/virtualbox/tasks/main.yml @@ -0,0 +1,7 @@ +--- +- include_tasks: debian_family.yml + when: ansible_os_family == 'Debian' + +- include_tasks: redhat_family.yml + when: ansible_os_family == "RedHat" +... \ No newline at end of file diff --git a/playbooks/roles/virtualbox/tasks/redhat_family.yml b/playbooks/roles/virtualbox/tasks/redhat_family.yml new file mode 100644 index 00000000..76bd56ab --- /dev/null +++ b/playbooks/roles/virtualbox/tasks/redhat_family.yml @@ -0,0 +1,17 @@ +--- +- name: Install the 'Development tools' package group + yum: + name: "@Development tools" + state: present + +- name: Install dependencies + yum: name={{ item }} state=present + with_items: + - kernel-devel + - deltarpm + +- copy: src=virtualbox_centos.repo dest=/etc/yum.repos.d/virtualbox.repo owner=root group=root mode=0644 force=no + +- name: Install VirtualBox + command: yum install -y VirtualBox-{{ virtualbox_version }} +... diff --git a/vm/ansible/roles/wkhtmltopdf/tasks/main.yml b/playbooks/roles/wkhtmltopdf/tasks/main.yml similarity index 50% rename from vm/ansible/roles/wkhtmltopdf/tasks/main.yml rename to playbooks/roles/wkhtmltopdf/tasks/main.yml index 52c95925..18ea0016 100644 --- a/vm/ansible/roles/wkhtmltopdf/tasks/main.yml +++ b/playbooks/roles/wkhtmltopdf/tasks/main.yml @@ -9,7 +9,7 @@ when: ansible_os_family == 'RedHat' - name: install base fonts - apt: name={{ item }} state=present + apt: name={{ item }} state=present force=yes with_items: - libxrender1 - libxext6 @@ -17,11 +17,24 @@ - xfonts-base when: ansible_os_family == 'Debian' +# wkhtmltopdf has been locked down to 0.12.3 intentionally since 0.12.4 has problems. +# I you want to upgrade try it on multiple platforms and then decide - name: download wkthmltox linux get_url: url=https://github.com/frappe/wkhtmltopdf/raw/master/wkhtmltox-0.12.3_linux-generic-{{ "amd64" if ansible_architecture == "x86_64" else "i386"}}.tar.xz dest=/tmp/wkhtmltox.tar.xz -- name: unarchive wkhtmltopdf - unarchive: src=/tmp/wkhtmltox.tar.xz dest=/tmp/wkhtmltox +- name: Creates directory + file: path=/tmp/wkhtmltox state=directory -- name: move to /usr/local/bin - command: creates="/usr/local/bin/wkhtmltopdf" mv /tmp/wkhtmltox/bin/wkhtmltopdf /usr/local/bin/wkhtmltopdf \ No newline at end of file +- name: unarchive wkhtmltopdf + unarchive: src=/tmp/wkhtmltox.tar.xz dest=/tmp + +- name: copy to /usr/local/bin + copy: src="/tmp/wkhtmltox/bin/wkhtmltopdf" dest="/usr/local/bin/wkhtmltopdf" + become: true + become_user: root + +- name: make wkhtmltopdf executable + file: path=/usr/local/bin/wkhtmltopdf mode="o+x" + become: true + become_user: root +... \ No newline at end of file diff --git a/playbooks/site.yml b/playbooks/site.yml new file mode 100644 index 00000000..839620ee --- /dev/null +++ b/playbooks/site.yml @@ -0,0 +1,48 @@ +--- +# This is the master playbook that deploys the whole Frappe and ERPNext stack + +- hosts: localhost + become: yes + become_user: root + roles: + - { role: common, tags: common } + - { role: locale, tags: locale } + - { role: swap, tags: swap, when: production } + - { role: logwatch, tags: logwatch, when: production } + - { role: bash_screen_wall, tags: bash_screen_wall, when: production } + - { role: frappe_selinux, tags: frappe_selinux, when: production } + - { role: dns_caching, tags: dns_caching, when: production } + - { role: ntpd, tags: ntpd, when: production } + - { role: mariadb, tags: mariadb } + - { role: wkhtmltopdf, tags: wkhtmltopdf } + - { role: nodejs, tags: nodejs } + - { role: psutil, tags: psutil } + - { role: redis, tags: redis } + - { role: supervisor, tags: supervisor, when: production } + - { role: nginx, tags: nginx, when: production } + - { role: fail2ban, tags: fail2ban, when: production } + tasks: + - name: Set hostname + hostname: name='{{ hostname }}' + when: hostname is defined and production + + - name: Start NTPD + service: name=ntpd state=started + when: ansible_distribution == 'CentOS' and production + + - name: Start NTPD + service: name=ntp state=started + when: ansible_distribution == 'Debian' or ansible_distribution == 'Ubuntu' and production + + - include_tasks: macosx.yml + when: ansible_distribution == 'MacOSX' + +- name: setup bench and dev environment + hosts: localhost + vars: + bench_repo_path: "/home/{{ frappe_user }}/.bench" + bench_path: "/home/{{ frappe_user }}/{{ bench_name }}" + roles: + # setup frappe-bench + - { role: bench, tags: "bench", when: not run_travis and not without_bench_setup } +... diff --git a/playbooks/vm_build.yml b/playbooks/vm_build.yml new file mode 100644 index 00000000..2be9639a --- /dev/null +++ b/playbooks/vm_build.yml @@ -0,0 +1,9 @@ +--- +- name: Install Packer + hosts: localhost + become: yes + become_user: root + roles: + - { role: virtualbox, tags: "virtualbox" } + - { role: packer, tags: "packer" } +... diff --git a/requirements.txt b/requirements.txt index 57eb2683..4365cc1d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,4 +6,4 @@ honcho psutil python-crontab semantic_version -GitPython==0.3.2.rc1 +GitPython==0.3.2.1 diff --git a/vm/Readme.md b/vm/Readme.md index d2a8cbc6..316b3c33 100644 --- a/vm/Readme.md +++ b/vm/Readme.md @@ -1,25 +1,37 @@ -### ERPNext VM Builder +## ERPNext VM Builder -#### Steps to build a vm image -* Install VirtualBox -* Place a `base.ova` ubuntu base image in the current directory. -* `./packer build vm.json` builds a new vm. +### Steps to build a VM Image -#### How it works +* `python build.py` builds a new Production VM, a Dev VM and a Dev Vagrant Box -Packer imports the base image in a virtual machine and boots it. It runs the following -* `scripts/install_ansible.sh` sets up ansible on the vm. -* The `ansible/vm.yml` playbook sets up the dependencies, installs a bench and sets up a site. It also puts it into production. -* `scripts/set_message.sh` sets welcome message (with update instructions) in the vm. -* `scripts/zerofree.sh` writes zero to all the free space in the disk, it shrinks the disk image. +### Requirements -#### For a build server +* Bench should be installed +* Ansible should be installed -Running the `build.py` script builds a vm and puts it in `~/public`. It also writes a `latest.json` file in `~/public` with filename of the latest build and its md5sum. -#### Packer binary +### How it works -The binary included in this tree is compiled (for linux amd64) with a fix for https://github.com/mitchellh/packer/issues/2447. We can remove it once a new version of packer is released. +Apart from the above the rest is handled by bench: +* Install prerequisites if not already installed + - virtualbox + - packer +* Cleanup + - Clean the required directories +* Build the VM using packer + - Packer downloads the mentioned Ubuntu iso, boots a virtual machine and preceeds the preseed.cfg file into it in order to setup a clean Ubuntu OS + - Then packer uses ssh to enter the virtual machine to execute the required commands + - `scripts/debian_family/install_ansible.sh` sets up ansible on the vm. + - Depending on the VM being built, the `vm-develop.json` or the `vm-production.json` is used + - `scripts/set_message.sh` sets welcome message (with update instructions) in the vm. + - `scripts/cleanup.sh` writes zeros to all the free space in the disk, it shrinks the disk image +* Set the correct permissions for the built Vagrant and Virtual Appliance Images +* Cleanup + - Delete the generated files from the required directories +* Restart nginx + + +Running the `build.py` script builds the VMs and puts them in `~/Public`. It also creates the md5 hashes for the same, and puts them in the same folder. diff --git a/vm/Vagrantfile b/vm/Vagrantfile new file mode 100644 index 00000000..19fe518f --- /dev/null +++ b/vm/Vagrantfile @@ -0,0 +1,73 @@ +# -*- mode: ruby -*- +# vi: set ft=ruby : + +# All Vagrant configuration is done below. The "2" in Vagrant.configure +# configures the configuration version (we support older styles for +# backwards compatibility). Please don't change it unless you know what +# you're doing. +Vagrant.configure(2) do |config| + # The most common configuration options are documented and commented below. + # For a complete reference, please see the online documentation at + # https://docs.vagrantup.com. + + # Every Vagrant development environment requires a box. You can search for + # boxes at https://atlas.hashicorp.com/search. + config.vm.box = "erpnext" + config.ssh.username = "frappe" + config.ssh.password = "frappe" + + # Disable automatic box update checking. If you disable this, then + # boxes will only be checked for updates when the user runs + # `vagrant box outdated`. This is not recommended. + # config.vm.box_check_update = false + + # Create a forwarded port mapping which allows access to a specific port + # within the machine from a port on the host machine. In the example below, + # accessing "localhost:8080" will access port 80 on the guest machine. + # config.vm.network "forwarded_port", guest: 80, host: 8080 + + # Create a private network, which allows host-only access to the machine + # using a specific IP. + # config.vm.network "private_network", ip: "192.168.33.10" + + # Create a public network, which generally matched to bridged network. + # Bridged networks make the machine appear as another physical device on + # your network. + # config.vm.network "public_network" + + # Share an additional folder to the guest VM. The first argument is + # the path on the host to the actual folder. The second argument is + # the path on the guest to mount the folder. And the optional third + # argument is a set of non-required options. + # config.vm.synced_folder "../data", "/vagrant_data" + + # Provider-specific configuration so you can fine-tune various + # backing providers for Vagrant. These expose provider-specific options. + # Example for VirtualBox: + # + # config.vm.provider "virtualbox" do |vb| + # # Display the VirtualBox GUI when booting the machine + # vb.gui = true + # + # # Customize the amount of memory on the VM: + # vb.memory = "1024" + # end + # + # View the documentation for the provider you are using for more + # information on available options. + + # Define a Vagrant Push strategy for pushing to Atlas. Other push strategies + # such as FTP and Heroku are also available. See the documentation at + # https://docs.vagrantup.com/v2/push/atlas.html for more information. + # config.push.define "atlas" do |push| + # push.app = "YOUR_ATLAS_USERNAME/YOUR_APPLICATION_NAME" + # end + + # Enable provisioning with a shell script. Additional provisioners such as + # Puppet, Chef, Ansible, Salt, and Docker are also available. Please see the + # documentation for more information about their specific syntax and use. + # config.vm.provision "shell", inline: <<-SHELL + # sudo apt-get update + # sudo apt-get install -y apache2 + # SHELL +end diff --git a/vm/ansible/base.yml b/vm/ansible/base.yml deleted file mode 100644 index 8206c13b..00000000 --- a/vm/ansible/base.yml +++ /dev/null @@ -1,104 +0,0 @@ ---- - - name: Apply basic roles - hosts: all - become: yes - become_user: root - vars: - - mysql_secure_installation: True - - mysql_conf_tpl: templates/mariadb_config.cnf - - nginx_conf_file: templates/nginx.conf - roles: - - mariadb - - nginx - - wkhtmltopdf - tasks: - - name: Install the 'Development tools' package group (Redhat) - yum: name="@Development tools" state=present - when: ansible_os_family == 'RedHat' - - name: Install packages - yum: name={{ item }} state=present - with_items: - - git - - nginx - - zlib-devel - - bzip2-devel - - openssl-devel - - postfix - - python-devel - - mariadb - - redis - - libXrender - - libXext - - cronie - - sudo - - which - - xorg-x11-fonts-75dpi - - xorg-x11-fonts-Type1 - - screen - - vim - - git - - nginx - - zlib-devel - - bzip2-devel - - openssl-devel - - postfix - - python-devel - - redis - - cronie - - sudo - - which - - supervisor - - nodejs - - npm - - libtiff-devel - - libjpeg-devel - - libzip-devel - - freetype-devel - - lcms2-devel - - libwebp-devel - - tcl-devel - - tk-devel - when: ansible_os_family == 'RedHat' - - name: Install packages - apt: pkg={{ item }} state=present - with_items: - - python-dev - - python-setuptools - - build-essential - - python-mysqldb - - git - - ntp - - vim - - screen - - htop - - mariadb-server - - mariadb-common - - libmariadbclient-dev - - libxslt1.1 - - libxslt1-dev - - redis-server - - libssl-dev - - libcrypto++-dev - - postfix - - nginx - - supervisor - - python-pip - - fontconfig - - libxrender1 - - libxext6 - - xfonts-75dpi - - xfonts-base - - nodejs - - npm - - libtiff5-dev - - libjpeg8-dev - - zlib1g-dev - - libfreetype6-dev - - liblcms2-dev - - libwebp-dev - - tcl8.6-dev - - tk8.6-dev - - python-tk - when: ansible_os_family == 'Debian' - - user: name="{{ frappe_user }}" - diff --git a/vm/ansible/library/bench.py b/vm/ansible/library/bench.py deleted file mode 100644 index cb3aa9e1..00000000 --- a/vm/ansible/library/bench.py +++ /dev/null @@ -1,84 +0,0 @@ -#!/usr/bin/python -import os -import subprocess -import json - - -def init_bench(path, frappe_branch): - if not frappe_branch: - frappe_branch = 'master' - subprocess.check_call("bench init {} --frappe-branch {}".format(path, frappe_branch), shell=True) - -def check_if_app_exists(app, bench_path): - return os.path.exists(os.path.join(bench_path, 'apps', app)) - -def check_if_site_exists(site, bench_path): - return os.path.exists(os.path.join(bench_path, 'sites', site)) - -def get_app(app, url, branch, bench_path): - subprocess.check_call("bench get-app {} {} --branch {}".format(app, url, branch), cwd=bench_path, shell=True) - -def install_site(site, mariadb_root_password, bench_path): - admin_password = site.get('admin_password') - site_name = site['name'] - subprocess.check_call("bench new-site {} --mariadb-root-password {} --admin-password {}".format(site_name, mariadb_root_password, admin_password), cwd=bench_path, shell=True) - - for app in site['apps']: - subprocess.check_call("bench --site {} install-app {}".format(site['name'], app), cwd=bench_path, shell=True) - - site_config_path = os.path.join(bench_path, 'sites', site_name, 'site_config.json') - with open(site_config_path) as f: - site_config = json.load(f) - site_config.update(site.get('site_config', {})) - with open(site_config_path, 'wb') as f: - json.dump(site_config, f) - -def main(): - module = AnsibleModule( - argument_spec=dict( - path=dict(required=True), - apps=dict(required=True), - sites=dict(required=True), - mariadb_root_password=dict(), - frappe_branch=dict(), - ), - supports_check_mode=True - ) - changed = False - - bench_path = module.params['path'] - if not os.path.exists(bench_path): - if module.check_mode: - module.exit_json(changed=True) - - init_bench(bench_path, module.params['frappe_branch']) - changed = True - - - for app in module.params['apps']: - if not check_if_app_exists(app['name'], bench_path): - if module.check_mode: - module.exit_json(changed=True) - - get_app(app['url'], app.get('branch', 'master'), bench_path) - changed = True - - for site in module.params['sites']: - if not check_if_site_exists(site['name'], bench_path): - if module.check_mode: - module.exit_json(changed=True) - - mariadb_root_password = module.params.get('mariadb_root_password') - if not mariadb_root_password: - module.fail_json(msg="MariaDB root password not passed") - - if not site.get('admin_password'): - module.fail_json(msg="Admin password not passed for {}".format(site['name'])) - - install_site(site, mariadb_root_password, bench_path) - changed = True - - module.exit_json(changed=changed) - -from ansible.module_utils.basic import * -main() diff --git a/vm/ansible/roles/mariadb/meta/main.yml b/vm/ansible/roles/mariadb/meta/main.yml deleted file mode 100644 index b2beef2c..00000000 --- a/vm/ansible/roles/mariadb/meta/main.yml +++ /dev/null @@ -1,19 +0,0 @@ ---- -galaxy_info: - author: "Attila van der Velde" - description: "Installs MariaDB" - company: "PCextreme B.V." - license: "license (MIT, BSD)" - min_ansible_version: 1.8 - platforms: - - name: EL - versions: - - 6 - - 7 - - name: Ubuntu - versions: - - trusty - categories: - - database:sql - -dependencies: [] diff --git a/vm/ansible/roles/mariadb/tasks/main.yml b/vm/ansible/roles/mariadb/tasks/main.yml deleted file mode 100644 index 16f09151..00000000 --- a/vm/ansible/roles/mariadb/tasks/main.yml +++ /dev/null @@ -1,18 +0,0 @@ ---- -- include: centos.yml - when: ansible_distribution == 'CentOS' and ansible_distribution_major_version|int >= 6 - -- include: ubuntu.yml - when: ansible_distribution == 'Ubuntu' and ansible_distribution_version == '14.04' - -- name: Add configuration - template: src={{ mysql_conf_tpl }} dest={{ mysql_conf_dir[ansible_distribution] }}/{{ mysql_conf_file }} owner=root group=root mode=0644 - when: mysql_conf_tpl != 'change_me' - notify: restart mysql - -- name: Start and enable service - service: name=mysql state=started enabled=yes - -- include: mysql_secure_installation.yml - debug: var=mysql_secure_installation - when: mysql_secure_installation and mysql_root_password is defined diff --git a/vm/ansible/roles/mariadb/vars/main.yml b/vm/ansible/roles/mariadb/vars/main.yml deleted file mode 100644 index 4248ca90..00000000 --- a/vm/ansible/roles/mariadb/vars/main.yml +++ /dev/null @@ -1,4 +0,0 @@ ---- -mysql_conf_dir: - "CentOS": /etc/my.cnf.d - "Ubuntu": /etc/mysql/conf.d diff --git a/vm/ansible/roles/nginx/.travis.yml b/vm/ansible/roles/nginx/.travis.yml deleted file mode 100644 index 24648b13..00000000 --- a/vm/ansible/roles/nginx/.travis.yml +++ /dev/null @@ -1,35 +0,0 @@ ---- -language: python -python: "2.7" - -env: - - SITE=test.yml - -before_install: - - sudo apt-get update -qq - - sudo apt-get install -y curl - -install: - # Install Ansible. - - pip install ansible - - # Add ansible.cfg to pick up roles path. - - "{ echo '[defaults]'; echo 'roles_path = ../'; } >> ansible.cfg" - -script: - # Check the role/playbook's syntax. - - "ansible-playbook -i tests/inventory tests/$SITE --syntax-check" - - # Run the role/playbook with ansible-playbook. - - "ansible-playbook -i tests/inventory tests/$SITE --connection=local --sudo" - - # Run the role/playbook again, checking to make sure it's idempotent. - - > - ansible-playbook -i tests/inventory tests/$SITE --connection=local --sudo - | grep -q 'changed=0.*failed=0' - && (echo 'Idempotence test: pass' && exit 0) - || (echo 'Idempotence test: fail' && exit 1) - - # TODO - get the test working. Probably need to add a virtual host. - # Request a page via Nginx, to make sure Nginx is running and responds. - # - "curl http://localhost/" diff --git a/vm/ansible/roles/nginx/tasks/setup-Debian.yml b/vm/ansible/roles/nginx/tasks/setup-Debian.yml deleted file mode 100644 index ced11b65..00000000 --- a/vm/ansible/roles/nginx/tasks/setup-Debian.yml +++ /dev/null @@ -1,6 +0,0 @@ ---- -- name: Ensure nginx is installed. - apt: - pkg: nginx - state: installed - default_release: "{{ nginx_default_release }}" diff --git a/vm/ansible/roles/nginx/templates/nginx.conf.j2 b/vm/ansible/roles/nginx/templates/nginx.conf.j2 deleted file mode 100644 index a43202ce..00000000 --- a/vm/ansible/roles/nginx/templates/nginx.conf.j2 +++ /dev/null @@ -1,51 +0,0 @@ -user {{ nginx_user }}; - -error_log {{ nginx_error_log }}; -pid /var/run/nginx.pid; - -worker_processes {{ nginx_worker_processes }}; - -events { - worker_connections {{ nginx_worker_connections }}; -} - -http { - include /etc/nginx/mime.types; - default_type application/octet-stream; - - server_names_hash_bucket_size 64; - - client_max_body_size {{ nginx_client_max_body_size }}; - - log_format main '$remote_addr - $remote_user [$time_local] "$request" ' - '$status $body_bytes_sent "$http_referer" ' - '"$http_user_agent" "$http_x_forwarded_for"'; - - access_log {{ nginx_access_log }}; - - sendfile {{ nginx_sendfile }}; - tcp_nopush {{ nginx_tcp_nopush }}; - tcp_nodelay {{ nginx_tcp_nodelay }}; - - keepalive_timeout {{ nginx_keepalive_timeout }}; - keepalive_requests {{ nginx_keepalive_requests }}; - - #gzip on; - -{% if nginx_proxy_cache_path %} - proxy_cache_path {{ nginx_proxy_cache_path }}; -{% endif %} - -{% for upstream in nginx_upstreams %} - upstream {{ upstream.name }} { -{% if upstream.strategy is defined %} - {{ upstream.strategy }}; -{% endif %} -{% for server in upstream.servers %} - server {{ server }}; -{% endfor %} - } -{% endfor %} - - include {{ nginx_vhost_path }}/*; -} diff --git a/vm/ansible/roles/swap/defaults/main.yml b/vm/ansible/roles/swap/defaults/main.yml deleted file mode 100644 index 3eac0ae0..00000000 --- a/vm/ansible/roles/swap/defaults/main.yml +++ /dev/null @@ -1 +0,0 @@ -swap_size_mb: 1024 \ No newline at end of file diff --git a/vm/ansible/roles/wkhtmltopdf/defaults/main.yml b/vm/ansible/roles/wkhtmltopdf/defaults/main.yml deleted file mode 100644 index 5c56e85f..00000000 --- a/vm/ansible/roles/wkhtmltopdf/defaults/main.yml +++ /dev/null @@ -1 +0,0 @@ -wkhtmltopdf_version: 0.12.3 \ No newline at end of file diff --git a/vm/ansible/templates/nginx.conf b/vm/ansible/templates/nginx.conf deleted file mode 100644 index 5360e904..00000000 --- a/vm/ansible/templates/nginx.conf +++ /dev/null @@ -1,44 +0,0 @@ -# For more information on configuration, see: -# * Official English Documentation: http://nginx.org/en/docs/ -# * Official Russian Documentation: http://nginx.org/ru/docs/ - -user www-data; -worker_processes 1; - -error_log /var/log/nginx/error.log; -#error_log /var/log/nginx/error.log notice; -#error_log /var/log/nginx/error.log info; - -pid /run/nginx.pid; - - -events { - worker_connections 1024; -} - - -http { - include /etc/nginx/mime.types; - default_type application/octet-stream; - - log_format main '$remote_addr - $remote_user [$time_local] "$request" ' - '$status $body_bytes_sent "$http_referer" ' - '"$http_user_agent" "$http_x_forwarded_for"'; - - access_log /var/log/nginx/access.log main; - - sendfile on; - #tcp_nopush on; - - #keepalive_timeout 0; - keepalive_timeout 65; - - #gzip on; - - index index.html index.htm; - - # Load modular configuration files from the /etc/nginx/conf.d directory. - # See http://nginx.org/en/docs/ngx_core_module.html#include - # for more information. - include /etc/nginx/conf.d/*.conf; -} diff --git a/vm/ansible/vars.yml b/vm/ansible/vars.yml deleted file mode 100644 index b082ce4a..00000000 --- a/vm/ansible/vars.yml +++ /dev/null @@ -1,2 +0,0 @@ ---- -mysql_root_password: frappe diff --git a/vm/ansible/vm.yml b/vm/ansible/vm.yml deleted file mode 100644 index 18357509..00000000 --- a/vm/ansible/vm.yml +++ /dev/null @@ -1,64 +0,0 @@ ---- -- include: base.yml -- name: Clone bench - hosts: all - user: "{{ frappe_user }}" - tasks: - - git: - repo=https://github.com/frappe/bench - dest=/home/frappe/bench-repo - depth=no - tags: - - bench_install - -- name: Install bench - hosts: all - become: yes - become_user: root - tasks: - - - easy_install: name=pip executable=easy_install-2.7 - - pip: name=/home/frappe/bench-repo extra_args='-e' - tags: - - bench_install - - -- name: Setup bench - hosts: all - become: yes - become_user: "{{ frappe_user }}" - become_method: sudo - gather_facts: false - tasks: - - bench: - path: /home/frappe/frappe-bench - frappe_branch: develop - apps: - - name: erpnext - url: https://github.com/frappe/erpnext - - name: erpnext_demo - url: https://github.com/frappe/erpnext_demo - mariadb_root_password: "{{ mysql_root_password }}" - sites: - - name: erpnext.vm - admin_password: "{{ admin_password or 'admin' }}" - apps: - - erpnext - tags: - - bench_setup - -- name: Setup Production - hosts: all - become: yes - become_user: root - become_method: sudo - tasks: - - shell: "bench setup sudoers {{ frappe_user }}" - args: - chdir: "/home/{{ frappe_user }}/frappe-bench" - creates: "/home/{{ frappe_user }}/frappe-bench/config/supervisor.conf" - - shell: "bench setup production {{ frappe_user }}" - args: - chdir: "/home/{{ frappe_user }}/frappe-bench" - creates: "/home/{{ frappe_user }}/frappe-bench/config/supervisor.conf" - - file: path="/home/{{ frappe_user }}/frappe-bench/logs/" recurse=yes owner="{{ frappe_user }}" diff --git a/vm/ansible/vm_vars/all.yml b/vm/ansible/vm_vars/all.yml deleted file mode 100644 index 322f16eb..00000000 --- a/vm/ansible/vm_vars/all.yml +++ /dev/null @@ -1,4 +0,0 @@ ---- -mysql_root_password: frappe -frappe_user: frappe -admin_password: admin diff --git a/vm/build.py b/vm/build.py index a3b47b7e..30cf7da6 100644 --- a/vm/build.py +++ b/vm/build.py @@ -1,50 +1,128 @@ """ Builds a vm and puts it in ~/public with a latest.json that has its filename and md5sum """ -import os -import subprocess -import json -import stat -OUTPUT_DIR = 'output-virtualbox-ovf' -PUBLIC_DIR = os.path.join(os.path.expanduser('~'), 'public') +# imports - standard imports +import os +import json +import stat +import errno +from shutil import rmtree +from distutils import spawn +from subprocess import check_output + +NEW_FILES = [] +BUILDS = ['Production', 'Developer'] +PUBLIC_DIR = os.path.join(os.path.expanduser('~'), 'Public') +SYMLINKS = ['ERPNext-Production.ova', 'ERPNext-Dev.ova', 'ERPNext-Vagrant.box', + 'ERPNext-Production.ova.md5', 'ERPNext-Dev.ova.md5', 'ERPNext-Vagrant.box.md5'] def main(): + install_virtualbox() + install_packer() + cleanup() build_vm() - update_latest() - move_to_public() + generate_md5_hashes() + generate_symlinks() + delete_old_vms() + move_current_vms() cleanup() -def build_vm(): - subprocess.check_call("./packer build vm.json", shell=True) +def install_virtualbox(): + if not spawn.find_executable("virtualbox"): + check_output(['bench', 'install', 'virtualbox']) -def move_to_public(): - src = get_filepath() - dest = os.path.join(PUBLIC_DIR, os.path.join(PUBLIC_DIR, get_filename())) +def install_packer(): + if not os.path.exists(os.path.join('/', 'opt', 'packer')): + check_output(['bench', 'install', 'packer']) + +def silent_remove(name, is_dir=False): + ''' + Method to safely remove a file or directory, + without throwing error if file doesn't exist + + By default takes in file as input, for directory: + is_dir = True + ''' + try: + if is_dir: + rmtree(name) + else: + os.remove(name) + except OSError as e: + if e.errno != errno.ENOENT: # errno.ENOENT = no such file or directory + raise # re-raise exception if a different error occurred + +def cleanup(): + silent_remove("Production Builds", is_dir=True) + silent_remove("Developer Builds", is_dir=True) + silent_remove("packer_virtualbox-iso_virtualbox-iso_md5.checksum") + +def build_vm(): + check_output(["/opt/packer", "build", "vm-production.json"]) + check_output(["/opt/packer", "build", "vm-develop.json"]) + +def md5(build, file): + return check_output("md5sum '{} Builds/{}'".format(build, file), shell=True).split()[0] + +def move_to_public(build, file): + NEW_FILES.append(file) + src = os.path.join('{} Builds/{}'.format(build, file)) + dest = os.path.join(PUBLIC_DIR, file) os.rename(src, dest) + # Make Public folder readable by others st = os.stat(dest) os.chmod(dest, st.st_mode | stat.S_IROTH) -def update_latest(): - with open(os.path.join(PUBLIC_DIR, "latest.json"), 'w') as f: - json.dump(get_latest(), f) +def generate_md5_hashes(): + for build in BUILDS: + for file in os.listdir('{} Builds'.format(build)): + if file.endswith(".ova") or file.endswith(".box"): + with open('{} Builds/{}.md5'.format(build, file), 'w') as f: + f.write(md5(build, file)) + move_to_public(build, file) + move_to_public(build, '{}.md5'.format(file)) -def get_latest(): - md5 = subprocess.check_output("md5sum {}".format(get_filepath()), shell=True).split()[0] - return { - "filename": get_filename(), - "md5": md5 - } +def generate_symlinks(): + for file in NEW_FILES: + if 'md5' in file: + if 'Vagrant' in file: + silent_remove(os.path.join(PUBLIC_DIR, 'ERPNext-Vagrant.box.md5')) + os.symlink(os.path.join(PUBLIC_DIR, file), + os.path.join(PUBLIC_DIR, 'ERPNext-Vagrant.box.md5')) + elif 'Production' in file: + silent_remove(os.path.join(PUBLIC_DIR, 'ERPNext-Production.ova.md5')) + os.symlink(os.path.join(PUBLIC_DIR, file), + os.path.join(PUBLIC_DIR, 'ERPNext-Production.ova.md5')) + else: # Develop + silent_remove(os.path.join(PUBLIC_DIR, 'ERPNext-Dev.ova.md5')) + os.symlink(os.path.join(PUBLIC_DIR, file), + os.path.join(PUBLIC_DIR, 'ERPNext-Dev.ova.md5')) + else: # ova/box files + if 'Vagrant' in file: + silent_remove(os.path.join(PUBLIC_DIR, 'ERPNext-Vagrant.box')) + os.symlink(os.path.join(PUBLIC_DIR, file), + os.path.join(PUBLIC_DIR, 'ERPNext-Vagrant.box')) + elif 'Production' in file: + silent_remove(os.path.join(PUBLIC_DIR, 'ERPNext-Production.ova')) + os.symlink(os.path.join(PUBLIC_DIR, file), + os.path.join(PUBLIC_DIR, 'ERPNext-Production.ova')) + else: # Develop + silent_remove(os.path.join(PUBLIC_DIR, 'ERPNext-Dev.ova')) + os.symlink(os.path.join(PUBLIC_DIR, file), + os.path.join(PUBLIC_DIR, 'ERPNext-Dev.ova')) -def get_filename(): - return os.listdir(OUTPUT_DIR)[0] +def delete_old_vms(): + silent_remove(os.path.join(PUBLIC_DIR, 'BACKUPS'), is_dir=True) -def get_filepath(): - filename = os.listdir(OUTPUT_DIR)[0] - return os.path.join(OUTPUT_DIR, filename) - -def cleanup(): - os.rmdir(OUTPUT_DIR) +def move_current_vms(): + os.mkdir(os.path.join(PUBLIC_DIR, 'BACKUPS')) + for file in os.listdir(PUBLIC_DIR): + if file in NEW_FILES or file in SYMLINKS or file == 'BACKUPS': + continue + src = os.path.join(PUBLIC_DIR, '{}'.format(file)) + dest = os.path.join(PUBLIC_DIR, 'BACKUPS/{}'.format(file)) + os.rename(src, dest) if __name__ == "__main__": main() diff --git a/vm/http/preseed.cfg b/vm/http/preseed.cfg new file mode 100644 index 00000000..87c4622e --- /dev/null +++ b/vm/http/preseed.cfg @@ -0,0 +1,41 @@ +choose-mirror-bin mirror/http/proxy string +choose-mirror-bin mirror/http/proxy string +d-i base-installer/kernel/override-image string linux-server +d-i clock-setup/utc boolean true +d-i clock-setup/utc-auto boolean true +d-i finish-install/reboot_in_progress note +d-i grub-installer/only_debian boolean true +d-i grub-installer/with_other_os boolean true +d-i partman-auto/disk string /dev/sda +d-i partman-auto-lvm/guided_size string max +d-i partman-auto/choose_recipe select atomic +d-i partman-auto/method string lvm +d-i partman-lvm/confirm boolean true +d-i partman-lvm/confirm boolean true +d-i partman-lvm/confirm_nooverwrite boolean true +d-i partman-lvm/device_remove_lvm boolean true +d-i partman/choose_partition select finish +d-i partman/confirm boolean true +d-i partman/confirm_nooverwrite boolean true +d-i partman/confirm_write_new_label boolean true +d-i pkgsel/include string openssh-server cryptsetup build-essential libssl-dev libreadline-dev zlib1g-dev linux-source dkms nfs-common +d-i pkgsel/install-language-support boolean false +d-i pkgsel/update-policy select none +d-i pkgsel/upgrade select full-upgrade +d-i time/zone string UTC +tasksel tasksel/first multiselect standard, ubuntu-server + +d-i console-setup/ask_detect boolean false +d-i keyboard-configuration/layoutcode string us +d-i keyboard-configuration/modelcode string pc105 +d-i debian-installer/locale string en_US + +# Create frappe user account. +d-i passwd/user-fullname string frappe +d-i passwd/username string frappe +d-i passwd/user-password password frappe +d-i passwd/user-password-again password frappe +d-i user-setup/allow-password-weak boolean true +d-i user-setup/encrypt-home boolean false +d-i passwd/user-default-groups frappe sudo +d-i passwd/user-uid string 900 diff --git a/vm/packer b/vm/packer deleted file mode 100755 index 26d83158..00000000 Binary files a/vm/packer and /dev/null differ diff --git a/vm/scripts/debian_family/cleanup.sh b/vm/scripts/debian_family/cleanup.sh new file mode 100644 index 00000000..d70c7a09 --- /dev/null +++ b/vm/scripts/debian_family/cleanup.sh @@ -0,0 +1,13 @@ +#!/bin/bash -eux + +# Apt cleanup +apt-get autoremove +apt-get clean +apt-get update + +# Zero out the rest of the free space using dd, then delete the written file. +dd if=/dev/zero of=/EMPTY bs=1M +rm -f /EMPTY + +# Add `sync` so Packer doesn't quit too early, before the large file is deleted. +sync \ No newline at end of file diff --git a/vm/scripts/debian_family/install_ansible.sh b/vm/scripts/debian_family/install_ansible.sh new file mode 100644 index 00000000..c173dd94 --- /dev/null +++ b/vm/scripts/debian_family/install_ansible.sh @@ -0,0 +1,10 @@ +#!/bin/bash -eux + +# Install Ansible repository. +apt -y update && apt-get -y upgrade +apt -y install software-properties-common +apt-add-repository ppa:ansible/ansible + +# Install Ansible. +apt -y update +apt -y install ansible diff --git a/vm/scripts/debian_family/install_erpnext_develop.sh b/vm/scripts/debian_family/install_erpnext_develop.sh new file mode 100644 index 00000000..4d3031af --- /dev/null +++ b/vm/scripts/debian_family/install_erpnext_develop.sh @@ -0,0 +1,8 @@ +#!/bin/bash -eux + +# Install base requirements. +apt-get install -y curl git wget vim python-dev gcc + +# Install ERPNext +wget https://raw.githubusercontent.com/frappe/bench/master/playbooks/install.py +sudo python install.py --develop --user frappe --mysql-root-password frappe --admin-password admin \ No newline at end of file diff --git a/vm/scripts/debian_family/install_erpnext_production.sh b/vm/scripts/debian_family/install_erpnext_production.sh new file mode 100644 index 00000000..c0e4ade0 --- /dev/null +++ b/vm/scripts/debian_family/install_erpnext_production.sh @@ -0,0 +1,8 @@ +#!/bin/bash -eux + +# Install base requirements. +apt-get install -y curl git wget vim python-dev gcc + +# Install ERPNext +wget https://raw.githubusercontent.com/frappe/bench/master/playbooks/install.py +sudo python install.py --production --user frappe --mysql-root-password frappe --admin-password admin \ No newline at end of file diff --git a/vm/scripts/debian_family/setup.sh b/vm/scripts/debian_family/setup.sh new file mode 100644 index 00000000..743300e1 --- /dev/null +++ b/vm/scripts/debian_family/setup.sh @@ -0,0 +1,8 @@ +#!/bin/bash -eux + +# Add frappe user to sudoers. +echo "frappe ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers +sed -i "s/^.*requiretty/#Defaults requiretty/" /etc/sudoers + +# Disable daily apt unattended updates. +echo 'APT::Periodic::Enable "0";' >> /etc/apt/apt.conf.d/10periodic diff --git a/vm/scripts/install_ansible.sh b/vm/scripts/install_ansible.sh deleted file mode 100644 index 76787e24..00000000 --- a/vm/scripts/install_ansible.sh +++ /dev/null @@ -1,4 +0,0 @@ -sudo apt-get install -y software-properties-common -sudo apt-add-repository -y ppa:ansible/ansible -sudo apt-get update -sudo apt-get install -y ansible diff --git a/vm/scripts/restart_supervisor.sh b/vm/scripts/restart_supervisor.sh new file mode 100644 index 00000000..0afad870 --- /dev/null +++ b/vm/scripts/restart_supervisor.sh @@ -0,0 +1,14 @@ +#! /bin/bash + + +# Write out current crontab +crontab -l > current_cron + +# Echo new cron into cron file +echo "@reboot sleep 20 && systemctl restart supervisor" >> current_cron + +# Install new cron file +crontab current_cron + +# Delete the temporary cron file +rm current_cron \ No newline at end of file diff --git a/vm/scripts/set_message_develop.sh b/vm/scripts/set_message_develop.sh new file mode 100644 index 00000000..a1e94353 --- /dev/null +++ b/vm/scripts/set_message_develop.sh @@ -0,0 +1,18 @@ +#! /bin/bash + +message=" + ERPNext Evaluation VM (built on `date +\"%B %d, %Y\"`) + + Please access ERPNext by going to http://localhost:8000 on the host system. + The username is \"Administrator\" and password is \"admin\" + + Do consider donating at https://frappe.io/buy + + To update, login as + username: frappe + password: frappe + cd frappe-bench + bench update +" +echo "$message" | sudo tee -a /etc/issue +echo "$message" | sudo tee -a /etc/motd diff --git a/vm/scripts/set_message.sh b/vm/scripts/set_message_production.sh similarity index 75% rename from vm/scripts/set_message.sh rename to vm/scripts/set_message_production.sh index 7212727c..74182ca6 100644 --- a/vm/scripts/set_message.sh +++ b/vm/scripts/set_message_production.sh @@ -1,9 +1,10 @@ #! /bin/bash message=" - ERPNext evaluation VM (built on `date +\"%B %d, %Y\"`) + ERPNext VM (built on `date +\"%B %d, %Y\"`) Please access ERPNext by going to http://localhost:8080 on the host system. + The username is \"Administrator\" and password is \"admin\" Do consider donating at https://frappe.io/buy diff --git a/vm/scripts/zerofree.sh b/vm/scripts/zerofree.sh deleted file mode 100644 index 17b06a27..00000000 --- a/vm/scripts/zerofree.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/sh -sudo apt-get clean -sudo dd if=/dev/zero of=/EMPTY bs=1M -sudo rm -f /EMPTY diff --git a/vm/vm-develop.json b/vm/vm-develop.json new file mode 100644 index 00000000..d5173b93 --- /dev/null +++ b/vm/vm-develop.json @@ -0,0 +1,98 @@ +{ + "builders": [{ + "vm_name": "ERPNext-Develop-{{isotime \"20060102150405\"}}", + "output_directory": "Developer Builds", + "type": "virtualbox-iso", + "boot_command": [ + "", + "", + "", + "", + "/install/vmlinuz", + " auto", + " console-setup/ask_detect=false", + " console-setup/layoutcode=us", + " console-setup/modelcode=pc105", + " debconf/frontend=noninteractive", + " debian-installer=en_US", + " fb=false", + " initrd=/install/initrd.gz", + " kbd-chooser/method=us", + " keyboard-configuration/layout=USA", + " keyboard-configuration/variant=USA", + " locale=en_US", + " netcfg/get_domain=vm", + " netcfg/get_hostname=ubuntu", + " grub-installer/bootdev=/dev/sda", + " noapic", + " preseed/url=http://{{ .HTTPIP }}:{{ .HTTPPort }}/preseed.cfg", + " -- ", + "" + ], + "boot_wait": "10s", + "format": "ova", + "guest_os_type": "Ubuntu_64", + "headless": true, + "iso_url": "http://releases.ubuntu.com/16.04/ubuntu-16.04.4-server-amd64.iso", + "iso_checksum": "6a7f31eb125a0b2908cf2333d7777c82", + "iso_checksum_type": "md5", + "ssh_username": "frappe", + "ssh_password": "frappe", + "ssh_port": 22, + "ssh_wait_timeout": "10000s", + "http_directory": "http", + "guest_additions_mode": "disable", + "virtualbox_version_file": ".vbox_version", + "guest_additions_path": "VBoxGuestAdditions_{{.Version}}.iso", + "export_opts": [ + "--vsys", "0", + "--product", "ERPNext", + "--producturl", "https://erpnext.com", + "--vendor", "Frappe Techonologies", + "--vendorurl", "https://frappe.io", + "--description", "ERPNext Evaluation VM" + ], + "shutdown_command": "echo 'frappe'|sudo -S shutdown -P now", + "vboxmanage": [ + [ "modifyvm", "{{.Name}}", "--memory", "1024" ], + [ "modifyvm", "{{.Name}}", "--cpus", "1" ], + [ "modifyvm", "{{.Name}}", "--audio", "none" ], + [ "modifyvm", "{{.Name}}", "--natpf1", "vm_ssh,tcp,,3022,,22" ], + [ "modifyvm", "{{.Name}}", "--natpf1", "vm_http,tcp,,8080,,80" ], + [ "modifyvm", "{{.Name}}", "--natpf1", "vm_http2,tcp,,8000,,8000" ] + ] + }], + "provisioners": [{ + "type": "shell", + "execute_command": "echo 'frappe' | {{.Vars}} sudo -S -E bash '{{.Path}}'", + "script": "scripts/debian_family/install_ansible.sh" + }, { + "type": "shell", + "execute_command": "echo 'frappe' | {{.Vars}} sudo -S -E bash '{{.Path}}'", + "script": "scripts/debian_family/setup.sh" + }, { + "type": "shell", + "execute_command": "echo 'frappe' | {{.Vars}} sudo -S -E bash '{{.Path}}'", + "script": "scripts/debian_family/install_erpnext_develop.sh" + }, { + "type": "shell", + "execute_command": "echo 'frappe' | {{.Vars}} sudo -S -E bash '{{.Path}}'", + "script": "scripts/debian_family/cleanup.sh" + }, { + "type": "shell", + "script": "scripts/set_message_develop.sh" + }, { + "type": "shell", + "execute_command": "echo 'frappe' | {{.Vars}} sudo -S -E bash '{{.Path}}'", + "script": "scripts/restart_supervisor.sh" + }], + "post-processors": [{ + "type": "checksum", + "checksum_types": ["md5"] + }, { + "type": "vagrant", + "keep_input_artifact": true, + "output": "Developer Builds/ERPNext-Vagrant-Develop-{{isotime \"20060102150405\"}}.box", + "vagrantfile_template": "Vagrantfile" + }] +} diff --git a/vm/vm-production.json b/vm/vm-production.json new file mode 100644 index 00000000..76d2c3be --- /dev/null +++ b/vm/vm-production.json @@ -0,0 +1,92 @@ +{ + "builders": [{ + "vm_name": "ERPNext-Production-{{isotime \"20060102150405\"}}", + "output_directory": "Production Builds", + "type": "virtualbox-iso", + "boot_command": [ + "", + "", + "", + "", + "/install/vmlinuz", + " auto", + " console-setup/ask_detect=false", + " console-setup/layoutcode=us", + " console-setup/modelcode=pc105", + " debconf/frontend=noninteractive", + " debian-installer=en_US", + " fb=false", + " initrd=/install/initrd.gz", + " kbd-chooser/method=us", + " keyboard-configuration/layout=USA", + " keyboard-configuration/variant=USA", + " locale=en_US", + " netcfg/get_domain=vm", + " netcfg/get_hostname=ubuntu", + " grub-installer/bootdev=/dev/sda", + " noapic", + " preseed/url=http://{{ .HTTPIP }}:{{ .HTTPPort }}/preseed.cfg", + " -- ", + "" + ], + "boot_wait": "10s", + "format": "ova", + "guest_os_type": "Ubuntu_64", + "headless": true, + "iso_url": "http://releases.ubuntu.com/16.04/ubuntu-16.04.4-server-amd64.iso", + "iso_checksum": "6a7f31eb125a0b2908cf2333d7777c82", + "iso_checksum_type": "md5", + "ssh_username": "frappe", + "ssh_password": "frappe", + "ssh_port": 22, + "ssh_wait_timeout": "10000s", + "http_directory": "http", + "guest_additions_mode": "disable", + "virtualbox_version_file": ".vbox_version", + "guest_additions_path": "VBoxGuestAdditions_{{.Version}}.iso", + "export_opts": [ + "--vsys", "0", + "--product", "ERPNext", + "--producturl", "https://erpnext.com", + "--vendor", "Frappe Techonologies", + "--vendorurl", "https://frappe.io", + "--description", "ERPNext Evaluation VM" + ], + "shutdown_command": "echo 'frappe'|sudo -S shutdown -P now", + "vboxmanage": [ + [ "modifyvm", "{{.Name}}", "--memory", "1024" ], + [ "modifyvm", "{{.Name}}", "--cpus", "1" ], + [ "modifyvm", "{{.Name}}", "--audio", "none" ], + [ "modifyvm", "{{.Name}}", "--natpf1", "vm_ssh,tcp,,3022,,22" ], + [ "modifyvm", "{{.Name}}", "--natpf1", "vm_http,tcp,,8080,,80" ] + ] + }], + "provisioners": [{ + "type": "shell", + "execute_command": "echo 'frappe' | {{.Vars}} sudo -S -E bash '{{.Path}}'", + "script": "scripts/debian_family/install_ansible.sh" + }, { + "type": "shell", + "execute_command": "echo 'frappe' | {{.Vars}} sudo -S -E bash '{{.Path}}'", + "script": "scripts/debian_family/setup.sh" + }, { + "type": "shell", + "execute_command": "echo 'frappe' | {{.Vars}} sudo -S -E bash '{{.Path}}'", + "script": "scripts/debian_family/install_erpnext_production.sh" + }, { + "type": "shell", + "execute_command": "echo 'frappe' | {{.Vars}} sudo -S -E bash '{{.Path}}'", + "script": "scripts/debian_family/cleanup.sh" + }, { + "type": "shell", + "script": "scripts/set_message_production.sh" + }, { + "type": "shell", + "execute_command": "echo 'frappe' | {{.Vars}} sudo -S -E bash '{{.Path}}'", + "script": "scripts/restart_supervisor.sh" + }], + "post-processors": [{ + "type": "checksum", + "checksum_types": ["md5"] + }] +} diff --git a/vm/vm.json b/vm/vm.json deleted file mode 100644 index 403d92ae..00000000 --- a/vm/vm.json +++ /dev/null @@ -1,42 +0,0 @@ -{ - "builders":[{ - "vm_name": "ERPNext-{{isotime \"020106\"}}", - "type": "virtualbox-ovf", - "format": "ova", - "headless": true, - "source_path": "base.ova", - "ssh_username": "frappe", - "ssh_password": "frappe", - "guest_additions_mode": "disable", - "ssh_port": 22, - "ssh_wait_timeout": "300s", - "export_opts": ["--vsys", "0", "--product", "ERPNext", "--producturl", "https://erpnext.com", "--vendor", "Frappe Techonologies", "--vendorurl", "https://frappe.io", "--description", "ERPNext Evaluation VM"], - "shutdown_command": " echo 'packer' | sudo shutdown -P -h now", - "vboxmanage": [ - [ "modifyvm", "{{.Name}}", "--memory", "512" ], - [ "modifyvm", "{{.Name}}", "--audio", "none" ], - [ "modifyvm", "{{.Name}}", "--cpus", "1" ], - [ "modifyvm", "{{.Name}}", "--natpf1", "vm_ssh,tcp,,3022,,22" ], - [ "modifyvm", "{{.Name}}", "--natpf1", "vm_http,tcp,,8080,,80" ] - ] - } - ], - "provisioners": [{ - "type": "shell", - "script": "scripts/install_ansible.sh" - }, { - "type": "ansible-local", - "playbook_file": "ansible/vm.yml", - "playbook_dir": "ansible", - "group_vars": "ansible/vm_vars", - "role_paths": [ - "ansible/roles" - ] - }, { - "type": "shell", - "script": "scripts/set_message.sh" - }, { - "type": "shell", - "script": "scripts/zerofree.sh" - }] -}