diff --git a/.travis.yml b/.travis.yml index 89dbafb2..9c42c989 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,11 +1,14 @@ language: python dist: trusty +group: deprecated-2017Q2 sudo: required python: - "2.7" install: + - sudo rm /etc/apt/sources.list.d/docker.list + - sudo pip install urllib3 pyOpenSSL ndg-httpsclient pyasn1 - sudo apt-get purge -y mysql-common mysql-server mysql-client - sudo apt-get install --only-upgrade -y git - mkdir -p ~/.bench diff --git a/README.md b/README.md index 0ea8c84c..487c3946 100755 --- a/README.md +++ b/README.md @@ -82,6 +82,7 @@ Note: Please do not remove the bench directory the above commands will create - This is an opinionated setup so it is best to setup on a blank server. - Works on Ubuntu 14.04 to 16.04, CentOS 7+, Debian 7 to 8 and MacOS X. - You may have to install Python 2.7 (eg on Ubuntu 16.04+) by running `apt-get install python-minimal` +- You may also have to install build-essential and python-setuptools by running `apt-get install build-essential python-setuptools` - This script will install the pre-requisites, install bench and setup an ERPNext site - Passwords for Frappe Administrator and MariaDB (root) will be asked - You can then login as **Administrator** with the Administrator password @@ -134,6 +135,13 @@ For production: --- +## Docker Install - For Developers (beta) + +1. For developer setup, you can also use the official [Frappé Docker](https://github.com/frappe/frappe_docker/). +2. The app, mariadb and redis run on individual containers +3. This setup supports multi-tenancy and exposes the frappe-bench volume as a external storage. +4. For more details, [read the instructions on the Frappé Docker README](https://github.com/frappe/frappe_docker/) + Help ==== diff --git a/bench/app.py b/bench/app.py index 84827ea8..84103b7b 100755 --- a/bench/app.py +++ b/bench/app.py @@ -253,7 +253,7 @@ def get_repo_dir(app, bench_path='.'): return os.path.join(bench_path, 'apps', app) def switch_branch(branch, apps=None, bench_path='.', upgrade=False, check_upgrade=True): - from .utils import update_requirements, backup_all_sites, patch_sites, build_assets, pre_upgrade, post_upgrade + from .utils import update_requirements, update_npm_packages, backup_all_sites, patch_sites, build_assets, pre_upgrade, post_upgrade from . import utils apps_dir = os.path.join(bench_path, 'apps') version_upgrade = (False,) @@ -293,6 +293,7 @@ def switch_branch(branch, apps=None, bench_path='.', upgrade=False, check_upgrad if version_upgrade[0] and upgrade: update_requirements() + update_npm_packages() pre_upgrade(version_upgrade[1], version_upgrade[2]) reload(utils) backup_all_sites() diff --git a/bench/commands/__init__.py b/bench/commands/__init__.py index e31debab..86caf0bf 100755 --- a/bench/commands/__init__.py +++ b/bench/commands/__init__.py @@ -37,7 +37,7 @@ bench_command.add_command(switch_to_develop) from bench.commands.utils import (start, restart, set_nginx_port, set_ssl_certificate, set_ssl_certificate_key, set_url_root, set_mariadb_host, set_default_site, download_translations, shell, backup_site, backup_all_sites, release, renew_lets_encrypt, - disable_production, bench_src) + disable_production, bench_src, prepare_staging) bench_command.add_command(start) bench_command.add_command(restart) bench_command.add_command(set_nginx_port) @@ -51,6 +51,7 @@ bench_command.add_command(shell) bench_command.add_command(backup_site) bench_command.add_command(backup_all_sites) bench_command.add_command(release) +bench_command.add_command(prepare_staging) bench_command.add_command(renew_lets_encrypt) bench_command.add_command(disable_production) bench_command.add_command(bench_src) diff --git a/bench/commands/make.py b/bench/commands/make.py index e8a0d893..56d51c6e 100755 --- a/bench/commands/make.py +++ b/bench/commands/make.py @@ -10,13 +10,15 @@ import click @click.option('--no-backups',is_flag=True, help="Run migrations for all sites in the bench") @click.option('--no-auto-update',is_flag=True, help="Build JS and CSS artifacts for the bench") @click.option('--verbose',is_flag=True, help="Verbose output during install") +@click.option('--skip-bench-mkdir', is_flag=True, help="Skip mkdir frappe-bench") +@click.option('--skip-redis-config-generation', is_flag=True, help="Skip redis config generation if already specifying the common-site-config file") def init(path, apps_path, frappe_path, frappe_branch, no_procfile, no_backups, - no_auto_update, clone_from, verbose): + no_auto_update, clone_from, verbose, skip_bench_mkdir, skip_redis_config_generation): "Create a new bench" from bench.utils import init init(path, apps_path=apps_path, no_procfile=no_procfile, no_backups=no_backups, no_auto_update=no_auto_update, frappe_path=frappe_path, frappe_branch=frappe_branch, - verbose=verbose, clone_from=clone_from) + verbose=verbose, clone_from=clone_from, skip_bench_mkdir=skip_bench_mkdir, skip_redis_config_generation=skip_redis_config_generation) click.echo('Bench {} initialized'.format(path)) @@ -44,5 +46,3 @@ def remove_app(app_name): "completely remove app from bench" from bench.app import remove_app remove_app(app_name) - - diff --git a/bench/commands/setup.py b/bench/commands/setup.py index 5c635933..1d83c750 100755 --- a/bench/commands/setup.py +++ b/bench/commands/setup.py @@ -118,6 +118,12 @@ def setup_socketio(): from bench.utils import setup_socketio setup_socketio() +@click.command('requirements') +def setup_requirements(): + "Setup python and node requirements" + from bench.utils import update_requirements, update_npm_packages + update_requirements() + update_npm_packages() @click.command('config') def setup_config(): @@ -155,17 +161,18 @@ def remove_domain(domain, site=None): remove_domain(site, domain, bench_path='.') @click.command('sync-domains') -@click.argument('domains') +@click.option('--domain', multiple=True) @click.option('--site', prompt=True) -def sync_domains(domains, site=None): +def sync_domains(domain=None, site=None): from bench.config.site_config import sync_domains if not site: print("Please specify site") sys.exit(1) - domains = json.loads(domains) - if not isinstance(domains, list): + try: + domains = list(map(str,domain)) + except Exception: print("Domains should be a json list of strings or dictionaries") sys.exit(1) @@ -186,6 +193,7 @@ setup.add_command(setup_backups) setup.add_command(setup_env) setup.add_command(setup_procfile) setup.add_command(setup_socketio) +setup.add_command(setup_requirements) setup.add_command(setup_config) setup.add_command(setup_fonts) setup.add_command(add_domain) diff --git a/bench/commands/update.py b/bench/commands/update.py index b3bcc383..d38e82e0 100755 --- a/bench/commands/update.py +++ b/bench/commands/update.py @@ -3,7 +3,7 @@ import sys, os from bench.config.common_site_config import get_config from bench.app import pull_all_apps, is_version_upgrade from bench.utils import (update_bench, validate_upgrade, pre_upgrade, post_upgrade, before_update, - update_requirements, backup_all_sites, patch_sites, build_assets, restart_supervisor_processes) + update_requirements, update_npm_packages, backup_all_sites, patch_sites, build_assets, restart_supervisor_processes) from bench import patches #TODO: Not DRY @@ -62,7 +62,8 @@ def update(pull=False, patch=False, build=False, bench=False, auto=False, restar _update(pull, patch, build, bench, auto, restart_supervisor, requirements, no_backup, upgrade, force=force, reset=reset) -def _update(pull=False, patch=False, build=False, update_bench=False, auto=False, restart_supervisor=False, requirements=False, no_backup=False, upgrade=False, bench_path='.', force=False, reset=False): +def _update(pull=False, patch=False, build=False, update_bench=False, auto=False, restart_supervisor=False, + requirements=False, no_backup=False, upgrade=False, bench_path='.', force=False, reset=False): conf = get_config(bench_path=bench_path) version_upgrade = is_version_upgrade(bench_path=bench_path) @@ -78,8 +79,8 @@ def _update(pull=False, patch=False, build=False, update_bench=False, auto=False pull_all_apps(bench_path=bench_path, reset=reset) if requirements: - print('Updating Python libraries...') update_requirements(bench_path=bench_path) + update_npm_packages(bench_path=bench_path) if upgrade and (version_upgrade[0] or (not version_upgrade[0] and force)): pre_upgrade(version_upgrade[1], version_upgrade[2], bench_path=bench_path) diff --git a/bench/commands/utils.py b/bench/commands/utils.py index f8ed4a15..3a3202c4 100644 --- a/bench/commands/utils.py +++ b/bench/commands/utils.py @@ -119,17 +119,24 @@ def backup_all_sites(): @click.command('release') @click.argument('app') @click.argument('bump-type', type=click.Choice(['major', 'minor', 'patch', 'stable', 'prerelease'])) -@click.option('--develop', default='develop') -@click.option('--master', default='master') +@click.option('--from-branch', default='develop') +@click.option('--to-branch', default='master') @click.option('--remote', default='upstream') @click.option('--owner', default='frappe') @click.option('--repo-name') -def release(app, bump_type, develop, master, owner, repo_name, remote): +def release(app, bump_type, from_branch, to_branch, owner, repo_name, remote): "Release app (internal to the Frappe team)" from bench.release import release - release(bench_path='.', app=app, bump_type=bump_type, develop=develop, master=master, + release(bench_path='.', app=app, bump_type=bump_type, from_branch=from_branch, to_branch=to_branch, remote=remote, owner=owner, repo_name=repo_name) +@click.command('prepare-staging') +@click.argument('app') +def prepare_staging(app): + """Prepare staging branch from develop branch""" + from bench.prepare_staging import prepare_staging + prepare_staging(bench_path='.', app=app) + @click.command('disable-production') def disable_production(): diff --git a/bench/config/common_site_config.py b/bench/config/common_site_config.py index 02edfc3b..f99b07cb 100644 --- a/bench/config/common_site_config.py +++ b/bench/config/common_site_config.py @@ -62,7 +62,7 @@ def update_config_for_frappe(config, bench_path): if key not in config: config[key] = "redis://localhost:{0}".format(ports[key]) - for key in ('webserver_port', 'socketio_port'): + for key in ('webserver_port', 'socketio_port', 'file_watcher_port'): if key not in config: config[key] = ports[key] @@ -75,6 +75,7 @@ def make_ports(bench_path): default_ports = { "webserver_port": 8000, "socketio_port": 9000, + "file_watcher_port": 6787, "redis_queue": 11000, "redis_socketio": 12000, "redis_cache": 13000 diff --git a/bench/config/lets_encrypt.py b/bench/config/lets_encrypt.py index d21ec580..f09a75bf 100755 --- a/bench/config/lets_encrypt.py +++ b/bench/config/lets_encrypt.py @@ -81,12 +81,12 @@ def run_certbot_and_setup_ssl(site, custom_domain, bench_path): def setup_crontab(): job_command = 'sudo service nginx stop && /opt/certbot-auto renew && sudo service nginx start' - user_crontab = CronTab() - if job_command not in str(user_crontab): - job = user_crontab.new(command=job_command, comment="Renew lets-encrypt every month") + system_crontab = CronTab(tabfile='/etc/crontab', user=True) + if job_command not in str(system_crontab): + job = system_crontab.new(command=job_command, comment="Renew lets-encrypt every month") job.every().month() job.enable() - user_crontab.write() + system_crontab.write() def create_dir_if_missing(path): diff --git a/bench/package.json b/bench/package.json new file mode 100644 index 00000000..31121283 --- /dev/null +++ b/bench/package.json @@ -0,0 +1,18 @@ +{ + "name": "frappe", + "description": "Default package.json for frappe apps", + "dependencies": { + "babel-core": "^6.24.1", + "babel-preset-babili": "0.0.12", + "babel-preset-es2015": "^6.24.1", + "babel-preset-es2016": "^6.24.1", + "babel-preset-es2017": "^6.24.1", + "chokidar": "^1.7.0", + "cookie": "^0.3.1", + "express": "^4.15.3", + "less": "^2.7.2", + "redis": "^2.7.1", + "socket.io": "^2.0.1", + "superagent": "^3.5.2" + } +} diff --git a/bench/prepare_staging.py b/bench/prepare_staging.py new file mode 100755 index 00000000..baa2a7c3 --- /dev/null +++ b/bench/prepare_staging.py @@ -0,0 +1,73 @@ +#! env python +import os +import git +import click +from .config.common_site_config import get_config + +github_username = None +github_password = None + +def prepare_staging(bench_path, app, remote='upstream'): + from .release import get_release_message + validate(bench_path) + + repo_path = os.path.join(bench_path, 'apps', app) + update_branches(repo_path, remote) + message = get_release_message(repo_path, from_branch='develop', to_branch='staging', remote=remote) + + if not message: + print('No commits to release') + return + + print() + print(message) + print() + + click.confirm('Do you want to continue?', abort=True) + + create_staging(repo_path) + push_commits(repo_path) + +def validate(bench_path): + from .release import validate + + config = get_config(bench_path) + validate(bench_path, config) + +def update_branches(repo_path, remote): + from .release import update_branch + update_branch(repo_path, 'staging', remote) + update_branch(repo_path, 'develop', remote) + + git.Repo(repo_path).git.checkout('develop') + +def create_staging(repo_path, from_branch='develop'): + from .release import handle_merge_error + + print('creating staging from', from_branch) + repo = git.Repo(repo_path) + g = repo.git + g.checkout('staging') + try: + g.merge(from_branch, '--no-ff') + except git.exc.GitCommandError as e: + handle_merge_error(e, source=from_branch, target='staging') + + g.checkout(from_branch) + try: + g.merge('staging') + except git.exc.GitCommandError as e: + handle_merge_error(e, source='staging', target=from_branch) + +def push_commits(repo_path, remote='upstream'): + print('pushing staging branch of', repo_path) + + repo = git.Repo(repo_path) + g = repo.git + + args = [ + 'develop:develop', + 'staging:staging' + ] + + print(g.push(remote, *args)) diff --git a/bench/release.py b/bench/release.py index caea605e..c58a446a 100755 --- a/bench/release.py +++ b/bench/release.py @@ -4,10 +4,8 @@ import os import sys import semantic_version import git -import json import requests import getpass -import argparse import re from requests.auth import HTTPBasicAuth import requests.exceptions @@ -15,30 +13,39 @@ from time import sleep from .config.common_site_config import get_config import click +branches_to_update = { + 'develop': [], + 'hotfix': ['develop'] +} + github_username = None github_password = None -def release(bench_path, app, bump_type, develop='develop', master='master', +def release(bench_path, app, bump_type, from_branch='develop', to_branch='master', remote='upstream', owner='frappe', repo_name=None): - validate(bench_path) - - bump(bench_path, app, bump_type, develop=develop, master=master, owner=owner, - repo_name=repo_name, remote=remote) - -def validate(bench_path): config = get_config(bench_path) + if not config.get('release_bench'): print('bench not configured to release') sys.exit(1) + if config.get('branches_to_update'): + branches_to_update.update(config.get('branches_to_update')) + + validate(bench_path, config) + + bump(bench_path, app, bump_type, from_branch=from_branch, to_branch=to_branch, owner=owner, + repo_name=repo_name, remote=remote) + +def validate(bench_path, config): global github_username, github_password github_username = config.get('github_username') github_password = config.get('github_password') if not github_username: - github_username = input('Username: ') + github_username = click.prompt('Username', type=str) if not github_password: github_password = getpass.getpass() @@ -46,12 +53,12 @@ def validate(bench_path): r = requests.get('https://api.github.com/user', auth=HTTPBasicAuth(github_username, github_password)) r.raise_for_status() -def bump(bench_path, app, bump_type, develop, master, remote, owner, repo_name=None): +def bump(bench_path, app, bump_type, from_branch, to_branch, remote, owner, repo_name=None): assert bump_type in ['minor', 'major', 'patch', 'stable', 'prerelease'] repo_path = os.path.join(bench_path, 'apps', app) - update_branches_and_check_for_changelog(repo_path, develop, master, remote=remote) - message = get_release_message(repo_path, develop=develop, master=master, remote=remote) + update_branches_and_check_for_changelog(repo_path, from_branch, to_branch, remote=remote) + message = get_release_message(repo_path, from_branch=from_branch, to_branch=to_branch, remote=remote) if not message: print('No commits to release') @@ -63,21 +70,22 @@ def bump(bench_path, app, bump_type, develop, master, remote, owner, repo_name=N click.confirm('Do you want to continue?', abort=True) - new_version = bump_repo(repo_path, bump_type, develop=develop, master=master) + new_version = bump_repo(repo_path, bump_type, from_branch=from_branch, to_branch=to_branch) commit_changes(repo_path, new_version) - tag_name = create_release(repo_path, new_version, develop=develop, master=master) - push_release(repo_path, develop=develop, master=master, remote=remote) + tag_name = create_release(repo_path, new_version, from_branch=from_branch, to_branch=to_branch) + push_release(repo_path, from_branch=from_branch, to_branch=to_branch, remote=remote) create_github_release(repo_path, tag_name, message, remote=remote, owner=owner, repo_name=repo_name) print('Released {tag} for {repo_path}'.format(tag=tag_name, repo_path=repo_path)) -def update_branches_and_check_for_changelog(repo_path, develop='develop', master='master', remote='upstream'): +def update_branches_and_check_for_changelog(repo_path, from_branch='develop', to_branch='master', remote='upstream'): - update_branch(repo_path, master, remote=remote) - update_branch(repo_path, develop, remote=remote) - if develop != 'develop': - update_branch(repo_path, 'develop', remote=remote) + update_branch(repo_path, to_branch, remote=remote) + update_branch(repo_path, from_branch, remote=remote) - git.Repo(repo_path).git.checkout(develop) + for branch in branches_to_update[from_branch]: + update_branch(repo_path, branch, remote=remote) + + git.Repo(repo_path).git.checkout(from_branch) check_for_unmerged_changelog(repo_path) def update_branch(repo_path, branch, remote): @@ -94,18 +102,18 @@ def check_for_unmerged_changelog(repo_path): if os.path.exists(current) and [f for f in os.listdir(current) if f != "readme.md"]: raise Exception("Unmerged change log! in " + repo_path) -def get_release_message(repo_path, develop='develop', master='master', remote='upstream'): - print('getting release message for', repo_path, 'comparing', master, '...', develop) +def get_release_message(repo_path, from_branch='develop', to_branch='master', remote='upstream'): + print('getting release message for', repo_path, 'comparing', to_branch, '...', from_branch) repo = git.Repo(repo_path) g = repo.git - log = g.log('{remote}/{master}..{remote}/{develop}'.format( - remote=remote, master=master, develop=develop), '--format=format:%s', '--no-merges') + log = g.log('{remote}/{to_branch}..{remote}/{from_branch}'.format( + remote=remote, to_branch=to_branch, from_branch=from_branch), '--format=format:%s', '--no-merges') if log: return "* " + log.replace('\n', '\n* ') -def bump_repo(repo_path, bump_type, develop='develop', master='master'): +def bump_repo(repo_path, bump_type, from_branch='develop', to_branch='master'): current_version = get_current_version(repo_path) new_version = get_bumped_version(current_version, bump_type) @@ -199,32 +207,32 @@ def commit_changes(repo_path, new_version): repo.index.add([os.path.join(app_name, '__init__.py')]) repo.index.commit('bumped to version {}'.format(new_version)) -def create_release(repo_path, new_version, develop='develop', master='master'): +def create_release(repo_path, new_version, from_branch='develop', to_branch='master'): print('creating release for version', new_version) repo = git.Repo(repo_path) g = repo.git - g.checkout(master) + g.checkout(to_branch) try: - g.merge(develop, '--no-ff') + g.merge(from_branch, '--no-ff') except git.exc.GitCommandError as e: - handle_merge_error(e, source=develop, target=master) + handle_merge_error(e, source=from_branch, target=to_branch) tag_name = 'v' + new_version repo.create_tag(tag_name, message='Release {}'.format(new_version)) - g.checkout(develop) + g.checkout(from_branch) try: - g.merge(master) + g.merge(to_branch) except git.exc.GitCommandError as e: - handle_merge_error(e, source=master, target=develop) + handle_merge_error(e, source=to_branch, target=from_branch) - if develop != 'develop': - print('merging master into develop') - g.checkout('develop') + for branch in branches_to_update[from_branch]: + print('merging master into', branch) + g.checkout(branch) try: - g.merge(master) + g.merge(to_branch) except git.exc.GitCommandError as e: - handle_merge_error(e, source=master, target='develop') + handle_merge_error(e, source=to_branch, target=branch) return tag_name @@ -236,18 +244,18 @@ def handle_merge_error(e, source, target): print('-'*80) click.confirm('Have you manually resolved the error?', abort=True) -def push_release(repo_path, develop='develop', master='master', remote='upstream'): - print('pushing branches', master, develop, 'of', repo_path) +def push_release(repo_path, from_branch='develop', to_branch='master', remote='upstream'): + print('pushing branches', to_branch, from_branch, 'of', repo_path) repo = git.Repo(repo_path) g = repo.git args = [ - '{master}:{master}'.format(master=master), - '{develop}:{develop}'.format(develop=develop) + '{to_branch}:{to_branch}'.format(to_branch=to_branch), + '{from_branch}:{from_branch}'.format(from_branch=from_branch) ] - if develop != 'develop': - print('pushing develop branch of', repo_path) - args.append('develop:develop') + for branch in branches_to_update[from_branch]: + print('pushing {0} branch of'.format(branch), repo_path) + args.append('{branch}:{branch}'.format(branch=branch)) args.append('--tags') diff --git a/bench/tests/test_init.py b/bench/tests/test_init.py index d1045a6f..b4c16b47 100755 --- a/bench/tests/test_init.py +++ b/bench/tests/test_init.py @@ -40,6 +40,7 @@ class TestBenchInit(unittest.TestCase): self.assert_common_site_config("test-bench-1", { "webserver_port": 8000, "socketio_port": 9000, + "file_watcher_port": 6787, "redis_queue": "redis://localhost:11000", "redis_socketio": "redis://localhost:12000", "redis_cache": "redis://localhost:13000" @@ -51,6 +52,7 @@ class TestBenchInit(unittest.TestCase): self.assert_common_site_config("test-bench-2", { "webserver_port": 8001, "socketio_port": 9001, + "file_watcher_port": 6788, "redis_queue": "redis://localhost:11001", "redis_socketio": "redis://localhost:12001", "redis_cache": "redis://localhost:13001" @@ -210,7 +212,7 @@ class TestBenchInit(unittest.TestCase): self.assert_exists(python_path, "site-packages", "pip") site_packages = os.listdir(os.path.join(python_path, "site-packages")) - self.assertTrue(any(package.startswith("MySQL_python-1.2.5") for package in site_packages)) + self.assertTrue(any(package.startswith("mysqlclient-1.3.10") for package in site_packages)) def assert_config(self, bench_name): for config, search_key in ( diff --git a/bench/utils.py b/bench/utils.py index bd82d835..b0859231 100755 --- a/bench/utils.py +++ b/bench/utils.py @@ -2,6 +2,8 @@ import os, sys, shutil, subprocess, logging, itertools, requests, json, platform from distutils.spawn import find_executable import bench from bench import env +from six import iteritems + class PatchError(Exception): pass @@ -25,21 +27,27 @@ def get_env_cmd(cmd, bench_path='.'): def init(path, apps_path=None, no_procfile=False, no_backups=False, no_auto_update=False, frappe_path=None, frappe_branch=None, wheel_cache_dir=None, - verbose=False, clone_from=None): + verbose=False, clone_from=None, skip_bench_mkdir=False, skip_redis_config_generation=False): from .app import get_app, install_apps_from_path from .config.common_site_config import make_config from .config import redis from .config.procfile import setup_procfile from bench.patches import set_all_patches_executed - if os.path.exists(path): - print('Directory {} already exists!'.format(path)) - raise Exception("Site directory already exists") - # sys.exit(1) + if(skip_bench_mkdir): + pass + else: + if os.path.exists(path): + print('Directory {} already exists!'.format(path)) + raise Exception("Site directory already exists") + os.makedirs(path) - os.makedirs(path) for dirname in folders_in_bench: - os.mkdir(os.path.join(path, dirname)) + try: + os.makedirs(os.path.join(path, dirname)) + except OSError, e: + if e.errno != os.errno.EEXIST: + pass setup_logging() @@ -61,11 +69,13 @@ def init(path, apps_path=None, no_procfile=False, no_backups=False, bench.set_frappe_version(bench_path=path) if bench.FRAPPE_VERSION > 5: - setup_socketio(bench_path=path) + update_npm_packages(bench_path=path) set_all_patches_executed(bench_path=path) build_assets(bench_path=path) - redis.generate_config(path) + + if not skip_redis_config_generation: + redis.generate_config(path) if not no_procfile: setup_procfile(path) @@ -119,7 +129,7 @@ def exec_cmd(cmd, cwd='.'): logger.info(cmd) - p = subprocess.Popen(cmd, cwd=cwd, shell=True, stdout=stdout, stderr=stderr) + p = subprocess.Popen(cmd, cwd=cwd, shell=True, stdout=stdout, stderr=stderr, universal_newlines=True) if async: return_code = print_output(p) @@ -134,10 +144,12 @@ def setup_env(bench_path='.'): exec_cmd('./env/bin/pip -q install --upgrade pip', cwd=bench_path) exec_cmd('./env/bin/pip -q install wheel', cwd=bench_path) # exec_cmd('./env/bin/pip -q install https://github.com/frappe/MySQLdb1/archive/MySQLdb-1.2.5-patched.tar.gz', cwd=bench_path) + exec_cmd('./env/bin/pip -q install six', cwd=bench_path) exec_cmd('./env/bin/pip -q install -e git+https://github.com/frappe/python-pdfkit.git#egg=pdfkit', cwd=bench_path) def setup_socketio(bench_path='.'): - exec_cmd("npm install socket.io redis express superagent cookie", cwd=bench_path) + exec_cmd("npm install socket.io redis express superagent cookie babel-core less chokidar \ + babel-cli babel-preset-es2015 babel-preset-es2016 babel-preset-es2017 babel-preset-babili", cwd=bench_path) def patch_sites(bench_path='.'): bench.set_frappe_version(bench_path=bench_path) @@ -352,6 +364,7 @@ def set_default_site(site, bench_path='.'): cwd=os.path.join(bench_path, 'sites')) def update_requirements(bench_path='.'): + print('Updating Python libraries...') pip = os.path.join(bench_path, 'env', 'bin', 'pip') # upgrade pip to latest @@ -367,6 +380,38 @@ def update_requirements(bench_path='.'): req_file = os.path.join(apps_dir, app, 'requirements.txt') install_requirements(pip, req_file) +def update_npm_packages(bench_path='.'): + print('Updating node libraries...') + apps_dir = os.path.join(bench_path, 'apps') + package_json = {} + + for app in os.listdir(apps_dir): + package_json_path = os.path.join(apps_dir, app, 'package.json') + + if os.path.exists(package_json_path): + with open(package_json_path, "r") as f: + app_package_json = json.loads(f.read()) + # package.json is usually a dict in a dict + for key, value in iteritems(app_package_json): + if not key in package_json: + package_json[key] = value + else: + if isinstance(value, dict): + package_json[key].update(value) + elif isinstance(value, list): + package_json[key].extend(value) + else: + package_json[key] = value + + if package_json is {}: + with open(os.path.join(os.path.dirname(__file__), 'package.json'), 'r') as f: + package_json = json.loads(f.read()) + + with open(os.path.join(bench_path, 'package.json'), 'w') as f: + f.write(json.dumps(package_json, indent=1, sort_keys=True)) + + exec_cmd('npm install', cwd=bench_path) + def install_requirements(pip, req_file): if os.path.exists(req_file): exec_cmd("{pip} install -q -r {req_file}".format(pip=pip, req_file=req_file)) diff --git a/docs/branch_details.md b/docs/branch_details.md new file mode 100644 index 00000000..413d939a --- /dev/null +++ b/docs/branch_details.md @@ -0,0 +1,13 @@ +### ERPNext/Frappe Branching + +#### Branch Description + - `develop` Branch: All new feature developments will go in develop branch + - `staging` Branch: This branch serves as a release candidate. Before a week, release team will pull the feature from develop branch to staging branch. + EG: if the feature is in 25 July's milestone then it should go in staging on 19th July. + - `master` Branch: Community release. + - `hotfix` Branch: mainly define for support issues. This will include bugs or any high priority task like security patches. + +#### Where to send PR? + - If you are working on a new feature, then PR should point to develop branch + - If you are working on support issue / bug / error report, then PR should point to hotfix brach + - While performing testing on Staging branch, if any fix needed then only send that fix PR to staging. \ No newline at end of file diff --git a/docs/contribution_guidelines.md b/docs/contribution_guidelines.md new file mode 100644 index 00000000..9d6ba368 --- /dev/null +++ b/docs/contribution_guidelines.md @@ -0,0 +1,46 @@ +# Contribution Guidelines + +### Introduction (for first timers) + +Thank you for your interesting in contributing to an open source project! Our world works on people taking initiative to contribute to the "commons" and contributing to open source means you are contributing to make things better for not only yourself, but everyone else too! So thank you for taking this initiative. + +Great projects also work because of great quality. Open source or not, the user really cares that things should work as they are advertised, and consistently. New features should follow the same pattern and so that users don't have to learn things again and again. + +Developers who maintain open source also expect that you follow certain guidelines. These guidelines ensure that developers are able quickly give feedback on your contribution and how to make it better. Most probably you might have to go back and change a few things, but it will be in th interest of making this process better for everyone. So do be prepared for some back and forth. + +Happy contributing! + +### Feedback Policy + +We will strive for a "Zero Pull Request Pending" policy, inspired by "Zero Inbox". This means, that if the pull request is good, it will be merged within a day and if it does not meet the requirements, it will be closed. + +### Design Guides + +Please read the following design guidelines carefully when contributing: + +1. [Form Design Guidelines](https://github.com/frappe/erpnext/wiki/Form-Design-Guidelines) +1. [How to break large contributions into smaller ones](https://github.com/frappe/erpnext/wiki/Cascading-Pull-Requests) + +### Pull Request Requirements + +1. **Test Cases:** Important to add test cases, even if its a very simple one that just calls the function. For UI, till we don't have Selenium testing setup, we need to see a screenshot / animated GIF. +1. **UX:** If your change involves user experience, add a screenshot / narration / animated GIF. +1. **Documentation:** Test Case must involve updating necessary documentation +1. **Explanation:** Include explanation if there is a design change, explain the use case and why this suggested change is better. If you are including a new library or replacing one, please give sufficient reference of why the suggested library is better. +1. **Demo:** Remember to update the demo script so that data related your feature is included in the demo. +1. **Failing Tests:** This is simple, you must make sure all automated tests are passing. +1. **Very Large Contribution:** It is very hard to accept and merge very large contributions, because there are too many lines of code to check and its implications can be large and unexpected. They way to contribute big features is to build them part by part. We can understand there are exceptions, but in most cases try and keep your pull-request to **30 lines of code** excluding tests and config files. **Use [Cascading Pull Requests](https://github.com/frappe/erpnext/wiki/Cascading-Pull-Requests)** for large features. +1. **Incomplete Contributions must be hidden:** If the contribution is WIP or incomplete - which will most likely be the case, you can send small PRs as long as the user is not exposed to unfinished functionality. This will ensure that your code does not have build or other collateral issues. But these features must remain completely hidden to the user. +1. **Incorrect Patches:** If your design involves schema change and you must include patches that update the data as per your new schema. +1. **Incorrect Naming:** The naming of variables, models, fields etc must be consistent as per the existing design and semantics used in the system. +1. **Translated Strings:** All user facing strings / text must be wrapped in the `__("")` function in javascript and `_("")` function in Python, so that it is shown as translated to the user. +1. **Deprecated API:** The API used in the pull request must be the latest recommended methods and usage of globals like `cur_frm` must be avoided. +1. **Whitespace and indentation:** The ERPNext and Frappe Project uses tabs (I know and we are sorry, but its too much effort to change it now and we don't want to lose the history). The indentation must be consistent whether you are writing Javascript or Python. Multi-line strings or expressions must also be consistently indented, not hanging like a bee hive at the end of the line. We just think the code looks a lot more stable that way. + +#### What if my Pull Request is closed? + +Don't worry, fix the problem and re-open it! + +#### Why do we follow this policy? + +This is because ERPNext is at a stage where it is being used by thousands of companies and introducing breaking changes can be harmful for everyone. Also we do not want to stop the speed of contributions and the best way to encourage contributors is to give fast feedback. \ No newline at end of file diff --git a/docs/release_policy.md b/docs/release_policy.md new file mode 100644 index 00000000..c105ed56 --- /dev/null +++ b/docs/release_policy.md @@ -0,0 +1,63 @@ +# Release Policy + +#### Definitions: + - `develop` Branch: All new feature developments will go in develop branch + - `staging` Branch: This branch serves as a release candidate. Before a week, release team will pull the feature from develop branch to staging branch. + EG: if the feature is in 25 July's milestone then it should go in staging on 19th July. + - `master` Branch: `master` branch serves as a stable branch. This will use as production deployment. + - `hotfix` Branch: mainly define for support issues. This will include bugs or any high priority task like security patches. + +#### Create release from staging +- On Tuesday, we will release from staging to master. + +- Versioning: Given a version number MAJOR.MINOR.PATCH, increment the: + - MAJOR version when you make incompatible API changes, + - MINOR version when you add functionality in a backwards-compatible manner, and + - PATCH version when you make backwards-compatible bug fixes. + +- Impact on branches: + - merge staging branch to master + - push merge commit back to staging branch + - push merge commit to develop branch + - push merge commit to hotfix branch + +- Use release command to create release, +``` usage: bench release APP patch|minor|major --from-branch staging ``` + +--- + +#### Create staging branch + +- On Wednesday morning, `develop` will be merge into `staging`. `staging` branch is a release candidate. All new features will first go from `develop` to `staging` and then `staging` to `master`. + +- Use the prepare-staging command to create staging branch +```usage: bench prepare-staging APP``` + +- Impact on branches? + - merge all commits from develop branch to staging + - push merge commit back to develop + +- QA will use staging for testing. + +- Deploy staging branch on frappe.io, erpnext.org, frappe.erpnext.com. + +- Only regression and security fixes can be cherry-picked into staging + +- Create a discuss post on what all new features or fixes going in next version. + +--- + +#### Create release from hotfix +- Depending on priority, hotfix release will take place. + +- Versioning: + - PATCH version when you make backwards-compatible bug fixes. + +- Impact on branches: + - merge hotfix branch to master + - push merge commit back to staging branch + - push merge commit to develop branch + - push merge commit to staging branch + +- Use release command to create release, +``` usage: bench release APP patch --from-branch hotfix ``` diff --git a/docs/releasing_frappe_erpext.md b/docs/releasing_frappe_erpext.md new file mode 100644 index 00000000..606c4c4d --- /dev/null +++ b/docs/releasing_frappe_erpext.md @@ -0,0 +1,39 @@ +# Releasing Frappe ERPNext + +* Make a new bench dedicated for releasing +``` +bench init release-bench --frappe-path git@github.com:frappe/frappe.git +``` + +* Get ERPNext in the release bench +``` +bench get-app erpnext git@github.com:frappe/erpnext.git +``` + +* Configure as release bench. Add this to the common_site_config.json +``` +"release_bench": true, +``` + +* Add branches to update in common_site_config.json +``` +"branches_to_update": { + "staging": ["develop", "hotfix"], + "hotfix": ["develop", "staging"] +} +``` + +* Use the release commands to release +``` +Usage: bench release [OPTIONS] APP BUMP_TYPE +``` + +* Arguments : + * _APP_ App name e.g [frappe|erpnext|yourapp] + * _BUMP_TYPE_ [major|minor|patch|stable|prerelease] +* Options: + * --from-branch git develop branch, default is develop + * --to-branch git master branch, default is master + * --remote git remote, default is upstream + * --owner git owner, default is frappe + * --repo-name git repo name if different from app name \ No newline at end of file diff --git a/playbooks/develop/create_user.yml b/playbooks/develop/create_user.yml index 306dabc1..2980ed4e 100755 --- a/playbooks/develop/create_user.yml +++ b/playbooks/develop/create_user.yml @@ -10,14 +10,24 @@ file: path: '/home/{{ frappe_user }}' mode: 'o+rx' + owner: '{{ frappe_user }}' + group: '{{ frappe_user }}' + recurse: yes when: ansible_distribution == 'Ubuntu' or ansible_distribution == 'CentOS' or ansible_distribution == 'Debian' - name: Set home folder perms file: path: '/Users/{{ frappe_user }}' mode: 'o+rx' + owner: '{{ frappe_user }}' + group: '{{ frappe_user }}' + recurse: yes when: ansible_distribution == 'MacOSX' - name: Set /tmp/.bench folder perms - command: 'chown -R {{ frappe_user }}:{{ frappe_user }} {{ repo_path }}' + file: + path: '{{ repo_path }}' + owner: '{{ frappe_user }}' + group: '{{ frappe_user }}' + recurse: yes when: ansible_distribution == 'Ubuntu' or ansible_distribution == 'CentOS' or ansible_distribution == 'Debian' \ No newline at end of file diff --git a/playbooks/develop/includes/setup_bench.yml b/playbooks/develop/includes/setup_bench.yml index f25cf9f0..7b768c4c 100644 --- a/playbooks/develop/includes/setup_bench.yml +++ b/playbooks/develop/includes/setup_bench.yml @@ -18,6 +18,12 @@ become: yes become_user: root + - name: Overwrite bench if required + file: + state: absent + path: "{{ bench_path }}" + when: overwrite + - name: Check whether bench exists stat: path="{{ bench_path }}" register: bench_stat diff --git a/playbooks/develop/includes/setup_dev_env.yml b/playbooks/develop/includes/setup_dev_env.yml index 3b54c2af..4e07360e 100644 --- a/playbooks/develop/includes/setup_dev_env.yml +++ b/playbooks/develop/includes/setup_dev_env.yml @@ -1,10 +1,4 @@ --- - # Setup Socketio - - name: setup procfile - command: bench setup socketio - args: - creates: "{{ bench_path }}/node_modules" - chdir: "{{ bench_path }}" # Setup Procfile - name: setup procfile diff --git a/playbooks/develop/includes/wkhtmltopdf.yml b/playbooks/develop/includes/wkhtmltopdf.yml index bdd7e419..ea4c0c28 100644 --- a/playbooks/develop/includes/wkhtmltopdf.yml +++ b/playbooks/develop/includes/wkhtmltopdf.yml @@ -1,7 +1,7 @@ --- - name: download wkthmltox linux - get_url: url=http://download.gna.org/wkhtmltopdf/0.12/0.12.3/wkhtmltox-0.12.3_linux-generic-{{ "amd64" if ansible_architecture == "x86_64" else "i386"}}.tar.xz dest=/tmp/wkhtmltox.tar.xz - + get_url: url=https://github.com/frappe/wkhtmltopdf/raw/master/wkhtmltox-0.12.3_linux-generic-{{ "amd64" if ansible_architecture == "x86_64" else "i386"}}.tar.xz dest=/tmp/wkhtmltox.tar.xz + - name: Creates directory file: path=/tmp/wkhtmltox state=directory diff --git a/playbooks/install.py b/playbooks/install.py index 657cf465..1c6f7406 100755 --- a/playbooks/install.py +++ b/playbooks/install.py @@ -61,7 +61,7 @@ def install_bench(args): # Restricting ansible version due to following bug in ansible 2.1 # https://github.com/ansible/ansible-modules-core/issues/3752 success = run_os_command({ - 'pip': "sudo pip install 'ansible==2.0.2.0'" + 'pip': "sudo pip install ansible" }) if not success: @@ -97,7 +97,7 @@ def install_bench(args): extra_vars.update(repo_path=repo_path) run_playbook('develop/create_user.yml', extra_vars=extra_vars) - extra_vars.update(get_passwords(args.run_travis or args.without_bench_setup)) + extra_vars.update(get_passwords(args)) if args.production: extra_vars.update(max_worker_connections=multiprocessing.cpu_count() * 1024) @@ -229,9 +229,31 @@ def could_not_install(package): def is_sudo_user(): return os.geteuid() == 0 -def get_passwords(ignore_prompt=False): + +def get_passwords(args): + """ + Returns a dict of passwords for further use + and creates passwords.txt in the bench user's home directory + """ + + ignore_prompt = args.run_travis or args.without_bench_setup + mysql_root_password, admin_password = '', '' + passwords_file_path = os.path.join(os.path.expanduser('~' + args.user), 'passwords.txt') + if not ignore_prompt: - mysql_root_password, admin_password = '', '' + # set passwords from existing passwords.txt + if os.path.isfile(passwords_file_path): + with open(passwords_file_path, 'r') as f: + passwords = json.load(f) + mysql_root_password, admin_password = passwords['mysql_root_password'], passwords['admin_password'] + + # set passwords from cli args + if args.mysql_root_password: + mysql_root_password = args.mysql_root_password + if args.admin_password: + admin_password = args.admin_password + + # prompt for passwords pass_set = True while pass_set: # mysql root password @@ -262,7 +284,6 @@ def get_passwords(ignore_prompt=False): } if not ignore_prompt: - passwords_file_path = os.path.join(os.path.expanduser('~'), 'passwords.txt') with open(passwords_file_path, 'w') as f: json.dump(passwords, f, indent=1) @@ -270,6 +291,7 @@ def get_passwords(ignore_prompt=False): return passwords + def get_extra_vars_json(extra_args): # We need to pass production as extra_vars to the playbook to execute conditionals in the # playbook. Extra variables can passed as json or key=value pair. Here, we will use JSON. @@ -334,6 +356,14 @@ def parse_commandline_args(): parser.add_argument('--without-bench-setup', dest='without_bench_setup', action='store_true', default=False, help=argparse.SUPPRESS) + + # whether to overwrite an existing bench + parser.add_argument('--overwrite', dest='overwrite', action='store_true', default=False, + help='Whether to overwrite an existing bench') + + # set passwords + parser.add_argument('--mysql-root-password', dest='mysql_root_password', help='Set mysql root password') + parser.add_argument('--admin-password', dest='admin_password', help='Set admin password') args = parser.parse_args() diff --git a/vm/ansible/roles/wkhtmltopdf/tasks/main.yml b/vm/ansible/roles/wkhtmltopdf/tasks/main.yml index fbd0c775..52c95925 100644 --- a/vm/ansible/roles/wkhtmltopdf/tasks/main.yml +++ b/vm/ansible/roles/wkhtmltopdf/tasks/main.yml @@ -18,7 +18,7 @@ when: ansible_os_family == 'Debian' - name: download wkthmltox linux - get_url: url=http://download.gna.org/wkhtmltopdf/0.12/0.12.3/wkhtmltox-0.12.3_linux-generic-{{ "amd64" if ansible_architecture == "x86_64" else "i386"}}.tar.xz dest=/tmp/wkhtmltox.tar.xz + get_url: url=https://github.com/frappe/wkhtmltopdf/raw/master/wkhtmltox-0.12.3_linux-generic-{{ "amd64" if ansible_architecture == "x86_64" else "i386"}}.tar.xz dest=/tmp/wkhtmltox.tar.xz - name: unarchive wkhtmltopdf unarchive: src=/tmp/wkhtmltox.tar.xz dest=/tmp/wkhtmltox