From bf56902cd7bccfdc544984216dd58c383ee132a7 Mon Sep 17 00:00:00 2001 From: shreyas Date: Fri, 27 May 2016 12:25:12 +0530 Subject: [PATCH 1/8] [Fix] Ansible installer fails to install wkhtmltopdf with newer ansible version --- playbooks/install.py | 79 ++++++++++++++++++++++++-------------------- 1 file changed, 44 insertions(+), 35 deletions(-) diff --git a/playbooks/install.py b/playbooks/install.py index 4ddf8b51..1596f0d3 100644 --- a/playbooks/install.py +++ b/playbooks/install.py @@ -45,8 +45,10 @@ def install_bench(args): 'pip': 'sudo pip install --upgrade setuptools' }) + # Restricting ansible version due to following bug in ansible 2.1 + # https://github.com/ansible/ansible-modules-core/issues/3752 success = run_os_command({ - 'pip': 'sudo pip install ansible' + 'pip': "sudo pip install 'ansible==2.0.2.0'" }) if not success: @@ -58,18 +60,21 @@ def install_bench(args): if is_sudo_user() and not args.user and not args.production: raise Exception('Please run this script as a non-root user with sudo privileges, but without using sudo or pass --user=USER') - # create user if not exists - run_playbook('develop/create_user.yml', user=args.user, extra_args=vars(args)) # args is namespace, but we would like to use it as dict in calling function, so use vars() + if args.production and not args.user: + args.user = 'frappe' + + extra_vars = get_extra_vars(vars(args)) + + # create user if not exists + run_playbook('develop/create_user.yml', extra_vars=extra_vars) + if args.develop: - run_playbook('develop/install.yml', sudo=True, user=args.user, extra_args=vars(args)) + run_playbook('develop/install.yml', sudo=True, extra_vars=extra_vars) elif args.production: - if not args.user: - args.user = 'frappe' - - run_playbook('production/install.yml', sudo=True, user=args.user, extra_args=vars(args)) + run_playbook('production/install.yml', sudo=True, extra_vars=extra_vars) shutil.rmtree(tmp_bench_repo) @@ -100,10 +105,11 @@ def clone_bench_repo(args): return 0 branch = args.bench_branch or 'develop' + repo_url = args.bench_repo_url or 'https://github.com/frappe/bench' success = run_os_command( - {'git': 'git clone https://github.com/frappe/bench {bench_repo} --depth 1 --branch {branch}'.format( - bench_repo=tmp_bench_repo, branch=branch)} + {'git': 'git clone {repo_url} {bench_repo} --depth 1 --branch {branch}'.format( + repo_url=repo_url, bench_repo=tmp_bench_repo, branch=branch)} ) return success @@ -162,46 +168,47 @@ def get_passwords(run_travis=False): 'admin_password': admin_password } -def get_extra_vars_json(extra_args, run_travis=False): - # We need to pass production as extra_vars to the playbook to execute conditionals in the - # playbook. Extra variables can passed as json or key=value pair. Here, we will use JSON. +def get_vars_json_path(extra_vars): json_path = os.path.join(os.path.abspath(os.path.expanduser('~')), 'extra_vars.json') - extra_vars = dict(extra_args.items()) - - # Get mysql root password and admin password - run_travis = extra_args.get('run_travis') - extra_vars.update(get_passwords(run_travis)) - - # Decide for branch to be cloned depending upon whether we setting up production - branch = 'master' if extra_args.get('production') else 'develop' - extra_vars.update(branch=branch) - - # Get max worker_connections in nginx. - # https://www.digitalocean.com/community/tutorials/how-to-optimize-nginx-configuration - # The amount of clients that can be served can be multiplied by the amount of cores. - extra_vars.update(max_worker_connections=multiprocessing.cpu_count() * 1024) with open(json_path, mode='w') as j: json.dump(extra_vars, j, indent=1, sort_keys=True) return ('@' + json_path) -def run_playbook(playbook_name, sudo=False, user=None, extra_args=None): - user = user or getpass.getuser() +def get_extra_vars(extra_args): + extra_vars = dict(extra_args.items()) + + run_travis = extra_args.get('run_travis') + extra_vars.update(get_passwords(run_travis)) + + branch = 'master' if extra_args.get('setup_production') else 'develop' + extra_vars.update(branch=branch) + + extra_vars.update(max_worker_connections=multiprocessing.cpu_count() * 1024) + + user = args.user or getpass.getuser() + if user == 'root': raise Exception('--user cannot be root') - extra_args['frappe_user'] = user - extra_vars = get_extra_vars_json(extra_args) + extra_vars['frappe_user'] = user - args = ['ansible-playbook', '-c', 'local', playbook_name, '-e', extra_vars] + return extra_vars + +def run_playbook(playbook_name, sudo=False, extra_vars=None): + args = ['ansible-playbook', '-c', 'local', playbook_name] + + if extra_vars: + args.extend(['-e', get_vars_json_path(extra_vars=extra_vars)]) + + if extra_vars.get('verbosity'): + args.append('-vvvv') if sudo: + user = extra_vars.get('user') args.extend(['--become', '--become-user={0}'.format(user)]) - if extra_args.get('verbosity'): - args.append('-vvvv') - success = subprocess.check_call(args, cwd=os.path.join(tmp_bench_repo, 'playbooks')) return success @@ -230,6 +237,8 @@ def parse_commandline_args(): parser.add_argument('--bench-branch', dest='bench_branch', help='Clone a particular branch of bench repository') + parser.add_argument('--bench-repo-url', dest='bench_repo_url', help='Clones bench repository from the given url') + # To enable testing of script using Travis, this should skip the prompt parser.add_argument('--run-travis', dest='run_travis', action='store_true', default=False, help=argparse.SUPPRESS) From ea07745bbb8038c7e1105ade70c7c9eca19148b5 Mon Sep 17 00:00:00 2001 From: Valmik Jangla Date: Thu, 19 May 2016 15:41:12 +0530 Subject: [PATCH 2/8] Added command for adding Let's Encrypt SSL to site --- bench/cli.py | 5 +- bench/commands/__init__.py | 4 +- bench/commands/setup.py | 9 +++ bench/commands/utils.py | 5 ++ bench/config/lets_encrypt.py | 95 ++++++++++++++++++++++++++ bench/config/procfile.py | 4 +- bench/config/production_setup.py | 8 +-- bench/config/templates/letsencrypt.cfg | 20 ++++++ requirements.txt | 1 + 9 files changed, 141 insertions(+), 10 deletions(-) create mode 100755 bench/config/lets_encrypt.py mode change 100644 => 100755 bench/config/procfile.py mode change 100644 => 100755 bench/config/production_setup.py create mode 100755 bench/config/templates/letsencrypt.cfg diff --git a/bench/cli.py b/bench/cli.py index 3a53cd40..14f72db6 100644 --- a/bench/cli.py +++ b/bench/cli.py @@ -47,9 +47,10 @@ def check_uid(): sys.exit(1) def cmd_requires_root(): - if len(sys.argv) > 2 and sys.argv[2] in ('production', 'sudoers'): + if len(sys.argv) > 2 and sys.argv[2] in ('production', 'sudoers', 'lets-encrypt'): return True - if len(sys.argv) > 2 and sys.argv[1] in ('patch',): + #Changed > to >=, unsure if will cause the apolcaypse + if len(sys.argv) >= 2 and sys.argv[1] in ('patch', 'renew-lets-encrypt'): return True def change_dir(): diff --git a/bench/commands/__init__.py b/bench/commands/__init__.py index d66c05c6..962ad199 100644 --- a/bench/commands/__init__.py +++ b/bench/commands/__init__.py @@ -30,7 +30,7 @@ bench_command.add_command(switch_to_v5) from bench.commands.utils import (start, restart, set_nginx_port, set_ssl_certificate, set_ssl_certificate_key, set_url_root, - set_mariadb_host, set_default_site, download_translations, shell, backup_site, backup_all_sites, release) + set_mariadb_host, set_default_site, download_translations, shell, backup_site, backup_all_sites, release, renew_lets_encrypt) bench_command.add_command(start) bench_command.add_command(restart) bench_command.add_command(set_nginx_port) @@ -44,7 +44,7 @@ bench_command.add_command(shell) bench_command.add_command(backup_site) bench_command.add_command(backup_all_sites) bench_command.add_command(release) - +bench_command.add_command(renew_lets_encrypt) from bench.commands.setup import setup bench_command.add_command(setup) diff --git a/bench/commands/setup.py b/bench/commands/setup.py index c62876a7..56b33609 100644 --- a/bench/commands/setup.py +++ b/bench/commands/setup.py @@ -63,6 +63,14 @@ def setup_env(): setup_env() +@click.command('lets-encrypt') +@click.argument('site') +def setup_letsencrypt(site): + "Setup lets-encrypt for site" + from bench.config.lets_encrypt import setup_letsencrypt + setup_letsencrypt(site, bench_path='.') + + @click.command('procfile') def setup_procfile(): "Setup Procfile for bench start" @@ -88,6 +96,7 @@ setup.add_command(setup_sudoers) setup.add_command(setup_nginx) setup.add_command(setup_supervisor) setup.add_command(setup_redis) +setup.add_command(setup_letsencrypt) setup.add_command(setup_production) setup.add_command(setup_auto_update) setup.add_command(setup_backups) diff --git a/bench/commands/utils.py b/bench/commands/utils.py index a3f8bdbf..78b697b9 100644 --- a/bench/commands/utils.py +++ b/bench/commands/utils.py @@ -76,6 +76,11 @@ def download_translations(): from bench.utils import download_translations_p download_translations_p() +@click.command('renew-lets-encrypt') +def renew_lets_encrypt(): + "Renew Let's Encrypt certificate" + from bench.config.lets_encrypt import renew_certs + renew_certs() @click.command() def shell(bench='.'): diff --git a/bench/config/lets_encrypt.py b/bench/config/lets_encrypt.py new file mode 100755 index 00000000..f82f5391 --- /dev/null +++ b/bench/config/lets_encrypt.py @@ -0,0 +1,95 @@ +import bench, os, click, errno, urllib +from bench.utils import exec_cmd, update_site_config, CommandFailedError +from bench.config.common_site_config import update_config as update_common_config +from bench.config.nginx import make_nginx_conf +from bench.config.production_setup import service +from bench.config.common_site_config import get_config +from crontab import CronTab + +def setup_letsencrypt(site, bench_path): + + site_path = os.path.join(bench_path, "sites", site, "site_config.json") + if not os.path.exists(os.path.dirname(site_path)): + print "No site named "+site + return + + click.confirm('Running this will stop the nginx service temporarily causing your sites to go offline\n' + 'Do you want to continue?', + abort=True) + + if not get_config(bench_path).get("dns_multitenant"): + print "You cannot setup SSL without DNS Multitenancy" + return + + create_config(site) + run_certbot_and_setup_ssl(site, bench_path) + setup_crontab() + + +def create_config(site): + config = bench.env.get_template('letsencrypt.cfg').render(domain=site) + config_path = '/etc/letsencrypt/configs/{site}.cfg'.format(site=site) + create_dir_if_missing(config_path) + + with open(config_path, 'w') as f: + f.write(config) + + +def run_certbot_and_setup_ssl(site, bench_path): + service('nginx', 'stop') + get_certbot() + + try: + exec_cmd("{path} --config /etc/letsencrypt/configs/{site}.cfg certonly".format(path=get_certbot_path(), site=site)) + except CommandFailedError: + service('nginx', 'start') + print "There was a problem trying to setup SSL for your site" + return + + ssl_path = "/etc/letsencrypt/live/{site}/".format(site=site) + + ssl_config = { "ssl_certificate": os.path.join(ssl_path, "fullchain.pem"), + "ssl_certificate_key": os.path.join(ssl_path, "privkey.pem") } + + update_site_config(site, ssl_config, bench=bench_path) + make_nginx_conf(bench_path) + + service('nginx', 'start') + + +def setup_crontab(): + job_command = 'sudo service nginx stop && /opt/certbot-auto renew && sudo service nginx start' + user_crontab = CronTab(user=True) + if job_command not in str(user_crontab): + job = user_crontab.new(command=job_command, comment="Renew lets-encrypt every month") + job.month.every(1) + job.enable() + user_crontab.write() + + +def create_dir_if_missing(path): + if not os.path.exists(os.path.dirname(path)): + os.makedirs(os.path.dirname(path)) + + +def get_certbot(): + certbot_path = get_certbot_path() + create_dir_if_missing(certbot_path) + + if not os.path.isfile(certbot_path): + urllib.urlretrieve ("https://dl.eff.org/certbot-auto", certbot_path) + os.chmod(certbot_path, 0744) + + +def get_certbot_path(): + return "/opt/certbot-auto" + + +def renew_certs(): + click.confirm('Running this will stop the nginx service temporarily causing your sites to go offline\n' + 'Do you want to continue?', + abort=True) + + service('nginx', 'start') + exec_cmd("{path} renew".format(path=get_certbot_path())) + service('nginx', 'stop') diff --git a/bench/config/procfile.py b/bench/config/procfile.py old mode 100644 new mode 100755 index 5fa47124..598d0a3a --- a/bench/config/procfile.py +++ b/bench/config/procfile.py @@ -1,6 +1,6 @@ import bench, os, click -from bench.utils import find_executable -from bench.app import use_rq +from bench.utils import find_executable +from bench.app import use_rq from bench.config.common_site_config import get_config def setup_procfile(bench_path, force=False): diff --git a/bench/config/production_setup.py b/bench/config/production_setup.py old mode 100644 new mode 100755 index e164f11b..4461224f --- a/bench/config/production_setup.py +++ b/bench/config/production_setup.py @@ -27,14 +27,14 @@ def setup_production(user, bench='.'): if os.environ.get('NO_SERVICE_RESTART'): return - restart_service('nginx') + service('nginx', 'restart') -def restart_service(service): +def service(service, option): if os.path.basename(get_program(['systemctl']) or '') == 'systemctl' and is_running_systemd(): - exec_cmd("{service_manager} restart {service}".format(service_manager='systemctl', service=service)) + exec_cmd("{service_manager} {option} {service}".format(service_manager='systemctl', option=option, service=service)) elif os.path.basename(get_program(['service']) or '') == 'service': - exec_cmd("{service_manager} {service} restart ".format(service_manager='service', service=service)) + exec_cmd("{service_manager} {service} {option} ".format(service_manager='service', service=service, option=option)) else: # look for 'service_manager' and 'service_manager_command' in environment service_manager = os.environ.get("BENCH_SERVICE_MANAGER") diff --git a/bench/config/templates/letsencrypt.cfg b/bench/config/templates/letsencrypt.cfg new file mode 100755 index 00000000..ba57815a --- /dev/null +++ b/bench/config/templates/letsencrypt.cfg @@ -0,0 +1,20 @@ +# This is an example of the kind of things you can do in a configuration file. +# All flags used by the client can be configured here. Run Certbot with +# "--help" to learn more about the available options. + +# Use a 4096 bit RSA key instead of 2048 +rsa-key-size = 4096 + +# Uncomment and update to register with the specified e-mail address +#email = email@domain.com + +# Uncomment and update to generate certificates for the specified +# domains. +domains = {{ domain }} + +# Uncomment to use a text interface instead of ncurses +text = True + +# Uncomment to use the standalone authenticator on port 443 +authenticator = standalone +standalone-supported-challenges = tls-sni-01 diff --git a/requirements.txt b/requirements.txt index 2629b148..f5a239a6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,5 +3,6 @@ jinja2 virtualenv requests honcho +python-crontab semantic_version GitPython==0.3.2.rc1 From 95eae6d3b474eaf06eb872bb7eb7c0896a342286 Mon Sep 17 00:00:00 2001 From: Revant Nandgaonkar Date: Sat, 30 Apr 2016 22:48:33 +0530 Subject: [PATCH 3/8] make bench release available for app developers --- bench/commands/utils.py | 7 ++++--- bench/release.py | 8 ++++---- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/bench/commands/utils.py b/bench/commands/utils.py index 78b697b9..8dc99b1e 100644 --- a/bench/commands/utils.py +++ b/bench/commands/utils.py @@ -116,13 +116,14 @@ def backup_all_sites(): @click.command('release') -@click.argument('app', type=click.Choice(['frappe', 'erpnext', 'erpnext_shopify', 'paypal_integration', 'schools'])) +@click.argument('app') @click.argument('bump-type', type=click.Choice(['major', 'minor', 'patch'])) +@click.argument('owner', default='frappe') @click.option('--develop', default='develop') @click.option('--master', default='master') -def release(app, bump_type, develop, master): +def release(app, bump_type, develop, master, owner): "Release app (internal to the Frappe team)" from bench.release import release repo = os.path.join('apps', app) - release(repo, bump_type, develop, master) + release(repo, bump_type, develop, master, owner) diff --git a/bench/release.py b/bench/release.py index e7cf03cd..c5a79fbe 100755 --- a/bench/release.py +++ b/bench/release.py @@ -27,7 +27,7 @@ repo_map = { 'schools': 'schools' } -def release(repo, bump_type, develop, master): +def release(repo, bump_type, develop, master, owner): if not get_config(".").get('release_bench'): print 'bench not configured to release' sys.exit(1) @@ -36,9 +36,9 @@ def release(repo, bump_type, develop, master): github_password = getpass.getpass() r = requests.get('https://api.github.com/user', auth=HTTPBasicAuth(github_username, github_password)) r.raise_for_status() - bump(repo, bump_type, develop=develop, master=master) + bump(repo, bump_type, develop=develop, master=master, owner=owner) -def bump(repo, bump_type, develop='develop', master='master', remote='upstream'): +def bump(repo, bump_type, develop='develop', master='master', remote='upstream', owner='frappe'): assert bump_type in ['minor', 'major', 'patch'] update_branches_and_check_for_changelog(repo, bump_type, develop=develop, master=master, remote=remote) message = get_release_message(repo, develop_branch=develop, master_branch=master) @@ -56,7 +56,7 @@ def bump(repo, bump_type, develop='develop', master='master', remote='upstream') commit_changes(repo, new_version) tag_name = create_release(repo, new_version, develop_branch=develop, master_branch=master) push_release(repo, develop_branch=develop, master_branch=master) - create_github_release('frappe', repo, tag_name, message) + create_github_release(owner, repo, tag_name, message) print 'Released {tag} for {repo}'.format(tag=tag_name, repo=repo) def update_branches_and_check_for_changelog(repo, bump_type, develop='develop', master='master', remote='upstream'): From 4a60de92684a737055b5f6e6629c8cebe7670dea Mon Sep 17 00:00:00 2001 From: Revant Nandgaonkar Date: Wed, 11 May 2016 12:16:27 +0530 Subject: [PATCH 4/8] removed repo_map and changed owner to option from argument --- bench/commands/utils.py | 3 +-- bench/release.py | 10 +--------- 2 files changed, 2 insertions(+), 11 deletions(-) diff --git a/bench/commands/utils.py b/bench/commands/utils.py index 8dc99b1e..075b55f1 100644 --- a/bench/commands/utils.py +++ b/bench/commands/utils.py @@ -118,12 +118,11 @@ def backup_all_sites(): @click.command('release') @click.argument('app') @click.argument('bump-type', type=click.Choice(['major', 'minor', 'patch'])) -@click.argument('owner', default='frappe') @click.option('--develop', default='develop') @click.option('--master', default='master') +@click.option('--owner', default='frappe') def release(app, bump_type, develop, master, owner): "Release app (internal to the Frappe team)" from bench.release import release repo = os.path.join('apps', app) release(repo, bump_type, develop, master, owner) - diff --git a/bench/release.py b/bench/release.py index c5a79fbe..37169b41 100755 --- a/bench/release.py +++ b/bench/release.py @@ -19,14 +19,6 @@ import click github_username = None github_password = None -repo_map = { - 'frappe': 'frappe', - 'erpnext': 'erpnext', - 'erpnext_shopify': 'erpnext_shopify', - 'paypal_integration': 'paypal_integration', - 'schools': 'schools' -} - def release(repo, bump_type, develop, master, owner): if not get_config(".").get('release_bench'): print 'bench not configured to release' @@ -207,7 +199,7 @@ def create_github_release(owner, repo, tag_name, log, gh_username=None, gh_passw raise Exception, "No credentials" gh_username = github_username gh_password = github_password - repo = repo_map[os.path.basename(repo)] + repo = os.path.basename(repo) data = { 'tag_name': tag_name, 'target_commitish': 'master', From 08e65f7881734fe0844c5af6f115e84eb4e2c449 Mon Sep 17 00:00:00 2001 From: Revant Nandgaonkar Date: Tue, 17 May 2016 14:41:52 +0530 Subject: [PATCH 5/8] added repo-name option to release command --- bench/commands/utils.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/bench/commands/utils.py b/bench/commands/utils.py index 075b55f1..8d1d6687 100644 --- a/bench/commands/utils.py +++ b/bench/commands/utils.py @@ -121,8 +121,9 @@ def backup_all_sites(): @click.option('--develop', default='develop') @click.option('--master', default='master') @click.option('--owner', default='frappe') -def release(app, bump_type, develop, master, owner): +@click.option('--repo-name') +def release(app, bump_type, develop, master, owner, repo_name): "Release app (internal to the Frappe team)" from bench.release import release - repo = os.path.join('apps', app) + repo = os.path.join('apps', repo_name) if repo_name else os.path.join('apps', app) release(repo, bump_type, develop, master, owner) From 40247bae0fbd1ad436644b16be37fbeca3f288ea Mon Sep 17 00:00:00 2001 From: Revant Nandgaonkar Date: Tue, 17 May 2016 18:40:21 +0530 Subject: [PATCH 6/8] added app argument to compare repo and app --- bench/commands/utils.py | 2 +- bench/release.py | 78 ++++++++++++++++++++++++++++++----------- 2 files changed, 58 insertions(+), 22 deletions(-) diff --git a/bench/commands/utils.py b/bench/commands/utils.py index 8d1d6687..a95461f8 100644 --- a/bench/commands/utils.py +++ b/bench/commands/utils.py @@ -126,4 +126,4 @@ def release(app, bump_type, develop, master, owner, repo_name): "Release app (internal to the Frappe team)" from bench.release import release repo = os.path.join('apps', repo_name) if repo_name else os.path.join('apps', app) - release(repo, bump_type, develop, master, owner) + release(repo, bump_type, develop, master, owner, app) diff --git a/bench/release.py b/bench/release.py index 37169b41..8fa852d8 100755 --- a/bench/release.py +++ b/bench/release.py @@ -19,7 +19,7 @@ import click github_username = None github_password = None -def release(repo, bump_type, develop, master, owner): +def release(repo, bump_type, develop, master, owner, app): if not get_config(".").get('release_bench'): print 'bench not configured to release' sys.exit(1) @@ -28,12 +28,14 @@ def release(repo, bump_type, develop, master, owner): github_password = getpass.getpass() r = requests.get('https://api.github.com/user', auth=HTTPBasicAuth(github_username, github_password)) r.raise_for_status() - bump(repo, bump_type, develop=develop, master=master, owner=owner) + + bump(repo, bump_type, develop=develop, master=master, owner=owner, app=app) + +def bump(repo, bump_type, develop='develop', master='master', remote='upstream', owner='frappe', app=None): -def bump(repo, bump_type, develop='develop', master='master', remote='upstream', owner='frappe'): assert bump_type in ['minor', 'major', 'patch'] - update_branches_and_check_for_changelog(repo, bump_type, develop=develop, master=master, remote=remote) - message = get_release_message(repo, develop_branch=develop, master_branch=master) + update_branches_and_check_for_changelog(repo, bump_type, develop=develop, master=master, remote=remote, app=app) + message = get_release_message(repo, develop_branch=develop, master_branch=master, app=app) if not message: print 'No commits to release' return @@ -44,25 +46,34 @@ def bump(repo, bump_type, develop='develop', master='master', remote='upstream', click.confirm('Do you want to continue?', abort=True) - new_version = bump_repo(repo, bump_type, develop=develop, master=master, remote=remote) - commit_changes(repo, new_version) - tag_name = create_release(repo, new_version, develop_branch=develop, master_branch=master) - push_release(repo, develop_branch=develop, master_branch=master) - create_github_release(owner, repo, tag_name, message) + new_version = bump_repo(repo, bump_type, develop=develop, master=master, remote=remote, app=app) + commit_changes(repo, new_version, app=app) + tag_name = create_release(repo, new_version, develop_branch=develop, master_branch=master, app=app) + push_release(repo, develop_branch=develop, master_branch=master, app=app) + create_github_release(owner, repo, tag_name, message, app) print 'Released {tag} for {repo}'.format(tag=tag_name, repo=repo) -def update_branches_and_check_for_changelog(repo, bump_type, develop='develop', master='master', remote='upstream'): - update_branch(repo, master, remote=remote) - update_branch(repo, develop, remote=remote) +def update_branches_and_check_for_changelog(repo, bump_type, develop='develop', master='master', remote='upstream', app=None): + + update_branch(repo, master, remote=remote, app=app) + update_branch(repo, develop, remote=remote, app=app) if develop != 'develop': - update_branch(repo, 'develop', remote=remote) - + update_branch(repo, 'develop', remote=remote, app=app) + + if os.path.basename(repo) != app: + repo = app + repo = os.path.join('apps', repo) + git.Repo(repo).git.checkout(develop) check_for_unmerged_changelog(repo) -def update_branch(repo_path, branch, remote='origin'): +def update_branch(repo_path, branch, remote='origin', app=None): print "updating local branch of", repo_path, 'using', remote + '/' + branch + if os.path.basename(repo_path) != app: + repo_path = app + repo_path = os.path.join('apps', repo_path) + repo = git.Repo(repo_path) g = repo.git g.fetch(remote) @@ -74,7 +85,13 @@ def check_for_unmerged_changelog(repo): if os.path.exists(current) and [f for f in os.listdir(current) if f != "readme.md"]: raise Exception("Unmerged change log! in " + repo) -def get_release_message(repo_path, develop_branch='develop', master_branch='master'): +def get_release_message(repo_path, develop_branch='develop', master_branch='master', app=None): + + if os.path.basename(repo_path) != app: + repo_path = app + repo_path = os.path.join('apps', repo_path) + + print 'getting release message for', repo_path, 'comparing', master_branch, '...', develop_branch repo = git.Repo(repo_path) g = repo.git @@ -83,7 +100,11 @@ def get_release_message(repo_path, develop_branch='develop', master_branch='mast return "* " + log.replace('\n', '\n* ') -def bump_repo(repo, bump_type, develop='develop', master='master', remote='upstream'): +def bump_repo(repo, bump_type, develop='develop', master='master', remote='upstream', app=None): + if os.path.basename(repo) != app: + repo = app + repo = os.path.join('apps', repo) + current_version = get_current_version(repo) new_version = get_bumped_version(current_version, bump_type) @@ -145,7 +166,11 @@ def set_filename_version(filename, version_number, pattern): with open(filename, 'w') as f: f.write(contents) -def commit_changes(repo_path, version): +def commit_changes(repo_path, version, app=None): + if os.path.basename(repo_path) != app: + repo_path = app + repo_path = os.path.join('apps', repo_path) + print 'committing version change to', repo_path repo = git.Repo(repo_path) @@ -155,7 +180,12 @@ def commit_changes(repo_path, version): repo.index.add([os.path.join(repo_name, 'hooks.py')]) repo.index.commit('bumped to version {}'.format(version)) -def create_release(repo_path, version, remote='origin', develop_branch='develop', master_branch='master'): +def create_release(repo_path, version, remote='origin', develop_branch='develop', master_branch='master', app=None): + + if os.path.basename(repo_path) != app: + repo_path = app + repo_path = os.path.join('apps', repo_path) + print 'creating release for version', version repo = git.Repo(repo_path) g = repo.git @@ -173,7 +203,12 @@ def create_release(repo_path, version, remote='origin', develop_branch='develop' return tag_name -def push_release(repo_path, develop_branch='develop', master_branch='master'): +def push_release(repo_path, develop_branch='develop', master_branch='master', app=None): + + if os.path.basename(repo_path) != app: + repo_path = app + repo_path = os.path.join('apps', repo_path) + print 'pushing branches', master_branch, develop_branch, 'of', repo_path repo = git.Repo(repo_path) g = repo.git @@ -191,6 +226,7 @@ def push_release(repo_path, develop_branch='develop', master_branch='master'): print g.push('upstream', *args) def create_github_release(owner, repo, tag_name, log, gh_username=None, gh_password=None): + print 'creating release on github' global github_username, github_password From 6a0a7223d27196121c7d8575bf90cdd10a9b4fcf Mon Sep 17 00:00:00 2001 From: Revant Nandgaonkar Date: Tue, 17 May 2016 21:47:40 +0530 Subject: [PATCH 7/8] compare app and repo to set repo --- bench/release.py | 31 +++++++++---------------------- 1 file changed, 9 insertions(+), 22 deletions(-) diff --git a/bench/release.py b/bench/release.py index 8fa852d8..630b8d3f 100755 --- a/bench/release.py +++ b/bench/release.py @@ -60,10 +60,8 @@ def update_branches_and_check_for_changelog(repo, bump_type, develop='develop', if develop != 'develop': update_branch(repo, 'develop', remote=remote, app=app) - if os.path.basename(repo) != app: - repo = app - repo = os.path.join('apps', repo) - + repo = os.path.join('apps', app) if os.path.basename(repo) != app else repo + git.Repo(repo).git.checkout(develop) check_for_unmerged_changelog(repo) @@ -87,11 +85,8 @@ def check_for_unmerged_changelog(repo): def get_release_message(repo_path, develop_branch='develop', master_branch='master', app=None): - if os.path.basename(repo_path) != app: - repo_path = app - repo_path = os.path.join('apps', repo_path) - - + repo_path = os.path.join('apps', app) if os.path.basename(repo_path) != app else repo_path + print 'getting release message for', repo_path, 'comparing', master_branch, '...', develop_branch repo = git.Repo(repo_path) g = repo.git @@ -101,10 +96,8 @@ def get_release_message(repo_path, develop_branch='develop', master_branch='mast def bump_repo(repo, bump_type, develop='develop', master='master', remote='upstream', app=None): - if os.path.basename(repo) != app: - repo = app - repo = os.path.join('apps', repo) - + repo = os.path.join('apps', app) if os.path.basename(repo) != app else repo + current_version = get_current_version(repo) new_version = get_bumped_version(current_version, bump_type) @@ -167,9 +160,7 @@ def set_filename_version(filename, version_number, pattern): f.write(contents) def commit_changes(repo_path, version, app=None): - if os.path.basename(repo_path) != app: - repo_path = app - repo_path = os.path.join('apps', repo_path) + repo_path = os.path.join('apps', app) if os.path.basename(repo_path) != app else repo_path print 'committing version change to', repo_path @@ -182,9 +173,7 @@ def commit_changes(repo_path, version, app=None): def create_release(repo_path, version, remote='origin', develop_branch='develop', master_branch='master', app=None): - if os.path.basename(repo_path) != app: - repo_path = app - repo_path = os.path.join('apps', repo_path) + repo_path = os.path.join('apps', app) if os.path.basename(repo_path) != app else repo_path print 'creating release for version', version repo = git.Repo(repo_path) @@ -205,9 +194,7 @@ def create_release(repo_path, version, remote='origin', develop_branch='develop' def push_release(repo_path, develop_branch='develop', master_branch='master', app=None): - if os.path.basename(repo_path) != app: - repo_path = app - repo_path = os.path.join('apps', repo_path) + repo_path = os.path.join('apps', app) if os.path.basename(repo_path) != app else repo_path print 'pushing branches', master_branch, develop_branch, 'of', repo_path repo = git.Repo(repo_path) From 7c86581897ab4c8d4fbdb26e909b5fc3a0f88530 Mon Sep 17 00:00:00 2001 From: Anand Doshi Date: Thu, 9 Jun 2016 15:39:54 +0530 Subject: [PATCH 8/8] [fix] release script --- bench/commands/utils.py | 9 +- bench/release.py | 225 +++++++++++++++++++++++----------------- setup.py | 3 +- 3 files changed, 137 insertions(+), 100 deletions(-) diff --git a/bench/commands/utils.py b/bench/commands/utils.py index a95461f8..9483bc0b 100644 --- a/bench/commands/utils.py +++ b/bench/commands/utils.py @@ -117,13 +117,14 @@ def backup_all_sites(): @click.command('release') @click.argument('app') -@click.argument('bump-type', type=click.Choice(['major', 'minor', 'patch'])) +@click.argument('bump-type', type=click.Choice(['major', 'minor', 'patch', 'stable', 'prerelease'])) @click.option('--develop', default='develop') @click.option('--master', default='master') +@click.option('--remote', default='upstream') @click.option('--owner', default='frappe') @click.option('--repo-name') -def release(app, bump_type, develop, master, owner, repo_name): +def release(app, bump_type, develop, master, owner, repo_name, remote): "Release app (internal to the Frappe team)" from bench.release import release - repo = os.path.join('apps', repo_name) if repo_name else os.path.join('apps', app) - release(repo, bump_type, develop, master, owner, app) + release(bench_path='.', app=app, bump_type=bump_type, develop=develop, master=master, + remote=remote, owner=owner, repo_name=repo_name) diff --git a/bench/release.py b/bench/release.py index 630b8d3f..4211bcd8 100755 --- a/bench/release.py +++ b/bench/release.py @@ -15,27 +15,36 @@ from time import sleep from .config.common_site_config import get_config import click - github_username = None github_password = None -def release(repo, bump_type, develop, master, owner, app): - if not get_config(".").get('release_bench'): +def release(bench_path, app, bump_type, develop='develop', master='master', + remote='upstream', owner='frappe', repo_name=None): + + validate(bench_path) + + bump(bench_path, app, bump_type, develop=develop, master=master, owner=owner, + repo_name=repo_name, remote=remote) + +def validate(bench_path): + if not get_config(bench_path).get('release_bench'): print 'bench not configured to release' sys.exit(1) + global github_username, github_password - github_username = raw_input('username:') + github_username = raw_input('Username: ') github_password = getpass.getpass() + r = requests.get('https://api.github.com/user', auth=HTTPBasicAuth(github_username, github_password)) r.raise_for_status() - - bump(repo, bump_type, develop=develop, master=master, owner=owner, app=app) -def bump(repo, bump_type, develop='develop', master='master', remote='upstream', owner='frappe', app=None): +def bump(bench_path, app, bump_type, develop, master, remote, owner, repo_name=None): + assert bump_type in ['minor', 'major', 'patch', 'stable', 'prerelease'] + + repo_path = os.path.join(bench_path, 'apps', app) + update_branches_and_check_for_changelog(repo_path, develop, master, remote=remote) + message = get_release_message(repo_path, develop=develop, master=master, remote=remote) - assert bump_type in ['minor', 'major', 'patch'] - update_branches_and_check_for_changelog(repo, bump_type, develop=develop, master=master, remote=remote, app=app) - message = get_release_message(repo, develop_branch=develop, master_branch=master, app=app) if not message: print 'No commits to release' return @@ -46,71 +55,63 @@ def bump(repo, bump_type, develop='develop', master='master', remote='upstream', click.confirm('Do you want to continue?', abort=True) - new_version = bump_repo(repo, bump_type, develop=develop, master=master, remote=remote, app=app) - commit_changes(repo, new_version, app=app) - tag_name = create_release(repo, new_version, develop_branch=develop, master_branch=master, app=app) - push_release(repo, develop_branch=develop, master_branch=master, app=app) - create_github_release(owner, repo, tag_name, message, app) - print 'Released {tag} for {repo}'.format(tag=tag_name, repo=repo) + new_version = bump_repo(repo_path, bump_type, develop=develop, master=master) + commit_changes(repo_path, new_version) + tag_name = create_release(repo_path, new_version, develop=develop, master=master) + push_release(repo_path, develop=develop, master=master, remote=remote) + create_github_release(repo_path, tag_name, message, remote=remote, owner=owner, repo_name=repo_name) + print 'Released {tag} for {repo_path}'.format(tag=tag_name, repo_path=repo_path) -def update_branches_and_check_for_changelog(repo, bump_type, develop='develop', master='master', remote='upstream', app=None): +def update_branches_and_check_for_changelog(repo_path, develop='develop', master='master', remote='upstream'): - update_branch(repo, master, remote=remote, app=app) - update_branch(repo, develop, remote=remote, app=app) + update_branch(repo_path, master, remote=remote) + update_branch(repo_path, develop, remote=remote) if develop != 'develop': - update_branch(repo, 'develop', remote=remote, app=app) + update_branch(repo_path, 'develop', remote=remote) - repo = os.path.join('apps', app) if os.path.basename(repo) != app else repo - - git.Repo(repo).git.checkout(develop) - check_for_unmerged_changelog(repo) + git.Repo(repo_path).git.checkout(develop) + check_for_unmerged_changelog(repo_path) -def update_branch(repo_path, branch, remote='origin', app=None): +def update_branch(repo_path, branch, remote): print "updating local branch of", repo_path, 'using', remote + '/' + branch - if os.path.basename(repo_path) != app: - repo_path = app - repo_path = os.path.join('apps', repo_path) - repo = git.Repo(repo_path) g = repo.git g.fetch(remote) g.checkout(branch) g.reset('--hard', remote+'/'+branch) -def check_for_unmerged_changelog(repo): - current = os.path.join(repo, os.path.basename(repo), 'change_log', 'current') +def check_for_unmerged_changelog(repo_path): + current = os.path.join(repo_path, os.path.basename(repo_path), 'change_log', 'current') if os.path.exists(current) and [f for f in os.listdir(current) if f != "readme.md"]: - raise Exception("Unmerged change log! in " + repo) + raise Exception("Unmerged change log! in " + repo_path) -def get_release_message(repo_path, develop_branch='develop', master_branch='master', app=None): +def get_release_message(repo_path, develop='develop', master='master', remote='upstream'): + print 'getting release message for', repo_path, 'comparing', master, '...', develop - repo_path = os.path.join('apps', app) if os.path.basename(repo_path) != app else repo_path - - print 'getting release message for', repo_path, 'comparing', master_branch, '...', develop_branch repo = git.Repo(repo_path) g = repo.git - log = g.log('upstream/{master_branch}..upstream/{develop_branch}'.format(master_branch=master_branch, develop_branch=develop_branch), '--format=format:%s', '--no-merges') + log = g.log('{remote}/{master}..{remote}/{develop}'.format( + remote=remote, master=master, develop=develop), '--format=format:%s', '--no-merges') + if log: return "* " + log.replace('\n', '\n* ') - -def bump_repo(repo, bump_type, develop='develop', master='master', remote='upstream', app=None): - repo = os.path.join('apps', app) if os.path.basename(repo) != app else repo - - current_version = get_current_version(repo) +def bump_repo(repo_path, bump_type, develop='develop', master='master'): + current_version = get_current_version(repo_path) new_version = get_bumped_version(current_version, bump_type) print 'bumping version from', current_version, 'to', new_version - set_version(repo, new_version) + set_version(repo_path, new_version) return new_version -def get_current_version(repo): - filename = os.path.join(repo, 'setup.py') +def get_current_version(repo_path): + # TODO clean this up! + filename = os.path.join(repo_path, os.path.basename(repo_path), '__init__.py') with open(filename) as f: contents = f.read() - match = re.search(r"^(\s*%s\s*=\s*['\\\"])(.+?)(['\"])(?sm)" % 'version', + match = re.search(r"^(\s*%s\s*=\s*['\\\"])(.+?)(['\"])(?sm)" % '__version__', contents) return match.group(2) @@ -119,27 +120,50 @@ def get_bumped_version(version, bump_type): if bump_type == 'minor': v.minor += 1 v.patch = 0 + v.prerelease = None + elif bump_type == 'major': v.major += 1 v.minor = 0 v.patch = 0 + v.prerelease = None + elif bump_type == 'patch': v.patch += 1 + v.prerelease = None + + elif bump_type == 'stable': + # remove pre-release tag + v.prerelease = None + + elif bump_type == 'prerelease': + if v.prerelease == None: + v.prerelease = ('beta',) + + elif len(v.prerelease)==1: + v.prerelease[1] = '1' + + else: + v.prerelease[1] = str(int(v.prerelease[1]) + 1) + return unicode(v) -def set_version(repo, version): - set_setuppy_version(repo, version) - set_versionpy_version(repo, version) - set_hooks_version(repo, version) +def set_version(repo_path, version): + set_filename_version(os.path.join(repo_path, os.path.basename(repo_path),'__init__.py'), version, '__version__') -def set_setuppy_version(repo, version): - set_filename_version(os.path.join(repo, 'setup.py'), version, 'version') + # TODO fix this + # set_setuppy_version(repo_path, version) + # set_versionpy_version(repo_path, version) + # set_hooks_version(repo_path, version) -def set_versionpy_version(repo, version): - set_filename_version(os.path.join(repo, os.path.basename(repo),'__version__.py'), version, '__version__') - -def set_hooks_version(repo, version): - set_filename_version(os.path.join(repo, os.path.basename(repo),'hooks.py'), version, 'app_version') +# def set_setuppy_version(repo_path, version): +# set_filename_version(os.path.join(repo_path, 'setup.py'), version, 'version') +# +# def set_versionpy_version(repo_path, version): +# set_filename_version(os.path.join(repo_path, os.path.basename(repo_path),'__version__.py'), version, '__version__') +# +# def set_hooks_version(repo_path, version): +# set_filename_version(os.path.join(repo_path, os.path.basename(repo_path),'hooks.py'), version, 'app_version') def set_filename_version(filename, version_number, pattern): changed = [] @@ -159,60 +183,70 @@ def set_filename_version(filename, version_number, pattern): with open(filename, 'w') as f: f.write(contents) -def commit_changes(repo_path, version, app=None): - repo_path = os.path.join('apps', app) if os.path.basename(repo_path) != app else repo_path - +def commit_changes(repo_path, new_version): print 'committing version change to', repo_path repo = git.Repo(repo_path) - repo_name = os.path.basename(repo_path) - repo.index.add(['setup.py']) - repo.index.add([os.path.join(repo_name, '__version__.py')]) - repo.index.add([os.path.join(repo_name, 'hooks.py')]) - repo.index.commit('bumped to version {}'.format(version)) + app_name = os.path.basename(repo_path) + repo.index.add([os.path.join(app_name, '__init__.py')]) + repo.index.commit('bumped to version {}'.format(new_version)) -def create_release(repo_path, version, remote='origin', develop_branch='develop', master_branch='master', app=None): - - repo_path = os.path.join('apps', app) if os.path.basename(repo_path) != app else repo_path - - print 'creating release for version', version +def create_release(repo_path, new_version, develop='develop', master='master'): + print 'creating release for version', new_version repo = git.Repo(repo_path) g = repo.git - g.checkout(master_branch) - g.merge(develop_branch, '--no-ff') - tag_name = 'v' + version - repo.create_tag(tag_name, message='Release {}'.format(version)) - g.checkout(develop_branch) - g.merge(master_branch) + g.checkout(master) + try: + g.merge(develop, '--no-ff') + except git.exc.GitCommandError, e: + handle_merge_error(e, source=develop, target=master) - if develop_branch != 'develop': + tag_name = 'v' + new_version + repo.create_tag(tag_name, message='Release {}'.format(new_version)) + g.checkout(develop) + + try: + g.merge(master) + except git.exc.GitCommandError, e: + handle_merge_error(e, source=master, target=develop) + + if develop != 'develop': print 'merging master into develop' g.checkout('develop') - g.merge(master_branch) + try: + g.merge(master) + except git.exc.GitCommandError, e: + handle_merge_error(e, source=master, target='develop') return tag_name -def push_release(repo_path, develop_branch='develop', master_branch='master', app=None): +def handle_merge_error(e, source, target): + print '-'*80 + print 'Error when merging {source} into {target}'.format(source=source, target=target) + print e + print 'You can open a new terminal, try to manually resolve the conflict/error and continue' + print '-'*80 + click.confirm('Have you manually resolved the error?', abort=True) - repo_path = os.path.join('apps', app) if os.path.basename(repo_path) != app else repo_path - - print 'pushing branches', master_branch, develop_branch, 'of', repo_path +def push_release(repo_path, develop='develop', master='master', remote='upstream'): + print 'pushing branches', master, develop, 'of', repo_path repo = git.Repo(repo_path) g = repo.git args = [ - '{master}:{master}'.format(master=master_branch), - '{develop}:{develop}'.format(develop=develop_branch) + '{master}:{master}'.format(master=master), + '{develop}:{develop}'.format(develop=develop) ] - - if develop_branch != 'develop': + + if develop != 'develop': print 'pushing develop branch of', repo_path args.append('develop:develop') - - args.append('--tags') - - print g.push('upstream', *args) -def create_github_release(owner, repo, tag_name, log, gh_username=None, gh_password=None): + args.append('--tags') + + print g.push(remote, *args) + +def create_github_release(repo_path, tag_name, message, remote='upstream', owner='frappe', repo_name=None, + gh_username=None, gh_password=None): print 'creating release on github' @@ -222,18 +256,21 @@ def create_github_release(owner, repo, tag_name, log, gh_username=None, gh_passw raise Exception, "No credentials" gh_username = github_username gh_password = github_password - repo = os.path.basename(repo) + + repo_name = repo_name or os.path.basename(repo_path) data = { 'tag_name': tag_name, 'target_commitish': 'master', 'name': 'Release ' + tag_name, - 'body': log, + 'body': message, 'draft': False, 'prerelease': False } for i in xrange(3): try: - r = requests.post('https://api.github.com/repos/{owner}/{repo}/releases'.format(owner=owner, repo=repo), auth=HTTPBasicAuth(gh_username, gh_password), data=json.dumps(data)) + r = requests.post('https://api.github.com/repos/{owner}/{repo_name}/releases'.format( + owner=owner, repo_name=repo_name), + auth=HTTPBasicAuth(gh_username, gh_password), data=json.dumps(data)) r.raise_for_status() break except requests.exceptions.HTTPError: diff --git a/setup.py b/setup.py index 5decf181..fc623809 100644 --- a/setup.py +++ b/setup.py @@ -1,7 +1,6 @@ from setuptools import setup, find_packages from pip.req import parse_requirements -import re -import ast +import re, ast # get version from __version__ variable in bench/__init__.py _version_re = re.compile(r'__version__\s+=\s+(.*)')