From 2fdbdbfcf974a7b24f9340604009453270578fdc Mon Sep 17 00:00:00 2001 From: Revant Nandgaonkar Date: Thu, 5 Apr 2018 17:29:14 +0530 Subject: [PATCH 01/49] systemd templated added --- .../frappe-bench-frappe-default-worker.service | 12 ++++++++++++ .../systemd/frappe-bench-frappe-long-worker.service | 12 ++++++++++++ .../systemd/frappe-bench-frappe-schedule.service | 12 ++++++++++++ .../frappe-bench-frappe-short-worker.service | 12 ++++++++++++ .../systemd/frappe-bench-frappe-web.service | 12 ++++++++++++ .../systemd/frappe-bench-node-socketio.service | 13 +++++++++++++ .../systemd/frappe-bench-redis-cache.service | 12 ++++++++++++ .../systemd/frappe-bench-redis-queue.service | 12 ++++++++++++ .../systemd/frappe-bench-redis-socketio.service | 12 ++++++++++++ .../templates/systemd/frappe-bench-redis.target | 6 ++++++ .../templates/systemd/frappe-bench-web.target | 6 ++++++ .../templates/systemd/frappe-bench-workers.target | 6 ++++++ bench/config/templates/systemd/frappe-bench.target | 6 ++++++ 13 files changed, 133 insertions(+) create mode 100644 bench/config/templates/systemd/frappe-bench-frappe-default-worker.service create mode 100644 bench/config/templates/systemd/frappe-bench-frappe-long-worker.service create mode 100644 bench/config/templates/systemd/frappe-bench-frappe-schedule.service create mode 100644 bench/config/templates/systemd/frappe-bench-frappe-short-worker.service create mode 100644 bench/config/templates/systemd/frappe-bench-frappe-web.service create mode 100644 bench/config/templates/systemd/frappe-bench-node-socketio.service create mode 100644 bench/config/templates/systemd/frappe-bench-redis-cache.service create mode 100644 bench/config/templates/systemd/frappe-bench-redis-queue.service create mode 100644 bench/config/templates/systemd/frappe-bench-redis-socketio.service create mode 100644 bench/config/templates/systemd/frappe-bench-redis.target create mode 100644 bench/config/templates/systemd/frappe-bench-web.target create mode 100644 bench/config/templates/systemd/frappe-bench-workers.target create mode 100644 bench/config/templates/systemd/frappe-bench.target diff --git a/bench/config/templates/systemd/frappe-bench-frappe-default-worker.service b/bench/config/templates/systemd/frappe-bench-frappe-default-worker.service new file mode 100644 index 00000000..d02bc4a4 --- /dev/null +++ b/bench/config/templates/systemd/frappe-bench-frappe-default-worker.service @@ -0,0 +1,12 @@ +[Unit] +Description="{{ bench_name }}-frappe-default-worker" +PartOf={{ bench_name }}-workers.target + +[Service] +User={{ user }} +Group={{ user }} +Restart=always +ExecStart={{ bench_cmd }} worker --queue default +StandardOutput=file:{{ bench_dir }}/logs/worker.log +StandardError=file:{{ bench_dir }}/logs/worker.error.log +WorkingDirectory={{ bench_dir }} diff --git a/bench/config/templates/systemd/frappe-bench-frappe-long-worker.service b/bench/config/templates/systemd/frappe-bench-frappe-long-worker.service new file mode 100644 index 00000000..35489739 --- /dev/null +++ b/bench/config/templates/systemd/frappe-bench-frappe-long-worker.service @@ -0,0 +1,12 @@ +[Unit] +Description="{{ bench_name }}-frappe-short-worker" +PartOf={{ bench_name }}-workers.target + +[Service] +User={{ user }} +Group={{ user }} +Restart=always +ExecStart={{ bench_cmd }} worker --queue long +StandardOutput=file:{{ bench_dir }}/logs/worker.log +StandardError=file:{{ bench_dir }}/logs/worker.error.log +WorkingDirectory={{ bench_dir }} diff --git a/bench/config/templates/systemd/frappe-bench-frappe-schedule.service b/bench/config/templates/systemd/frappe-bench-frappe-schedule.service new file mode 100644 index 00000000..da66c6e8 --- /dev/null +++ b/bench/config/templates/systemd/frappe-bench-frappe-schedule.service @@ -0,0 +1,12 @@ +[Unit] +Description="{{ bench_name }}-frappe-schedule" +PartOf={{ bench_name }}-workers.target + +[Service] +User={{ user }} +Group={{ user }} +Restart=always +ExecStart={{ bench_cmd }} schedule +StandardOutput=file:{{ bench_dir }}/logs/schedule.log +StandardError=file:{{ bench_dir }}/logs/schedule.error.log +WorkingDirectory={{ bench_dir }} diff --git a/bench/config/templates/systemd/frappe-bench-frappe-short-worker.service b/bench/config/templates/systemd/frappe-bench-frappe-short-worker.service new file mode 100644 index 00000000..42843c56 --- /dev/null +++ b/bench/config/templates/systemd/frappe-bench-frappe-short-worker.service @@ -0,0 +1,12 @@ +[Unit] +Description="{{ bench }}-frappe-short-worker" +PartOf={{ bench }}-workers.target + +[Service] +User=revant +Group=revant +Restart=always +ExecStart={{ bench_cmd }} worker --queue short +StandardOutput=file:{{ bench_dir }}/logs/worker.log +StandardError=file:{{ bench_dir }}/logs/worker.error.log +WorkingDirectory={{ bench_dir }} diff --git a/bench/config/templates/systemd/frappe-bench-frappe-web.service b/bench/config/templates/systemd/frappe-bench-frappe-web.service new file mode 100644 index 00000000..bb2f0e38 --- /dev/null +++ b/bench/config/templates/systemd/frappe-bench-frappe-web.service @@ -0,0 +1,12 @@ +[Unit] +Description="{{ bench_name }}-frappe-web" +PartOf={{ bench_name }}-web.target + +[Service] +User={{ user }} +Group={{ user }} +Restart=always +ExecStart={{ bench_dir }}/env/bin/gunicorn -b 127.0.0.1:{{ webserver_port }} -w {{ gunicorn_workers }} -t {{ http_timeout }} frappe.app:application --preload +StandardOutput=file:{{ bench_dir }}/logs/web.log +StandardError=file:{{ bench_dir }}/logs/web.error.log +WorkingDirectory={{ sites_dir }} diff --git a/bench/config/templates/systemd/frappe-bench-node-socketio.service b/bench/config/templates/systemd/frappe-bench-node-socketio.service new file mode 100644 index 00000000..ed49b4a4 --- /dev/null +++ b/bench/config/templates/systemd/frappe-bench-node-socketio.service @@ -0,0 +1,13 @@ +[Unit] +After={{ bench_name }}-frappe-web.service +Description="{{ bench_name }}-node-socketio" +PartOf={{ bench_name }}-web.target + +[Service] +User={{ user }} +Group={{ user }} +Restart=always +ExecStart={{ node }} {{ bench_dir }}/apps/frappe/socketio.js +StandardOutput=file:{{ bench_dir }}/logs/node-socketio.log +StandardError=file:{{ bench_dir }}/logs/node-socketio.error.log +WorkingDirectory={{ bench_dir }} diff --git a/bench/config/templates/systemd/frappe-bench-redis-cache.service b/bench/config/templates/systemd/frappe-bench-redis-cache.service new file mode 100644 index 00000000..025f749e --- /dev/null +++ b/bench/config/templates/systemd/frappe-bench-redis-cache.service @@ -0,0 +1,12 @@ +[Unit] +Description="{{ bench_name }}-redis-cache" +PartOf={{ bench_name }}-redis.target + +[Service] +User={{ user }} +Group={{ user }} +Restart=always +ExecStart={{ redis_server }} {{ redis_cache_config }} +StandardOutput=file:{{ bench_dir }}/logs/redis-cache.log +StandardError=file:{{ bench_dir }}/logs/redis-cache.error.log +WorkingDirectory={{ sites_dir }} diff --git a/bench/config/templates/systemd/frappe-bench-redis-queue.service b/bench/config/templates/systemd/frappe-bench-redis-queue.service new file mode 100644 index 00000000..1d773da7 --- /dev/null +++ b/bench/config/templates/systemd/frappe-bench-redis-queue.service @@ -0,0 +1,12 @@ +[Unit] +Description="{{ bench_name }}-redis-queue" +PartOf={{ bench_name }}-redis.target + +[Service] +User={{ user }} +Group={{ user }} +Restart=always +ExecStart={{ redis_server }} {{ redis_queue_config }} +StandardOutput=file:{{ bench_dir }}/logs/redis-queue.log +StandardError=file:{{ bench_dir }}/logs/redis-queue.error.log +WorkingDirectory={{ sites_dir }} diff --git a/bench/config/templates/systemd/frappe-bench-redis-socketio.service b/bench/config/templates/systemd/frappe-bench-redis-socketio.service new file mode 100644 index 00000000..3eeafdb9 --- /dev/null +++ b/bench/config/templates/systemd/frappe-bench-redis-socketio.service @@ -0,0 +1,12 @@ +[Unit] +Description="{{ bench_name }}-redis-socketio" +PartOf={{ bench_name }}-redis.target + +[Service] +User={{ user }} +Group={{ user }} +Restart=always +ExecStart={{ redis_server }} {{ redis_socketio_config }} +StandardOutput=file:{{ bench_dir }}/logs/redis-socketio.log +StandardError=file:{{ bench_dir }}/logs/redis-socketio.error.log +WorkingDirectory={{ sites_dir }} diff --git a/bench/config/templates/systemd/frappe-bench-redis.target b/bench/config/templates/systemd/frappe-bench-redis.target new file mode 100644 index 00000000..5bc14734 --- /dev/null +++ b/bench/config/templates/systemd/frappe-bench-redis.target @@ -0,0 +1,6 @@ +[Unit] +After=network.target +Wants={{ bench_name }}-redis-cache.service {{ bench_name }}-redis-queue.service {{ bench_name }}-redis-socketio.service + +[Install] +WantedBy=multi-user.target diff --git a/bench/config/templates/systemd/frappe-bench-web.target b/bench/config/templates/systemd/frappe-bench-web.target new file mode 100644 index 00000000..63a4b449 --- /dev/null +++ b/bench/config/templates/systemd/frappe-bench-web.target @@ -0,0 +1,6 @@ +[Unit] +After=network.target +Wants={{ bench_name }}-frappe-web.service {{ bench_name }}-node-socketio.service + +[Install] +WantedBy=multi-user.target diff --git a/bench/config/templates/systemd/frappe-bench-workers.target b/bench/config/templates/systemd/frappe-bench-workers.target new file mode 100644 index 00000000..336521ff --- /dev/null +++ b/bench/config/templates/systemd/frappe-bench-workers.target @@ -0,0 +1,6 @@ +[Unit] +After=network.target +Wants={{ bench_name }}-frappe-default-worker.service {{ bench_name }}-frappe-short-worker.service {{ bench_name }}-frappe-long-worker.service {{ bench_name }}-frappe-schedule.service + +[Install] +WantedBy=multi-user.target diff --git a/bench/config/templates/systemd/frappe-bench.target b/bench/config/templates/systemd/frappe-bench.target new file mode 100644 index 00000000..7d768233 --- /dev/null +++ b/bench/config/templates/systemd/frappe-bench.target @@ -0,0 +1,6 @@ +[Unit] +After=network.target +Requires={{ bench_name }}-web.target {{ bench_name }}-workers.target {{ bench_name }}-redis.target + +[Install] +WantedBy=multi-user.target From bfc77c297d81c4f873bd22ce59d240f1455fd469 Mon Sep 17 00:00:00 2001 From: Ameya Shenoy Date: Thu, 5 Apr 2018 18:52:28 +0530 Subject: [PATCH 02/49] ammend and redo --- bench/config/systemd.py | 44 +++++++++++++++++++++++++++++++++++++++++ bench/utils.py | 29 +++++++++++++++++++++++++++ 2 files changed, 73 insertions(+) create mode 100644 bench/config/systemd.py diff --git a/bench/config/systemd.py b/bench/config/systemd.py new file mode 100644 index 00000000..2d8ce713 --- /dev/null +++ b/bench/config/systemd.py @@ -0,0 +1,44 @@ +import os, getpass, click +import bench + +def generate_supervisor_config(bench_path, user=None, yes=False): + from bench.app import get_current_frappe_version, use_rq + from bench.utils import get_bench_name, find_executable + from bench.config.common_site_config import get_config, update_config, get_gunicorn_workers + + template = bench.env.get_template('systemd/frappe-bench.target') + if not user: + user = getpass.getuser() + + config = get_config(bench_path=bench_path) + + bench_dir = os.path.abspath(bench_path) + + config = template.render(**{ + "bench_dir": bench_dir, + "sites_dir": os.path.join(bench_dir, 'sites'), + "user": user, + "frappe_version": get_current_frappe_version(bench_path), + "use_rq": use_rq(bench_path), + "http_timeout": config.get("http_timeout", 120), + "redis_server": find_executable('redis-server'), + "node": find_executable('node') or find_executable('nodejs'), + "redis_cache_config": os.path.join(bench_dir, 'config', 'systemd/frappe-bench-redis-cache.service'), + "redis_socketio_config": os.path.join(bench_dir, 'config', 'systemd/frappe-bench-redis-socketio.service'), + "redis_queue_config": os.path.join(bench_dir, 'config', 'systemd/frappe-bench-redis-queue.service'), + "webserver_port": config.get('webserver_port', 8000), + "gunicorn_workers": config.get('gunicorn_workers', get_gunicorn_workers()["gunicorn_workers"]), + "bench_name": get_bench_name(bench_path), + "background_workers": config.get('background_workers') or 1, + "bench_cmd": find_executable('bench') + }) + + conf_path = os.path.join(bench_path, 'config', 'supervisor.conf') + if not yes and os.path.exists(conf_path): + click.confirm('supervisor.conf already exists and this will overwrite it. Do you want to continue?', + abort=True) + + with open(conf_path, 'w') as f: + f.write(config) + + update_config({'restart_supervisor_on_update': True}, bench_path=bench_path) diff --git a/bench/utils.py b/bench/utils.py index 029e9ec2..c958a3b3 100755 --- a/bench/utils.py +++ b/bench/utils.py @@ -426,6 +426,35 @@ def restart_supervisor_processes(bench_path='.', web_workers=False): exec_cmd('sudo supervisorctl restart {group}'.format(group=group), cwd=bench_path) +def restart_systemcd_processes(bench_path='.', web_workers=False): + from .config.common_site_config import get_config + conf = get_config(bench_path=bench_path) + bench_name = get_bench_name(bench_path) + + cmd = conf.get('systemd_restart_cmd') + if cmd: + exec_cmd(cmd, cwd=bench_path) + + else: + systemd_status = subprocess.check_output(['sudo', 'systemctl', 'status'], cwd=bench_path) + systemd_status = safe_decode(supervisor_status) + + if web_workers and '{bench_name}-web:'.format(bench_name=bench_name) in systemd_status: + group = '{bench_name}-web: '.format(bench_name=bench_name) + + elif '{bench_name}-workers:'.format(bench_name=bench_name) in systemd_status: + group = '{bench_name}-workers: {bench_name}-web:'.format(bench_name=bench_name) + + # backward compatibility + elif '{bench_name}-processes:'.format(bench_name=bench_name) in systemd_status: + group = '{bench_name}-processes:'.format(bench_name=bench_name) + + # backward compatibility + else: + group = 'frappe:' + + exec_cmd('sudo systemctl restart {group}'.format(group=group), cwd=bench_path) + def set_default_site(site, bench_path='.'): if not site in get_sites(bench_path=bench_path): raise Exception("Site not in bench") From 6a5226e071f181ff699308572e303cc413afd3f5 Mon Sep 17 00:00:00 2001 From: Revant Nandgaonkar Date: Wed, 11 Apr 2018 19:32:15 +0530 Subject: [PATCH 03/49] generate frappe-bench.target --- bench/commands/setup.py | 11 +++++- bench/config/systemd.py | 77 ++++++++++++++++++++++++++++++++++------- 2 files changed, 74 insertions(+), 14 deletions(-) diff --git a/bench/commands/setup.py b/bench/commands/setup.py index c7a8d12d..90101d7f 100755 --- a/bench/commands/setup.py +++ b/bench/commands/setup.py @@ -233,6 +233,14 @@ def setup_nginx_proxy_jail(**kwargs): from bench.utils import run_playbook run_playbook('roles/fail2ban/tasks/configure_nginx_jail.yml', extra_vars=kwargs) +@click.command('systemd') +@click.option('--user') +@click.option('--yes', help='Yes to regeneration of systemd config files', is_flag=True, default=False) +def setup_systemd(user=None, yes=False): + "generate configs for systemd with an optional user argument" + from bench.config.systemd import generate_systemd_config + generate_systemd_config(bench_path=".", user=user, yes=yes) + setup.add_command(setup_sudoers) setup.add_command(setup_nginx) setup.add_command(reload_nginx) @@ -254,4 +262,5 @@ setup.add_command(sync_domains) setup.add_command(setup_firewall) setup.add_command(set_ssh_port) setup.add_command(setup_roles) -setup.add_command(setup_nginx_proxy_jail) \ No newline at end of file +setup.add_command(setup_nginx_proxy_jail) +setup.add_command(setup_systemd) diff --git a/bench/config/systemd.py b/bench/config/systemd.py index 2d8ce713..b5579f43 100644 --- a/bench/config/systemd.py +++ b/bench/config/systemd.py @@ -1,20 +1,20 @@ import os, getpass, click import bench -def generate_supervisor_config(bench_path, user=None, yes=False): +def generate_systemd_config(bench_path, user=None, yes=False): from bench.app import get_current_frappe_version, use_rq from bench.utils import get_bench_name, find_executable from bench.config.common_site_config import get_config, update_config, get_gunicorn_workers - template = bench.env.get_template('systemd/frappe-bench.target') if not user: user = getpass.getuser() config = get_config(bench_path=bench_path) bench_dir = os.path.abspath(bench_path) + bench_name = get_bench_name(bench_path) - config = template.render(**{ + bench_info = { "bench_dir": bench_dir, "sites_dir": os.path.join(bench_dir, 'sites'), "user": user, @@ -23,22 +23,73 @@ def generate_supervisor_config(bench_path, user=None, yes=False): "http_timeout": config.get("http_timeout", 120), "redis_server": find_executable('redis-server'), "node": find_executable('node') or find_executable('nodejs'), - "redis_cache_config": os.path.join(bench_dir, 'config', 'systemd/frappe-bench-redis-cache.service'), - "redis_socketio_config": os.path.join(bench_dir, 'config', 'systemd/frappe-bench-redis-socketio.service'), - "redis_queue_config": os.path.join(bench_dir, 'config', 'systemd/frappe-bench-redis-queue.service'), + "redis_cache_config": os.path.join(bench_dir, 'config', 'redis_cache.conf'), + "redis_socketio_config": os.path.join(bench_dir, 'config', 'redis_socketio.conf'), + "redis_queue_config": os.path.join(bench_dir, 'config', 'redis_queue.conf'), "webserver_port": config.get('webserver_port', 8000), "gunicorn_workers": config.get('gunicorn_workers', get_gunicorn_workers()["gunicorn_workers"]), "bench_name": get_bench_name(bench_path), "background_workers": config.get('background_workers') or 1, "bench_cmd": find_executable('bench') - }) + } - conf_path = os.path.join(bench_path, 'config', 'supervisor.conf') - if not yes and os.path.exists(conf_path): - click.confirm('supervisor.conf already exists and this will overwrite it. Do you want to continue?', + if not yes: + click.confirm('current systemd configuration will be overwritten. Do you want to continue?', abort=True) - with open(conf_path, 'w') as f: - f.write(config) + setup_systemd_directory(bench_path) + setup_main_config(bench_info, bench_path) + # setup_workers_config(bench_info) + # setup_web_config(bench_info) + # setup_redis_config(bench_info) - update_config({'restart_supervisor_on_update': True}, bench_path=bench_path) + update_config({'restart_systemd_on_update': True}, bench_path=bench_path) + +def setup_systemd_directory(bench_path): + if not os.path.exists(os.path.join(bench_path, 'config', 'systemd')): + os.makedirs(os.path.join(bench_path, 'config', 'systemd')) + +def setup_main_config(bench_info, bench_path): + # Main config + bench_template = bench.env.get_template('systemd/frappe-bench.target') + bench_config = bench_template.render(**bench_info) + bench_config_path = os.path.join(bench_path, 'config', 'systemd' , bench_info.get("bench_name") + '.target') + + with open(bench_config_path, 'w') as f: + f.write(bench_config) + +def setup_workers_config(bench_info, bench_path): + # Worker Group + bench_workers_target_template = bench.env.get_template('systemd/frappe-bench-workers.target') + bench_default_worker_template = bench.env.get_template('systemd/frappe-bench-frappe-default-worker.service') + bench_short_worker_template = bench.env.get_template('systemd/frappe-bench-frappe-short-worker.service') + bench_long_worker_template = bench.env.get_template('systemd/frappe-bench-frappe-long-worker.service') + bench_schedule_worker_template = bench.env.get_template('systemd/frappe-bench-frappe-schedule.service') + + bench_workers_target_config = bench_workers_target_template.render(**bench_info) + bench_default_worker_config = bench_default_worker_template.render(**bench_info) + bench_short_worker_config = bench_short_worker_template.render(**bench_info) + bench_long_worker_config = bench_long_worker_template.render(**bench_info) + bench_schedule_worker_config = bench_schedule_worker_template.render(**bench_info) + +def setup_web_config(bench_info, bench_path): + # Web Group + bench_web_target_template = bench.env.get_template('systemd/frappe-bench-web.target') + bench_web_service_template = bench.env.get_template('systemd/frappe-bench-frappe-web.service') + bench_node_socketio_template = bench.env.get_template('systemd/frappe-bench-node-socketio.service') + + bench_web_target_config = bench_web_target_template.render(**bench_info) + bench_web_service_config = bench_web_service_template.render(**bench_info) + bench_node_socketio_config = bench_node_socketio_template.render(**bench_info) + +def setup_redis_config(bench_info, bench_path): + # Redis Group + bench_redis_target_template = bench.env.get_template('systemd/frappe-bench-redis.target') + bench_redis_cache_template = bench.env.get_template('systemd/frappe-bench-redis-cache.service') + bench_redis_queue_template = bench.env.get_template('systemd/frappe-bench-redis-queue.service') + bench_redis_socketio_template = bench.env.get_template('systemd/frappe-bench-redis-socketio.service') + + bench_redis_target_config = bench_redis_target_template.render(**bench_info) + bench_redis_cache_config = bench_redis_cache_template.render(**bench_info) + bench_redis_queue_config = bench_redis_queue_template.render(**bench_info) + bench_redis_socketio_config = bench_redis_socketio_template.render(**bench_info) From 1b3c98d0ff7256779c8e0b71a370de0b3a42d225 Mon Sep 17 00:00:00 2001 From: Revant Nandgaonkar Date: Wed, 11 Apr 2018 21:29:06 +0530 Subject: [PATCH 04/49] bench setup systemd generates all configs --- bench/config/systemd.py | 57 ++++++++++++++++++++++++++++++++++++++--- 1 file changed, 54 insertions(+), 3 deletions(-) diff --git a/bench/config/systemd.py b/bench/config/systemd.py index b5579f43..0ba80e52 100644 --- a/bench/config/systemd.py +++ b/bench/config/systemd.py @@ -39,9 +39,9 @@ def generate_systemd_config(bench_path, user=None, yes=False): setup_systemd_directory(bench_path) setup_main_config(bench_info, bench_path) - # setup_workers_config(bench_info) - # setup_web_config(bench_info) - # setup_redis_config(bench_info) + setup_workers_config(bench_info, bench_path) + setup_web_config(bench_info, bench_path) + setup_redis_config(bench_info, bench_path) update_config({'restart_systemd_on_update': True}, bench_path=bench_path) @@ -72,6 +72,27 @@ def setup_workers_config(bench_info, bench_path): bench_long_worker_config = bench_long_worker_template.render(**bench_info) bench_schedule_worker_config = bench_schedule_worker_template.render(**bench_info) + bench_workers_target_config_path = os.path.join(bench_path, 'config', 'systemd' , bench_info.get("bench_name") + '-workers.target') + bench_default_worker_config_path = os.path.join(bench_path, 'config', 'systemd' , bench_info.get("bench_name") + '-frappe-default-worker.service') + bench_short_worker_config_path = os.path.join(bench_path, 'config', 'systemd' , bench_info.get("bench_name") + '-frappe-short-worker.service') + bench_long_worker_config_path = os.path.join(bench_path, 'config', 'systemd' , bench_info.get("bench_name") + '-frappe-long-worker.service') + bench_schedule_worker_config_path = os.path.join(bench_path, 'config', 'systemd' , bench_info.get("bench_name") + '-frappe-schedule.service') + + with open(bench_workers_target_config_path, 'w') as f: + f.write(bench_workers_target_config) + + with open(bench_default_worker_config_path, 'w') as f: + f.write(bench_default_worker_config) + + with open(bench_short_worker_config_path, 'w') as f: + f.write(bench_short_worker_config) + + with open(bench_long_worker_config_path, 'w') as f: + f.write(bench_long_worker_config) + + with open(bench_schedule_worker_config_path, 'w') as f: + f.write(bench_schedule_worker_config) + def setup_web_config(bench_info, bench_path): # Web Group bench_web_target_template = bench.env.get_template('systemd/frappe-bench-web.target') @@ -82,6 +103,19 @@ def setup_web_config(bench_info, bench_path): bench_web_service_config = bench_web_service_template.render(**bench_info) bench_node_socketio_config = bench_node_socketio_template.render(**bench_info) + bench_web_target_config_path = os.path.join(bench_path, 'config', 'systemd' , bench_info.get("bench_name") + '-web.target') + bench_web_service_config_path = os.path.join(bench_path, 'config', 'systemd' , bench_info.get("bench_name") + '-frappe-web.service') + bench_node_socketio_config_path = os.path.join(bench_path, 'config', 'systemd' , bench_info.get("bench_name") + '-node-socketio.service') + + with open(bench_web_target_config_path, 'w') as f: + f.write(bench_web_target_config) + + with open(bench_web_service_config_path, 'w') as f: + f.write(bench_web_service_config) + + with open(bench_node_socketio_config_path, 'w') as f: + f.write(bench_node_socketio_config) + def setup_redis_config(bench_info, bench_path): # Redis Group bench_redis_target_template = bench.env.get_template('systemd/frappe-bench-redis.target') @@ -93,3 +127,20 @@ def setup_redis_config(bench_info, bench_path): bench_redis_cache_config = bench_redis_cache_template.render(**bench_info) bench_redis_queue_config = bench_redis_queue_template.render(**bench_info) bench_redis_socketio_config = bench_redis_socketio_template.render(**bench_info) + + bench_redis_target_config_path = os.path.join(bench_path, 'config', 'systemd' , bench_info.get("bench_name") + '-redis.target') + bench_redis_cache_config_path = os.path.join(bench_path, 'config', 'systemd' , bench_info.get("bench_name") + '-redis-cache.service') + bench_redis_queue_config_path = os.path.join(bench_path, 'config', 'systemd' , bench_info.get("bench_name") + '-redis-queue.service') + bench_redis_socketio_config_path = os.path.join(bench_path, 'config', 'systemd' , bench_info.get("bench_name") + '-redis-socketio.service') + + with open(bench_redis_target_config_path, 'w') as f: + f.write(bench_redis_target_config) + + with open(bench_redis_cache_config_path, 'w') as f: + f.write(bench_redis_cache_config) + + with open(bench_redis_queue_config_path, 'w') as f: + f.write(bench_redis_queue_config) + + with open(bench_redis_socketio_config_path, 'w') as f: + f.write(bench_redis_socketio_config) From 624e8bdf635b7eda4dc56403e4048ea2c8384edd Mon Sep 17 00:00:00 2001 From: Revant Nandgaonkar Date: Wed, 11 Apr 2018 22:25:47 +0530 Subject: [PATCH 05/49] restart systemd on update --- bench/app.py | 8 ++++++-- bench/commands/config.py | 7 +++++++ bench/commands/utils.py | 11 +++++++---- bench/config/common_site_config.py | 1 + 4 files changed, 21 insertions(+), 6 deletions(-) diff --git a/bench/app.py b/bench/app.py index 443156f3..436700c3 100755 --- a/bench/app.py +++ b/bench/app.py @@ -1,6 +1,7 @@ import os from .utils import (exec_cmd, get_frappe, check_git_for_shallow_clone, build_assets, - restart_supervisor_processes, get_cmd_output, run_frappe_cmd, CommandFailedError) + restart_supervisor_processes, get_cmd_output, run_frappe_cmd, CommandFailedError, + restart_systemd_processes) from .config.common_site_config import get_config import logging @@ -110,6 +111,8 @@ def get_app(git_url, branch=None, bench_path='.', build_asset_files=True, verbos conf = get_config(bench_path=bench_path) if conf.get('restart_supervisor_on_update'): restart_supervisor_processes(bench_path=bench_path) + if conf.get('restart_systemd_on_update'): + restart_systemd_processes(bench_path=bench_path) def new_app(app, bench_path='.'): # For backwards compatibility @@ -160,7 +163,8 @@ def remove_app(app, bench_path='.'): run_frappe_cmd("build", bench_path=bench_path) if get_config(bench_path).get('restart_supervisor_on_update'): restart_supervisor_processes(bench_path=bench_path) - + if get_config(bench_path).get('restart_systemd_on_update'): + restart_systemd_processes(bench_path=bench_path) def pull_all_apps(bench_path='.', reset=False): '''Check all apps if there no local changes, pull''' diff --git a/bench/commands/config.py b/bench/commands/config.py index cd94985a..a7a0f3ec 100644 --- a/bench/commands/config.py +++ b/bench/commands/config.py @@ -23,6 +23,12 @@ def config_restart_supervisor_on_update(state): state = True if state == 'on' else False update_config({'restart_supervisor_on_update': state}) +@click.command('restart_systemd_on_update') +@click.argument('state', type=click.Choice(['on', 'off'])) +def config_restart_systemd_on_update(state): + "Enable/Disable auto restart of systemd units" + state = True if state == 'on' else False + update_config({'restart_systemd_on_update': state}) @click.command('update_bench_on_update') @click.argument('state', type=click.Choice(['on', 'off'])) @@ -112,6 +118,7 @@ def remove_common_config(keys): config.add_command(config_auto_update) config.add_command(config_update_bench_on_update) config.add_command(config_restart_supervisor_on_update) +config.add_command(config_restart_systemd_on_update) config.add_command(config_dns_multitenant) config.add_command(config_serve_default_site) config.add_command(config_http_timeout) diff --git a/bench/commands/utils.py b/bench/commands/utils.py index 3a3202c4..9445def6 100644 --- a/bench/commands/utils.py +++ b/bench/commands/utils.py @@ -14,10 +14,13 @@ def start(no_dev, concurrency): @click.command('restart') @click.option('--web', is_flag=True, default=False) def restart(web): - "Restart supervisor processes" - from bench.utils import restart_supervisor_processes - restart_supervisor_processes(bench_path='.', web_workers=web) - + "Restart supervisor processes or systemd units" + from bench.utils import restart_supervisor_processes, restart_systemd_processes + from bench.config.common_site_config import get_config + if get_config('.').get('restart_supervisor_on_update') == True: + restart_supervisor_processes(bench_path='.', web_workers=web) + if get_config('.').get('restart_systemd_on_update'): + restart_systemd_processes(bench_path='.', web_workers=web) @click.command('set-nginx-port') @click.argument('site') diff --git a/bench/config/common_site_config.py b/bench/config/common_site_config.py index f99b07cb..d0cbddb2 100644 --- a/bench/config/common_site_config.py +++ b/bench/config/common_site_config.py @@ -7,6 +7,7 @@ except ImportError: default_config = { 'restart_supervisor_on_update': False, + 'restart_systemd_on_update': False, 'auto_update': False, 'serve_default_site': True, 'rebase_on_pull': False, From b94a5e63b92baf04e4a5027449be3e9eb5eebe44 Mon Sep 17 00:00:00 2001 From: Revant Nandgaonkar Date: Wed, 11 Apr 2018 22:26:49 +0530 Subject: [PATCH 06/49] bench update restarts systemd if specified --- bench/commands/update.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/bench/commands/update.py b/bench/commands/update.py index 5de03cbe..0060ebdd 100755 --- a/bench/commands/update.py +++ b/bench/commands/update.py @@ -3,7 +3,8 @@ import sys, os from bench.config.common_site_config import get_config from bench.app import pull_all_apps, is_version_upgrade from bench.utils import (update_bench, validate_upgrade, pre_upgrade, post_upgrade, before_update, - update_requirements, update_node_packages, backup_all_sites, patch_sites, build_assets, restart_supervisor_processes) + update_requirements, update_node_packages, backup_all_sites, patch_sites, build_assets, + restart_supervisor_processes, restart_systemd_processes) from bench import patches #TODO: Not DRY @@ -14,11 +15,12 @@ from bench import patches @click.option('--bench',is_flag=True, help="Update bench") @click.option('--requirements',is_flag=True, help="Update requirements") @click.option('--restart-supervisor',is_flag=True, help="restart supervisor processes after update") +@click.option('--restart-systemd',is_flag=True, help="restart systemd units after update") @click.option('--auto',is_flag=True) @click.option('--no-backup',is_flag=True) @click.option('--force',is_flag=True) @click.option('--reset', is_flag=True, help="Hard resets git branch's to their new states overriding any changes and overriding rebase on pull") -def update(pull=False, patch=False, build=False, bench=False, auto=False, restart_supervisor=False, requirements=False, no_backup=False, force=False, reset=False): +def update(pull=False, patch=False, build=False, bench=False, auto=False, restart_supervisor=False, restart_systemd=False, requirements=False, no_backup=False, force=False, reset=False): "Update bench" if not (pull or patch or build or bench or requirements): @@ -55,10 +57,10 @@ def update(pull=False, patch=False, build=False, bench=False, auto=False, restar print("This would take significant time to migrate and might break custom apps.") click.confirm('Do you want to continue?', abort=True) - _update(pull, patch, build, bench, auto, restart_supervisor, requirements, no_backup, force=force, reset=reset) + _update(pull, patch, build, bench, auto, restart_supervisor, restart_systemd, requirements, no_backup, force=force, reset=reset) def _update(pull=False, patch=False, build=False, update_bench=False, auto=False, restart_supervisor=False, - requirements=False, no_backup=False, bench_path='.', force=False, reset=False): + restart_systemd=False, requirements=False, no_backup=False, bench_path='.', force=False, reset=False): conf = get_config(bench_path=bench_path) version_upgrade = is_version_upgrade(bench_path=bench_path) @@ -94,6 +96,8 @@ def _update(pull=False, patch=False, build=False, update_bench=False, auto=False post_upgrade(version_upgrade[1], version_upgrade[2], bench_path=bench_path) if restart_supervisor or conf.get('restart_supervisor_on_update'): restart_supervisor_processes(bench_path=bench_path) + if restart_systemd or conf.get('restart_systemd_on_update'): + restart_systemd_processes(bench_path=bench_path) print("_"*80) print("Bench: Deployment tool for Frappe and ERPNext (https://erpnext.org).") From 9f7725f6ed2e0551d2983a6740509d7c7186edb9 Mon Sep 17 00:00:00 2001 From: Revant Nandgaonkar Date: Wed, 11 Apr 2018 22:27:54 +0530 Subject: [PATCH 07/49] TODO: restart_systemd_processes --- bench/utils.py | 43 ++++++++++++++++++++++--------------------- 1 file changed, 22 insertions(+), 21 deletions(-) diff --git a/bench/utils.py b/bench/utils.py index 8240058c..88cd1768 100755 --- a/bench/utils.py +++ b/bench/utils.py @@ -397,34 +397,35 @@ def restart_supervisor_processes(bench_path='.', web_workers=False): exec_cmd('sudo supervisorctl restart {group}'.format(group=group), cwd=bench_path) -def restart_systemcd_processes(bench_path='.', web_workers=False): - from .config.common_site_config import get_config - conf = get_config(bench_path=bench_path) - bench_name = get_bench_name(bench_path) +def restart_systemd_processes(bench_path='.', web_workers=False): + print("Restarting . . .") + # from .config.common_site_config import get_config + # conf = get_config(bench_path=bench_path) + # bench_name = get_bench_name(bench_path) - cmd = conf.get('systemd_restart_cmd') - if cmd: - exec_cmd(cmd, cwd=bench_path) + # cmd = conf.get('systemd_restart_cmd') + # if cmd: + # exec_cmd(cmd, cwd=bench_path) - else: - systemd_status = subprocess.check_output(['sudo', 'systemctl', 'status'], cwd=bench_path) - systemd_status = safe_decode(supervisor_status) + # else: + # systemd_status = subprocess.check_output(['sudo', 'systemctl', 'status'], cwd=bench_path) + # systemd_status = safe_decode(supervisor_status) - if web_workers and '{bench_name}-web:'.format(bench_name=bench_name) in systemd_status: - group = '{bench_name}-web: '.format(bench_name=bench_name) + # if web_workers and '{bench_name}-web:'.format(bench_name=bench_name) in systemd_status: + # group = '{bench_name}-web: '.format(bench_name=bench_name) - elif '{bench_name}-workers:'.format(bench_name=bench_name) in systemd_status: - group = '{bench_name}-workers: {bench_name}-web:'.format(bench_name=bench_name) + # elif '{bench_name}-workers:'.format(bench_name=bench_name) in systemd_status: + # group = '{bench_name}-workers: {bench_name}-web:'.format(bench_name=bench_name) - # backward compatibility - elif '{bench_name}-processes:'.format(bench_name=bench_name) in systemd_status: - group = '{bench_name}-processes:'.format(bench_name=bench_name) + # # backward compatibility + # elif '{bench_name}-processes:'.format(bench_name=bench_name) in systemd_status: + # group = '{bench_name}-processes:'.format(bench_name=bench_name) - # backward compatibility - else: - group = 'frappe:' + # # backward compatibility + # else: + # group = 'frappe:' - exec_cmd('sudo systemctl restart {group}'.format(group=group), cwd=bench_path) + # exec_cmd('sudo systemctl restart {group}'.format(group=group), cwd=bench_path) def set_default_site(site, bench_path='.'): if not site in get_sites(bench_path=bench_path): From b5f180ae9fc92160205b917bf169672cf912c755 Mon Sep 17 00:00:00 2001 From: Revant Nandgaonkar Date: Thu, 12 Apr 2018 16:05:29 +0530 Subject: [PATCH 08/49] Fix short worker template --- .../systemd/frappe-bench-frappe-short-worker.service | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/bench/config/templates/systemd/frappe-bench-frappe-short-worker.service b/bench/config/templates/systemd/frappe-bench-frappe-short-worker.service index 42843c56..79d266fe 100644 --- a/bench/config/templates/systemd/frappe-bench-frappe-short-worker.service +++ b/bench/config/templates/systemd/frappe-bench-frappe-short-worker.service @@ -1,10 +1,10 @@ [Unit] -Description="{{ bench }}-frappe-short-worker" -PartOf={{ bench }}-workers.target +Description="{{ bench_name }}-frappe-short-worker" +PartOf={{ bench_name }}-workers.target [Service] -User=revant -Group=revant +User={{ user }} +Group={{ user }} Restart=always ExecStart={{ bench_cmd }} worker --queue short StandardOutput=file:{{ bench_dir }}/logs/worker.log From b285a688d7cd09740cd1dfa6f5fb75ba7798bd0d Mon Sep 17 00:00:00 2001 From: Revant Nandgaonkar Date: Thu, 12 Apr 2018 17:04:17 +0530 Subject: [PATCH 09/49] systemd for bench bench setup systemd bench restart --systemd bench setup systemd --stop --- bench/commands/setup.py | 5 +++-- bench/commands/utils.py | 8 +++++--- bench/config/systemd.py | 7 ++++++- bench/utils.py | 33 +++++---------------------------- 4 files changed, 19 insertions(+), 34 deletions(-) diff --git a/bench/commands/setup.py b/bench/commands/setup.py index bc663777..ba31ea7c 100755 --- a/bench/commands/setup.py +++ b/bench/commands/setup.py @@ -258,10 +258,11 @@ def setup_nginx_proxy_jail(**kwargs): @click.command('systemd') @click.option('--user') @click.option('--yes', help='Yes to regeneration of systemd config files', is_flag=True, default=False) -def setup_systemd(user=None, yes=False): +@click.option('--stop', help='Stop bench services', is_flag=True, default=False) +def setup_systemd(user=None, yes=False, stop=False): "generate configs for systemd with an optional user argument" from bench.config.systemd import generate_systemd_config - generate_systemd_config(bench_path=".", user=user, yes=yes) + generate_systemd_config(bench_path=".", user=user, yes=yes, stop=stop) setup.add_command(setup_sudoers) setup.add_command(setup_nginx) diff --git a/bench/commands/utils.py b/bench/commands/utils.py index 9445def6..c10fca08 100644 --- a/bench/commands/utils.py +++ b/bench/commands/utils.py @@ -13,13 +13,15 @@ def start(no_dev, concurrency): @click.command('restart') @click.option('--web', is_flag=True, default=False) -def restart(web): +@click.option('--supervisor', is_flag=True, default=False) +@click.option('--systemd', is_flag=True, default=False) +def restart(web, supervisor, systemd): "Restart supervisor processes or systemd units" from bench.utils import restart_supervisor_processes, restart_systemd_processes from bench.config.common_site_config import get_config - if get_config('.').get('restart_supervisor_on_update') == True: + if get_config('.').get('restart_supervisor_on_update') or supervisor: restart_supervisor_processes(bench_path='.', web_workers=web) - if get_config('.').get('restart_systemd_on_update'): + if get_config('.').get('restart_systemd_on_update') or systemd: restart_systemd_processes(bench_path='.', web_workers=web) @click.command('set-nginx-port') diff --git a/bench/config/systemd.py b/bench/config/systemd.py index 0ba80e52..9686ad56 100644 --- a/bench/config/systemd.py +++ b/bench/config/systemd.py @@ -1,7 +1,7 @@ import os, getpass, click import bench -def generate_systemd_config(bench_path, user=None, yes=False): +def generate_systemd_config(bench_path, user=None, yes=False, stop=False): from bench.app import get_current_frappe_version, use_rq from bench.utils import get_bench_name, find_executable from bench.config.common_site_config import get_config, update_config, get_gunicorn_workers @@ -14,6 +14,11 @@ def generate_systemd_config(bench_path, user=None, yes=False): bench_dir = os.path.abspath(bench_path) bench_name = get_bench_name(bench_path) + if stop: + from bench.utils import exec_cmd + exec_cmd('sudo systemctl stop -- $(systemctl show -p Requires {bench_name}.target | cut -d= -f2)'.format(bench_name=bench_name)) + return + bench_info = { "bench_dir": bench_dir, "sites_dir": os.path.join(bench_dir, 'sites'), diff --git a/bench/utils.py b/bench/utils.py index 88cd1768..8c55171f 100755 --- a/bench/utils.py +++ b/bench/utils.py @@ -398,34 +398,11 @@ def restart_supervisor_processes(bench_path='.', web_workers=False): exec_cmd('sudo supervisorctl restart {group}'.format(group=group), cwd=bench_path) def restart_systemd_processes(bench_path='.', web_workers=False): - print("Restarting . . .") - # from .config.common_site_config import get_config - # conf = get_config(bench_path=bench_path) - # bench_name = get_bench_name(bench_path) - - # cmd = conf.get('systemd_restart_cmd') - # if cmd: - # exec_cmd(cmd, cwd=bench_path) - - # else: - # systemd_status = subprocess.check_output(['sudo', 'systemctl', 'status'], cwd=bench_path) - # systemd_status = safe_decode(supervisor_status) - - # if web_workers and '{bench_name}-web:'.format(bench_name=bench_name) in systemd_status: - # group = '{bench_name}-web: '.format(bench_name=bench_name) - - # elif '{bench_name}-workers:'.format(bench_name=bench_name) in systemd_status: - # group = '{bench_name}-workers: {bench_name}-web:'.format(bench_name=bench_name) - - # # backward compatibility - # elif '{bench_name}-processes:'.format(bench_name=bench_name) in systemd_status: - # group = '{bench_name}-processes:'.format(bench_name=bench_name) - - # # backward compatibility - # else: - # group = 'frappe:' - - # exec_cmd('sudo systemctl restart {group}'.format(group=group), cwd=bench_path) + from .config.common_site_config import get_config + conf = get_config(bench_path=bench_path) + bench_name = get_bench_name(bench_path) + exec_cmd('sudo systemctl stop -- $(systemctl show -p Requires {bench_name}.target | cut -d= -f2)'.format(bench_name=bench_name)) + exec_cmd('sudo systemctl start {bench_name}.target'.format(bench_name=bench_name)) def set_default_site(site, bench_path='.'): if not site in get_sites(bench_path=bench_path): From 47b88fb722fe6f7d7bdbd72f53321ec19cf15d36 Mon Sep 17 00:00:00 2001 From: Revant Nandgaonkar Date: Fri, 13 Apr 2018 00:26:52 +0530 Subject: [PATCH 10/49] symlinks creation/deletion --- bench/commands/setup.py | 7 ++-- bench/config/systemd.py | 73 ++++++++++++++++++++++++++++++++++++++--- 2 files changed, 73 insertions(+), 7 deletions(-) diff --git a/bench/commands/setup.py b/bench/commands/setup.py index ba31ea7c..e29f8af4 100755 --- a/bench/commands/setup.py +++ b/bench/commands/setup.py @@ -259,10 +259,13 @@ def setup_nginx_proxy_jail(**kwargs): @click.option('--user') @click.option('--yes', help='Yes to regeneration of systemd config files', is_flag=True, default=False) @click.option('--stop', help='Stop bench services', is_flag=True, default=False) -def setup_systemd(user=None, yes=False, stop=False): +@click.option('--create-symlinks', help='Create Symlinks', is_flag=True, default=False) +@click.option('--delete-symlinks', help='Delete Symlinks', is_flag=True, default=False) +def setup_systemd(user=None, yes=False, stop=False, create_symlinks=False, delete_symlinks=False): "generate configs for systemd with an optional user argument" from bench.config.systemd import generate_systemd_config - generate_systemd_config(bench_path=".", user=user, yes=yes, stop=stop) + generate_systemd_config(bench_path=".", user=user, yes=yes, + stop=stop, create_symlinks=create_symlinks, delete_symlinks=delete_symlinks) setup.add_command(setup_sudoers) setup.add_command(setup_nginx) diff --git a/bench/config/systemd.py b/bench/config/systemd.py index 9686ad56..bc343dfd 100644 --- a/bench/config/systemd.py +++ b/bench/config/systemd.py @@ -1,10 +1,13 @@ import os, getpass, click import bench +from bench.utils import exec_cmd +from bench.app import get_current_frappe_version, use_rq +from bench.utils import get_bench_name, find_executable +from bench.config.common_site_config import get_config, update_config, get_gunicorn_workers -def generate_systemd_config(bench_path, user=None, yes=False, stop=False): - from bench.app import get_current_frappe_version, use_rq - from bench.utils import get_bench_name, find_executable - from bench.config.common_site_config import get_config, update_config, get_gunicorn_workers +def generate_systemd_config(bench_path, user=None, yes=False, + stop=False, create_symlinks=False, + delete_symlinks=False): if not user: user = getpass.getuser() @@ -15,10 +18,17 @@ def generate_systemd_config(bench_path, user=None, yes=False, stop=False): bench_name = get_bench_name(bench_path) if stop: - from bench.utils import exec_cmd exec_cmd('sudo systemctl stop -- $(systemctl show -p Requires {bench_name}.target | cut -d= -f2)'.format(bench_name=bench_name)) return + if create_symlinks: + _create_symlinks(bench_path) + return + + if delete_symlinks: + _delete_symlinks(bench_path) + return + bench_info = { "bench_dir": bench_dir, "sites_dir": os.path.join(bench_dir, 'sites'), @@ -149,3 +159,56 @@ def setup_redis_config(bench_info, bench_path): with open(bench_redis_socketio_config_path, 'w') as f: f.write(bench_redis_socketio_config) + +def _create_symlinks(bench_path): + bench_dir = os.path.abspath(bench_path) + etc_systemd_system = os.path.join('/', 'etc', 'systemd', 'system') + config_path = os.path.join(bench_dir, 'config', 'systemd') + unit_files = get_unit_files(bench_dir) + for unit_file in unit_files: + filename = "".join(unit_file) + # if '-worker' in unit_file[0] and unit_file[1] == '.service': + # unit_file.insert(1, '@') + + exec_cmd('sudo ln -s {config_path}/{unit_file} {etc_systemd_system}/{unit_file_init}'.format( + config_path=config_path, + etc_systemd_system=etc_systemd_system, + unit_file=filename, + unit_file_init="".join(unit_file) + )) + exec_cmd('sudo systemctl daemon-reload') + +def _delete_symlinks(bench_path): + bench_dir = os.path.abspath(bench_path) + etc_systemd_system = os.path.join('/', 'etc', 'systemd', 'system') + config_path = os.path.join(bench_dir, 'config', 'systemd') + unit_files = get_unit_files(bench_dir) + for unit_file in unit_files: + # if '-worker' in unit_file[0] and unit_file[1] == '.service': + # unit_file.insert(1, '@') + + exec_cmd('sudo rm {etc_systemd_system}/{unit_file_init}'.format( + config_path=config_path, + etc_systemd_system=etc_systemd_system, + unit_file_init="".join(unit_file) + )) + exec_cmd('sudo systemctl daemon-reload') + +def get_unit_files(bench_path): + bench_name = get_bench_name(bench_path) + unit_files = [ + [bench_name, ".target"], + [bench_name+"-workers", ".target"], + [bench_name+"-web", ".target"], + [bench_name+"-redis", ".target"], + [bench_name+"-frappe-default-worker", ".service"], + [bench_name+"-frappe-short-worker", ".service"], + [bench_name+"-frappe-long-worker", ".service"], + [bench_name+"-frappe-schedule", ".service"], + [bench_name+"-frappe-web", ".service"], + [bench_name+"-node-socketio", ".service"], + [bench_name+"-redis-cache", ".service"], + [bench_name+"-redis-queue", ".service"], + [bench_name+"-redis-socketio", ".service"], + ] + return unit_files From b13773d806894c4ff21d618aa0b7cced813b10e1 Mon Sep 17 00:00:00 2001 From: Revant Nandgaonkar Date: Mon, 16 Apr 2018 19:49:44 +0530 Subject: [PATCH 11/49] initialize number of processes from template --- bench/config/systemd.py | 31 +++++++++++-------- ...frappe-bench-frappe-default-worker.service | 2 +- .../frappe-bench-frappe-long-worker.service | 2 +- .../frappe-bench-frappe-short-worker.service | 2 +- .../systemd/frappe-bench-workers.target | 2 +- 5 files changed, 22 insertions(+), 17 deletions(-) diff --git a/bench/config/systemd.py b/bench/config/systemd.py index bc343dfd..856d1083 100644 --- a/bench/config/systemd.py +++ b/bench/config/systemd.py @@ -29,6 +29,17 @@ def generate_systemd_config(bench_path, user=None, yes=False, _delete_symlinks(bench_path) return + number_of_workers = config.get('background_workers') or 1 + background_workers = [] + for i in range(number_of_workers): + background_workers.append(get_bench_name(bench_path) + "-frappe-default-worker@" + str(i+1) + ".service") + + for i in range(number_of_workers): + background_workers.append(get_bench_name(bench_path) + "-frappe-short-worker@" + str(i+1) + ".service") + + for i in range(number_of_workers): + background_workers.append(get_bench_name(bench_path) + "-frappe-long-worker@" + str(i+1) + ".service") + bench_info = { "bench_dir": bench_dir, "sites_dir": os.path.join(bench_dir, 'sites'), @@ -44,7 +55,7 @@ def generate_systemd_config(bench_path, user=None, yes=False, "webserver_port": config.get('webserver_port', 8000), "gunicorn_workers": config.get('gunicorn_workers', get_gunicorn_workers()["gunicorn_workers"]), "bench_name": get_bench_name(bench_path), - "background_workers": config.get('background_workers') or 1, + "worker_target_wants": " ".join(background_workers), "bench_cmd": find_executable('bench') } @@ -88,9 +99,9 @@ def setup_workers_config(bench_info, bench_path): bench_schedule_worker_config = bench_schedule_worker_template.render(**bench_info) bench_workers_target_config_path = os.path.join(bench_path, 'config', 'systemd' , bench_info.get("bench_name") + '-workers.target') - bench_default_worker_config_path = os.path.join(bench_path, 'config', 'systemd' , bench_info.get("bench_name") + '-frappe-default-worker.service') - bench_short_worker_config_path = os.path.join(bench_path, 'config', 'systemd' , bench_info.get("bench_name") + '-frappe-short-worker.service') - bench_long_worker_config_path = os.path.join(bench_path, 'config', 'systemd' , bench_info.get("bench_name") + '-frappe-long-worker.service') + bench_default_worker_config_path = os.path.join(bench_path, 'config', 'systemd' , bench_info.get("bench_name") + '-frappe-default-worker@.service') + bench_short_worker_config_path = os.path.join(bench_path, 'config', 'systemd' , bench_info.get("bench_name") + '-frappe-short-worker@.service') + bench_long_worker_config_path = os.path.join(bench_path, 'config', 'systemd' , bench_info.get("bench_name") + '-frappe-long-worker@.service') bench_schedule_worker_config_path = os.path.join(bench_path, 'config', 'systemd' , bench_info.get("bench_name") + '-frappe-schedule.service') with open(bench_workers_target_config_path, 'w') as f: @@ -167,9 +178,6 @@ def _create_symlinks(bench_path): unit_files = get_unit_files(bench_dir) for unit_file in unit_files: filename = "".join(unit_file) - # if '-worker' in unit_file[0] and unit_file[1] == '.service': - # unit_file.insert(1, '@') - exec_cmd('sudo ln -s {config_path}/{unit_file} {etc_systemd_system}/{unit_file_init}'.format( config_path=config_path, etc_systemd_system=etc_systemd_system, @@ -184,9 +192,6 @@ def _delete_symlinks(bench_path): config_path = os.path.join(bench_dir, 'config', 'systemd') unit_files = get_unit_files(bench_dir) for unit_file in unit_files: - # if '-worker' in unit_file[0] and unit_file[1] == '.service': - # unit_file.insert(1, '@') - exec_cmd('sudo rm {etc_systemd_system}/{unit_file_init}'.format( config_path=config_path, etc_systemd_system=etc_systemd_system, @@ -201,9 +206,9 @@ def get_unit_files(bench_path): [bench_name+"-workers", ".target"], [bench_name+"-web", ".target"], [bench_name+"-redis", ".target"], - [bench_name+"-frappe-default-worker", ".service"], - [bench_name+"-frappe-short-worker", ".service"], - [bench_name+"-frappe-long-worker", ".service"], + [bench_name+"-frappe-default-worker@", ".service"], + [bench_name+"-frappe-short-worker@", ".service"], + [bench_name+"-frappe-long-worker@", ".service"], [bench_name+"-frappe-schedule", ".service"], [bench_name+"-frappe-web", ".service"], [bench_name+"-node-socketio", ".service"], diff --git a/bench/config/templates/systemd/frappe-bench-frappe-default-worker.service b/bench/config/templates/systemd/frappe-bench-frappe-default-worker.service index d02bc4a4..a666313c 100644 --- a/bench/config/templates/systemd/frappe-bench-frappe-default-worker.service +++ b/bench/config/templates/systemd/frappe-bench-frappe-default-worker.service @@ -1,5 +1,5 @@ [Unit] -Description="{{ bench_name }}-frappe-default-worker" +Description="{{ bench_name }}-frappe-default-worker %I" PartOf={{ bench_name }}-workers.target [Service] diff --git a/bench/config/templates/systemd/frappe-bench-frappe-long-worker.service b/bench/config/templates/systemd/frappe-bench-frappe-long-worker.service index 35489739..fe81fa1b 100644 --- a/bench/config/templates/systemd/frappe-bench-frappe-long-worker.service +++ b/bench/config/templates/systemd/frappe-bench-frappe-long-worker.service @@ -1,5 +1,5 @@ [Unit] -Description="{{ bench_name }}-frappe-short-worker" +Description="{{ bench_name }}-frappe-short-worker %I" PartOf={{ bench_name }}-workers.target [Service] diff --git a/bench/config/templates/systemd/frappe-bench-frappe-short-worker.service b/bench/config/templates/systemd/frappe-bench-frappe-short-worker.service index 79d266fe..f27e2f6b 100644 --- a/bench/config/templates/systemd/frappe-bench-frappe-short-worker.service +++ b/bench/config/templates/systemd/frappe-bench-frappe-short-worker.service @@ -1,5 +1,5 @@ [Unit] -Description="{{ bench_name }}-frappe-short-worker" +Description="{{ bench_name }}-frappe-short-worker %I" PartOf={{ bench_name }}-workers.target [Service] diff --git a/bench/config/templates/systemd/frappe-bench-workers.target b/bench/config/templates/systemd/frappe-bench-workers.target index 336521ff..270bf200 100644 --- a/bench/config/templates/systemd/frappe-bench-workers.target +++ b/bench/config/templates/systemd/frappe-bench-workers.target @@ -1,6 +1,6 @@ [Unit] After=network.target -Wants={{ bench_name }}-frappe-default-worker.service {{ bench_name }}-frappe-short-worker.service {{ bench_name }}-frappe-long-worker.service {{ bench_name }}-frappe-schedule.service +Wants={{ worker_target_wants }} [Install] WantedBy=multi-user.target From 8ee06bf45cd54f9c76c8d1a4ce95e36818628eb7 Mon Sep 17 00:00:00 2001 From: tundebabzy Date: Wed, 25 Apr 2018 18:07:12 +0100 Subject: [PATCH 12/49] fix no procfile found --- MANIFEST.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MANIFEST.in b/MANIFEST.in index 898efefa..e84c5e61 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -4,4 +4,4 @@ recursive-include bench *.conf recursive-include bench *.py recursive-include bench *.txt recursive-include bench *.json -recursive-include bench/templates * +recursive-include bench/config/templates * From f62341dee9d6976aa89216ca9d89774e2e37fdc3 Mon Sep 17 00:00:00 2001 From: Ameya Shenoy Date: Mon, 30 Apr 2018 16:07:37 +0530 Subject: [PATCH 13/49] modifications to setup production for systemd --- bench/config/production_setup.py | 31 +++++++++++++++++++++---------- bench/config/supervisor.py | 1 + bench/config/systemd.py | 1 + 3 files changed, 23 insertions(+), 10 deletions(-) diff --git a/bench/config/production_setup.py b/bench/config/production_setup.py index afde3ff8..026d78bc 100755 --- a/bench/config/production_setup.py +++ b/bench/config/production_setup.py @@ -1,10 +1,18 @@ from bench.utils import get_program, exec_cmd, get_cmd_output, fix_prod_setup_perms, get_bench_name, find_executable, CommandFailedError from bench.config.supervisor import generate_supervisor_config +from bench.config.systemd import generate_systemd_config from bench.config.nginx import make_nginx_conf +from bench.config.common_site_config import get_config import os, subprocess def setup_production(user, bench_path='.', yes=False): - generate_supervisor_config(bench_path=bench_path, user=user, yes=yes) + if get_config(bench_path).get('restart_supervisor_on_update') and get_config(bench_path).get('restart_systemd_on_update'): + raise Exception("You cannot use supervisor and systemd at the same time. Modify your common_site_config accordingly." ) + + if get_config(bench_path).get('restart_systemd_on_update'): + generate_systemd_config(bench_path=bench_path, user=user, yes=yes) + else: + generate_supervisor_config(bench_path=bench_path, user=user, yes=yes) make_nginx_conf(bench_path=bench_path, yes=yes) fix_prod_setup_perms(bench_path, frappe_user=user) remove_default_nginx_configs() @@ -12,18 +20,20 @@ def setup_production(user, bench_path='.', yes=False): bench_name = get_bench_name(bench_path) nginx_conf = '/etc/nginx/conf.d/{bench_name}.conf'.format(bench_name=bench_name) - supervisor_conf_extn = "ini" if is_centos7() else "conf" - supervisor_conf = os.path.join(get_supervisor_confdir(), '{bench_name}.{extn}'.format( - bench_name=bench_name, extn=supervisor_conf_extn)) + if get_config(bench_path).get('restart_supervisor_on_update'): + supervisor_conf_extn = "ini" if is_centos7() else "conf" + supervisor_conf = os.path.join(get_supervisor_confdir(), '{bench_name}.{extn}'.format( + bench_name=bench_name, extn=supervisor_conf_extn)) - # Check if symlink exists, If not then create it. - if not os.path.islink(supervisor_conf): - os.symlink(os.path.abspath(os.path.join(bench_path, 'config', 'supervisor.conf')), supervisor_conf) + # Check if symlink exists, If not then create it. + if not os.path.islink(supervisor_conf): + os.symlink(os.path.abspath(os.path.join(bench_path, 'config', 'supervisor.conf')), supervisor_conf) if not os.path.islink(nginx_conf): os.symlink(os.path.abspath(os.path.join(bench_path, 'config', 'nginx.conf')), nginx_conf) - reload_supervisor() + if get_config(bench_path).get('restart_supervisor_on_update'): + reload_supervisor() if os.environ.get('NO_SERVICE_RESTART'): return @@ -41,8 +51,9 @@ def disable_production(bench_path='.'): if os.path.islink(supervisor_conf): os.unlink(supervisor_conf) - exec_cmd('sudo supervisorctl reread') - exec_cmd('sudo supervisorctl update') + if get_config(bench_path).get('restart_supervisor_on_update'): + exec_cmd('sudo supervisorctl reread') + exec_cmd('sudo supervisorctl update') # nginx nginx_conf = '/etc/nginx/conf.d/{bench_name}.conf'.format(bench_name=bench_name) diff --git a/bench/config/supervisor.py b/bench/config/supervisor.py index 27396b15..60db0018 100644 --- a/bench/config/supervisor.py +++ b/bench/config/supervisor.py @@ -42,4 +42,5 @@ def generate_supervisor_config(bench_path, user=None, yes=False): f.write(config) update_config({'restart_supervisor_on_update': True}, bench_path=bench_path) + update_config({'restart_systemd_on_update': False}, bench_path=bench_path) diff --git a/bench/config/systemd.py b/bench/config/systemd.py index 856d1083..72bed1f2 100644 --- a/bench/config/systemd.py +++ b/bench/config/systemd.py @@ -70,6 +70,7 @@ def generate_systemd_config(bench_path, user=None, yes=False, setup_redis_config(bench_info, bench_path) update_config({'restart_systemd_on_update': True}, bench_path=bench_path) + update_config({'restart_supervisor_on_update': False}, bench_path=bench_path) def setup_systemd_directory(bench_path): if not os.path.exists(os.path.join(bench_path, 'config', 'systemd')): From 09d4414dc7394d21a31ca82467d22dcc23d8abe4 Mon Sep 17 00:00:00 2001 From: Ameya Shenoy Date: Mon, 30 Apr 2018 19:48:56 +0530 Subject: [PATCH 14/49] systemctl doesnt start services 2 layers deep --- bench/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bench/utils.py b/bench/utils.py index f0bb0c63..7b079d35 100755 --- a/bench/utils.py +++ b/bench/utils.py @@ -403,7 +403,7 @@ def restart_systemd_processes(bench_path='.', web_workers=False): conf = get_config(bench_path=bench_path) bench_name = get_bench_name(bench_path) exec_cmd('sudo systemctl stop -- $(systemctl show -p Requires {bench_name}.target | cut -d= -f2)'.format(bench_name=bench_name)) - exec_cmd('sudo systemctl start {bench_name}.target'.format(bench_name=bench_name)) + exec_cmd('sudo systemctl start -- $(systemctl show -p Requires {bench_name}.target | cut -d= -f2)'.format(bench_name=bench_name)) def set_default_site(site, bench_path='.'): if not site in get_sites(bench_path=bench_path): From 168ea9a0b171f957a1604a0631ec55af1be525d2 Mon Sep 17 00:00:00 2001 From: Ameya Shenoy Date: Tue, 15 May 2018 13:42:19 +0530 Subject: [PATCH 15/49] added installation of bench-manager to readme --- README.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/README.md b/README.md index 9cfff9e4..f10cbe90 100755 --- a/README.md +++ b/README.md @@ -144,6 +144,18 @@ For production: --- +## Bench Manager +Bench Manager is a graphical user interface to emulate the functionalities of Frappe Bench. Like the command line utility it helps you install apps, manage multiple sites, update apps and much more, in the form of a frappe website. + +``` +$ bench setup manager +``` + +What all it does: +1. Create new site bench-manager.local +2. Gets the `bench_manager` app from https://github.com/frappe/bench_manager if it doesn't exist already +3. Installs the bench_manager app on the site bench-manager.local + ## Docker Install - For Developers (beta) 1. For developer setup, you can also use the official [Frappé Docker](https://github.com/frappe/frappe_docker/). From 2591da06225a1d1e8a5c6b00f75e116a6f428f52 Mon Sep 17 00:00:00 2001 From: Ameya Shenoy Date: Tue, 15 May 2018 16:18:02 +0530 Subject: [PATCH 16/49] [fix] update fails if no remote found - The bench update used to fail if there was no remote found for the app. Fixed by adding check if remote exists. If it doesn't the app is added to excluded_apps.txt and not considered for an update - [minor] optimized get_excluded_apps list by calling it only once, as opposed to getting it for every app --- bench/app.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/bench/app.py b/bench/app.py index 663b6a34..28e35342 100755 --- a/bench/app.py +++ b/bench/app.py @@ -222,14 +222,19 @@ Here are your choices: wait for them to be merged in the core.'''.format(app)) sys.exit(1) + excluded_apps = get_excluded_apps() for app in get_apps(bench_path=bench_path): - excluded_apps = get_excluded_apps() if app in excluded_apps: print("Skipping pull for app {}".format(app)) continue app_dir = get_repo_dir(app, bench_path=bench_path) if os.path.exists(os.path.join(app_dir, '.git')): remote = get_remote(app) + if not remote: + # remote is False, i.e. remote doesn't exist, add the app to excluded_apps.txt + add_to_excluded_apps_txt(app, bench_path=bench_path) + print("Skipping pull for app {}, since remote doesn't exist, and adding it to excluded apps".format(app)) + continue logger.info('pulling {0}'.format(app)) if reset: exec_cmd("git fetch --all", cwd=app_dir) @@ -276,12 +281,13 @@ def get_remote(app, bench_path='.'): stderr=subprocess.STDOUT) contents = contents.decode('utf-8') if re.findall('upstream[\s]+', contents): - remote = 'upstream' + return 'upstream' + elif not contents: + # if contents is an empty string => remote doesn't exist + return False else: # get the first remote - remote = contents.splitlines()[0].split()[0] - - return remote + return contents.splitlines()[0].split()[0] def use_rq(bench_path): bench_path = os.path.abspath(bench_path) From 615240247a4497c88f8d7731f17be9e7f4b12baf Mon Sep 17 00:00:00 2001 From: Ameya Shenoy Date: Wed, 30 May 2018 16:38:27 +0000 Subject: [PATCH 17/49] PR and Issue Templates updated --- .github/ISSUE_TEMPLATE/bug_report.md | 68 +++++++++++++++++++++++ .github/ISSUE_TEMPLATE/feature_request.md | 21 +++++++ .github/PULL_REQUEST_TEMPLATE.md | 28 ++++++++++ 3 files changed, 117 insertions(+) create mode 100644 .github/ISSUE_TEMPLATE/bug_report.md create mode 100644 .github/ISSUE_TEMPLATE/feature_request.md create mode 100644 .github/PULL_REQUEST_TEMPLATE.md diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 00000000..aa3a104e --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,68 @@ +--- +name: Bug report +about: Create a report to help us improve + +--- + +Issue: Bug report + +Our project, as you've probably heard, is getting really popular and truth is we're getting a bit overwhelmed by the activity surrounding it. There are just too many issues for us to manage properly. + +**Do the checklist before filing an issue:** + - [ ] Have a usage question? Ask your question on [Discuss Forum](https://discuss.erpnext.com). We use [Discuss Forum](https://discuss.erpnext.com) for usage question and GitHub for bugs. + - [ ] Can you replicate the issue? + - [ ] Is this something you can debug and fix? Send a pull request! Bug fixes and documentation fixes are welcome + +**Describe the bug** :chart_with_downwards_trend: +A clear and concise description of what the bug is. + +**To Reproduce** :page_with_curl: +Steps to reproduce the behavior: +1. Go to '...' +2. Click on '....' +3. Scroll down to '....' +4. See error + +**Expected behavior** :chart_with_upwards_trend: +A clear and concise description of what you expected to happen. + +**Screenshots** :crystal_ball: +If applicable, add screenshots to help explain your problem. + +**Desktop (please complete the following information):** :cyclone: + - OS: + - [ ] Linux + - [ ] macOS + - [ ] Windows + - [ ] Others? Please mention: + - Browser: + - [ ] Safari + - [ ] Chrome + - [ ] Firefox + - [ ] Other? Please mention: + +**Smartphone (please complete the following information):** :iphone: :computer: + - Device: + - [ ] iPhone + - [ ] Android + - Browser: + - [ ] Safari + - [ ] Chrome + - [ ] Firefox + - [ ] Other? Please mention: + +**Version Information** +- Which branch are you on? + - [ ] `master` :star2: + - [ ] `develop` :fire: +- Frappe Version: +- ERPNext Version: + +**Additional context** :page_facing_up: +Add any other context about the problem here. + +**Possible Solution** :bookmark_tabs: +Any idea what might be causing the issue. Or if you have a proposed solution to the problem, + +**Please don't be intimidated by the long list of options you've fill. Try to fill out as much as you can. Remember, the more the information the easier it is for us to replicate and fix the issue** :grin: + diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 00000000..e09e5e95 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,21 @@ +--- +name: Feature request +about: Suggest an idea for this project + +--- + +Issue: Feature Request + +**Is your feature request related to a problem? Please describe.** +A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] + +**Describe the solution you'd like** +A clear and concise description of what you want to happen. + +**Describe alternatives you've considered** +A clear and concise description of any alternative solutions or features you've considered. + +**Additional context** +Add any other context or screenshots about the feature request here. + + diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 00000000..8e2bd857 --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,28 @@ +Pull-Request + +- [ ] Have you followed the guidelines in our Contributing document? +- [ ] Have you checked to ensure there aren't other open [Pull Requests](../pulls) for the same update/change? +- [ ] Have you lint your code locally prior to submission? +- [ ] Have you successfully run tests with your changes locally? +- [ ] Does your commit message have an explanation for your changes and why you'd like us to include them? +- [ ] Docs have been added / updated +- [ ] Tests for the changes have been added (for bug fixes / features) +- [ ] Did you modify the existing test cases? If yes, why? + +--- + +What type of a PR is this? + +- [ ] Changes to Existing Features +- [ ] New Feature Submissions +- [ ] Bug Fix +- [ ] Breaking Change + +--- + +- Motivation and Context (What existing problem does the pull request solve): +- Related Issue: +- Screenshots (if applicable, remember, a picture tells a thousand words): + +**Please don't be intimidated by the long list of options you've fill. Try to fill out as much as you can. Remember, the more the information the easier it is for us to test and get your pull request merged** :grin: + From 2913f6d64882bb9b11297eb6e6e9b8cf0a0b1ba3 Mon Sep 17 00:00:00 2001 From: Ameya Shenoy Date: Sat, 9 Jun 2018 08:48:20 +0000 Subject: [PATCH 18/49] mariadb fixes --- playbooks/roles/mariadb/README.md | 13 +++--- playbooks/roles/mariadb/defaults/main.yml | 1 - playbooks/roles/mariadb/handlers/main.yml | 1 - playbooks/roles/mariadb/tasks/centos.yml | 7 +++- playbooks/roles/mariadb/tasks/main.yml | 10 +++-- .../tasks/mysql_secure_installation.yml | 42 ++++--------------- .../tasks/{ubuntu.yml => ubuntu-trusty.yml} | 14 +++---- .../roles/mariadb/tasks/ubuntu-xenial.yml | 27 ++++++++++++ .../mariadb/templates/mariadb_centos.repo.j2 | 2 +- .../mariadb/templates/mariadb_ubuntu.list.j2 | 2 +- playbooks/roles/mariadb/vars/main.yml | 3 +- 11 files changed, 60 insertions(+), 62 deletions(-) rename playbooks/roles/mariadb/tasks/{ubuntu.yml => ubuntu-trusty.yml} (63%) create mode 100644 playbooks/roles/mariadb/tasks/ubuntu-xenial.yml diff --git a/playbooks/roles/mariadb/README.md b/playbooks/roles/mariadb/README.md index bc872db5..72e42382 100644 --- a/playbooks/roles/mariadb/README.md +++ b/playbooks/roles/mariadb/README.md @@ -7,6 +7,8 @@ Installs MariaDB ``` CentOS 6 & 7 Ubuntu 14.04 +Ubuntu 16.04 +Debain 9 ``` ## Post install @@ -22,7 +24,7 @@ None MariaDB version: ``` -mariadb_version: 10.0 +mariadb_version: 10.2 ``` Configuration template: @@ -52,13 +54,10 @@ None ``` - hosts: servers roles: - - { role: pcextreme.mariadb } + - { role: mariadb } ``` -## License +## Credits -MIT / BSD +- [Attila van der Velde](https://github.com/vdvm) -## Author Information - -Created by [Attila van der Velde](https://github.com/vdvm) diff --git a/playbooks/roles/mariadb/defaults/main.yml b/playbooks/roles/mariadb/defaults/main.yml index fce5bc00..78831c10 100644 --- a/playbooks/roles/mariadb/defaults/main.yml +++ b/playbooks/roles/mariadb/defaults/main.yml @@ -5,4 +5,3 @@ mysql_conf_tpl: change_me mysql_conf_file: settings.cnf mysql_secure_installation: false -... \ No newline at end of file diff --git a/playbooks/roles/mariadb/handlers/main.yml b/playbooks/roles/mariadb/handlers/main.yml index 7072dbd9..3755d8ce 100644 --- a/playbooks/roles/mariadb/handlers/main.yml +++ b/playbooks/roles/mariadb/handlers/main.yml @@ -1,4 +1,3 @@ --- - name: restart mysql service: name=mysql state=restarted -... \ No newline at end of file diff --git a/playbooks/roles/mariadb/tasks/centos.yml b/playbooks/roles/mariadb/tasks/centos.yml index 2b5ced27..c069aacd 100644 --- a/playbooks/roles/mariadb/tasks/centos.yml +++ b/playbooks/roles/mariadb/tasks/centos.yml @@ -6,5 +6,8 @@ yum: name={{ item }} enablerepo=mariadb state=present with_items: - MariaDB-server - - MySQL-python # required for secure_install -... + - MariaDB-client + +- name: Install MySQLdb Python package for secure installations. + yum: name=MySQL-python state=present + when: mysql_secure_installation and mysql_root_password is defined diff --git a/playbooks/roles/mariadb/tasks/main.yml b/playbooks/roles/mariadb/tasks/main.yml index 8d3e6ea0..6802e7ad 100644 --- a/playbooks/roles/mariadb/tasks/main.yml +++ b/playbooks/roles/mariadb/tasks/main.yml @@ -1,10 +1,12 @@ --- +- include: centos.yml + when: ansible_distribution == 'CentOS' and ansible_distribution_major_version|int >= 6 -- include_tasks: centos.yml - when: ansible_distribution == 'CentOS' +- include: ubuntu-trusty.yml + when: ansible_distribution == 'Ubuntu' and ansible_distribution_version == '14.04' -- include_tasks: ubuntu.yml - when: ansible_distribution == 'Ubuntu' +- include: ubuntu-xenial.yml + when: ansible_distribution == 'Ubuntu' and ansible_distribution_version == '16.04' - name: Add configuration template: src={{ mysql_conf_tpl }} dest={{ mysql_conf_dir[ansible_distribution] }}/{{ mysql_conf_file }} owner=root group=root mode=0644 diff --git a/playbooks/roles/mariadb/tasks/mysql_secure_installation.yml b/playbooks/roles/mariadb/tasks/mysql_secure_installation.yml index 621ab4b0..27d1ca80 100644 --- a/playbooks/roles/mariadb/tasks/mysql_secure_installation.yml +++ b/playbooks/roles/mariadb/tasks/mysql_secure_installation.yml @@ -1,33 +1,22 @@ --- -# Set root password -# UPDATE mysql.user SET Password=PASSWORD('mysecret') WHERE User='root'; -# FLUSH PRIVILEGES; - debug: msg: "{{ mysql_root_password }}" -- name: Set root Password - mysql_user: name=root host={{ item }} password={{ mysql_root_password }} state=present - with_items: - - localhost - ignore_errors: yes - +# create root .my.cnf config file - name: Add .my.cnf template: src=my.cnf.j2 dest=/root/.my.cnf owner=root group=root mode=0600 -- name: display .my.cnf - command: cat /root/.my.cnf - register: details - -- debug: - msg: "{{ details.stdout_lines }}" +# Set root password +# UPDATE mysql.user SET Password=PASSWORD('mysecret') WHERE User='root'; +# FLUSH PRIVILEGES; - name: Set root Password - mysql_user: name=root host={{ item }} password={{ mysql_root_password }} state=present + mysql_user: login_password={{ mysql_root_password }} check_implicit_admin=yes name=root host={{ item }} password={{ mysql_root_password }} state=present with_items: + - localhost - 127.0.0.1 - ::1 - when: run_travis is not defined - name: Reload privilege tables command: 'mysql -ne "{{ item }}"' @@ -53,7 +42,7 @@ - name: Remove test database and access to it command: 'mysql -ne "{{ item }}"' with_items: - - DROP DATABASE if exists test + - DROP DATABASE IF EXISTS test - DELETE FROM mysql.db WHERE Db='test' OR Db='test\\_%' changed_when: False when: run_travis is not defined @@ -64,20 +53,3 @@ - FLUSH PRIVILEGES changed_when: False when: run_travis is not defined - -- name: add launchagents folder mac - file: path=~/Library/LaunchAgents state=directory - when: ansible_distribution == 'MacOSX' - -- name: add mysql to mac startup - file: src=/usr/local/opt/mariadb/homebrew.mxcl.mariadb.plist path=~/Library/LaunchAgents/homebrew.mxcl.mariadb.plist state=link force=yes - when: ansible_distribution == 'MacOSX' - -- name: stop mysql mac - command: launchctl unload ~/Library/LaunchAgents/homebrew.mxcl.mariadb.plist - when: ansible_distribution == 'MacOSX' - -- name: start mysql mac - command: launchctl load ~/Library/LaunchAgents/homebrew.mxcl.mariadb.plist - when: ansible_distribution == 'MacOSX' -... diff --git a/playbooks/roles/mariadb/tasks/ubuntu.yml b/playbooks/roles/mariadb/tasks/ubuntu-trusty.yml similarity index 63% rename from playbooks/roles/mariadb/tasks/ubuntu.yml rename to playbooks/roles/mariadb/tasks/ubuntu-trusty.yml index 5abec54b..56cd0b66 100644 --- a/playbooks/roles/mariadb/tasks/ubuntu.yml +++ b/playbooks/roles/mariadb/tasks/ubuntu-trusty.yml @@ -6,12 +6,6 @@ - name: Add repo key apt_key: id=1BB943DB url=http://keyserver.ubuntu.com/pks/lookup?op=get&search=0xCBCB082A1BB943DB state=present register: mariadb_key - when: ansible_distribution_version | version_compare('16.04', 'lt') - -- name: Add apt key for mariadb for Ubuntu >= 16.04 - apt_key: id=C74CD1D8 url=http://keyserver.ubuntu.com/pks/lookup?op=get&search=0xF1656F24C74CD1D8 state=present - register: mariadb_key - when: ansible_distribution_version | version_compare('16.04', 'ge') - name: Update apt cache apt: update_cache=yes @@ -25,5 +19,9 @@ apt: pkg={{ item }} state=present with_items: - mariadb-server - - python-mysqldb # required to set the MySQL password using ansible -... + - mariadb-client + - libmariadbclient18 + +- name: Install MySQLdb Python package for secure installations. + apt: pkg=python-mysqldb state=present + when: mysql_secure_installation and mysql_root_password is defined diff --git a/playbooks/roles/mariadb/tasks/ubuntu-xenial.yml b/playbooks/roles/mariadb/tasks/ubuntu-xenial.yml new file mode 100644 index 00000000..8e834ee8 --- /dev/null +++ b/playbooks/roles/mariadb/tasks/ubuntu-xenial.yml @@ -0,0 +1,27 @@ +--- +- name: Add repo file + template: src=mariadb_ubuntu.list.j2 dest=/etc/apt/sources.list.d/mariadb.list owner=root group=root mode=0644 + register: mariadb_list + +- name: Add repo key + apt_key: id=C74CD1D8 url=http://keyserver.ubuntu.com/pks/lookup?op=get&search=0xF1656F24C74CD1D8 state=present + register: mariadb_key + +- name: Update apt cache + apt: update_cache=yes + when: mariadb_list.changed == True or mariadb_key.changed == True + +- name: Unattended package installation + shell: export DEBIAN_FRONTEND=noninteractive + changed_when: false + +- name: Install MariaDB + apt: pkg={{ item }} state=present + with_items: + - mariadb-server + - mariadb-client + - libmariadbclient18 + +- name: Install MySQLdb Python package for secure installations. + apt: pkg=python-mysqldb state=present + when: mysql_secure_installation and mysql_root_password is defined diff --git a/playbooks/roles/mariadb/templates/mariadb_centos.repo.j2 b/playbooks/roles/mariadb/templates/mariadb_centos.repo.j2 index 0a025305..64738cc1 100644 --- a/playbooks/roles/mariadb/templates/mariadb_centos.repo.j2 +++ b/playbooks/roles/mariadb/templates/mariadb_centos.repo.j2 @@ -1,4 +1,4 @@ -# MariaDB {{ mariadb_version }} CentOS {{ ansible_distribution_major_version|int }} repository list +# MariaDB CentOS {{ ansible_distribution_major_version|int }} repository list # http://mariadb.org/mariadb/repositories/ [mariadb] name = MariaDB diff --git a/playbooks/roles/mariadb/templates/mariadb_ubuntu.list.j2 b/playbooks/roles/mariadb/templates/mariadb_ubuntu.list.j2 index dca4e70b..981b4d6b 100644 --- a/playbooks/roles/mariadb/templates/mariadb_ubuntu.list.j2 +++ b/playbooks/roles/mariadb/templates/mariadb_ubuntu.list.j2 @@ -1,4 +1,4 @@ -# MariaDB {{ mariadb_version }} Ubuntu {{ ansible_distribution_release | title }} repository list +# MariaDB Ubuntu {{ ansible_distribution_release | title }} repository list # http://mariadb.org/mariadb/repositories/ deb http://ams2.mirrors.digitalocean.com/mariadb/repo/{{ mariadb_version }}/ubuntu {{ ansible_distribution_release | lower }} main deb-src http://ams2.mirrors.digitalocean.com/mariadb/repo/{{ mariadb_version }}/ubuntu {{ ansible_distribution_release | lower }} main diff --git a/playbooks/roles/mariadb/vars/main.yml b/playbooks/roles/mariadb/vars/main.yml index b7bf4bcf..df81b3b7 100644 --- a/playbooks/roles/mariadb/vars/main.yml +++ b/playbooks/roles/mariadb/vars/main.yml @@ -3,7 +3,6 @@ mysql_conf_dir: "CentOS": /etc/my.cnf.d "Ubuntu": /etc/mysql/conf.d "Debian": /etc/mysql/conf.d - "MacOSX": /usr/local/etc/my.cnf.d mysql_conf_tpl: files/mariadb_config.cnf mysql_secure_installation: True -... \ No newline at end of file +... From c565269acfb7e13d3239da65b13d7304147d5535 Mon Sep 17 00:00:00 2001 From: Ameya Shenoy Date: Sat, 9 Jun 2018 10:09:44 +0000 Subject: [PATCH 19/49] cryptography urllib3 fixes --- playbooks/install.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playbooks/install.py b/playbooks/install.py index 529e473a..5fb99e59 100644 --- a/playbooks/install.py +++ b/playbooks/install.py @@ -36,7 +36,7 @@ def install_bench(args): # secure pip installation if find_executable('pip'): run_os_command({ - 'pip': 'sudo pip install --upgrade setuptools pip==9.0.3' + 'pip': 'sudo pip install --upgrade setuptools urllib3 requests cryptography pip==9.0.3' }) else: From bb2453f7e243d550a0c94bd789b41bbc3a19161c Mon Sep 17 00:00:00 2001 From: Ameya Shenoy Date: Sat, 9 Jun 2018 10:13:23 +0000 Subject: [PATCH 20/49] requests urllib3 crypto fixes --- playbooks/install.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playbooks/install.py b/playbooks/install.py index 5fb99e59..4c311586 100644 --- a/playbooks/install.py +++ b/playbooks/install.py @@ -51,7 +51,7 @@ def install_bench(args): if success: run_os_command({ - 'pip': 'sudo pip install --upgrade pip==9.0.3 setuptools', + 'pip': 'sudo pip install --upgrade setuptools urllib3 requests cryptography pip==9.0.3' }) success = run_os_command({ From e61227c973d85f6e91d22f723f17a00e94487b1e Mon Sep 17 00:00:00 2001 From: Ameya Shenoy Date: Sat, 9 Jun 2018 10:23:12 +0000 Subject: [PATCH 21/49] cryto --- playbooks/install.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playbooks/install.py b/playbooks/install.py index 5fb99e59..255fb0b3 100644 --- a/playbooks/install.py +++ b/playbooks/install.py @@ -55,7 +55,7 @@ def install_bench(args): }) success = run_os_command({ - 'pip': "sudo pip install ansible" + 'pip': "sudo pip install --upgrade urllib3 requests cryptography ansible" }) if not success: From 4559eba08ffd1911616bbf04b26659e9ff1a0387 Mon Sep 17 00:00:00 2001 From: Ameya Shenoy Date: Tue, 12 Jun 2018 12:44:07 +0530 Subject: [PATCH 22/49] added bench manager to readme --- README.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/README.md b/README.md index 9cfff9e4..724ec338 100755 --- a/README.md +++ b/README.md @@ -144,6 +144,22 @@ For production: --- +## Bench Manger (GUI for Bench) + +Bench Manager is a graphical user interface to emulate the functionalities of Frappé Bench. Like the command line utility it helps you install apps, manage multiple sites, update apps and much more. + +### Installation + +``` +$ bench setup manager +``` + +What all it does: + +1. Create new site bench-manager.local +2. Gets the bench_manager app from https://github.com/frappe/bench_manager if it doesn't exist already +3. Installs the bench_manager app on the site bench-manager.local + ## Docker Install - For Developers (beta) 1. For developer setup, you can also use the official [Frappé Docker](https://github.com/frappe/frappe_docker/). From 8c5d7837163e9f71911e3e2b7501300f9069ed0b Mon Sep 17 00:00:00 2001 From: Prateeksha Singh Date: Thu, 14 Jun 2018 09:10:55 +0530 Subject: [PATCH 23/49] [install] update frappe,erpnext docs apps to be installed (#677) --- bench/app.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/bench/app.py b/bench/app.py index 663b6a34..6cd495f2 100755 --- a/bench/app.py +++ b/bench/app.py @@ -75,7 +75,7 @@ def add_to_excluded_apps_txt(app, bench_path='.'): if app == 'frappe': raise ValueError('Frappe app cannot be excludeed from update') if app not in os.listdir('apps'): - raise ValueError('The app {} does not exist'.format(app)) + raise ValueError('The app {} does not exist'.format(app)) apps = get_excluded_apps(bench_path=bench_path) if app not in apps: apps.append(app) @@ -130,6 +130,22 @@ def get_app(git_url, branch=None, bench_path='.', build_asset_files=True, verbos apps_path = os.path.join(os.path.abspath(bench_path), 'apps') os.rename(os.path.join(apps_path, repo_name), os.path.join(apps_path, app_name)) + docs_app = '' + docs_app_url = '' + app_docs_map = { + 'frappe': 'frappe/frappe_io', + 'erpnext': 'erpnext/foundation' + } + if repo_name in app_docs_map.keys(): + docs_app = app_docs_map[repo_name] + docs_app_url = 'https://github.com/{docs_app}'.format(docs_app=docs_app) + print('Getting docs app for ' + app_name + '' + docs_app) + exec_cmd("git clone {git_url} {branch} {shallow_clone} --origin upstream".format( + git_url=docs_app_url, + shallow_clone=shallow_clone, + branch=branch), + cwd=os.path.join(bench_path, 'apps')) + print('installing', app_name) install_app(app=app_name, bench_path=bench_path, verbose=verbose) From 49d3887d76fe869aaffe121818b878be2cb39143 Mon Sep 17 00:00:00 2001 From: Rushabh Mehta Date: Thu, 14 Jun 2018 09:27:08 +0530 Subject: [PATCH 24/49] [ask] for changelog --- bench/release.py | 1 + 1 file changed, 1 insertion(+) diff --git a/bench/release.py b/bench/release.py index 6025d4a7..f0c0a65f 100755 --- a/bench/release.py +++ b/bench/release.py @@ -59,6 +59,7 @@ def confirm_testing(): print('') print('') click.confirm('Is manual testing done?', abort = True) + click.confirm('Have you added the change log?', abort = True) def bump(bench_path, app, bump_type, from_branch, to_branch, remote, owner, repo_name=None): assert bump_type in ['minor', 'major', 'patch', 'stable', 'prerelease'] From c342a82aa17215da94ee7ea7e8eccce14c08b039 Mon Sep 17 00:00:00 2001 From: Rushabh Mehta Date: Thu, 14 Jun 2018 10:37:11 +0530 Subject: [PATCH 25/49] better way to install docs --- bench/app.py | 21 ++++++--------------- 1 file changed, 6 insertions(+), 15 deletions(-) diff --git a/bench/app.py b/bench/app.py index 6cd495f2..9c7e5999 100755 --- a/bench/app.py +++ b/bench/app.py @@ -130,21 +130,12 @@ def get_app(git_url, branch=None, bench_path='.', build_asset_files=True, verbos apps_path = os.path.join(os.path.abspath(bench_path), 'apps') os.rename(os.path.join(apps_path, repo_name), os.path.join(apps_path, app_name)) - docs_app = '' - docs_app_url = '' - app_docs_map = { - 'frappe': 'frappe/frappe_io', - 'erpnext': 'erpnext/foundation' - } - if repo_name in app_docs_map.keys(): - docs_app = app_docs_map[repo_name] - docs_app_url = 'https://github.com/{docs_app}'.format(docs_app=docs_app) - print('Getting docs app for ' + app_name + '' + docs_app) - exec_cmd("git clone {git_url} {branch} {shallow_clone} --origin upstream".format( - git_url=docs_app_url, - shallow_clone=shallow_clone, - branch=branch), - cwd=os.path.join(bench_path, 'apps')) + # get apps for docs + if repo_name=='frappe': + get_app('https://github.com/frappe/frappe_io') + + if repo_name=='erpnext': + get_app('https://github.com/erpnext/foundation') print('installing', app_name) install_app(app=app_name, bench_path=bench_path, verbose=verbose) From 8fc3a87a37fbdde79e02af0aab3196e127d5231a Mon Sep 17 00:00:00 2001 From: Rushabh Mehta Date: Thu, 14 Jun 2018 10:52:17 +0530 Subject: [PATCH 26/49] better way to install docs --- bench/app.py | 33 ++++++++++++++++++--------------- 1 file changed, 18 insertions(+), 15 deletions(-) diff --git a/bench/app.py b/bench/app.py index 9c7e5999..437a864c 100755 --- a/bench/app.py +++ b/bench/app.py @@ -91,7 +91,8 @@ def remove_from_excluded_apps_txt(app, bench_path='.'): apps.remove(app) return write_excluded_apps_txt(apps, bench_path=bench_path) -def get_app(git_url, branch=None, bench_path='.', build_asset_files=True, verbose=False): +def get_app(git_url, branch=None, bench_path='.', build_asset_files=True, verbose=False, + postprocess = True): # from bench.utils import check_url try: from urlparse import urljoin @@ -130,23 +131,25 @@ def get_app(git_url, branch=None, bench_path='.', build_asset_files=True, verbos apps_path = os.path.join(os.path.abspath(bench_path), 'apps') os.rename(os.path.join(apps_path, repo_name), os.path.join(apps_path, app_name)) - # get apps for docs - if repo_name=='frappe': - get_app('https://github.com/frappe/frappe_io') - - if repo_name=='erpnext': - get_app('https://github.com/erpnext/foundation') - print('installing', app_name) install_app(app=app_name, bench_path=bench_path, verbose=verbose) - if build_asset_files: - build_assets(bench_path=bench_path) - conf = get_config(bench_path=bench_path) - if conf.get('restart_supervisor_on_update'): - restart_supervisor_processes(bench_path=bench_path) - if conf.get('restart_systemd_on_update'): - restart_systemd_processes(bench_path=bench_path) + if postprocess: + # get apps for docs + if repo_name=='frappe': + get_app('https://github.com/frappe/frappe_io', postprocess = False) + + if repo_name=='erpnext': + get_app('https://github.com/erpnext/foundation', post_process = False) + + if build_asset_files: + build_assets(bench_path=bench_path) + conf = get_config(bench_path=bench_path) + + if conf.get('restart_supervisor_on_update'): + restart_supervisor_processes(bench_path=bench_path) + if conf.get('restart_systemd_on_update'): + restart_systemd_processes(bench_path=bench_path) def new_app(app, bench_path='.'): # For backwards compatibility From 9f917f8464a79e93ebced72b4c793cecd93fde75 Mon Sep 17 00:00:00 2001 From: Rushabh Mehta Date: Thu, 14 Jun 2018 11:24:04 +0530 Subject: [PATCH 27/49] better way to install docs --- bench/app.py | 6 ++++-- bench/utils.py | 7 ++++--- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/bench/app.py b/bench/app.py index 437a864c..9e8e1ca9 100755 --- a/bench/app.py +++ b/bench/app.py @@ -137,10 +137,12 @@ def get_app(git_url, branch=None, bench_path='.', build_asset_files=True, verbos if postprocess: # get apps for docs if repo_name=='frappe': - get_app('https://github.com/frappe/frappe_io', postprocess = False) + get_app('https://github.com/frappe/frappe_io', bench_path = bench_path, + branch= 'master', postprocess = False) if repo_name=='erpnext': - get_app('https://github.com/erpnext/foundation', post_process = False) + get_app('https://github.com/erpnext/foundation', bench_path = bench_path, + branch= 'master', post_process = False) if build_asset_files: build_assets(bench_path=bench_path) diff --git a/bench/utils.py b/bench/utils.py index 7b079d35..36adb6ba 100755 --- a/bench/utils.py +++ b/bench/utils.py @@ -142,7 +142,8 @@ def exec_cmd(cmd, cwd='.'): logger.info(cmd) - p = subprocess.Popen(cmd, cwd=cwd, shell=True, stdout=stdout, stderr=stderr, universal_newlines=True) + p = subprocess.Popen(cmd, cwd=cwd, shell=True, stdout=stdout, stderr=stderr, + universal_newlines=True) if async: return_code = print_output(p) @@ -381,7 +382,7 @@ def restart_supervisor_processes(bench_path='.', web_workers=False): else: supervisor_status = subprocess.check_output(['sudo', 'supervisorctl', 'status'], cwd=bench_path) supervisor_status = safe_decode(supervisor_status) - + if web_workers and '{bench_name}-web:'.format(bench_name=bench_name) in supervisor_status: group = '{bench_name}-web: '.format(bench_name=bench_name) @@ -415,7 +416,7 @@ def update_requirements(bench_path='.'): print('Updating Python libraries...') pip = os.path.join(bench_path, 'env', 'bin', 'pip') - # pip 10 seems to have a few problems associated with it, temporary freeze pip at 9.0.3 + # pip 10 seems to have a few problems associated with it, temporary freeze pip at 9.0.3 exec_cmd("{pip} install --upgrade pip==9.0.3".format(pip=pip)) apps_dir = os.path.join(bench_path, 'apps') From c1a8be47915f9ec55fbecc383c4c62c7214e389c Mon Sep 17 00:00:00 2001 From: Rushabh Mehta Date: Thu, 14 Jun 2018 13:29:37 +0530 Subject: [PATCH 28/49] better way to install docs --- bench/app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bench/app.py b/bench/app.py index 9e8e1ca9..8f156430 100755 --- a/bench/app.py +++ b/bench/app.py @@ -142,7 +142,7 @@ def get_app(git_url, branch=None, bench_path='.', build_asset_files=True, verbos if repo_name=='erpnext': get_app('https://github.com/erpnext/foundation', bench_path = bench_path, - branch= 'master', post_process = False) + branch= 'master', postprocess = False) if build_asset_files: build_assets(bench_path=bench_path) From e6defad00313d311f987c651b1afe3df6a1f2a62 Mon Sep 17 00:00:00 2001 From: Ameya Shenoy Date: Tue, 19 Jun 2018 00:32:45 +0530 Subject: [PATCH 29/49] removed psutil as a requirement --- bench/config/nginx.py | 3 +-- bench/config/redis.py | 4 +--- requirements.txt | 1 - 3 files changed, 2 insertions(+), 6 deletions(-) diff --git a/bench/config/nginx.py b/bench/config/nginx.py index d3ba4c97..6bbfa9ad 100644 --- a/bench/config/nginx.py +++ b/bench/config/nginx.py @@ -215,7 +215,6 @@ def get_error_pages(): def get_limit_conn_shared_memory(): """Allocate 2 percent of total virtual memory as shared memory for nginx limit_conn_zone""" - import psutil - total_vm = (psutil.virtual_memory().total) / (1024 * 1024) # in MB + total_vm = (os.sysconf('SC_PAGE_SIZE') * os.sysconf('SC_PHYS_PAGES')) / (1024 * 1024) # in MB return int(0.02 * total_vm) diff --git a/bench/config/redis.py b/bench/config/redis.py index fe102135..6a32927e 100644 --- a/bench/config/redis.py +++ b/bench/config/redis.py @@ -67,9 +67,7 @@ def get_redis_version(): return float('{major}.{minor}'.format(major=version.major, minor=version.minor)) def get_max_redis_memory(): - import psutil - - total_virtual_mem = psutil.virtual_memory().total/(pow(1024, 2)) + total_virtual_mem = os.sysconf('SC_PAGE_SIZE') * os.sysconf('SC_PHYS_PAGES')/(pow(1024, 2)) max_memory = int(total_virtual_mem * 0.05) # Max memory for redis is 5% of virtual memory if max_memory < 50: diff --git a/requirements.txt b/requirements.txt index 4365cc1d..9b2c5b30 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,7 +3,6 @@ jinja2 virtualenv requests honcho -psutil python-crontab semantic_version GitPython==0.3.2.1 From 8d2b702888a19201144f56e3f8144ce7955b5664 Mon Sep 17 00:00:00 2001 From: Rushabh Mehta Date: Thu, 21 Jun 2018 11:45:43 +0530 Subject: [PATCH 30/49] [minor] python3 fix and fix for #663 (#664) --- bench/patches/v4/update_node.py | 2 +- bench/utils.py | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/bench/patches/v4/update_node.py b/bench/patches/v4/update_node.py index b6bab199..ffbe280c 100644 --- a/bench/patches/v4/update_node.py +++ b/bench/patches/v4/update_node.py @@ -8,7 +8,7 @@ def execute(bench_path): if node_exec: - result = subprocess.check_output([node_exec, '-v']) + result = subprocess.check_output([node_exec, '-v']).decode() else: click.echo(''' No node executable was found on your machine. diff --git a/bench/utils.py b/bench/utils.py index 36adb6ba..8d622d1f 100755 --- a/bench/utils.py +++ b/bench/utils.py @@ -94,6 +94,11 @@ def init(path, apps_path=None, no_procfile=False, no_backups=False, setup_backups(bench_path=path) if not no_auto_update: setup_auto_update(bench_path=path) + copy_patches_txt(bench_path) + +def copy_patches_txt(bench_path): + shutil.copy(os.path.join(os.path.dirname(os.path.abspath(__file__)), 'patches', 'patches.txt'), + os.path.join(bench_path, 'patches.txt')) def clone_apps_from(bench_path, clone_from, update_app=True): from .app import install_app From f1e9ab0f9fc0c211addf78a6681684d0044473c9 Mon Sep 17 00:00:00 2001 From: Ameya Shenoy Date: Thu, 21 Jun 2018 15:28:12 +0530 Subject: [PATCH 31/49] typo fix --- bench/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bench/utils.py b/bench/utils.py index 8d622d1f..7aedfd0f 100755 --- a/bench/utils.py +++ b/bench/utils.py @@ -94,7 +94,7 @@ def init(path, apps_path=None, no_procfile=False, no_backups=False, setup_backups(bench_path=path) if not no_auto_update: setup_auto_update(bench_path=path) - copy_patches_txt(bench_path) + copy_patches_txt(path) def copy_patches_txt(bench_path): shutil.copy(os.path.join(os.path.dirname(os.path.abspath(__file__)), 'patches', 'patches.txt'), From 2395e242cbc905d3cd84e40f8c0bac59448dbfc2 Mon Sep 17 00:00:00 2001 From: Ameya Shenoy Date: Tue, 26 Jun 2018 14:06:49 +0530 Subject: [PATCH 32/49] get frappe_theme for foundation --- bench/app.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bench/app.py b/bench/app.py index 8f156430..a69231e7 100755 --- a/bench/app.py +++ b/bench/app.py @@ -141,6 +141,8 @@ def get_app(git_url, branch=None, bench_path='.', build_asset_files=True, verbos branch= 'master', postprocess = False) if repo_name=='erpnext': + get_app('https://github.com/frappe/frappe_theme', bench_path = bench_path, + branch= 'master', postprocess = False) get_app('https://github.com/erpnext/foundation', bench_path = bench_path, branch= 'master', postprocess = False) From c33cff5fd82ec1468891c2b8b97511f1d18a86f7 Mon Sep 17 00:00:00 2001 From: Prateeksha Singh Date: Tue, 26 Jun 2018 17:08:01 +0530 Subject: [PATCH 33/49] Revert "get frappe_theme for foundation" --- bench/app.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/bench/app.py b/bench/app.py index a69231e7..8f156430 100755 --- a/bench/app.py +++ b/bench/app.py @@ -141,8 +141,6 @@ def get_app(git_url, branch=None, bench_path='.', build_asset_files=True, verbos branch= 'master', postprocess = False) if repo_name=='erpnext': - get_app('https://github.com/frappe/frappe_theme', bench_path = bench_path, - branch= 'master', postprocess = False) get_app('https://github.com/erpnext/foundation', bench_path = bench_path, branch= 'master', postprocess = False) From 4aaae8b791103ef2886f20ba761c9f6ce1186d7f Mon Sep 17 00:00:00 2001 From: Saurabh Date: Fri, 29 Jun 2018 11:36:22 +0530 Subject: [PATCH 34/49] Setup Wildcard SSL --- bench/commands/setup.py | 9 +++++++++ bench/config/lets_encrypt.py | 33 ++++++++++++++++++++++++++++++++- 2 files changed, 41 insertions(+), 1 deletion(-) diff --git a/bench/commands/setup.py b/bench/commands/setup.py index 1fe014e8..88b07bca 100755 --- a/bench/commands/setup.py +++ b/bench/commands/setup.py @@ -128,6 +128,15 @@ def setup_letsencrypt(site, custom_domain, non_interactive): setup_letsencrypt(site, custom_domain, bench_path='.', interactive=not non_interactive) +@click.command('wildcard-ssl') +@click.argument('domain') +@click.option('--email') +def setup_wildcard_ssl(domain, email): + ''' Setup wildcard ssl certificate ''' + from bench.config.lets_encrypt import setup_wildcard_ssl + setup_wildcard_ssl(domain, email, bench_path='.') + + @click.command('procfile') def setup_procfile(): "Setup Procfile for bench start" diff --git a/bench/config/lets_encrypt.py b/bench/config/lets_encrypt.py index c6dceb16..26a7e682 100755 --- a/bench/config/lets_encrypt.py +++ b/bench/config/lets_encrypt.py @@ -1,5 +1,5 @@ import bench, os, click, errno -from bench.utils import exec_cmd, CommandFailedError +from bench.utils import exec_cmd, CommandFailedError, update_common_site_config from bench.config.site_config import update_site_config, remove_domain, get_domains from bench.config.nginx import make_nginx_conf from bench.config.production_setup import service @@ -116,3 +116,34 @@ def renew_certs(): service('nginx', 'stop') exec_cmd("{path} renew".format(path=get_certbot_path())) service('nginx', 'start') + + +def setup_wildcard_ssl(domain, email, bench_path): + get_certbot() + + email_param = '' + if email: + email_param = '--email {0}'.format(email) + + try: + exec_cmd("{path} certonly --manual --preferred-challenges=dns {email_param} \ + --server https://acme-v02.api.letsencrypt.org/directory \ + --agree-tos -d {domain}".format(path=get_certbot_path(), domain=domain, + email_param=email_param)) + + except CommandFailedError: + print("There was a problem trying to setup SSL") + return + + ssl_path = "/etc/letsencrypt/live/{domain}/".format(site=domain) + ssl_config = { + "wildcard": { + "domain": domain, + "ssl_certificate": os.path.join(ssl_path, "fullchain.pem"), + "ssl_certificate_key": os.path.join(ssl_path, "privkey.pem") + } + } + + update_common_site_config(ssl_config) + make_nginx_conf(bench_path) + service('nginx', 'restart') From ab6cc5164f4419a85b0b79df8c29deb238148af9 Mon Sep 17 00:00:00 2001 From: Saurabh Date: Fri, 29 Jun 2018 13:37:03 +0530 Subject: [PATCH 35/49] check if dns mutitenancy is active --- bench/config/lets_encrypt.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/bench/config/lets_encrypt.py b/bench/config/lets_encrypt.py index 26a7e682..6484fac4 100755 --- a/bench/config/lets_encrypt.py +++ b/bench/config/lets_encrypt.py @@ -121,6 +121,10 @@ def renew_certs(): def setup_wildcard_ssl(domain, email, bench_path): get_certbot() + if not get_config(bench_path).get("dns_multitenant"): + print("You cannot setup SSL without DNS Multitenancy") + return + email_param = '' if email: email_param = '--email {0}'.format(email) From 2da047cd90f87c71e860df0c716647d4ec8980ba Mon Sep 17 00:00:00 2001 From: Saurabh Date: Fri, 29 Jun 2018 13:46:40 +0530 Subject: [PATCH 36/49] add command to generate wildcard ssl --- bench/cli.py | 2 +- bench/commands/setup.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/bench/cli.py b/bench/cli.py index e6ad9984..c314d90a 100755 --- a/bench/cli.py +++ b/bench/cli.py @@ -48,7 +48,7 @@ def check_uid(): def cmd_requires_root(): if len(sys.argv) > 2 and sys.argv[2] in ('production', 'sudoers', 'lets-encrypt', 'fonts', - 'print', 'firewall', 'ssh-port', 'role', 'fail2ban'): + 'print', 'firewall', 'ssh-port', 'role', 'fail2ban', 'wildcard-ssl'): return True if len(sys.argv) >= 2 and sys.argv[1] in ('patch', 'renew-lets-encrypt', 'disable-production', 'install'): diff --git a/bench/commands/setup.py b/bench/commands/setup.py index 88b07bca..d3aec204 100755 --- a/bench/commands/setup.py +++ b/bench/commands/setup.py @@ -288,6 +288,7 @@ setup.add_command(reload_nginx) setup.add_command(setup_supervisor) setup.add_command(setup_redis) setup.add_command(setup_letsencrypt) +setup.add_command(setup_wildcard_ssl) setup.add_command(setup_production) setup.add_command(setup_auto_update) setup.add_command(setup_backups) From cb2c51684bd73811804784d79f83da02b468de97 Mon Sep 17 00:00:00 2001 From: Saurabh Date: Fri, 29 Jun 2018 13:51:42 +0530 Subject: [PATCH 37/49] typo fix --- bench/config/lets_encrypt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bench/config/lets_encrypt.py b/bench/config/lets_encrypt.py index 6484fac4..25b6c388 100755 --- a/bench/config/lets_encrypt.py +++ b/bench/config/lets_encrypt.py @@ -139,7 +139,7 @@ def setup_wildcard_ssl(domain, email, bench_path): print("There was a problem trying to setup SSL") return - ssl_path = "/etc/letsencrypt/live/{domain}/".format(site=domain) + ssl_path = "/etc/letsencrypt/live/{domain}/".format(domain=domain) ssl_config = { "wildcard": { "domain": domain, From b81904b8c86486511bfc3bd89aef8f0bcc4838d0 Mon Sep 17 00:00:00 2001 From: Saurabh Date: Fri, 29 Jun 2018 13:57:11 +0530 Subject: [PATCH 38/49] prepend wildcard character to domain --- bench/config/lets_encrypt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bench/config/lets_encrypt.py b/bench/config/lets_encrypt.py index 25b6c388..c0361dc7 100755 --- a/bench/config/lets_encrypt.py +++ b/bench/config/lets_encrypt.py @@ -132,7 +132,7 @@ def setup_wildcard_ssl(domain, email, bench_path): try: exec_cmd("{path} certonly --manual --preferred-challenges=dns {email_param} \ --server https://acme-v02.api.letsencrypt.org/directory \ - --agree-tos -d {domain}".format(path=get_certbot_path(), domain=domain, + --agree-tos -d *.{domain}".format(path=get_certbot_path(), domain=domain, email_param=email_param)) except CommandFailedError: From 9f150c4c352c5fff9956e120b7090d44fde250bf Mon Sep 17 00:00:00 2001 From: Saurabh Date: Fri, 29 Jun 2018 14:24:15 +0530 Subject: [PATCH 39/49] check for wildcard character in domain if not exists prepend it --- bench/config/lets_encrypt.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/bench/config/lets_encrypt.py b/bench/config/lets_encrypt.py index c0361dc7..18525e7b 100755 --- a/bench/config/lets_encrypt.py +++ b/bench/config/lets_encrypt.py @@ -119,12 +119,22 @@ def renew_certs(): def setup_wildcard_ssl(domain, email, bench_path): - get_certbot() + + def _get_domain_name(domain): + if domain.split('.')[0] != '*': + domain = '*.{0}'.foramt(domain) + return domain + + domain = _get_domain_name(domain) + + return if not get_config(bench_path).get("dns_multitenant"): print("You cannot setup SSL without DNS Multitenancy") return + get_certbot() + email_param = '' if email: email_param = '--email {0}'.format(email) @@ -132,7 +142,7 @@ def setup_wildcard_ssl(domain, email, bench_path): try: exec_cmd("{path} certonly --manual --preferred-challenges=dns {email_param} \ --server https://acme-v02.api.letsencrypt.org/directory \ - --agree-tos -d *.{domain}".format(path=get_certbot_path(), domain=domain, + --agree-tos -d {domain}".format(path=get_certbot_path(), domain=domain, email_param=email_param)) except CommandFailedError: From 179dfc1685976c357952f0b9e82e1580810ae252 Mon Sep 17 00:00:00 2001 From: Saurabh Date: Fri, 29 Jun 2018 14:27:28 +0530 Subject: [PATCH 40/49] typo fix --- bench/config/lets_encrypt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bench/config/lets_encrypt.py b/bench/config/lets_encrypt.py index 18525e7b..abb61a10 100755 --- a/bench/config/lets_encrypt.py +++ b/bench/config/lets_encrypt.py @@ -122,7 +122,7 @@ def setup_wildcard_ssl(domain, email, bench_path): def _get_domain_name(domain): if domain.split('.')[0] != '*': - domain = '*.{0}'.foramt(domain) + domain = '*.{0}'.format(domain) return domain domain = _get_domain_name(domain) From 8ddfbd3e61d24db4920e64e4964b353d187f6221 Mon Sep 17 00:00:00 2001 From: Saurabh Date: Fri, 29 Jun 2018 14:31:48 +0530 Subject: [PATCH 41/49] debug --- bench/config/lets_encrypt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bench/config/lets_encrypt.py b/bench/config/lets_encrypt.py index abb61a10..0cbd4697 100755 --- a/bench/config/lets_encrypt.py +++ b/bench/config/lets_encrypt.py @@ -126,7 +126,7 @@ def setup_wildcard_ssl(domain, email, bench_path): return domain domain = _get_domain_name(domain) - + print("domain", domain) return if not get_config(bench_path).get("dns_multitenant"): From 51f38c028989b42f2f0e456ff5ec0ea2afc66c74 Mon Sep 17 00:00:00 2001 From: Saurabh Date: Fri, 29 Jun 2018 14:35:28 +0530 Subject: [PATCH 42/49] fix wildcard char --- bench/config/lets_encrypt.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/bench/config/lets_encrypt.py b/bench/config/lets_encrypt.py index 0cbd4697..3cdfef71 100755 --- a/bench/config/lets_encrypt.py +++ b/bench/config/lets_encrypt.py @@ -124,16 +124,13 @@ def setup_wildcard_ssl(domain, email, bench_path): if domain.split('.')[0] != '*': domain = '*.{0}'.format(domain) return domain - - domain = _get_domain_name(domain) - print("domain", domain) - return if not get_config(bench_path).get("dns_multitenant"): print("You cannot setup SSL without DNS Multitenancy") return get_certbot() + domain = _get_domain_name(domain) email_param = '' if email: From 8f39df5c34a7fa40ac27e6e1c3aa12e6de7dc55b Mon Sep 17 00:00:00 2001 From: Saurabh Date: Sat, 30 Jun 2018 10:38:59 +0530 Subject: [PATCH 43/49] Include base domain while generating ssl certificate if not excluded explicitly --- bench/commands/setup.py | 3 ++- bench/config/lets_encrypt.py | 25 ++++++++++++++++++------- 2 files changed, 20 insertions(+), 8 deletions(-) diff --git a/bench/commands/setup.py b/bench/commands/setup.py index d3aec204..25e0fc5e 100755 --- a/bench/commands/setup.py +++ b/bench/commands/setup.py @@ -131,7 +131,8 @@ def setup_letsencrypt(site, custom_domain, non_interactive): @click.command('wildcard-ssl') @click.argument('domain') @click.option('--email') -def setup_wildcard_ssl(domain, email): +@click.option('--exclude-base-domain', default=False, is_flag=True, help="SSL Certificate not applicable for base domain") +def setup_wildcard_ssl(domain, email, exclude_base_domain): ''' Setup wildcard ssl certificate ''' from bench.config.lets_encrypt import setup_wildcard_ssl setup_wildcard_ssl(domain, email, bench_path='.') diff --git a/bench/config/lets_encrypt.py b/bench/config/lets_encrypt.py index 3cdfef71..a225754c 100755 --- a/bench/config/lets_encrypt.py +++ b/bench/config/lets_encrypt.py @@ -118,19 +118,27 @@ def renew_certs(): service('nginx', 'start') -def setup_wildcard_ssl(domain, email, bench_path): +def setup_wildcard_ssl(domain, email, bench_path, exclude_base_domain): - def _get_domain_name(domain): - if domain.split('.')[0] != '*': - domain = '*.{0}'.format(domain) - return domain + def _get_domains(domain): + domain_list = [domain] + + if not domain.startswith('*.'): + # add wildcard caracter to domain if missing + domain_list.append('*.{0}'.format(domain)) + else: + # include base domain based on flag + if not exclude_base_domain: + domain_list.append(domain.replace('*.', '')) + + return domain_list if not get_config(bench_path).get("dns_multitenant"): print("You cannot setup SSL without DNS Multitenancy") return get_certbot() - domain = _get_domain_name(domain) + domain_list = _get_domains(domain.strip()) email_param = '' if email: @@ -139,7 +147,7 @@ def setup_wildcard_ssl(domain, email, bench_path): try: exec_cmd("{path} certonly --manual --preferred-challenges=dns {email_param} \ --server https://acme-v02.api.letsencrypt.org/directory \ - --agree-tos -d {domain}".format(path=get_certbot_path(), domain=domain, + --agree-tos -d {domain}".format(path=get_certbot_path(), domain=' -d '.join(domain_list), email_param=email_param)) except CommandFailedError: @@ -156,5 +164,8 @@ def setup_wildcard_ssl(domain, email, bench_path): } update_common_site_config(ssl_config) + setup_crontab() + make_nginx_conf(bench_path) service('nginx', 'restart') + \ No newline at end of file From 5c980dce10779935f0d46d3deae4431fc4ce1235 Mon Sep 17 00:00:00 2001 From: Saurabh Date: Mon, 2 Jul 2018 11:44:11 +0530 Subject: [PATCH 44/49] Test --- bench/config/lets_encrypt.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/bench/config/lets_encrypt.py b/bench/config/lets_encrypt.py index a225754c..6139504f 100755 --- a/bench/config/lets_encrypt.py +++ b/bench/config/lets_encrypt.py @@ -144,6 +144,13 @@ def setup_wildcard_ssl(domain, email, bench_path, exclude_base_domain): if email: email_param = '--email {0}'.format(email) + + print("{path} certonly --manual --preferred-challenges=dns {email_param} \ + --server https://acme-v02.api.letsencrypt.org/directory \ + --agree-tos -d {domain}".format(path=get_certbot_path(), domain=' -d '.join(domain_list), + email_param=email_param)) + return + try: exec_cmd("{path} certonly --manual --preferred-challenges=dns {email_param} \ --server https://acme-v02.api.letsencrypt.org/directory \ @@ -167,5 +174,6 @@ def setup_wildcard_ssl(domain, email, bench_path, exclude_base_domain): setup_crontab() make_nginx_conf(bench_path) + print("Restrting Nginx service") service('nginx', 'restart') \ No newline at end of file From 364cd8fe1b43a44f2c0bc9eb8ed6f17e183d6323 Mon Sep 17 00:00:00 2001 From: Saurabh Date: Mon, 2 Jul 2018 11:49:51 +0530 Subject: [PATCH 45/49] [fix][args] add param exclude base domain --- bench/commands/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bench/commands/setup.py b/bench/commands/setup.py index 25e0fc5e..8039be26 100755 --- a/bench/commands/setup.py +++ b/bench/commands/setup.py @@ -135,7 +135,7 @@ def setup_letsencrypt(site, custom_domain, non_interactive): def setup_wildcard_ssl(domain, email, exclude_base_domain): ''' Setup wildcard ssl certificate ''' from bench.config.lets_encrypt import setup_wildcard_ssl - setup_wildcard_ssl(domain, email, bench_path='.') + setup_wildcard_ssl(domain, email, bench_path='.', exclude_base_domain) @click.command('procfile') From 41e227cff1e32d5c89d5437bff310ab0f479c2be Mon Sep 17 00:00:00 2001 From: Saurabh Date: Mon, 2 Jul 2018 11:56:13 +0530 Subject: [PATCH 46/49] [fix] syntax fixes --- bench/commands/setup.py | 2 +- bench/config/lets_encrypt.py | 7 ------- 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/bench/commands/setup.py b/bench/commands/setup.py index 8039be26..d88ee80f 100755 --- a/bench/commands/setup.py +++ b/bench/commands/setup.py @@ -135,7 +135,7 @@ def setup_letsencrypt(site, custom_domain, non_interactive): def setup_wildcard_ssl(domain, email, exclude_base_domain): ''' Setup wildcard ssl certificate ''' from bench.config.lets_encrypt import setup_wildcard_ssl - setup_wildcard_ssl(domain, email, bench_path='.', exclude_base_domain) + setup_wildcard_ssl(domain, email, bench_path='.', exclude_base_domain=exclude_base_domain) @click.command('procfile') diff --git a/bench/config/lets_encrypt.py b/bench/config/lets_encrypt.py index 6139504f..9088766e 100755 --- a/bench/config/lets_encrypt.py +++ b/bench/config/lets_encrypt.py @@ -144,13 +144,6 @@ def setup_wildcard_ssl(domain, email, bench_path, exclude_base_domain): if email: email_param = '--email {0}'.format(email) - - print("{path} certonly --manual --preferred-challenges=dns {email_param} \ - --server https://acme-v02.api.letsencrypt.org/directory \ - --agree-tos -d {domain}".format(path=get_certbot_path(), domain=' -d '.join(domain_list), - email_param=email_param)) - return - try: exec_cmd("{path} certonly --manual --preferred-challenges=dns {email_param} \ --server https://acme-v02.api.letsencrypt.org/directory \ From 14f56d11773104836f58cc663f2dc26c066d556e Mon Sep 17 00:00:00 2001 From: Saurabh Date: Mon, 2 Jul 2018 12:00:58 +0530 Subject: [PATCH 47/49] check exclusion expicitly --- bench/config/lets_encrypt.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/bench/config/lets_encrypt.py b/bench/config/lets_encrypt.py index 9088766e..4ec0f851 100755 --- a/bench/config/lets_encrypt.py +++ b/bench/config/lets_encrypt.py @@ -128,8 +128,10 @@ def setup_wildcard_ssl(domain, email, bench_path, exclude_base_domain): domain_list.append('*.{0}'.format(domain)) else: # include base domain based on flag - if not exclude_base_domain: - domain_list.append(domain.replace('*.', '')) + domain_list.append(domain.replace('*.', '')) + + if exclude_base_domain: + domain_list.remove(domain.replace('*.', '')) return domain_list From f90f7e141efae9304b73ae73ed4501fdf431b634 Mon Sep 17 00:00:00 2001 From: Aditya Hase Date: Mon, 2 Jul 2018 16:24:53 +0530 Subject: [PATCH 48/49] Add single quotes around password variables in ansible script As mentioned in #657 admin and mysql root password with whitespaces aren't recognized properly by shell adding single quotes around them fixes this issue. --- playbooks/roles/bench/tasks/setup_erpnext.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playbooks/roles/bench/tasks/setup_erpnext.yml b/playbooks/roles/bench/tasks/setup_erpnext.yml index 2e49219c..5dee30b3 100644 --- a/playbooks/roles/bench/tasks/setup_erpnext.yml +++ b/playbooks/roles/bench/tasks/setup_erpnext.yml @@ -15,7 +15,7 @@ register: site_folder - name: Create a new site - command: "bench new-site {{ site }} --admin-password {{ admin_password }} --mariadb-root-password {{ mysql_root_password }}" + command: "bench new-site {{ site }} --admin-password '{{ admin_password }}' --mariadb-root-password '{{ mysql_root_password }}'" args: chdir: "{{ bench_path }}" when: not site_folder.stat.exists From 297f0d4bc54a295e8ad84982e08ad64850d917d1 Mon Sep 17 00:00:00 2001 From: Ameya Shenoy Date: Tue, 3 Jul 2018 12:58:52 +0530 Subject: [PATCH 49/49] modified mode install method --- .travis.yml | 2 -- playbooks/roles/nodejs/defaults/main.yml | 3 +++ playbooks/roles/nodejs/tasks/debian_family.yml | 14 +------------- playbooks/roles/nodejs/tasks/main.yml | 6 ++++++ playbooks/roles/nodejs/tasks/redhat_family.yml | 15 +-------------- playbooks/site.yml | 2 +- 6 files changed, 12 insertions(+), 30 deletions(-) create mode 100644 playbooks/roles/nodejs/defaults/main.yml diff --git a/.travis.yml b/.travis.yml index e4b562b7..ae45db62 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,5 @@ language: python dist: trusty -group: deprecated-2017Q2 -sudo: required python: - "2.7" diff --git a/playbooks/roles/nodejs/defaults/main.yml b/playbooks/roles/nodejs/defaults/main.yml new file mode 100644 index 00000000..f991faa1 --- /dev/null +++ b/playbooks/roles/nodejs/defaults/main.yml @@ -0,0 +1,3 @@ +--- +node_version: 8 +... \ No newline at end of file diff --git a/playbooks/roles/nodejs/tasks/debian_family.yml b/playbooks/roles/nodejs/tasks/debian_family.yml index c64bac55..a9930529 100644 --- a/playbooks/roles/nodejs/tasks/debian_family.yml +++ b/playbooks/roles/nodejs/tasks/debian_family.yml @@ -1,17 +1,5 @@ --- -- name: Add apt key for node repo - apt_key: - url: https://keyserver.ubuntu.com/pks/lookup?op=get&fingerprint=on&search=0x1655A0AB68576280 - id: "68576280" - state: present - -- name: Add repo - apt_repository: - repo: "deb [arch=amd64,i386] https://deb.nodesource.com/node_8.x {{ ansible_distribution_release }} main" - state: present - register: node_repo - -- name: Install nodejs 8.x +- name: Install nodejs {{ node_version }} apt: name: nodejs state: present diff --git a/playbooks/roles/nodejs/tasks/main.yml b/playbooks/roles/nodejs/tasks/main.yml index 85379727..6ff013d7 100644 --- a/playbooks/roles/nodejs/tasks/main.yml +++ b/playbooks/roles/nodejs/tasks/main.yml @@ -1,6 +1,12 @@ --- # Install's prerequisites, like fonts, image libraries, vim, screen, python-dev and gcc +- name: 'Add Node.js PPA' + tags: 'nodejs' + become: 'yes' + become_method: 'sudo' + shell: "curl --silent --location https://deb.nodesource.com/setup_{{ node_version }}.x | bash -" + - include_tasks: debian_family.yml when: ansible_os_family == 'Debian' diff --git a/playbooks/roles/nodejs/tasks/redhat_family.yml b/playbooks/roles/nodejs/tasks/redhat_family.yml index 15af3846..1c3d8fb1 100644 --- a/playbooks/roles/nodejs/tasks/redhat_family.yml +++ b/playbooks/roles/nodejs/tasks/redhat_family.yml @@ -1,18 +1,5 @@ --- -- name: Import Node source RPM key - rpm_key: - key: https://rpm.nodesource.com/pub/el/NODESOURCE-GPG-SIGNING-KEY-EL - state: present - when: ansible_os_family == 'RedHat' - -- name: Add Node Repo for RedHat - yum: - name: 'https://rpm.nodesource.com/pub_8.x/el/{{ ansible_distribution_major_version }}/{{ ansible_architecture }}/nodesource-release-el{{ ansible_distribution_major_version }}-1.noarch.rpm' - state: present - update_cache: yes - when: ansible_os_family == 'RedHat' - -- name: Install node v8 +- name: Install node v{{ node_version }} yum: name=nodejs state=present when: ansible_os_family == 'RedHat' ... \ No newline at end of file diff --git a/playbooks/site.yml b/playbooks/site.yml index 839620ee..ca7cb51a 100644 --- a/playbooks/site.yml +++ b/playbooks/site.yml @@ -7,6 +7,7 @@ roles: - { role: common, tags: common } - { role: locale, tags: locale } + - { role: nodejs, tags: nodejs } - { role: swap, tags: swap, when: production } - { role: logwatch, tags: logwatch, when: production } - { role: bash_screen_wall, tags: bash_screen_wall, when: production } @@ -15,7 +16,6 @@ - { role: ntpd, tags: ntpd, when: production } - { role: mariadb, tags: mariadb } - { role: wkhtmltopdf, tags: wkhtmltopdf } - - { role: nodejs, tags: nodejs } - { role: psutil, tags: psutil } - { role: redis, tags: redis } - { role: supervisor, tags: supervisor, when: production }