fix: make semantic changes to commands

* add missing __main__ call to commands.py
* remove unnecessary imports
* fix backup WITH_FILES logic
* follow python semantics (?)

Signed-off-by: Chinmay D. Pai <chinmaydpai@gmail.com>

build/common/commands/auto_migrate.py

@@ -9,10 +9,12 @@ from check_connection import get_config
 APP_VERSIONS_JSON_FILE = 'app_versions.json'
 APPS_TXT_FILE = 'apps.txt'
 
+
 def save_version_file(versions):
     with open(APP_VERSIONS_JSON_FILE, 'w') as f:
         return json.dump(versions, f, indent=1, sort_keys=True)
 
+
 def get_apps():
     apps = []
     try:
@@ -24,40 +26,43 @@ def get_apps():
     except FileNotFoundError as exception:
         print(exception)
         exit(1)
-    except:
+    except Exception:
         print(APPS_TXT_FILE+" is not valid")
         exit(1)
 
     return apps
 
+
 def get_container_versions(apps):
     versions = {}
     for app in apps:
         try:
             version = __import__(app).__version__
-            versions.update({app:version})
+            versions.update({app: version})
-        except:
+        except Exception:
             pass
 
         try:
-            path = os.path.join('..','apps', app)
+            path = os.path.join('..', 'apps', app)
             repo = git.Repo(path)
             commit_hash = repo.head.object.hexsha
-            versions.update({app+'_git_hash':commit_hash})
+            versions.update({app+'_git_hash': commit_hash})
-        except:
+        except Exception:
             pass
 
     return versions
 
+
 def get_version_file():
     versions = None
     try:
         with open(APP_VERSIONS_JSON_FILE) as versions_file:
             versions = json.load(versions_file)
-    except:
+    except Exception:
         pass
     return versions
 
+
 def main():
     is_ready = False
     apps = get_apps()
@@ -76,7 +81,7 @@ def main():
         version_file_hash = None
         container_hash = None
 
-        repo = git.Repo(os.path.join('..','apps',app))
+        repo = git.Repo(os.path.join('..', 'apps', app))
         branch = repo.active_branch.name
 
         if branch == 'develop':
@@ -105,5 +110,6 @@ def main():
         version_file = container_versions
         save_version_file(version_file)
 
+
 if __name__ == "__main__":
     main()

build/common/commands/background.py

@@ -1,10 +1,11 @@
-import frappe
 from frappe.utils.scheduler import start_scheduler
 
+
 def main():
     print("Starting background scheduler . . .")
     start_scheduler()
     exit(0)
 
+
 if __name__ == "__main__":
     main()

build/common/commands/backup.py

@@ -1,7 +1,8 @@
-import os, frappe, compileall, re
+import os
+import frappe
 from frappe.utils.backups import scheduled_backup
-from frappe.utils import now
+from frappe.utils import cint, get_sites, now
-from frappe.utils import get_sites
+
 
 def backup(sites, with_files=False):
     for site in sites:
@@ -20,13 +21,15 @@ def backup(sites, with_files=False):
             print("private files backup taken -", odb.backup_path_private_files, "- on", now())
         frappe.destroy()
 
+
 def main():
     installed_sites = ":".join(get_sites())
     sites = os.environ.get("SITES", installed_sites).split(":")
-    with_files=True if os.environ.get("WITH_FILES") else False
+    with_files = cint(os.environ.get("WITH_FILES"))
 
     backup(sites, with_files)
     exit(0)
 
+
 if __name__ == "__main__":
     main()

build/common/commands/check_connection.py

@@ -1,4 +1,6 @@
-import socket, os, json, time
+import socket
+import json
+import time
 from six.moves.urllib.parse import urlparse
 
 COMMON_SITE_CONFIG_FILE = 'common_site_config.json'
@@ -8,6 +10,7 @@ REDIS_SOCKETIO_KEY = 'redis_socketio'
 DB_HOST_KEY = 'db_host'
 DB_PORT = 3306
 
+
 def is_open(ip, port, timeout=30):
     s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
     s.settimeout(timeout)
@@ -15,16 +18,17 @@ def is_open(ip, port, timeout=30):
         s.connect((ip, int(port)))
         s.shutdown(socket.SHUT_RDWR)
         return True
-    except:
+    except Exception:
         return False
     finally:
         s.close()
 
+
 def check_host(ip, port, retry=10, delay=3, print_attempt=True):
     ipup = False
     for i in range(retry):
         if print_attempt:
-            print("Attempt {i} to connect to {ip}:{port}".format(ip=ip,port=port,i=i+1))
+            print("Attempt {i} to connect to {ip}:{port}".format(ip=ip, port=port, i=i+1))
         if is_open(ip, port):
             ipup = True
             break
@@ -32,6 +36,7 @@ def check_host(ip, port, retry=10, delay=3, print_attempt=True):
             time.sleep(delay)
     return ipup
 
+
 # Check connection to servers
 def get_config():
     config = None
@@ -41,11 +46,12 @@ def get_config():
     except FileNotFoundError as exception:
         print(exception)
         exit(1)
-    except:
+    except Exception:
         print(COMMON_SITE_CONFIG_FILE+" is not valid")
         exit(1)
     return config
 
+
 # Check mariadb
 def check_mariadb(retry=10, delay=3, print_attempt=True):
     config = get_config()
@@ -60,11 +66,12 @@ def check_mariadb(retry=10, delay=3, print_attempt=True):
         print("Connection to MariaDB timed out")
         exit(1)
 
+
 # Check redis queue
 def check_redis_queue(retry=10, delay=3, print_attempt=True):
     check_redis_queue = False
     config = get_config()
-    redis_queue_url = urlparse(config.get(REDIS_QUEUE_KEY,"redis://redis-queue:6379")).netloc
+    redis_queue_url = urlparse(config.get(REDIS_QUEUE_KEY, "redis://redis-queue:6379")).netloc
     redis_queue, redis_queue_port = redis_queue_url.split(":")
     check_redis_queue = check_host(
         redis_queue,
@@ -76,11 +83,12 @@ def check_redis_queue(retry=10, delay=3, print_attempt=True):
         print("Connection to redis queue timed out")
         exit(1)
 
+
 # Check redis cache
 def check_redis_cache(retry=10, delay=3, print_attempt=True):
     check_redis_cache = False
     config = get_config()
-    redis_cache_url = urlparse(config.get(REDIS_CACHE_KEY,"redis://redis-cache:6379")).netloc
+    redis_cache_url = urlparse(config.get(REDIS_CACHE_KEY, "redis://redis-cache:6379")).netloc
     redis_cache, redis_cache_port = redis_cache_url.split(":")
     check_redis_cache = check_host(
         redis_cache,
@@ -92,11 +100,12 @@ def check_redis_cache(retry=10, delay=3, print_attempt=True):
         print("Connection to redis cache timed out")
         exit(1)
 
+
 # Check redis socketio
 def check_redis_socketio(retry=10, delay=3, print_attempt=True):
     check_redis_socketio = False
     config = get_config()
-    redis_socketio_url = urlparse(config.get(REDIS_SOCKETIO_KEY,"redis://redis-socketio:6379")).netloc
+    redis_socketio_url = urlparse(config.get(REDIS_SOCKETIO_KEY, "redis://redis-socketio:6379")).netloc
     redis_socketio, redis_socketio_port = redis_socketio_url.split(":")
     check_redis_socketio = check_host(
         redis_socketio,
@@ -108,6 +117,7 @@ def check_redis_socketio(retry=10, delay=3, print_attempt=True):
         print("Connection to redis socketio timed out")
         exit(1)
 
+
 # Get site_config.json
 def get_site_config(site_name):
     site_config = None
@@ -115,6 +125,7 @@ def get_site_config(site_name):
         site_config = json.load(site_config_file)
     return site_config
 
+
 def main():
     check_mariadb()
     check_redis_queue()
@@ -122,5 +133,6 @@ def main():
     check_redis_socketio()
     print('Connections OK')
 
+
 if __name__ == "__main__":
     main()

build/common/commands/console.py

@@ -20,6 +20,11 @@ def console(site):
     print("Apps in this namespace:\n{}".format(", ".join(all_apps)))
     IPython.embed(display_banner="", header="")
 
+
 def main():
     site = sys.argv[-1]
     console(site)
+
+
+if __name__ == "__main__":
+    main()

build/common/commands/doctor.py

@@ -1,7 +1,3 @@
-import frappe
-import json
-import redis
-from rq import Worker
 from check_connection import (
     check_mariadb,
     check_redis_cache,
@@ -9,6 +5,7 @@ from check_connection import (
     check_redis_socketio,
 )
 
+
 def main():
     check_mariadb(retry=1, delay=0, print_attempt=False)
     print("MariaDB Connected")
@@ -22,5 +19,6 @@ def main():
     print("Health check successful")
     exit(0)
 
+
 if __name__ == "__main__":
     main()

build/common/commands/migrate.py

@@ -1,29 +1,34 @@
-import os, frappe, compileall, re, json
+import os
+import frappe
+import json
 
 from frappe.migrate import migrate
-from frappe.utils import get_sites
+from frappe.utils import cint, get_sites
 from check_connection import get_config
 
+
 def save_config(config):
     with open('common_site_config.json', 'w') as f:
         return json.dump(config, f, indent=1, sort_keys=True)
 
+
 def set_maintenance_mode(enable=True):
     conf = get_config()
 
     if enable:
-        conf.update({ "maintenance_mode": 1, "pause_scheduler": 1 })
+        conf.update({"maintenance_mode": 1, "pause_scheduler": 1})
         save_config(conf)
 
     if not enable:
-        conf.update({ "maintenance_mode": 0, "pause_scheduler": 0 })
+        conf.update({"maintenance_mode": 0, "pause_scheduler": 0})
         save_config(conf)
 
+
 def migrate_sites(maintenance_mode=False):
     installed_sites = ":".join(get_sites())
     sites = os.environ.get("SITES", installed_sites).split(":")
     if not maintenance_mode:
-        maintenance_mode = True if os.environ.get("MAINTENANCE_MODE") else False
+        maintenance_mode = cint(os.environ.get("MAINTENANCE_MODE"))
 
     if maintenance_mode:
         set_maintenance_mode(True)
@@ -40,9 +45,11 @@ def migrate_sites(maintenance_mode=False):
     if maintenance_mode:
         set_maintenance_mode(False)
 
+
 def main():
     migrate_sites()
     exit(0)
 
+
 if __name__ == "__main__":
     main()

build/common/commands/new.py

@@ -1,8 +1,10 @@
-import os, frappe, json
+import os
+import frappe
 
 from frappe.commands.site import _new_site
 from check_connection import get_config, get_site_config
 
+
 def get_password(env_var, default=None):
     return os.environ.get(env_var) or _get_password_from_secret(f"{env_var}_FILE") or default
 
@@ -71,11 +73,12 @@ def main():
     os.system(command)
 
     # Grant permission to database
-    command = mysql_command + "\"GRANT ALL PRIVILEGES ON \`{db_name}\`.* TO '{db_name}'@'%'; FLUSH PRIVILEGES;\"".format(
+    command = mysql_command + "\"GRANT ALL PRIVILEGES ON `{db_name}`.* TO '{db_name}'@'%'; FLUSH PRIVILEGES;\"".format(
         db_name=site_config.get('db_name')
     )
     os.system(command)
     exit(0)
 
+
 if __name__ == "__main__":
     main()

build/common/commands/push_backup.py

@@ -8,6 +8,7 @@ from frappe.utils import get_sites
 
 DATE_FORMAT = "%Y%m%d_%H%M%S"
 
+
 def get_file_ext():
     return {
         "database": "-database.sql.gz",
@@ -15,6 +16,7 @@ def get_file_ext():
         "public_files": "-files.tar"
     }
 
+
 def get_backup_details(sitename):
     backup_details = dict()
     file_ext = get_file_ext()
@@ -41,6 +43,7 @@ def get_backup_details(sitename):
 
     return backup_details
 
+
 def get_s3_config():
     check_environment_variables()
     bucket = os.environ.get('BUCKET_NAME')
@@ -55,31 +58,33 @@ def get_s3_config():
 
     return conn, bucket
 
+
 def check_environment_variables():
-    if not 'BUCKET_NAME' in os.environ:
+    if 'BUCKET_NAME' not in os.environ:
         print('Variable BUCKET_NAME not set')
         exit(1)
 
-    if not 'ACCESS_KEY_ID' in os.environ:
+    if 'ACCESS_KEY_ID' not in os.environ:
         print('Variable ACCESS_KEY_ID not set')
         exit(1)
 
-    if not 'SECRET_ACCESS_KEY' in os.environ:
+    if 'SECRET_ACCESS_KEY' not in os.environ:
         print('Variable SECRET_ACCESS_KEY not set')
         exit(1)
 
-    if not 'ENDPOINT_URL' in os.environ:
+    if 'ENDPOINT_URL' not in os.environ:
         print('Variable ENDPOINT_URL not set')
         exit(1)
 
-    if not 'BUCKET_DIR' in os.environ:
+    if 'BUCKET_DIR' not in os.environ:
         print('Variable BUCKET_DIR not set')
         exit(1)
 
-    if not 'REGION' in os.environ:
+    if 'REGION' not in os.environ:
         print('Variable REGION not set')
         exit(1)
 
+
 def upload_file_to_s3(filename, folder, conn, bucket):
 
     destpath = os.path.join(folder, os.path.basename(filename))
@@ -91,6 +96,7 @@ def upload_file_to_s3(filename, folder, conn, bucket):
         print("Error uploading: %s" % (e))
         exit(1)
 
+
 def delete_old_backups(limit, bucket, site_name):
     all_backups = list()
     all_backup_dates = list()
@@ -156,6 +162,7 @@ def delete_old_backups(limit, bucket, site_name):
                         print('Deleteing ' + obj.key)
                         s3.Object(bucket.name, obj.key).delete()
 
+
 def main():
     details = dict()
     sites = get_sites()
@@ -184,5 +191,6 @@ def main():
     print('push-backup complete')
     exit(0)
 
+
 if __name__ == "__main__":
     main()

build/common/commands/restore_backup.py

@@ -8,10 +8,10 @@ import boto3
 from new import get_password
 from push_backup import DATE_FORMAT, check_environment_variables
 from frappe.utils import get_sites, random_string
-from frappe.commands.site import _new_site
 from frappe.installer import make_conf, get_conf_params, make_site_dirs
 from check_connection import get_site_config, get_config
 
+
 def list_directories(path):
     directories = []
     for name in os.listdir(path):
@@ -19,25 +19,25 @@ def list_directories(path):
             directories.append(name)
     return directories
 
+
 def get_backup_dir():
     return os.path.join(
         os.path.expanduser('~'),
         'backups'
     )
 
+
 def decompress_db(files_base, site):
     database_file = files_base + '-database.sql.gz'
-    config = get_config()
-    site_config = get_site_config(site)
-    db_root_user = os.environ.get('DB_ROOT_USER', 'root')
     command = 'gunzip -c {database_file} > {database_extract}'.format(
         database_file=database_file,
-        database_extract=database_file.replace('.gz','')
+        database_extract=database_file.replace('.gz', '')
     )
 
     print('Extract Database GZip for site {}'.format(site))
     os.system(command)
 
+
 def restore_database(files_base, site):
     db_root_password = get_password('MYSQL_ROOT_PASSWORD')
     if not db_root_password:
@@ -60,13 +60,13 @@ def restore_database(files_base, site):
     )
 
     # drop db if exists for clean restore
-    drop_database = mysql_command + "\"DROP DATABASE IF EXISTS \`{db_name}\`;\"".format(
+    drop_database = mysql_command + "\"DROP DATABASE IF EXISTS `{db_name}`;\"".format(
         db_name=site_config.get('db_name')
     )
     os.system(drop_database)
 
     # create db
-    create_database = mysql_command + "\"CREATE DATABASE IF NOT EXISTS \`{db_name}\`;\"".format(
+    create_database = mysql_command + "\"CREATE DATABASE IF NOT EXISTS `{db_name}`;\"".format(
         db_name=site_config.get('db_name')
     )
     os.system(create_database)
@@ -86,7 +86,7 @@ def restore_database(files_base, site):
     os.system(set_user_password)
 
     # grant db privileges to user
-    grant_privileges = mysql_command + "\"GRANT ALL PRIVILEGES ON \`{db_name}\`.* TO '{db_name}'@'%'; FLUSH PRIVILEGES;\"".format(
+    grant_privileges = mysql_command + "\"GRANT ALL PRIVILEGES ON `{db_name}`.* TO '{db_name}'@'%'; FLUSH PRIVILEGES;\"".format(
         db_name=site_config.get('db_name')
     )
     os.system(grant_privileges)
@@ -96,12 +96,13 @@ def restore_database(files_base, site):
         db_host=config.get('db_host'),
         db_password=db_root_password,
         db_name=site_config.get('db_name'),
-        database_file=database_file.replace('.gz',''),
+        database_file=database_file.replace('.gz', ''),
     )
 
     print('Restoring database for site: {}'.format(site))
     os.system(command)
 
+
 def restore_files(files_base):
     public_files = files_base + '-files.tar'
     # extract tar
@@ -109,12 +110,14 @@ def restore_files(files_base):
     print('Extracting {}'.format(public_files))
     public_tar.extractall()
 
+
 def restore_private_files(files_base):
     private_files = files_base + '-private-files.tar'
     private_tar = tarfile.open(private_files)
     print('Extracting {}'.format(private_files))
     private_tar.extractall()
 
+
 def pull_backup_from_s3():
     check_environment_variables()
 
@@ -134,8 +137,8 @@ def pull_backup_from_s3():
     # Change directory to /home/frappe/backups
     os.chdir(get_backup_dir())
 
-    for obj in bucket.objects.filter(Prefix = bucket_dir):
+    for obj in bucket.objects.filter(Prefix=bucket_dir):
-        backup_file = obj.key.replace(os.path.join(bucket_dir,''),'')
+        backup_file = obj.key.replace(os.path.join(bucket_dir, ''), '')
         if not os.path.exists(os.path.dirname(backup_file)):
             os.makedirs(os.path.dirname(backup_file))
         print('Downloading {}'.format(backup_file))
@@ -143,6 +146,7 @@ def pull_backup_from_s3():
 
     os.chdir(os.path.join(os.path.expanduser('~'), 'frappe-bench', 'sites'))
 
+
 def main():
     backup_dir = get_backup_dir()
 
@@ -150,8 +154,8 @@ def main():
         pull_backup_from_s3()
 
     for site in list_directories(backup_dir):
-        site_slug = site.replace('.','_')
+        site_slug = site.replace('.', '_')
-        backups = [datetime.datetime.strptime(backup, DATE_FORMAT) for backup in list_directories(os.path.join(backup_dir,site))]
+        backups = [datetime.datetime.strptime(backup, DATE_FORMAT) for backup in list_directories(os.path.join(backup_dir, site))]
         latest_backup = max(backups).strftime(DATE_FORMAT)
         files_base = os.path.join(backup_dir, site, latest_backup, '')
         files_base += latest_backup + '-' + site_slug
@@ -164,8 +168,6 @@ def main():
             if not mariadb_root_password:
                 print('Variable MYSQL_ROOT_PASSWORD not set')
                 exit(1)
-            mariadb_root_username = os.environ.get('DB_ROOT_USER', 'root')
-            database_file = files_base + '-database.sql.gz'
 
             site_config = get_conf_params(
                 db_name='_' + hashlib.sha1(site.encode()).hexdigest()[:16],
@@ -186,5 +188,6 @@ def main():
 
     exit(0)
 
+
 if __name__ == "__main__":
     main()

build/common/commands/worker.py

@@ -1,10 +1,12 @@
-import os, frappe
+import os
 from frappe.utils.background_jobs import start_worker
 
+
 def main():
     queue = os.environ.get("WORKER_TYPE", "default")
     start_worker(queue, False)
     exit(0)
 
+
 if __name__ == "__main__":
     main()