refactor: add utils and constants

.gitignore

@@ -10,3 +10,6 @@ development
 !development/README.md
 deploy_key
 deploy_key.pub
+
+# Pycharm
+.idea

build/common/commands/auto_migrate.py

@@ -1,66 +1,15 @@
 import os
-import json
 import semantic_version
 import git
 
 from migrate import migrate_sites
-from check_connection import get_config
+from utils import (
-
+    save_version_file,
-APP_VERSIONS_JSON_FILE = 'app_versions.json'
+    get_apps,
-APPS_TXT_FILE = 'apps.txt'
+    get_container_versions,
-
+    get_version_file,
-
+    get_config
-def save_version_file(versions):
+)
-    with open(APP_VERSIONS_JSON_FILE, 'w') as f:
-        return json.dump(versions, f, indent=1, sort_keys=True)
-
-
-def get_apps():
-    apps = []
-    try:
-        with open(APPS_TXT_FILE) as apps_file:
-            for app in apps_file.readlines():
-                if app.strip():
-                    apps.append(app.strip())
-
-    except FileNotFoundError as exception:
-        print(exception)
-        exit(1)
-    except Exception:
-        print(APPS_TXT_FILE+" is not valid")
-        exit(1)
-
-    return apps
-
-
-def get_container_versions(apps):
-    versions = {}
-    for app in apps:
-        try:
-            version = __import__(app).__version__
-            versions.update({app: version})
-        except Exception:
-            pass
-
-        try:
-            path = os.path.join('..', 'apps', app)
-            repo = git.Repo(path)
-            commit_hash = repo.head.object.hexsha
-            versions.update({app+'_git_hash': commit_hash})
-        except Exception:
-            pass
-
-    return versions
-
-
-def get_version_file():
-    versions = None
-    try:
-        with open(APP_VERSIONS_JSON_FILE) as versions_file:
-            versions = json.load(versions_file)
-    except Exception:
-        pass
-    return versions
 
 
 def main():

build/common/commands/check_connection.py

@@ -1,15 +1,15 @@
 import socket
-import json
 import time
 from six.moves.urllib.parse import urlparse
-
+from utils import get_config
-COMMON_SITE_CONFIG_FILE = 'common_site_config.json'
+from constants import (
-REDIS_QUEUE_KEY = 'redis_queue'
+    REDIS_QUEUE_KEY,
-REDIS_CACHE_KEY = 'redis_cache'
+    REDIS_CACHE_KEY,
-REDIS_SOCKETIO_KEY = 'redis_socketio'
+    REDIS_SOCKETIO_KEY,
-DB_HOST_KEY = 'db_host'
+    DB_HOST_KEY,
-DB_PORT_KEY = 'db_port'
+    DB_PORT_KEY,
-DB_PORT = 3306
+    DB_PORT
+)
 
 
 def is_open(ip, port, timeout=30):
@@ -38,21 +38,6 @@ def check_host(ip, port, retry=10, delay=3, print_attempt=True):
     return ipup
 
 
-# Check connection to servers
-def get_config():
-    config = None
-    try:
-        with open(COMMON_SITE_CONFIG_FILE) as config_file:
-            config = json.load(config_file)
-    except FileNotFoundError as exception:
-        print(exception)
-        exit(1)
-    except Exception:
-        print(COMMON_SITE_CONFIG_FILE+" is not valid")
-        exit(1)
-    return config
-
-
 # Check service
 def check_service(
     retry=10,
@@ -133,14 +118,6 @@ def check_redis_socketio(retry=10, delay=3, print_attempt=True):
         exit(1)
 
 
-# Get site_config.json
-def get_site_config(site_name):
-    site_config = None
-    with open('{site_name}/site_config.json'.format(site_name=site_name)) as site_config_file:
-        site_config = json.load(site_config_file)
-    return site_config
-
-
 def main():
     check_service()
     check_redis_queue()

build/common/commands/constants.py

@@ -0,0 +1,10 @@
+REDIS_QUEUE_KEY = 'redis_queue'
+REDIS_CACHE_KEY = 'redis_cache'
+REDIS_SOCKETIO_KEY = 'redis_socketio'
+DB_HOST_KEY = 'db_host'
+DB_PORT_KEY = 'db_port'
+DB_PORT = 3306
+APP_VERSIONS_JSON_FILE = 'app_versions.json'
+APPS_TXT_FILE = 'apps.txt'
+COMMON_SITE_CONFIG_FILE = 'common_site_config.json'
+DATE_FORMAT = "%Y%m%d_%H%M%S"

build/common/commands/migrate.py

@@ -1,14 +1,8 @@
 import os
 import frappe
-import json
 
 from frappe.utils import cint, get_sites
-from check_connection import get_config, COMMON_SITE_CONFIG_FILE
+from utils import get_config, save_config
-
-
-def save_config(config):
-    with open(COMMON_SITE_CONFIG_FILE, 'w') as f:
-        return json.dump(config, f, indent=1, sort_keys=True)
 
 
 def set_maintenance_mode(enable=True):

build/common/commands/new.py

@@ -4,30 +4,13 @@ import semantic_version
 
 from frappe.commands.site import _new_site
 from frappe.installer import update_site_config
-from check_connection import get_config, get_site_config, COMMON_SITE_CONFIG_FILE
+from constants import COMMON_SITE_CONFIG_FILE
-from utils import run_command
+from utils import (
-
+    run_command,
-
+    get_config,
-def get_password(env_var, default=None):
+    get_site_config,
-    return os.environ.get(env_var) or _get_password_from_secret(f"{env_var}_FILE") or default
+    get_password,
-
+)
-
-def _get_password_from_secret(env_var):
-    """Fetches the secret value from the docker secret file
-    usually located inside /run/secrets/
-    Arguments:
-        env_var {str} -- Name of the environment variable
-        containing the path to the secret file.
-    Returns:
-        [str] -- Secret value
-    """
-    passwd = None
-    secret_file_path = os.environ.get(env_var)
-    if secret_file_path:
-        with open(secret_file_path) as secret_file:
-            passwd = secret_file.read().strip()
-
-    return passwd
 
 
 def main():

build/common/commands/push_backup.py

@@ -5,8 +5,12 @@ import boto3
 import datetime
 from glob import glob
 from frappe.utils import get_sites
-
+from constants import DATE_FORMAT
-DATE_FORMAT = "%Y%m%d_%H%M%S"
+from utils import (
+    get_s3_config,
+    upload_file_to_s3,
+    check_s3_environment_variables,
+)
 
 
 def get_file_ext():
@@ -45,64 +49,11 @@ def get_backup_details(sitename):
     return backup_details
 
 
-def get_s3_config():
-    check_environment_variables()
-    bucket = os.environ.get('BUCKET_NAME')
-
-    conn = boto3.client(
-        's3',
-        region_name=os.environ.get('REGION'),
-        aws_access_key_id=os.environ.get('ACCESS_KEY_ID'),
-        aws_secret_access_key=os.environ.get('SECRET_ACCESS_KEY'),
-        endpoint_url=os.environ.get('ENDPOINT_URL')
-    )
-
-    return conn, bucket
-
-
-def check_environment_variables():
-    if 'BUCKET_NAME' not in os.environ:
-        print('Variable BUCKET_NAME not set')
-        exit(1)
-
-    if 'ACCESS_KEY_ID' not in os.environ:
-        print('Variable ACCESS_KEY_ID not set')
-        exit(1)
-
-    if 'SECRET_ACCESS_KEY' not in os.environ:
-        print('Variable SECRET_ACCESS_KEY not set')
-        exit(1)
-
-    if 'ENDPOINT_URL' not in os.environ:
-        print('Variable ENDPOINT_URL not set')
-        exit(1)
-
-    if 'BUCKET_DIR' not in os.environ:
-        print('Variable BUCKET_DIR not set')
-        exit(1)
-
-    if 'REGION' not in os.environ:
-        print('Variable REGION not set')
-        exit(1)
-
-
-def upload_file_to_s3(filename, folder, conn, bucket):
-
-    destpath = os.path.join(folder, os.path.basename(filename))
-    try:
-        print("Uploading file:", filename)
-        conn.upload_file(filename, bucket, destpath)
-
-    except Exception as e:
-        print("Error uploading: %s" % (e))
-        exit(1)
-
-
 def delete_old_backups(limit, bucket, site_name):
     all_backups = list()
     all_backup_dates = list()
     backup_limit = int(limit)
-    check_environment_variables()
+    check_s3_environment_variables()
     bucket_dir = os.environ.get('BUCKET_DIR')
     oldest_backup_date = None
 

build/common/commands/restore_backup.py

@@ -1,25 +1,27 @@
 import os
 import datetime
-import json
 import tarfile
 import hashlib
 import frappe
 import boto3
 
-from new import get_password
-from push_backup import DATE_FORMAT, check_environment_variables
-from utils import run_command
 from frappe.utils import get_sites, random_string
-from frappe.installer import make_conf, get_conf_params, make_site_dirs, update_site_config
+from frappe.installer import (
-from check_connection import get_site_config, get_config, COMMON_SITE_CONFIG_FILE
+    make_conf,
-
+    get_conf_params,
-
+    make_site_dirs,
-def list_directories(path):
+    update_site_config
-    directories = []
+)
-    for name in os.listdir(path):
+from constants import COMMON_SITE_CONFIG_FILE, DATE_FORMAT
-        if os.path.isdir(os.path.join(path, name)):
+from utils import (
-            directories.append(name)
+    run_command,
-    return directories
+    list_directories,
+    set_key_in_site_config,
+    get_site_config,
+    get_config,
+    get_password,
+    check_s3_environment_variables,
+)
 
 
 def get_backup_dir():
@@ -74,23 +76,6 @@ def restore_database(files_base, site_config_path, site):
         set_key_in_site_config('encryption_key', site, site_config_path)
 
 
-def set_key_in_site_config(key, site, site_config_path):
-    site_config = get_site_config_from_path(site_config_path)
-    value = site_config.get(key)
-    if value:
-        print('Set {key} in site config for site: {site}'.format(key=key, site=site))
-        update_site_config(key, value,
-                            site_config_path=os.path.join(os.getcwd(), site, "site_config.json"))
-
-
-def get_site_config_from_path(site_config_path):
-    site_config = dict()
-    if os.path.exists(site_config_path):
-        with open(site_config_path, 'r') as sc:
-            site_config = json.load(sc)
-    return site_config
-
-
 def restore_files(files_base):
     public_files = files_base + '-files.tar'
     # extract tar
@@ -107,7 +92,7 @@ def restore_private_files(files_base):
 
 
 def pull_backup_from_s3():
-    check_environment_variables()
+    check_s3_environment_variables()
 
     # https://stackoverflow.com/a/54672690
     s3 = boto3.resource(

build/common/commands/utils.py

@@ -1,5 +1,15 @@
+import json
+import os
 import subprocess
+import boto3
+import git
 
+from frappe.installer import update_site_config
+from constants import (
+    APP_VERSIONS_JSON_FILE,
+    APPS_TXT_FILE,
+    COMMON_SITE_CONFIG_FILE
+)
 
 def run_command(command, stdout=None, stdin=None, stderr=None):
     stdout = stdout or subprocess.PIPE
@@ -13,3 +23,182 @@ def run_command(command, stdout=None, stdin=None, stderr=None):
         print(f"stdout:\n{out}")
         print(f"\nstderr:\n{error}")
         exit(process.returncode)
+
+
+def save_version_file(versions):
+    with open(APP_VERSIONS_JSON_FILE, 'w') as f:
+        return json.dump(versions, f, indent=1, sort_keys=True)
+
+
+def get_apps():
+    apps = []
+    try:
+        with open(APPS_TXT_FILE) as apps_file:
+            for app in apps_file.readlines():
+                if app.strip():
+                    apps.append(app.strip())
+
+    except FileNotFoundError as exception:
+        print(exception)
+        exit(1)
+    except Exception:
+        print(APPS_TXT_FILE + " is not valid")
+        exit(1)
+
+    return apps
+
+
+def get_container_versions(apps):
+    versions = {}
+    for app in apps:
+        try:
+            version = __import__(app).__version__
+            versions.update({app: version})
+        except Exception:
+            pass
+
+        try:
+            path = os.path.join('..', 'apps', app)
+            repo = git.Repo(path)
+            commit_hash = repo.head.object.hexsha
+            versions.update({app+'_git_hash': commit_hash})
+        except Exception:
+            pass
+
+    return versions
+
+
+def get_version_file():
+    versions = None
+    try:
+        with open(APP_VERSIONS_JSON_FILE) as versions_file:
+            versions = json.load(versions_file)
+    except Exception:
+        pass
+    return versions
+
+
+def get_config():
+    config = None
+    try:
+        with open(COMMON_SITE_CONFIG_FILE) as config_file:
+            config = json.load(config_file)
+    except FileNotFoundError as exception:
+        print(exception)
+        exit(1)
+    except Exception:
+        print(COMMON_SITE_CONFIG_FILE + " is not valid")
+        exit(1)
+    return config
+
+
+def get_site_config(site_name):
+    site_config = None
+    with open('{site_name}/site_config.json'.format(site_name=site_name)) as site_config_file:
+        site_config = json.load(site_config_file)
+    return site_config
+
+
+def save_config(config):
+    with open(COMMON_SITE_CONFIG_FILE, 'w') as f:
+        return json.dump(config, f, indent=1, sort_keys=True)
+
+
+def get_password(env_var, default=None):
+    return os.environ.get(env_var) or get_password_from_secret(f"{env_var}_FILE") or default
+
+
+def get_password_from_secret(env_var):
+    """Fetches the secret value from the docker secret file
+    usually located inside /run/secrets/
+    Arguments:
+        env_var {str} -- Name of the environment variable
+        containing the path to the secret file.
+    Returns:
+        [str] -- Secret value
+    """
+    passwd = None
+    secret_file_path = os.environ.get(env_var)
+    if secret_file_path:
+        with open(secret_file_path) as secret_file:
+            passwd = secret_file.read().strip()
+
+    return passwd
+
+
+def get_s3_config():
+    check_s3_environment_variables()
+    bucket = os.environ.get('BUCKET_NAME')
+
+    conn = boto3.client(
+        's3',
+        region_name=os.environ.get('REGION'),
+        aws_access_key_id=os.environ.get('ACCESS_KEY_ID'),
+        aws_secret_access_key=os.environ.get('SECRET_ACCESS_KEY'),
+        endpoint_url=os.environ.get('ENDPOINT_URL')
+    )
+
+    return conn, bucket
+
+
+def upload_file_to_s3(filename, folder, conn, bucket):
+
+    destpath = os.path.join(folder, os.path.basename(filename))
+    try:
+        print("Uploading file:", filename)
+        conn.upload_file(filename, bucket, destpath)
+
+    except Exception as e:
+        print("Error uploading: %s" % (e))
+        exit(1)
+
+
+def list_directories(path):
+    directories = []
+    for name in os.listdir(path):
+        if os.path.isdir(os.path.join(path, name)):
+            directories.append(name)
+    return directories
+
+
+def get_site_config_from_path(site_config_path):
+    site_config = dict()
+    if os.path.exists(site_config_path):
+        with open(site_config_path, 'r') as sc:
+            site_config = json.load(sc)
+    return site_config
+
+
+def set_key_in_site_config(key, site, site_config_path):
+    site_config = get_site_config_from_path(site_config_path)
+    value = site_config.get(key)
+    if value:
+        print('Set {key} in site config for site: {site}'.format(key=key, site=site))
+        update_site_config(key, value,
+                            site_config_path=os.path.join(os.getcwd(), site, "site_config.json"))
+
+
+def check_s3_environment_variables():
+    if 'BUCKET_NAME' not in os.environ:
+        print('Variable BUCKET_NAME not set')
+        exit(1)
+
+    if 'ACCESS_KEY_ID' not in os.environ:
+        print('Variable ACCESS_KEY_ID not set')
+        exit(1)
+
+    if 'SECRET_ACCESS_KEY' not in os.environ:
+        print('Variable SECRET_ACCESS_KEY not set')
+        exit(1)
+
+    if 'ENDPOINT_URL' not in os.environ:
+        print('Variable ENDPOINT_URL not set')
+        exit(1)
+
+    if 'BUCKET_DIR' not in os.environ:
+        print('Variable BUCKET_DIR not set')
+        exit(1)
+
+    if 'REGION' not in os.environ:
+        print('Variable REGION not set')
+        exit(1)

build/erpnext-nginx/install_app.sh

@@ -26,7 +26,9 @@ yarn install --production=true
 mkdir -p /home/frappe/frappe-bench/sites/assets/${APP_NAME}
 cp -R /home/frappe/frappe-bench/apps/${APP_NAME}/${APP_NAME}/public/* /home/frappe/frappe-bench/sites/assets/${APP_NAME}
 
-echo "rsync -a --delete /var/www/html/assets/${APP_NAME} /assets" > /rsync
+# Add frappe and all the apps available under in frappe-bench here
+echo "rsync -a --delete /var/www/html/assets/frappe /assets" > /rsync
+echo "rsync -a --delete /var/www/html/assets/${APP_NAME} /assets" >> /rsync
 chmod +x /rsync
 
 rm /home/frappe/frappe-bench/sites/apps.txt