Merge pull request #162 from revant/feat-auto-migrate

feat: auto migrate

build/common/commands/auto_migrate.py

@@ -0,0 +1,110 @@
+import os
+import json
+import semantic_version
+import git
+
+from migrate import migrate_sites
+from check_connection import get_config
+
+APP_VERSIONS_JSON_FILE = 'app_versions.json'
+APPS_TXT_FILE = 'apps.txt'
+
+def save_version_file(versions):
+    with open(APP_VERSIONS_JSON_FILE, 'w') as f:
+        return json.dump(versions, f, indent=1, sort_keys=True)
+
+def get_apps():
+    apps = []
+    try:
+        with open(APPS_TXT_FILE) as apps_file:
+            for app in apps_file.readlines():
+                if app.strip():
+                    apps.append(app.strip())
+
+    except FileNotFoundError as exception:
+        print(exception)
+        exit(1)
+    except:
+        print(APPS_TXT_FILE+" is not valid")
+        exit(1)
+
+    return apps
+
+def get_container_versions(apps):
+    versions = {}
+    for app in apps:
+        try:
+            version = __import__(app).__version__
+            versions.update({app:version})
+        except:
+            pass
+
+        try:
+            path = os.path.join('..','apps', app)
+            repo = git.Repo(path)
+            commit_hash = repo.head.object.hexsha
+            versions.update({app+'_git_hash':commit_hash})
+        except:
+            pass
+
+    return versions
+
+def get_version_file():
+    versions = None
+    try:
+        with open(APP_VERSIONS_JSON_FILE) as versions_file:
+            versions = json.load(versions_file)
+    except:
+        pass
+    return versions
+
+def main():
+    is_ready = False
+    apps = get_apps()
+
+    container_versions = get_container_versions(apps)
+
+    version_file = get_version_file()
+
+    if not version_file:
+        version_file = container_versions
+        save_version_file(version_file)
+        is_ready = True
+
+    for app in apps:
+        container_version = None
+        file_version = None
+        version_file_hash = None
+        container_hash = None
+
+        repo = git.Repo(os.path.join('..','apps',app))
+        branch = repo.active_branch.name
+
+        if branch == 'develop':
+            version_file_hash = version_file.get(app+'_git_hash')
+            container_hash = container_versions.get(app+'_git_hash')
+            if container_hash and version_file_hash:
+                if container_hash != version_file_hash:
+                    is_ready = True
+                    break
+
+        if version_file.get(app):
+            file_version = semantic_version.Version(version_file.get(app))
+
+        if container_versions.get(app):
+            container_version = semantic_version.Version(container_versions.get(app))
+
+        if file_version and container_version:
+            if container_version > file_version:
+                is_ready = True
+                break
+
+    config = get_config()
+
+    if is_ready and config.get('maintenance_mode') != 1:
+        migrate_sites(maintenance_mode=True)
+        version_file = container_versions
+        save_version_file(version_file)
+
+if __name__ == "__main__":
+    main()

build/common/commands/background.py

@@ -1,7 +1,10 @@
 import frappe
 from frappe.utils.scheduler import start_scheduler
 
-print("Starting background scheduler . . .")
-start_scheduler()
+def main():
+    print("Starting background scheduler . . .")
+    start_scheduler()
+    exit(0)
 
-exit(0)
+if __name__ == "__main__":
+    main()

build/common/commands/backup.py

@@ -20,10 +20,13 @@ def backup(sites, with_files=False):
             print("private files backup taken -", odb.backup_path_private_files, "- on", now())
         frappe.destroy()
 
-installed_sites = ":".join(get_sites())
-sites = os.environ.get("SITES", installed_sites).split(":")
-with_files=True if os.environ.get("WITH_FILES") else False
+def main():
+    installed_sites = ":".join(get_sites())
+    sites = os.environ.get("SITES", installed_sites).split(":")
+    with_files=True if os.environ.get("WITH_FILES") else False
 
-backup(sites, with_files)
+    backup(sites, with_files)
+    exit(0)
 
-exit(0)
+if __name__ == "__main__":
+    main()

build/common/commands/check_connection.py

@@ -108,6 +108,13 @@ def check_redis_socketio(retry=10, delay=3, print_attempt=True):
         print("Connection to redis socketio timed out")
         exit(1)
 
+# Get site_config.json
+def get_site_config(site_name):
+    site_config = None
+    with open('{site_name}/site_config.json'.format(site_name=site_name)) as site_config_file:
+        site_config = json.load(site_config_file)
+    return site_config
+
 def main():
     check_mariadb()
     check_redis_queue()
@@ -116,4 +123,4 @@ def main():
     print('Connections OK')
 
 if __name__ == "__main__":
-   main()
+    main()

build/common/commands/console.py

@@ -20,6 +20,6 @@ def console(site):
     print("Apps in this namespace:\n{}".format(", ".join(all_apps)))
     IPython.embed(display_banner="", header="")
 
-
-site = sys.argv[-1]
-console(site)
+def main():
+    site = sys.argv[-1]
+    console(site)

build/common/commands/doctor.py

@@ -23,4 +23,4 @@ def main():
     exit(0)
 
 if __name__ == "__main__":
-   main()
+    main()

build/common/commands/migrate.py

@@ -2,12 +2,7 @@ import os, frappe, compileall, re, json
 
 from frappe.migrate import migrate
 from frappe.utils import get_sites
-
-def get_config():
-    config = None
-    with open('common_site_config.json') as config_file:
-        config = json.load(config_file)
-    return config
+from check_connection import get_config
 
 def save_config(config):
     with open('common_site_config.json', 'w') as f:
@@ -24,24 +19,30 @@ def set_maintenance_mode(enable=True):
         conf.update({ "maintenance_mode": 0, "pause_scheduler": 0 })
         save_config(conf)
 
+def migrate_sites(maintenance_mode=False):
+    installed_sites = ":".join(get_sites())
+    sites = os.environ.get("SITES", installed_sites).split(":")
+    if not maintenance_mode:
+        maintenance_mode = True if os.environ.get("MAINTENANCE_MODE") else False
 
-installed_sites = ":".join(get_sites())
-sites = os.environ.get("SITES", installed_sites).split(":")
-maintenance_mode = True if os.environ.get("MAINTENANCE_MODE") else False
+    if maintenance_mode:
+        set_maintenance_mode(True)
 
-if maintenance_mode:
-    set_maintenance_mode(True)
+    for site in sites:
+        print('Migrating', site)
+        frappe.init(site=site)
+        frappe.connect()
+        try:
+            migrate()
+        finally:
+            frappe.destroy()
 
-for site in sites:
-    print('Migrating', site)
-    frappe.init(site=site)
-    frappe.connect()
-    try:
-        migrate()
-    finally:
-        frappe.destroy()
+    if maintenance_mode:
+        set_maintenance_mode(False)
 
-if maintenance_mode:
-    set_maintenance_mode(False)
+def main():
+    migrate_sites()
+    exit(0)
 
-exit(0)
+if __name__ == "__main__":
+    main()

build/common/commands/new.py

@@ -1,68 +1,68 @@
 import os, frappe, json
 
 from frappe.commands.site import _new_site
+from check_connection import get_config, get_site_config
 
-site_name = os.environ.get("SITE_NAME", 'site1.localhost')
-mariadb_root_username = os.environ.get("DB_ROOT_USER", 'root')
-mariadb_root_password = os.environ.get("MYSQL_ROOT_PASSWORD", 'admin')
-force = True if os.environ.get("FORCE", None) else False
-install_apps = os.environ.get("INSTALL_APPS", None)
-install_apps = install_apps.split(',') if install_apps else []
-frappe.init(site_name, new_site=True)
+def main():
+    site_name = os.environ.get("SITE_NAME", 'site1.localhost')
+    mariadb_root_username = os.environ.get("DB_ROOT_USER", 'root')
+    mariadb_root_password = os.environ.get("MYSQL_ROOT_PASSWORD", 'admin')
+    force = True if os.environ.get("FORCE", None) else False
+    install_apps = os.environ.get("INSTALL_APPS", None)
+    install_apps = install_apps.split(',') if install_apps else []
+    frappe.init(site_name, new_site=True)
 
-_new_site(
-    None,
-    site_name,
-    mariadb_root_username=mariadb_root_username,
-    mariadb_root_password=mariadb_root_password,
-    admin_password=os.environ.get("ADMIN_PASSWORD", 'admin'),
-    verbose=True,
-    install_apps=install_apps,
-    source_sql=None,
-    force=force,
-    reinstall=False,
-)
+    _new_site(
+        None,
+        site_name,
+        mariadb_root_username=mariadb_root_username,
+        mariadb_root_password=mariadb_root_password,
+        admin_password=os.environ.get("ADMIN_PASSWORD", 'admin'),
+        verbose=True,
+        install_apps=install_apps,
+        source_sql=None,
+        force=force,
+        reinstall=False,
+    )
 
-config = None
-with open('common_site_config.json') as config_file:
-    config = json.load(config_file)
+    config = get_config()
 
-site_config = None
-with open('{site_name}/site_config.json'.format(site_name=site_name)) as site_config_file:
-    site_config = json.load(site_config_file)
+    site_config = get_site_config(site_name)
 
-# update User's host to '%' required to connect from any container
-command = 'mysql -h{db_host} -u{mariadb_root_username} -p{mariadb_root_password} -e '.format(
-    db_host=config.get('db_host'),
-    mariadb_root_username=mariadb_root_username,
-    mariadb_root_password=mariadb_root_password
-)
-command += "\"UPDATE mysql.user SET Host = '%' where User = '{db_name}'; FLUSH PRIVILEGES;\"".format(
-    db_name=site_config.get('db_name')
-)
-os.system(command)
+    # update User's host to '%' required to connect from any container
+    command = 'mysql -h{db_host} -u{mariadb_root_username} -p{mariadb_root_password} -e '.format(
+        db_host=config.get('db_host'),
+        mariadb_root_username=mariadb_root_username,
+        mariadb_root_password=mariadb_root_password
+    )
+    command += "\"UPDATE mysql.user SET Host = '%' where User = '{db_name}'; FLUSH PRIVILEGES;\"".format(
+        db_name=site_config.get('db_name')
+    )
+    os.system(command)
 
-# Set db password
-command = 'mysql -h{db_host} -u{mariadb_root_username} -p{mariadb_root_password} -e '.format(
-    db_host=config.get('db_host'),
-    mariadb_root_username=mariadb_root_username,
-    mariadb_root_password=mariadb_root_password
-)
-command += "\"SET PASSWORD FOR '{db_name}'@'%' = PASSWORD('{db_password}'); FLUSH PRIVILEGES;\"".format(
-    db_name=site_config.get('db_name'),
-    db_password=site_config.get('db_password')
-)
-os.system(command)
+    # Set db password
+    command = 'mysql -h{db_host} -u{mariadb_root_username} -p{mariadb_root_password} -e '.format(
+        db_host=config.get('db_host'),
+        mariadb_root_username=mariadb_root_username,
+        mariadb_root_password=mariadb_root_password
+    )
+    command += "\"SET PASSWORD FOR '{db_name}'@'%' = PASSWORD('{db_password}'); FLUSH PRIVILEGES;\"".format(
+        db_name=site_config.get('db_name'),
+        db_password=site_config.get('db_password')
+    )
+    os.system(command)
 
-# Grant permission to database
-command = 'mysql -h{db_host} -u{mariadb_root_username} -p{mariadb_root_password} -e '.format(
-    db_host=config.get('db_host'),
-    mariadb_root_username=mariadb_root_username,
-    mariadb_root_password=mariadb_root_password
-)
-command += "\"GRANT ALL PRIVILEGES ON \`{db_name}\`.* TO '{db_name}'@'%'; FLUSH PRIVILEGES;\"".format(
-    db_name=site_config.get('db_name')
-)
-os.system(command)
+    # Grant permission to database
+    command = 'mysql -h{db_host} -u{mariadb_root_username} -p{mariadb_root_password} -e '.format(
+        db_host=config.get('db_host'),
+        mariadb_root_username=mariadb_root_username,
+        mariadb_root_password=mariadb_root_password
+    )
+    command += "\"GRANT ALL PRIVILEGES ON \`{db_name}\`.* TO '{db_name}'@'%'; FLUSH PRIVILEGES;\"".format(
+        db_name=site_config.get('db_name')
+    )
+    os.system(command)
+    exit(0)
 
-exit(0)
+if __name__ == "__main__":
+    main()

build/common/commands/worker.py

@@ -1,7 +1,10 @@
 import os, frappe
 from frappe.utils.background_jobs import start_worker
 
-queue = os.environ.get("WORKER_TYPE", "default")
-start_worker(queue, False)
+def main():
+    queue = os.environ.get("WORKER_TYPE", "default")
+    start_worker(queue, False)
+    exit(0)
 
-exit(0)
+if __name__ == "__main__":
+    main()

build/common/worker/docker-entrypoint.sh

@@ -76,6 +76,10 @@ if [ "$1" = 'start' ]; then
     export FRAPPE_PORT=8000
   fi
 
+  if [[ ! -z "$AUTO_MIGRATE" ]]; then
+    su frappe -c ". /home/frappe/frappe-bench/env/bin/activate \
+      && python /home/frappe/frappe-bench/commands/auto_migrate.py"
+  fi
 
   if [[ -z "$RUN_AS_ROOT" ]]; then
     su frappe -c ". /home/frappe/frappe-bench/env/bin/activate \

installation/docker-compose-erpnext.yml

@@ -37,6 +37,7 @@ services:
       - REDIS_QUEUE=redis-queue:6379
       - REDIS_SOCKETIO=redis-socketio:6379
       - SOCKETIO_PORT=9000
+      - AUTO_MIGRATE=1
     volumes:
       - ./sites:/home/frappe/frappe-bench/sites:rw
       - assets-vol:/home/frappe/frappe-bench/sites/assets:rw

installation/docker-compose-frappe.yml

@@ -37,6 +37,7 @@ services:
       - REDIS_QUEUE=redis-queue:6379
       - REDIS_SOCKETIO=redis-socketio:6379
       - SOCKETIO_PORT=9000
+      - AUTO_MIGRATE=1
     volumes:
       - ./sites:/home/frappe/frappe-bench/sites:rw
       - assets-vol:/home/frappe/frappe-bench/sites/assets:rw