diff --git a/build/common/commands/auto_migrate.py b/build/common/commands/auto_migrate.py new file mode 100644 index 00000000..55124f0b --- /dev/null +++ b/build/common/commands/auto_migrate.py @@ -0,0 +1,110 @@ +import os +import json +import semantic_version +import git + +from migrate import migrate_sites +from check_connection import get_config + +APP_VERSIONS_JSON_FILE = 'app_versions.json' +APPS_TXT_FILE = 'apps.txt' + +def save_version_file(versions): + with open(APP_VERSIONS_JSON_FILE, 'w') as f: + return json.dump(versions, f, indent=1, sort_keys=True) + +def get_apps(): + apps = [] + try: + with open(APPS_TXT_FILE) as apps_file: + for app in apps_file.readlines(): + if app.strip(): + apps.append(app.strip()) + + except FileNotFoundError as exception: + print(exception) + exit(1) + except: + print(APPS_TXT_FILE+" is not valid") + exit(1) + + return apps + +def get_container_versions(apps): + versions = {} + for app in apps: + try: + version = __import__(app).__version__ + versions.update({app:version}) + except: + pass + + try: + path = os.path.join('..','apps', app) + repo = git.Repo(path) + commit_hash = repo.head.object.hexsha + versions.update({app+'_git_hash':commit_hash}) + except: + pass + + return versions + +def get_version_file(): + versions = None + try: + with open(APP_VERSIONS_JSON_FILE) as versions_file: + versions = json.load(versions_file) + except: + pass + return versions + +def main(): + is_ready = False + apps = get_apps() + + container_versions = get_container_versions(apps) + + version_file = get_version_file() + + if not version_file: + version_file = container_versions + save_version_file(version_file) + is_ready = True + + for app in apps: + container_version = None + file_version = None + version_file_hash = None + container_hash = None + + repo = git.Repo(os.path.join('..','apps',app)) + branch = repo.active_branch.name + + if branch == 'develop': + version_file_hash = version_file.get(app+'_git_hash') + container_hash = container_versions.get(app+'_git_hash') + if container_hash and version_file_hash: + if container_hash != version_file_hash: + is_ready = True + break + + if version_file.get(app): + file_version = semantic_version.Version(version_file.get(app)) + + if container_versions.get(app): + container_version = semantic_version.Version(container_versions.get(app)) + + if file_version and container_version: + if container_version > file_version: + is_ready = True + break + + config = get_config() + + if is_ready and config.get('maintenance_mode') != 1: + migrate_sites(maintenance_mode=True) + version_file = container_versions + save_version_file(version_file) + +if __name__ == "__main__": + main() diff --git a/build/common/commands/background.py b/build/common/commands/background.py index 7065efd9..fb008ba9 100644 --- a/build/common/commands/background.py +++ b/build/common/commands/background.py @@ -1,7 +1,10 @@ import frappe from frappe.utils.scheduler import start_scheduler -print("Starting background scheduler . . .") -start_scheduler() +def main(): + print("Starting background scheduler . . .") + start_scheduler() + exit(0) -exit(0) +if __name__ == "__main__": + main() diff --git a/build/common/commands/backup.py b/build/common/commands/backup.py index e6fbcbb6..9fa01cfd 100644 --- a/build/common/commands/backup.py +++ b/build/common/commands/backup.py @@ -20,10 +20,13 @@ def backup(sites, with_files=False): print("private files backup taken -", odb.backup_path_private_files, "- on", now()) frappe.destroy() -installed_sites = ":".join(get_sites()) -sites = os.environ.get("SITES", installed_sites).split(":") -with_files=True if os.environ.get("WITH_FILES") else False +def main(): + installed_sites = ":".join(get_sites()) + sites = os.environ.get("SITES", installed_sites).split(":") + with_files=True if os.environ.get("WITH_FILES") else False -backup(sites, with_files) + backup(sites, with_files) + exit(0) -exit(0) +if __name__ == "__main__": + main() diff --git a/build/common/commands/check_connection.py b/build/common/commands/check_connection.py index 80fb7dbd..b44ea12b 100644 --- a/build/common/commands/check_connection.py +++ b/build/common/commands/check_connection.py @@ -108,6 +108,13 @@ def check_redis_socketio(retry=10, delay=3, print_attempt=True): print("Connection to redis socketio timed out") exit(1) +# Get site_config.json +def get_site_config(site_name): + site_config = None + with open('{site_name}/site_config.json'.format(site_name=site_name)) as site_config_file: + site_config = json.load(site_config_file) + return site_config + def main(): check_mariadb() check_redis_queue() @@ -116,4 +123,4 @@ def main(): print('Connections OK') if __name__ == "__main__": - main() + main() diff --git a/build/common/commands/console.py b/build/common/commands/console.py index 20b3fd34..a9ade863 100644 --- a/build/common/commands/console.py +++ b/build/common/commands/console.py @@ -20,6 +20,6 @@ def console(site): print("Apps in this namespace:\n{}".format(", ".join(all_apps))) IPython.embed(display_banner="", header="") - -site = sys.argv[-1] -console(site) +def main(): + site = sys.argv[-1] + console(site) diff --git a/build/common/commands/doctor.py b/build/common/commands/doctor.py index 36e71411..98f2509f 100644 --- a/build/common/commands/doctor.py +++ b/build/common/commands/doctor.py @@ -23,4 +23,4 @@ def main(): exit(0) if __name__ == "__main__": - main() + main() diff --git a/build/common/commands/migrate.py b/build/common/commands/migrate.py index be382bea..7f8a1602 100644 --- a/build/common/commands/migrate.py +++ b/build/common/commands/migrate.py @@ -2,12 +2,7 @@ import os, frappe, compileall, re, json from frappe.migrate import migrate from frappe.utils import get_sites - -def get_config(): - config = None - with open('common_site_config.json') as config_file: - config = json.load(config_file) - return config +from check_connection import get_config def save_config(config): with open('common_site_config.json', 'w') as f: @@ -24,24 +19,30 @@ def set_maintenance_mode(enable=True): conf.update({ "maintenance_mode": 0, "pause_scheduler": 0 }) save_config(conf) +def migrate_sites(maintenance_mode=False): + installed_sites = ":".join(get_sites()) + sites = os.environ.get("SITES", installed_sites).split(":") + if not maintenance_mode: + maintenance_mode = True if os.environ.get("MAINTENANCE_MODE") else False -installed_sites = ":".join(get_sites()) -sites = os.environ.get("SITES", installed_sites).split(":") -maintenance_mode = True if os.environ.get("MAINTENANCE_MODE") else False + if maintenance_mode: + set_maintenance_mode(True) -if maintenance_mode: - set_maintenance_mode(True) + for site in sites: + print('Migrating', site) + frappe.init(site=site) + frappe.connect() + try: + migrate() + finally: + frappe.destroy() -for site in sites: - print('Migrating', site) - frappe.init(site=site) - frappe.connect() - try: - migrate() - finally: - frappe.destroy() + if maintenance_mode: + set_maintenance_mode(False) -if maintenance_mode: - set_maintenance_mode(False) +def main(): + migrate_sites() + exit(0) -exit(0) +if __name__ == "__main__": + main() diff --git a/build/common/commands/new.py b/build/common/commands/new.py index c951e63e..b603458c 100644 --- a/build/common/commands/new.py +++ b/build/common/commands/new.py @@ -1,68 +1,68 @@ import os, frappe, json from frappe.commands.site import _new_site +from check_connection import get_config, get_site_config -site_name = os.environ.get("SITE_NAME", 'site1.localhost') -mariadb_root_username = os.environ.get("DB_ROOT_USER", 'root') -mariadb_root_password = os.environ.get("MYSQL_ROOT_PASSWORD", 'admin') -force = True if os.environ.get("FORCE", None) else False -install_apps = os.environ.get("INSTALL_APPS", None) -install_apps = install_apps.split(',') if install_apps else [] -frappe.init(site_name, new_site=True) +def main(): + site_name = os.environ.get("SITE_NAME", 'site1.localhost') + mariadb_root_username = os.environ.get("DB_ROOT_USER", 'root') + mariadb_root_password = os.environ.get("MYSQL_ROOT_PASSWORD", 'admin') + force = True if os.environ.get("FORCE", None) else False + install_apps = os.environ.get("INSTALL_APPS", None) + install_apps = install_apps.split(',') if install_apps else [] + frappe.init(site_name, new_site=True) -_new_site( - None, - site_name, - mariadb_root_username=mariadb_root_username, - mariadb_root_password=mariadb_root_password, - admin_password=os.environ.get("ADMIN_PASSWORD", 'admin'), - verbose=True, - install_apps=install_apps, - source_sql=None, - force=force, - reinstall=False, -) + _new_site( + None, + site_name, + mariadb_root_username=mariadb_root_username, + mariadb_root_password=mariadb_root_password, + admin_password=os.environ.get("ADMIN_PASSWORD", 'admin'), + verbose=True, + install_apps=install_apps, + source_sql=None, + force=force, + reinstall=False, + ) -config = None -with open('common_site_config.json') as config_file: - config = json.load(config_file) + config = get_config() -site_config = None -with open('{site_name}/site_config.json'.format(site_name=site_name)) as site_config_file: - site_config = json.load(site_config_file) + site_config = get_site_config(site_name) -# update User's host to '%' required to connect from any container -command = 'mysql -h{db_host} -u{mariadb_root_username} -p{mariadb_root_password} -e '.format( - db_host=config.get('db_host'), - mariadb_root_username=mariadb_root_username, - mariadb_root_password=mariadb_root_password -) -command += "\"UPDATE mysql.user SET Host = '%' where User = '{db_name}'; FLUSH PRIVILEGES;\"".format( - db_name=site_config.get('db_name') -) -os.system(command) + # update User's host to '%' required to connect from any container + command = 'mysql -h{db_host} -u{mariadb_root_username} -p{mariadb_root_password} -e '.format( + db_host=config.get('db_host'), + mariadb_root_username=mariadb_root_username, + mariadb_root_password=mariadb_root_password + ) + command += "\"UPDATE mysql.user SET Host = '%' where User = '{db_name}'; FLUSH PRIVILEGES;\"".format( + db_name=site_config.get('db_name') + ) + os.system(command) -# Set db password -command = 'mysql -h{db_host} -u{mariadb_root_username} -p{mariadb_root_password} -e '.format( - db_host=config.get('db_host'), - mariadb_root_username=mariadb_root_username, - mariadb_root_password=mariadb_root_password -) -command += "\"SET PASSWORD FOR '{db_name}'@'%' = PASSWORD('{db_password}'); FLUSH PRIVILEGES;\"".format( - db_name=site_config.get('db_name'), - db_password=site_config.get('db_password') -) -os.system(command) + # Set db password + command = 'mysql -h{db_host} -u{mariadb_root_username} -p{mariadb_root_password} -e '.format( + db_host=config.get('db_host'), + mariadb_root_username=mariadb_root_username, + mariadb_root_password=mariadb_root_password + ) + command += "\"SET PASSWORD FOR '{db_name}'@'%' = PASSWORD('{db_password}'); FLUSH PRIVILEGES;\"".format( + db_name=site_config.get('db_name'), + db_password=site_config.get('db_password') + ) + os.system(command) -# Grant permission to database -command = 'mysql -h{db_host} -u{mariadb_root_username} -p{mariadb_root_password} -e '.format( - db_host=config.get('db_host'), - mariadb_root_username=mariadb_root_username, - mariadb_root_password=mariadb_root_password -) -command += "\"GRANT ALL PRIVILEGES ON \`{db_name}\`.* TO '{db_name}'@'%'; FLUSH PRIVILEGES;\"".format( - db_name=site_config.get('db_name') -) -os.system(command) + # Grant permission to database + command = 'mysql -h{db_host} -u{mariadb_root_username} -p{mariadb_root_password} -e '.format( + db_host=config.get('db_host'), + mariadb_root_username=mariadb_root_username, + mariadb_root_password=mariadb_root_password + ) + command += "\"GRANT ALL PRIVILEGES ON \`{db_name}\`.* TO '{db_name}'@'%'; FLUSH PRIVILEGES;\"".format( + db_name=site_config.get('db_name') + ) + os.system(command) + exit(0) -exit(0) +if __name__ == "__main__": + main() diff --git a/build/common/commands/worker.py b/build/common/commands/worker.py index 6ec0bcbf..c810adf9 100644 --- a/build/common/commands/worker.py +++ b/build/common/commands/worker.py @@ -1,7 +1,10 @@ import os, frappe from frappe.utils.background_jobs import start_worker -queue = os.environ.get("WORKER_TYPE", "default") -start_worker(queue, False) +def main(): + queue = os.environ.get("WORKER_TYPE", "default") + start_worker(queue, False) + exit(0) -exit(0) +if __name__ == "__main__": + main() diff --git a/build/common/worker/docker-entrypoint.sh b/build/common/worker/docker-entrypoint.sh index bf8a90c2..26cb9ef1 100755 --- a/build/common/worker/docker-entrypoint.sh +++ b/build/common/worker/docker-entrypoint.sh @@ -76,6 +76,10 @@ if [ "$1" = 'start' ]; then export FRAPPE_PORT=8000 fi + if [[ ! -z "$AUTO_MIGRATE" ]]; then + su frappe -c ". /home/frappe/frappe-bench/env/bin/activate \ + && python /home/frappe/frappe-bench/commands/auto_migrate.py" + fi if [[ -z "$RUN_AS_ROOT" ]]; then su frappe -c ". /home/frappe/frappe-bench/env/bin/activate \ diff --git a/installation/docker-compose-erpnext.yml b/installation/docker-compose-erpnext.yml index f932a512..8e2cbd9e 100644 --- a/installation/docker-compose-erpnext.yml +++ b/installation/docker-compose-erpnext.yml @@ -37,6 +37,7 @@ services: - REDIS_QUEUE=redis-queue:6379 - REDIS_SOCKETIO=redis-socketio:6379 - SOCKETIO_PORT=9000 + - AUTO_MIGRATE=1 volumes: - ./sites:/home/frappe/frappe-bench/sites:rw - assets-vol:/home/frappe/frappe-bench/sites/assets:rw diff --git a/installation/docker-compose-frappe.yml b/installation/docker-compose-frappe.yml index df1e7c43..f8365225 100644 --- a/installation/docker-compose-frappe.yml +++ b/installation/docker-compose-frappe.yml @@ -37,6 +37,7 @@ services: - REDIS_QUEUE=redis-queue:6379 - REDIS_SOCKETIO=redis-socketio:6379 - SOCKETIO_PORT=9000 + - AUTO_MIGRATE=1 volumes: - ./sites:/home/frappe/frappe-bench/sites:rw - assets-vol:/home/frappe/frappe-bench/sites/assets:rw