feat: resotre backups from volume or cloud

build/common/commands/restore_backup.py

@@ -0,0 +1,175 @@
+import os
+import datetime
+import tarfile
+import hashlib
+import frappe
+import boto3
+
+from push_backup import DATE_FORMAT, check_environment_variables
+from frappe.utils import get_sites, random_string
+from frappe.commands.site import _new_site
+from frappe.installer import make_conf, get_conf_params
+from check_connection import get_site_config, get_config
+
+def list_directories(path):
+    directories = []
+    for name in os.listdir(path):
+        if os.path.isdir(os.path.join(path, name)):
+            directories.append(name)
+    return directories
+
+def get_backup_dir():
+    return os.path.join(
+        os.path.expanduser('~'),
+        'backups'
+    )
+
+def decompress_db(files_base, site):
+    database_file = files_base + '-database.sql.gz'
+    config = get_config()
+    site_config = get_site_config(site)
+    db_root_user = os.environ.get('DB_ROOT_USER', 'root')
+    command = 'gunzip -c {database_file} > {database_extract}'.format(
+        database_file=database_file,
+        database_extract=database_file.replace('.gz','')
+    )
+
+    print('Extract Database GZip for site {}'.format(site))
+    os.system(command)
+
+def restore_database(files_base, site):
+    db_root_password = os.environ.get('MYSQL_ROOT_PASSWORD')
+    if not db_root_password:
+        print('Variable MYSQL_ROOT_PASSWORD not set')
+        exit(1)
+
+    db_root_user = os.environ.get("DB_ROOT_USER", 'root')
+    # restore database
+
+    database_file = files_base + '-database.sql.gz'
+    decompress_db(files_base, site)
+    config = get_config()
+    site_config = get_site_config(site)
+
+    # mysql command prefix
+    mysql_command = 'mysql -u{db_root_user} -h{db_host} -p{db_password} -e '.format(
+        db_root_user=db_root_user,
+        db_host=config.get('db_host'),
+        db_password=db_root_password
+    )
+
+    # create db
+    create_database = mysql_command + "\"CREATE DATABASE IF NOT EXISTS \`{db_name}\`;\"".format(
+        db_name=site_config.get('db_name')
+    )
+    os.system(create_database)
+
+    # create user
+    create_user = mysql_command + "\"CREATE USER IF NOT EXISTS \'{db_name}\'@\'%\' IDENTIFIED BY \'{db_password}\'; FLUSH PRIVILEGES;\"".format(
+        db_name=site_config.get('db_name'),
+        db_password=site_config.get('db_password')
+    )
+    os.system(create_user)
+
+    # grant db privileges to user
+    grant_privileges = mysql_command + "\"GRANT ALL PRIVILEGES ON \`{db_name}\`.* TO '{db_name}'@'%'; FLUSH PRIVILEGES;\"".format(
+        db_name=site_config.get('db_name')
+    )
+    os.system(grant_privileges)
+
+    command = "mysql -u{db_root_user} -h{db_host} -p{db_password} '{db_name}' < {database_file}".format(
+        db_root_user=db_root_user,
+        db_host=config.get('db_host'),
+        db_password=db_root_password,
+        db_name=site_config.get('db_name'),
+        database_file=database_file.replace('.gz',''),
+    )
+
+    print('Restoring database for site: {}'.format(site))
+    os.system(command)
+
+def restore_files(files_base):
+    public_files = files_base + '-files.tar'
+    # extract tar
+    public_tar = tarfile.open(public_files)
+    print('Extracting {}'.format(public_files))
+    public_tar.extractall()
+
+def restore_private_files(files_base):
+    private_files = files_base + '-private-files.tar'
+    private_tar = tarfile.open(private_files)
+    print('Extracting {}'.format(private_files))
+    private_tar.extractall()
+
+def pull_backup_from_s3():
+    check_environment_variables()
+
+    # https://stackoverflow.com/a/54672690
+    s3 = boto3.resource(
+        's3',
+        aws_access_key_id=os.environ.get('ACCESS_KEY_ID'),
+        aws_secret_access_key=os.environ.get('SECRET_ACCESS_KEY'),
+        endpoint_url=os.environ.get('ENDPOINT_URL')
+    )
+
+    bucket_dir = os.environ.get('BUCKET_DIR')
+    bucket_name = os.environ.get('BUCKET_NAME')
+    bucket = s3.Bucket(bucket_name)
+
+    # Change directory to /home/frappe/backups
+    os.chdir(get_backup_dir())
+
+    for obj in bucket.objects.filter(Prefix = bucket_dir):
+        backup_file = obj.key.replace(os.path.join(bucket_dir,''),'')
+        if not os.path.exists(os.path.dirname(backup_file)):
+            os.makedirs(os.path.dirname(backup_file))
+        print('Downloading {}'.format(backup_file))
+        bucket.download_file(obj.key, backup_file)
+
+    os.chdir(os.path.join(os.path.expanduser('~'), 'frappe-bench', 'sites'))
+
+def main():
+    backup_dir = get_backup_dir()
+
+    if len(list_directories(backup_dir)) == 0:
+        pull_backup_from_s3()
+
+    for site in list_directories(backup_dir):
+        site_slug = site.replace('.','_')
+        backups = [datetime.datetime.strptime(backup, DATE_FORMAT) for backup in list_directories(os.path.join(backup_dir,site))]
+        latest_backup = max(backups).strftime(DATE_FORMAT)
+        files_base = os.path.join(backup_dir, site, latest_backup, '')
+        files_base += latest_backup + '-' + site_slug
+        if site in get_sites():
+            restore_database(files_base, site)
+            restore_private_files(files_base)
+            restore_files(files_base)
+        else:
+            mariadb_root_password = os.environ.get('MYSQL_ROOT_PASSWORD')
+            if not mariadb_root_password:
+                print('Variable MYSQL_ROOT_PASSWORD not set')
+                exit(1)
+            mariadb_root_username = os.environ.get('DB_ROOT_USER', 'root')
+            database_file = files_base + '-database.sql.gz'
+
+            site_config = get_conf_params(
+                db_name='_' + hashlib.sha1(site.encode()).hexdigest()[:16],
+                db_password=random_string(16)
+            )
+
+            frappe.local.site = site
+            frappe.local.sites_path = os.getcwd()
+            frappe.local.site_path = os.getcwd() + '/' + site
+            make_conf(
+                db_name=site_config.get('db_name'),
+                db_password=site_config.get('db_name'),
+            )
+
+            restore_database(files_base, site)
+            restore_private_files(files_base)
+            restore_files(files_base)
+
+    exit(0)
+
+if __name__ == "__main__":
+    main()

build/common/worker/docker-entrypoint.sh

@@ -181,6 +181,12 @@ elif [ "$1" = 'push-backup' ]; then
     && python /home/frappe/frappe-bench/commands/push_backup.py"
   exit
 
+elif [ "$1" = 'restore-backup' ]; then
+
+  su frappe -c ". /home/frappe/frappe-bench/env/bin/activate \
+    && python /home/frappe/frappe-bench/commands/restore_backup.py"
+  exit
+
 else
 
   exec su frappe -c "$@"