2
0
mirror of https://github.com/frappe/frappe_docker.git synced 2024-11-09 23:00:56 +00:00

fix: make semantic changes to commands

* add missing __main__ call to commands.py
* remove unnecessary imports
* fix backup WITH_FILES logic
* follow python semantics (?)

Signed-off-by: Chinmay D. Pai <chinmaydpai@gmail.com>
This commit is contained in:
Chinmay D. Pai 2020-04-29 01:45:59 +05:30
parent 5a5a79f206
commit 884a82d814
No known key found for this signature in database
GPG Key ID: 75507BE256F40CED
11 changed files with 103 additions and 55 deletions

View File

@ -9,10 +9,12 @@ from check_connection import get_config
APP_VERSIONS_JSON_FILE = 'app_versions.json'
APPS_TXT_FILE = 'apps.txt'
def save_version_file(versions):
with open(APP_VERSIONS_JSON_FILE, 'w') as f:
return json.dump(versions, f, indent=1, sort_keys=True)
def get_apps():
apps = []
try:
@ -24,40 +26,43 @@ def get_apps():
except FileNotFoundError as exception:
print(exception)
exit(1)
except:
except Exception:
print(APPS_TXT_FILE+" is not valid")
exit(1)
return apps
def get_container_versions(apps):
versions = {}
for app in apps:
try:
version = __import__(app).__version__
versions.update({app:version})
except:
versions.update({app: version})
except Exception:
pass
try:
path = os.path.join('..','apps', app)
path = os.path.join('..', 'apps', app)
repo = git.Repo(path)
commit_hash = repo.head.object.hexsha
versions.update({app+'_git_hash':commit_hash})
except:
versions.update({app+'_git_hash': commit_hash})
except Exception:
pass
return versions
def get_version_file():
versions = None
try:
with open(APP_VERSIONS_JSON_FILE) as versions_file:
versions = json.load(versions_file)
except:
except Exception:
pass
return versions
def main():
is_ready = False
apps = get_apps()
@ -76,7 +81,7 @@ def main():
version_file_hash = None
container_hash = None
repo = git.Repo(os.path.join('..','apps',app))
repo = git.Repo(os.path.join('..', 'apps', app))
branch = repo.active_branch.name
if branch == 'develop':
@ -105,5 +110,6 @@ def main():
version_file = container_versions
save_version_file(version_file)
if __name__ == "__main__":
main()

View File

@ -1,10 +1,11 @@
import frappe
from frappe.utils.scheduler import start_scheduler
def main():
print("Starting background scheduler . . .")
start_scheduler()
exit(0)
if __name__ == "__main__":
main()

View File

@ -1,7 +1,8 @@
import os, frappe, compileall, re
import os
import frappe
from frappe.utils.backups import scheduled_backup
from frappe.utils import now
from frappe.utils import get_sites
from frappe.utils import cint, get_sites, now
def backup(sites, with_files=False):
for site in sites:
@ -20,13 +21,15 @@ def backup(sites, with_files=False):
print("private files backup taken -", odb.backup_path_private_files, "- on", now())
frappe.destroy()
def main():
installed_sites = ":".join(get_sites())
sites = os.environ.get("SITES", installed_sites).split(":")
with_files=True if os.environ.get("WITH_FILES") else False
with_files = cint(os.environ.get("WITH_FILES"))
backup(sites, with_files)
exit(0)
if __name__ == "__main__":
main()

View File

@ -1,4 +1,6 @@
import socket, os, json, time
import socket
import json
import time
from six.moves.urllib.parse import urlparse
COMMON_SITE_CONFIG_FILE = 'common_site_config.json'
@ -8,6 +10,7 @@ REDIS_SOCKETIO_KEY = 'redis_socketio'
DB_HOST_KEY = 'db_host'
DB_PORT = 3306
def is_open(ip, port, timeout=30):
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.settimeout(timeout)
@ -15,16 +18,17 @@ def is_open(ip, port, timeout=30):
s.connect((ip, int(port)))
s.shutdown(socket.SHUT_RDWR)
return True
except:
except Exception:
return False
finally:
s.close()
def check_host(ip, port, retry=10, delay=3, print_attempt=True):
ipup = False
for i in range(retry):
if print_attempt:
print("Attempt {i} to connect to {ip}:{port}".format(ip=ip,port=port,i=i+1))
print("Attempt {i} to connect to {ip}:{port}".format(ip=ip, port=port, i=i+1))
if is_open(ip, port):
ipup = True
break
@ -32,6 +36,7 @@ def check_host(ip, port, retry=10, delay=3, print_attempt=True):
time.sleep(delay)
return ipup
# Check connection to servers
def get_config():
config = None
@ -41,11 +46,12 @@ def get_config():
except FileNotFoundError as exception:
print(exception)
exit(1)
except:
except Exception:
print(COMMON_SITE_CONFIG_FILE+" is not valid")
exit(1)
return config
# Check mariadb
def check_mariadb(retry=10, delay=3, print_attempt=True):
config = get_config()
@ -60,11 +66,12 @@ def check_mariadb(retry=10, delay=3, print_attempt=True):
print("Connection to MariaDB timed out")
exit(1)
# Check redis queue
def check_redis_queue(retry=10, delay=3, print_attempt=True):
check_redis_queue = False
config = get_config()
redis_queue_url = urlparse(config.get(REDIS_QUEUE_KEY,"redis://redis-queue:6379")).netloc
redis_queue_url = urlparse(config.get(REDIS_QUEUE_KEY, "redis://redis-queue:6379")).netloc
redis_queue, redis_queue_port = redis_queue_url.split(":")
check_redis_queue = check_host(
redis_queue,
@ -76,11 +83,12 @@ def check_redis_queue(retry=10, delay=3, print_attempt=True):
print("Connection to redis queue timed out")
exit(1)
# Check redis cache
def check_redis_cache(retry=10, delay=3, print_attempt=True):
check_redis_cache = False
config = get_config()
redis_cache_url = urlparse(config.get(REDIS_CACHE_KEY,"redis://redis-cache:6379")).netloc
redis_cache_url = urlparse(config.get(REDIS_CACHE_KEY, "redis://redis-cache:6379")).netloc
redis_cache, redis_cache_port = redis_cache_url.split(":")
check_redis_cache = check_host(
redis_cache,
@ -92,11 +100,12 @@ def check_redis_cache(retry=10, delay=3, print_attempt=True):
print("Connection to redis cache timed out")
exit(1)
# Check redis socketio
def check_redis_socketio(retry=10, delay=3, print_attempt=True):
check_redis_socketio = False
config = get_config()
redis_socketio_url = urlparse(config.get(REDIS_SOCKETIO_KEY,"redis://redis-socketio:6379")).netloc
redis_socketio_url = urlparse(config.get(REDIS_SOCKETIO_KEY, "redis://redis-socketio:6379")).netloc
redis_socketio, redis_socketio_port = redis_socketio_url.split(":")
check_redis_socketio = check_host(
redis_socketio,
@ -108,6 +117,7 @@ def check_redis_socketio(retry=10, delay=3, print_attempt=True):
print("Connection to redis socketio timed out")
exit(1)
# Get site_config.json
def get_site_config(site_name):
site_config = None
@ -115,6 +125,7 @@ def get_site_config(site_name):
site_config = json.load(site_config_file)
return site_config
def main():
check_mariadb()
check_redis_queue()
@ -122,5 +133,6 @@ def main():
check_redis_socketio()
print('Connections OK')
if __name__ == "__main__":
main()

View File

@ -20,6 +20,11 @@ def console(site):
print("Apps in this namespace:\n{}".format(", ".join(all_apps)))
IPython.embed(display_banner="", header="")
def main():
site = sys.argv[-1]
console(site)
if __name__ == "__main__":
main()

View File

@ -1,7 +1,3 @@
import frappe
import json
import redis
from rq import Worker
from check_connection import (
check_mariadb,
check_redis_cache,
@ -9,6 +5,7 @@ from check_connection import (
check_redis_socketio,
)
def main():
check_mariadb(retry=1, delay=0, print_attempt=False)
print("MariaDB Connected")
@ -22,5 +19,6 @@ def main():
print("Health check successful")
exit(0)
if __name__ == "__main__":
main()

View File

@ -1,29 +1,34 @@
import os, frappe, compileall, re, json
import os
import frappe
import json
from frappe.migrate import migrate
from frappe.utils import get_sites
from frappe.utils import cint, get_sites
from check_connection import get_config
def save_config(config):
with open('common_site_config.json', 'w') as f:
return json.dump(config, f, indent=1, sort_keys=True)
def set_maintenance_mode(enable=True):
conf = get_config()
if enable:
conf.update({ "maintenance_mode": 1, "pause_scheduler": 1 })
conf.update({"maintenance_mode": 1, "pause_scheduler": 1})
save_config(conf)
if not enable:
conf.update({ "maintenance_mode": 0, "pause_scheduler": 0 })
conf.update({"maintenance_mode": 0, "pause_scheduler": 0})
save_config(conf)
def migrate_sites(maintenance_mode=False):
installed_sites = ":".join(get_sites())
sites = os.environ.get("SITES", installed_sites).split(":")
if not maintenance_mode:
maintenance_mode = True if os.environ.get("MAINTENANCE_MODE") else False
maintenance_mode = cint(os.environ.get("MAINTENANCE_MODE"))
if maintenance_mode:
set_maintenance_mode(True)
@ -40,9 +45,11 @@ def migrate_sites(maintenance_mode=False):
if maintenance_mode:
set_maintenance_mode(False)
def main():
migrate_sites()
exit(0)
if __name__ == "__main__":
main()

View File

@ -1,17 +1,19 @@
import os, frappe, json
import os
import frappe
from frappe.commands.site import _new_site
from check_connection import get_config, get_site_config
def get_password(env_var, default=None):
return os.environ.get(env_var) or _get_password_from_secret(f"{env_var}_FILE") or default
def _get_password_from_secret(env_var):
"""Fetches the secret value from the docker secret file
"""Fetches the secret value from the docker secret file
usually located inside /run/secrets/
Arguments:
env_var {str} -- Name of the environment variable
env_var {str} -- Name of the environment variable
containing the path to the secret file.
Returns:
[str] -- Secret value
@ -71,11 +73,12 @@ def main():
os.system(command)
# Grant permission to database
command = mysql_command + "\"GRANT ALL PRIVILEGES ON \`{db_name}\`.* TO '{db_name}'@'%'; FLUSH PRIVILEGES;\"".format(
command = mysql_command + "\"GRANT ALL PRIVILEGES ON `{db_name}`.* TO '{db_name}'@'%'; FLUSH PRIVILEGES;\"".format(
db_name=site_config.get('db_name')
)
os.system(command)
exit(0)
if __name__ == "__main__":
main()

View File

@ -8,6 +8,7 @@ from frappe.utils import get_sites
DATE_FORMAT = "%Y%m%d_%H%M%S"
def get_file_ext():
return {
"database": "-database.sql.gz",
@ -15,6 +16,7 @@ def get_file_ext():
"public_files": "-files.tar"
}
def get_backup_details(sitename):
backup_details = dict()
file_ext = get_file_ext()
@ -41,6 +43,7 @@ def get_backup_details(sitename):
return backup_details
def get_s3_config():
check_environment_variables()
bucket = os.environ.get('BUCKET_NAME')
@ -55,31 +58,33 @@ def get_s3_config():
return conn, bucket
def check_environment_variables():
if not 'BUCKET_NAME' in os.environ:
if 'BUCKET_NAME' not in os.environ:
print('Variable BUCKET_NAME not set')
exit(1)
if not 'ACCESS_KEY_ID' in os.environ:
if 'ACCESS_KEY_ID' not in os.environ:
print('Variable ACCESS_KEY_ID not set')
exit(1)
if not 'SECRET_ACCESS_KEY' in os.environ:
if 'SECRET_ACCESS_KEY' not in os.environ:
print('Variable SECRET_ACCESS_KEY not set')
exit(1)
if not 'ENDPOINT_URL' in os.environ:
if 'ENDPOINT_URL' not in os.environ:
print('Variable ENDPOINT_URL not set')
exit(1)
if not 'BUCKET_DIR' in os.environ:
if 'BUCKET_DIR' not in os.environ:
print('Variable BUCKET_DIR not set')
exit(1)
if not 'REGION' in os.environ:
if 'REGION' not in os.environ:
print('Variable REGION not set')
exit(1)
def upload_file_to_s3(filename, folder, conn, bucket):
destpath = os.path.join(folder, os.path.basename(filename))
@ -91,6 +96,7 @@ def upload_file_to_s3(filename, folder, conn, bucket):
print("Error uploading: %s" % (e))
exit(1)
def delete_old_backups(limit, bucket, site_name):
all_backups = list()
all_backup_dates = list()
@ -156,6 +162,7 @@ def delete_old_backups(limit, bucket, site_name):
print('Deleteing ' + obj.key)
s3.Object(bucket.name, obj.key).delete()
def main():
details = dict()
sites = get_sites()
@ -184,5 +191,6 @@ def main():
print('push-backup complete')
exit(0)
if __name__ == "__main__":
main()

View File

@ -8,10 +8,10 @@ import boto3
from new import get_password
from push_backup import DATE_FORMAT, check_environment_variables
from frappe.utils import get_sites, random_string
from frappe.commands.site import _new_site
from frappe.installer import make_conf, get_conf_params, make_site_dirs
from check_connection import get_site_config, get_config
def list_directories(path):
directories = []
for name in os.listdir(path):
@ -19,25 +19,25 @@ def list_directories(path):
directories.append(name)
return directories
def get_backup_dir():
return os.path.join(
os.path.expanduser('~'),
'backups'
)
def decompress_db(files_base, site):
database_file = files_base + '-database.sql.gz'
config = get_config()
site_config = get_site_config(site)
db_root_user = os.environ.get('DB_ROOT_USER', 'root')
command = 'gunzip -c {database_file} > {database_extract}'.format(
database_file=database_file,
database_extract=database_file.replace('.gz','')
database_extract=database_file.replace('.gz', '')
)
print('Extract Database GZip for site {}'.format(site))
os.system(command)
def restore_database(files_base, site):
db_root_password = get_password('MYSQL_ROOT_PASSWORD')
if not db_root_password:
@ -60,13 +60,13 @@ def restore_database(files_base, site):
)
# drop db if exists for clean restore
drop_database = mysql_command + "\"DROP DATABASE IF EXISTS \`{db_name}\`;\"".format(
drop_database = mysql_command + "\"DROP DATABASE IF EXISTS `{db_name}`;\"".format(
db_name=site_config.get('db_name')
)
os.system(drop_database)
# create db
create_database = mysql_command + "\"CREATE DATABASE IF NOT EXISTS \`{db_name}\`;\"".format(
create_database = mysql_command + "\"CREATE DATABASE IF NOT EXISTS `{db_name}`;\"".format(
db_name=site_config.get('db_name')
)
os.system(create_database)
@ -86,7 +86,7 @@ def restore_database(files_base, site):
os.system(set_user_password)
# grant db privileges to user
grant_privileges = mysql_command + "\"GRANT ALL PRIVILEGES ON \`{db_name}\`.* TO '{db_name}'@'%'; FLUSH PRIVILEGES;\"".format(
grant_privileges = mysql_command + "\"GRANT ALL PRIVILEGES ON `{db_name}`.* TO '{db_name}'@'%'; FLUSH PRIVILEGES;\"".format(
db_name=site_config.get('db_name')
)
os.system(grant_privileges)
@ -96,12 +96,13 @@ def restore_database(files_base, site):
db_host=config.get('db_host'),
db_password=db_root_password,
db_name=site_config.get('db_name'),
database_file=database_file.replace('.gz',''),
database_file=database_file.replace('.gz', ''),
)
print('Restoring database for site: {}'.format(site))
os.system(command)
def restore_files(files_base):
public_files = files_base + '-files.tar'
# extract tar
@ -109,12 +110,14 @@ def restore_files(files_base):
print('Extracting {}'.format(public_files))
public_tar.extractall()
def restore_private_files(files_base):
private_files = files_base + '-private-files.tar'
private_tar = tarfile.open(private_files)
print('Extracting {}'.format(private_files))
private_tar.extractall()
def pull_backup_from_s3():
check_environment_variables()
@ -134,8 +137,8 @@ def pull_backup_from_s3():
# Change directory to /home/frappe/backups
os.chdir(get_backup_dir())
for obj in bucket.objects.filter(Prefix = bucket_dir):
backup_file = obj.key.replace(os.path.join(bucket_dir,''),'')
for obj in bucket.objects.filter(Prefix=bucket_dir):
backup_file = obj.key.replace(os.path.join(bucket_dir, ''), '')
if not os.path.exists(os.path.dirname(backup_file)):
os.makedirs(os.path.dirname(backup_file))
print('Downloading {}'.format(backup_file))
@ -143,6 +146,7 @@ def pull_backup_from_s3():
os.chdir(os.path.join(os.path.expanduser('~'), 'frappe-bench', 'sites'))
def main():
backup_dir = get_backup_dir()
@ -150,8 +154,8 @@ def main():
pull_backup_from_s3()
for site in list_directories(backup_dir):
site_slug = site.replace('.','_')
backups = [datetime.datetime.strptime(backup, DATE_FORMAT) for backup in list_directories(os.path.join(backup_dir,site))]
site_slug = site.replace('.', '_')
backups = [datetime.datetime.strptime(backup, DATE_FORMAT) for backup in list_directories(os.path.join(backup_dir, site))]
latest_backup = max(backups).strftime(DATE_FORMAT)
files_base = os.path.join(backup_dir, site, latest_backup, '')
files_base += latest_backup + '-' + site_slug
@ -164,8 +168,6 @@ def main():
if not mariadb_root_password:
print('Variable MYSQL_ROOT_PASSWORD not set')
exit(1)
mariadb_root_username = os.environ.get('DB_ROOT_USER', 'root')
database_file = files_base + '-database.sql.gz'
site_config = get_conf_params(
db_name='_' + hashlib.sha1(site.encode()).hexdigest()[:16],
@ -186,5 +188,6 @@ def main():
exit(0)
if __name__ == "__main__":
main()

View File

@ -1,10 +1,12 @@
import os, frappe
import os
from frappe.utils.background_jobs import start_worker
def main():
queue = os.environ.get("WORKER_TYPE", "default")
start_worker(queue, False)
exit(0)
if __name__ == "__main__":
main()