mirror of
https://github.com/frappe/frappe_docker.git
synced 2024-11-08 14:21:05 +00:00
Merge pull request #218 from Thunderbottom/commands-semantic-fix
fix: make semantic changes to commands
This commit is contained in:
Revant Nandgaonkar
GitHub
commit
361c0f6047
@ -9,10 +9,12 @@ from check_connection import get_config
|
||||
APP_VERSIONS_JSON_FILE = 'app_versions.json'
|
||||
APPS_TXT_FILE = 'apps.txt'
|
||||
|
||||
|
||||
def save_version_file(versions):
|
||||
with open(APP_VERSIONS_JSON_FILE, 'w') as f:
|
||||
return json.dump(versions, f, indent=1, sort_keys=True)
|
||||
|
||||
|
||||
def get_apps():
|
||||
apps = []
|
||||
try:
|
||||
@ -24,40 +26,43 @@ def get_apps():
|
||||
except FileNotFoundError as exception:
|
||||
print(exception)
|
||||
exit(1)
|
||||
except:
|
||||
except Exception:
|
||||
print(APPS_TXT_FILE+" is not valid")
|
||||
exit(1)
|
||||
|
||||
return apps
|
||||
|
||||
|
||||
def get_container_versions(apps):
|
||||
versions = {}
|
||||
for app in apps:
|
||||
try:
|
||||
version = __import__(app).__version__
|
||||
versions.update({app:version})
|
||||
except:
|
||||
versions.update({app: version})
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
try:
|
||||
path = os.path.join('..','apps', app)
|
||||
path = os.path.join('..', 'apps', app)
|
||||
repo = git.Repo(path)
|
||||
commit_hash = repo.head.object.hexsha
|
||||
versions.update({app+'_git_hash':commit_hash})
|
||||
except:
|
||||
versions.update({app+'_git_hash': commit_hash})
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return versions
|
||||
|
||||
|
||||
def get_version_file():
|
||||
versions = None
|
||||
try:
|
||||
with open(APP_VERSIONS_JSON_FILE) as versions_file:
|
||||
versions = json.load(versions_file)
|
||||
except:
|
||||
except Exception:
|
||||
pass
|
||||
return versions
|
||||
|
||||
|
||||
def main():
|
||||
is_ready = False
|
||||
apps = get_apps()
|
||||
@ -76,7 +81,7 @@ def main():
|
||||
version_file_hash = None
|
||||
container_hash = None
|
||||
|
||||
repo = git.Repo(os.path.join('..','apps',app))
|
||||
repo = git.Repo(os.path.join('..', 'apps', app))
|
||||
branch = repo.active_branch.name
|
||||
|
||||
if branch == 'develop':
|
||||
@ -105,5 +110,6 @@ def main():
|
||||
version_file = container_versions
|
||||
save_version_file(version_file)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
||||
@ -1,10 +1,11 @@
|
||||
import frappe
|
||||
from frappe.utils.scheduler import start_scheduler
|
||||
|
||||
|
||||
def main():
|
||||
print("Starting background scheduler . . .")
|
||||
start_scheduler()
|
||||
exit(0)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
||||
@ -1,7 +1,8 @@
|
||||
import os, frappe, compileall, re
|
||||
import os
|
||||
import frappe
|
||||
from frappe.utils.backups import scheduled_backup
|
||||
from frappe.utils import now
|
||||
from frappe.utils import get_sites
|
||||
from frappe.utils import cint, get_sites, now
|
||||
|
||||
|
||||
def backup(sites, with_files=False):
|
||||
for site in sites:
|
||||
@ -20,13 +21,15 @@ def backup(sites, with_files=False):
|
||||
print("private files backup taken -", odb.backup_path_private_files, "- on", now())
|
||||
frappe.destroy()
|
||||
|
||||
|
||||
def main():
|
||||
installed_sites = ":".join(get_sites())
|
||||
sites = os.environ.get("SITES", installed_sites).split(":")
|
||||
with_files=True if os.environ.get("WITH_FILES") else False
|
||||
with_files = cint(os.environ.get("WITH_FILES"))
|
||||
|
||||
backup(sites, with_files)
|
||||
exit(0)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
||||
@ -1,4 +1,6 @@
|
||||
import socket, os, json, time
|
||||
import socket
|
||||
import json
|
||||
import time
|
||||
from six.moves.urllib.parse import urlparse
|
||||
|
||||
COMMON_SITE_CONFIG_FILE = 'common_site_config.json'
|
||||
@ -8,6 +10,7 @@ REDIS_SOCKETIO_KEY = 'redis_socketio'
|
||||
DB_HOST_KEY = 'db_host'
|
||||
DB_PORT = 3306
|
||||
|
||||
|
||||
def is_open(ip, port, timeout=30):
|
||||
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
||||
s.settimeout(timeout)
|
||||
@ -15,16 +18,17 @@ def is_open(ip, port, timeout=30):
|
||||
s.connect((ip, int(port)))
|
||||
s.shutdown(socket.SHUT_RDWR)
|
||||
return True
|
||||
except:
|
||||
except Exception:
|
||||
return False
|
||||
finally:
|
||||
s.close()
|
||||
|
||||
|
||||
def check_host(ip, port, retry=10, delay=3, print_attempt=True):
|
||||
ipup = False
|
||||
for i in range(retry):
|
||||
if print_attempt:
|
||||
print("Attempt {i} to connect to {ip}:{port}".format(ip=ip,port=port,i=i+1))
|
||||
print("Attempt {i} to connect to {ip}:{port}".format(ip=ip, port=port, i=i+1))
|
||||
if is_open(ip, port):
|
||||
ipup = True
|
||||
break
|
||||
@ -32,6 +36,7 @@ def check_host(ip, port, retry=10, delay=3, print_attempt=True):
|
||||
time.sleep(delay)
|
||||
return ipup
|
||||
|
||||
|
||||
# Check connection to servers
|
||||
def get_config():
|
||||
config = None
|
||||
@ -41,11 +46,12 @@ def get_config():
|
||||
except FileNotFoundError as exception:
|
||||
print(exception)
|
||||
exit(1)
|
||||
except:
|
||||
except Exception:
|
||||
print(COMMON_SITE_CONFIG_FILE+" is not valid")
|
||||
exit(1)
|
||||
return config
|
||||
|
||||
|
||||
# Check mariadb
|
||||
def check_mariadb(retry=10, delay=3, print_attempt=True):
|
||||
config = get_config()
|
||||
@ -60,11 +66,12 @@ def check_mariadb(retry=10, delay=3, print_attempt=True):
|
||||
print("Connection to MariaDB timed out")
|
||||
exit(1)
|
||||
|
||||
|
||||
# Check redis queue
|
||||
def check_redis_queue(retry=10, delay=3, print_attempt=True):
|
||||
check_redis_queue = False
|
||||
config = get_config()
|
||||
redis_queue_url = urlparse(config.get(REDIS_QUEUE_KEY,"redis://redis-queue:6379")).netloc
|
||||
redis_queue_url = urlparse(config.get(REDIS_QUEUE_KEY, "redis://redis-queue:6379")).netloc
|
||||
redis_queue, redis_queue_port = redis_queue_url.split(":")
|
||||
check_redis_queue = check_host(
|
||||
redis_queue,
|
||||
@ -76,11 +83,12 @@ def check_redis_queue(retry=10, delay=3, print_attempt=True):
|
||||
print("Connection to redis queue timed out")
|
||||
exit(1)
|
||||
|
||||
|
||||
# Check redis cache
|
||||
def check_redis_cache(retry=10, delay=3, print_attempt=True):
|
||||
check_redis_cache = False
|
||||
config = get_config()
|
||||
redis_cache_url = urlparse(config.get(REDIS_CACHE_KEY,"redis://redis-cache:6379")).netloc
|
||||
redis_cache_url = urlparse(config.get(REDIS_CACHE_KEY, "redis://redis-cache:6379")).netloc
|
||||
redis_cache, redis_cache_port = redis_cache_url.split(":")
|
||||
check_redis_cache = check_host(
|
||||
redis_cache,
|
||||
@ -92,11 +100,12 @@ def check_redis_cache(retry=10, delay=3, print_attempt=True):
|
||||
print("Connection to redis cache timed out")
|
||||
exit(1)
|
||||
|
||||
|
||||
# Check redis socketio
|
||||
def check_redis_socketio(retry=10, delay=3, print_attempt=True):
|
||||
check_redis_socketio = False
|
||||
config = get_config()
|
||||
redis_socketio_url = urlparse(config.get(REDIS_SOCKETIO_KEY,"redis://redis-socketio:6379")).netloc
|
||||
redis_socketio_url = urlparse(config.get(REDIS_SOCKETIO_KEY, "redis://redis-socketio:6379")).netloc
|
||||
redis_socketio, redis_socketio_port = redis_socketio_url.split(":")
|
||||
check_redis_socketio = check_host(
|
||||
redis_socketio,
|
||||
@ -108,6 +117,7 @@ def check_redis_socketio(retry=10, delay=3, print_attempt=True):
|
||||
print("Connection to redis socketio timed out")
|
||||
exit(1)
|
||||
|
||||
|
||||
# Get site_config.json
|
||||
def get_site_config(site_name):
|
||||
site_config = None
|
||||
@ -115,6 +125,7 @@ def get_site_config(site_name):
|
||||
site_config = json.load(site_config_file)
|
||||
return site_config
|
||||
|
||||
|
||||
def main():
|
||||
check_mariadb()
|
||||
check_redis_queue()
|
||||
@ -122,5 +133,6 @@ def main():
|
||||
check_redis_socketio()
|
||||
print('Connections OK')
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
||||
@ -20,6 +20,11 @@ def console(site):
|
||||
print("Apps in this namespace:\n{}".format(", ".join(all_apps)))
|
||||
IPython.embed(display_banner="", header="")
|
||||
|
||||
|
||||
def main():
|
||||
site = sys.argv[-1]
|
||||
console(site)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
||||
@ -1,7 +1,3 @@
|
||||
import frappe
|
||||
import json
|
||||
import redis
|
||||
from rq import Worker
|
||||
from check_connection import (
|
||||
check_mariadb,
|
||||
check_redis_cache,
|
||||
@ -9,6 +5,7 @@ from check_connection import (
|
||||
check_redis_socketio,
|
||||
)
|
||||
|
||||
|
||||
def main():
|
||||
check_mariadb(retry=1, delay=0, print_attempt=False)
|
||||
print("MariaDB Connected")
|
||||
@ -22,5 +19,6 @@ def main():
|
||||
print("Health check successful")
|
||||
exit(0)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
||||
@ -1,29 +1,34 @@
|
||||
import os, frappe, compileall, re, json
|
||||
import os
|
||||
import frappe
|
||||
import json
|
||||
|
||||
from frappe.migrate import migrate
|
||||
from frappe.utils import get_sites
|
||||
from frappe.utils import cint, get_sites
|
||||
from check_connection import get_config
|
||||
|
||||
|
||||
def save_config(config):
|
||||
with open('common_site_config.json', 'w') as f:
|
||||
return json.dump(config, f, indent=1, sort_keys=True)
|
||||
|
||||
|
||||
def set_maintenance_mode(enable=True):
|
||||
conf = get_config()
|
||||
|
||||
if enable:
|
||||
conf.update({ "maintenance_mode": 1, "pause_scheduler": 1 })
|
||||
conf.update({"maintenance_mode": 1, "pause_scheduler": 1})
|
||||
save_config(conf)
|
||||
|
||||
if not enable:
|
||||
conf.update({ "maintenance_mode": 0, "pause_scheduler": 0 })
|
||||
conf.update({"maintenance_mode": 0, "pause_scheduler": 0})
|
||||
save_config(conf)
|
||||
|
||||
|
||||
def migrate_sites(maintenance_mode=False):
|
||||
installed_sites = ":".join(get_sites())
|
||||
sites = os.environ.get("SITES", installed_sites).split(":")
|
||||
if not maintenance_mode:
|
||||
maintenance_mode = True if os.environ.get("MAINTENANCE_MODE") else False
|
||||
maintenance_mode = cint(os.environ.get("MAINTENANCE_MODE"))
|
||||
|
||||
if maintenance_mode:
|
||||
set_maintenance_mode(True)
|
||||
@ -40,9 +45,11 @@ def migrate_sites(maintenance_mode=False):
|
||||
if maintenance_mode:
|
||||
set_maintenance_mode(False)
|
||||
|
||||
|
||||
def main():
|
||||
migrate_sites()
|
||||
exit(0)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
||||
@ -1,17 +1,19 @@
|
||||
import os, frappe, json
|
||||
import os
|
||||
import frappe
|
||||
|
||||
from frappe.commands.site import _new_site
|
||||
from check_connection import get_config, get_site_config
|
||||
|
||||
|
||||
def get_password(env_var, default=None):
|
||||
return os.environ.get(env_var) or _get_password_from_secret(f"{env_var}_FILE") or default
|
||||
|
||||
|
||||
def _get_password_from_secret(env_var):
|
||||
"""Fetches the secret value from the docker secret file
|
||||
"""Fetches the secret value from the docker secret file
|
||||
usually located inside /run/secrets/
|
||||
Arguments:
|
||||
env_var {str} -- Name of the environment variable
|
||||
env_var {str} -- Name of the environment variable
|
||||
containing the path to the secret file.
|
||||
Returns:
|
||||
[str] -- Secret value
|
||||
@ -77,5 +79,6 @@ def main():
|
||||
os.system(command)
|
||||
exit(0)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
||||
@ -8,6 +8,7 @@ from frappe.utils import get_sites
|
||||
|
||||
DATE_FORMAT = "%Y%m%d_%H%M%S"
|
||||
|
||||
|
||||
def get_file_ext():
|
||||
return {
|
||||
"database": "-database.sql.gz",
|
||||
@ -15,6 +16,7 @@ def get_file_ext():
|
||||
"public_files": "-files.tar"
|
||||
}
|
||||
|
||||
|
||||
def get_backup_details(sitename):
|
||||
backup_details = dict()
|
||||
file_ext = get_file_ext()
|
||||
@ -41,6 +43,7 @@ def get_backup_details(sitename):
|
||||
|
||||
return backup_details
|
||||
|
||||
|
||||
def get_s3_config():
|
||||
check_environment_variables()
|
||||
bucket = os.environ.get('BUCKET_NAME')
|
||||
@ -55,31 +58,33 @@ def get_s3_config():
|
||||
|
||||
return conn, bucket
|
||||
|
||||
|
||||
def check_environment_variables():
|
||||
if not 'BUCKET_NAME' in os.environ:
|
||||
if 'BUCKET_NAME' not in os.environ:
|
||||
print('Variable BUCKET_NAME not set')
|
||||
exit(1)
|
||||
|
||||
if not 'ACCESS_KEY_ID' in os.environ:
|
||||
if 'ACCESS_KEY_ID' not in os.environ:
|
||||
print('Variable ACCESS_KEY_ID not set')
|
||||
exit(1)
|
||||
|
||||
if not 'SECRET_ACCESS_KEY' in os.environ:
|
||||
if 'SECRET_ACCESS_KEY' not in os.environ:
|
||||
print('Variable SECRET_ACCESS_KEY not set')
|
||||
exit(1)
|
||||
|
||||
if not 'ENDPOINT_URL' in os.environ:
|
||||
if 'ENDPOINT_URL' not in os.environ:
|
||||
print('Variable ENDPOINT_URL not set')
|
||||
exit(1)
|
||||
|
||||
if not 'BUCKET_DIR' in os.environ:
|
||||
if 'BUCKET_DIR' not in os.environ:
|
||||
print('Variable BUCKET_DIR not set')
|
||||
exit(1)
|
||||
|
||||
if not 'REGION' in os.environ:
|
||||
if 'REGION' not in os.environ:
|
||||
print('Variable REGION not set')
|
||||
exit(1)
|
||||
|
||||
|
||||
def upload_file_to_s3(filename, folder, conn, bucket):
|
||||
|
||||
destpath = os.path.join(folder, os.path.basename(filename))
|
||||
@ -91,6 +96,7 @@ def upload_file_to_s3(filename, folder, conn, bucket):
|
||||
print("Error uploading: %s" % (e))
|
||||
exit(1)
|
||||
|
||||
|
||||
def delete_old_backups(limit, bucket, site_name):
|
||||
all_backups = list()
|
||||
all_backup_dates = list()
|
||||
@ -156,6 +162,7 @@ def delete_old_backups(limit, bucket, site_name):
|
||||
print('Deleteing ' + obj.key)
|
||||
s3.Object(bucket.name, obj.key).delete()
|
||||
|
||||
|
||||
def main():
|
||||
details = dict()
|
||||
sites = get_sites()
|
||||
@ -184,5 +191,6 @@ def main():
|
||||
print('push-backup complete')
|
||||
exit(0)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
||||
@ -8,10 +8,10 @@ import boto3
|
||||
from new import get_password
|
||||
from push_backup import DATE_FORMAT, check_environment_variables
|
||||
from frappe.utils import get_sites, random_string
|
||||
from frappe.commands.site import _new_site
|
||||
from frappe.installer import make_conf, get_conf_params, make_site_dirs
|
||||
from check_connection import get_site_config, get_config
|
||||
|
||||
|
||||
def list_directories(path):
|
||||
directories = []
|
||||
for name in os.listdir(path):
|
||||
@ -19,25 +19,25 @@ def list_directories(path):
|
||||
directories.append(name)
|
||||
return directories
|
||||
|
||||
|
||||
def get_backup_dir():
|
||||
return os.path.join(
|
||||
os.path.expanduser('~'),
|
||||
'backups'
|
||||
)
|
||||
|
||||
|
||||
def decompress_db(files_base, site):
|
||||
database_file = files_base + '-database.sql.gz'
|
||||
config = get_config()
|
||||
site_config = get_site_config(site)
|
||||
db_root_user = os.environ.get('DB_ROOT_USER', 'root')
|
||||
command = 'gunzip -c {database_file} > {database_extract}'.format(
|
||||
database_file=database_file,
|
||||
database_extract=database_file.replace('.gz','')
|
||||
database_extract=database_file.replace('.gz', '')
|
||||
)
|
||||
|
||||
print('Extract Database GZip for site {}'.format(site))
|
||||
os.system(command)
|
||||
|
||||
|
||||
def restore_database(files_base, site):
|
||||
db_root_password = get_password('MYSQL_ROOT_PASSWORD')
|
||||
if not db_root_password:
|
||||
@ -96,12 +96,13 @@ def restore_database(files_base, site):
|
||||
db_host=config.get('db_host'),
|
||||
db_password=db_root_password,
|
||||
db_name=site_config.get('db_name'),
|
||||
database_file=database_file.replace('.gz',''),
|
||||
database_file=database_file.replace('.gz', ''),
|
||||
)
|
||||
|
||||
print('Restoring database for site: {}'.format(site))
|
||||
os.system(command)
|
||||
|
||||
|
||||
def restore_files(files_base):
|
||||
public_files = files_base + '-files.tar'
|
||||
# extract tar
|
||||
@ -109,12 +110,14 @@ def restore_files(files_base):
|
||||
print('Extracting {}'.format(public_files))
|
||||
public_tar.extractall()
|
||||
|
||||
|
||||
def restore_private_files(files_base):
|
||||
private_files = files_base + '-private-files.tar'
|
||||
private_tar = tarfile.open(private_files)
|
||||
print('Extracting {}'.format(private_files))
|
||||
private_tar.extractall()
|
||||
|
||||
|
||||
def pull_backup_from_s3():
|
||||
check_environment_variables()
|
||||
|
||||
@ -134,8 +137,8 @@ def pull_backup_from_s3():
|
||||
# Change directory to /home/frappe/backups
|
||||
os.chdir(get_backup_dir())
|
||||
|
||||
for obj in bucket.objects.filter(Prefix = bucket_dir):
|
||||
backup_file = obj.key.replace(os.path.join(bucket_dir,''),'')
|
||||
for obj in bucket.objects.filter(Prefix=bucket_dir):
|
||||
backup_file = obj.key.replace(os.path.join(bucket_dir, ''), '')
|
||||
if not os.path.exists(os.path.dirname(backup_file)):
|
||||
os.makedirs(os.path.dirname(backup_file))
|
||||
print('Downloading {}'.format(backup_file))
|
||||
@ -143,6 +146,7 @@ def pull_backup_from_s3():
|
||||
|
||||
os.chdir(os.path.join(os.path.expanduser('~'), 'frappe-bench', 'sites'))
|
||||
|
||||
|
||||
def main():
|
||||
backup_dir = get_backup_dir()
|
||||
|
||||
@ -150,8 +154,8 @@ def main():
|
||||
pull_backup_from_s3()
|
||||
|
||||
for site in list_directories(backup_dir):
|
||||
site_slug = site.replace('.','_')
|
||||
backups = [datetime.datetime.strptime(backup, DATE_FORMAT) for backup in list_directories(os.path.join(backup_dir,site))]
|
||||
site_slug = site.replace('.', '_')
|
||||
backups = [datetime.datetime.strptime(backup, DATE_FORMAT) for backup in list_directories(os.path.join(backup_dir, site))]
|
||||
latest_backup = max(backups).strftime(DATE_FORMAT)
|
||||
files_base = os.path.join(backup_dir, site, latest_backup, '')
|
||||
files_base += latest_backup + '-' + site_slug
|
||||
@ -164,8 +168,6 @@ def main():
|
||||
if not mariadb_root_password:
|
||||
print('Variable MYSQL_ROOT_PASSWORD not set')
|
||||
exit(1)
|
||||
mariadb_root_username = os.environ.get('DB_ROOT_USER', 'root')
|
||||
database_file = files_base + '-database.sql.gz'
|
||||
|
||||
site_config = get_conf_params(
|
||||
db_name='_' + hashlib.sha1(site.encode()).hexdigest()[:16],
|
||||
@ -186,5 +188,6 @@ def main():
|
||||
|
||||
exit(0)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
||||
@ -1,10 +1,12 @@
|
||||
import os, frappe
|
||||
import os
|
||||
from frappe.utils.background_jobs import start_worker
|
||||
|
||||
|
||||
def main():
|
||||
queue = os.environ.get("WORKER_TYPE", "default")
|
||||
start_worker(queue, False)
|
||||
exit(0)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
||||
Loading…
Reference in New Issue
Block a user