Don't depend on boto3 while testing (do it in backend)

tests/_check_backup_files.py

@@ -0,0 +1,43 @@
+import os
+
+import boto3
+
+
+def main() -> int:
+    resource = boto3.resource(
+        service_name="s3",
+        endpoint_url="http://minio:9000",
+        region_name="us-east-1",
+        aws_access_key_id=os.getenv("MINIO_ACCESS_KEY"),
+        aws_secret_access_key=os.getenv("MINIO_SECRET_KEY"),
+    )
+    bucket = resource.Bucket("frappe")
+    db = False
+    config = False
+    private_files = False
+    public_files = False
+    for obj in bucket.objects.all():
+        if obj.key.endswith("database.sql.gz"):
+            db = True
+        elif obj.key.endswith("site_config_backup.json"):
+            config = True
+        elif obj.key.endswith("private-files.tar"):
+            private_files = True
+        elif obj.key.endswith("files.tar"):
+            public_files = True
+
+    exc = lambda type_: Exception(f"Didn't push {type_} backup")
+    if not db:
+        raise exc("database")
+    if not config:
+        raise exc("site config")
+    if not private_files:
+        raise exc("private files")
+    if not public_files:
+        raise exc("public files")
+    print("All files was pushed to S3!")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

tests/_create_bucket.py

@@ -0,0 +1,19 @@
+import os
+
+import boto3
+
+
+def main() -> int:
+    resource = boto3.resource(
+        service_name="s3",
+        endpoint_url="http://minio:9000",
+        region_name="us-east-1",
+        aws_access_key_id=os.getenv("MINIO_ACCESS_KEY"),
+        aws_secret_access_key=os.getenv("MINIO_SECRET_KEY"),
+    )
+    resource.create_bucket(Bucket="frappe")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

tests/main.py

@@ -9,8 +9,6 @@ from typing import Any, Callable, Optional
 from urllib.error import HTTPError
 from urllib.request import Request, urlopen
 
-import boto3
-
 CI = os.getenv("CI")
 SITE_NAME = "tests"
 BACKEND_SERVICES = (
@@ -194,12 +192,12 @@ def check_api():
 @log("Check if Frappe can connect to services in backends")
 def ping_frappe_connections_in_backends():
     for service in BACKEND_SERVICES:
-        docker_compose("cp", f"tests/ping_frappe_connections.py", f"{service}:/tmp/")
+        docker_compose("cp", "tests/_ping_frappe_connections.py", f"{service}:/tmp/")
         docker_compose(
             "exec",
             service,
             "/home/frappe/frappe-bench/env/bin/python",
-            f"/tmp/ping_frappe_connections.py",
+            f"/tmp/_ping_frappe_connections.py",
         )
 
 
@@ -225,17 +223,6 @@ def check_files():
     )
 
 
-def get_s3_resource():
-    return boto3.resource(
-        service_name="s3",
-        endpoint_url="http://127.0.0.1:9000",
-        region_name="us-east-1",
-        aws_access_key_id=MINIO_ACCESS_KEY,
-        aws_secret_access_key=MINIO_SECRET_KEY,
-        use_ssl=False,
-    )
-
-
 @log("Prepare S3 server")
 def prepare_s3_server():
     run(
@@ -250,13 +237,21 @@ def prepare_s3_server():
         f"MINIO_SECRET_KEY={MINIO_SECRET_KEY}",
         "--network",
         "test_default",
-        "--publish",
-        "9000:9000",
         "minio/minio",
         "server",
         "/data",
     )
-    get_s3_resource().create_bucket(Bucket="frappe")
+    docker_compose("cp", "tests/_create_bucket.py", "backend:/tmp")
+    docker_compose(
+        "exec",
+        "-e",
+        f"MINIO_ACCESS_KEY={MINIO_ACCESS_KEY}",
+        "-e",
+        f"MINIO_SECRET_KEY={MINIO_SECRET_KEY}",
+        "backend",
+        "/home/frappe/frappe-bench/env/bin/python",
+        "/tmp/_create_bucket.py",
+    )
 
 
 @log("Push backup to S3")
@@ -285,31 +280,17 @@ def push_backup_to_s3():
 
 @log("Check backup in S3")
 def check_backup_in_s3():
-    bucket = get_s3_resource().Bucket("frappe")
+    docker_compose("cp", "tests/_check_backup_files.py", "backend:/tmp")
-    db = False
+    docker_compose(
-    config = False
+        "exec",
-    private_files = False
+        "-e",
-    public_files = False
+        f"MINIO_ACCESS_KEY={MINIO_ACCESS_KEY}",
-    for obj in bucket.objects.all():
+        "-e",
-        if obj.key.endswith("database.sql.gz"):
+        f"MINIO_SECRET_KEY={MINIO_SECRET_KEY}",
-            db = True
+        "backend",
-        elif obj.key.endswith("site_config_backup.json"):
+        "/home/frappe/frappe-bench/env/bin/python",
-            config = True
+        "/tmp/_check_backup_files.py",
-        elif obj.key.endswith("private-files.tar"):
+    )
-            private_files = True
-        elif obj.key.endswith("files.tar"):
-            public_files = True
-
-    exc = lambda type_: Exception(f"Didn't push {type_} backup")
-    if not db:
-        raise exc("database")
-    if not config:
-        raise exc("site config")
-    if not private_files:
-        raise exc("private files")
-    if not public_files:
-        raise exc("public files")
-    print("All files was pushed to S3!")
 
 
 @log("Stop S3 container")