Use pytest (#705)

* Use pytest for tests

* Pin black

* Update CI

* Rename test_main to test_frappe_docker

* Force project name "test"

.github/workflows/docker-build-push.yml

@@ -34,19 +34,11 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v3
 
-      - name: Setup Python
-        uses: actions/setup-python@v3
-        with:
-          python-version: 3.9
-
       - name: Setup Buildx
         uses: docker/setup-buildx-action@v1
         with:
           driver-opts: network=host
 
-      - name: Install Docker Compose v2
-        uses: ndeloof/install-compose-action@4a33bc31f327b8231c4f343f6fba704fedc0fa23
-
       - name: Get latest versions
         run: python3 ./.github/scripts/get_latest_tags.py --repo ${{ inputs.repo }} --version ${{ inputs.version }}
 
@@ -57,8 +49,21 @@ jobs:
         env:
           REGISTRY_USER: localhost:5000/frappe
 
+      - name: Setup Python
+        uses: actions/setup-python@v3
+        with:
+          python-version: 3.9
+
+      - name: Install Docker Compose v2
+        uses: ndeloof/install-compose-action@4a33bc31f327b8231c4f343f6fba704fedc0fa23
+
+      - name: Install dependencies
+        run: |
+          python -m venv venv
+          venv/bin/pip install -r requirements-test.txt
+
       - name: Test
-        run: python3 tests/main.py
+        run: pytest
 
       - name: Login
         if: ${{ inputs.push }}

requirements-dev.txt

@@ -1,2 +1,3 @@
 frappe @ git+git://github.com/frappe/frappe.git
 boto3-stubs[s3]
+black==22.1.0

requirements-test.txt

@@ -0,0 +1 @@
+pytest==7.1.0

setup.cfg

@@ -7,3 +7,6 @@ known_third_party = frappe
 
 [codespell]
 skip = images/bench/Dockerfile
+
+[tool:pytest]
+addopts = -s --exitfirst

tests/_check_connections.py

@@ -0,0 +1,50 @@
+import asyncio
+import json
+import socket
+from typing import Any, Iterable
+
+Address = tuple[str, int]
+
+
+async def wait_for_port(address: Address) -> None:
+    # From https://github.com/clarketm/wait-for-it
+    while True:
+        try:
+            _, writer = await asyncio.open_connection(*address)
+            writer.close()
+            await writer.wait_closed()
+            break
+        except (socket.gaierror, ConnectionError, OSError, TypeError):
+            pass
+        await asyncio.sleep(0.1)
+
+
+def get_redis_url(addr: str) -> Address:
+    result = addr.replace("redis://", "")
+    result = result.split("/")[0]
+    parts = result.split(":")
+    assert len(parts) == 2
+    return parts[0], int(parts[1])
+
+
+def get_addresses(config: dict[str, Any]) -> Iterable[Address]:
+    yield (config["db_host"], config["db_port"])
+    for key in ("redis_cache", "redis_queue", "redis_socketio"):
+        yield get_redis_url(config[key])
+
+
+async def async_main(addresses: set[Address]) -> None:
+    tasks = [asyncio.wait_for(wait_for_port(addr), timeout=5) for addr in addresses]
+    await asyncio.gather(*tasks)
+
+
+def main() -> int:
+    with open("/home/frappe/frappe-bench/sites/common_site_config.json") as f:
+        config = json.load(f)
+    addresses = set(get_addresses(config))
+    asyncio.run(async_main(addresses))
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

tests/conftest.py

@@ -0,0 +1,152 @@
+import os
+import shutil
+import subprocess
+from dataclasses import dataclass
+from pathlib import Path
+
+import pytest
+
+from tests.utils import CI, Compose
+
+
+def _add_version_var(name: str, env_path: Path):
+    if not os.getenv(name):
+        return
+
+    if (gh_env := os.getenv("GITHUB_ENV")) and os.environ[name] == "develop":
+        with open(gh_env, "a") as f:
+            f.write(f"\n{name}=latest")
+
+        os.environ[name] = "latest"
+
+    with open(env_path, "a") as f:
+        f.write(f"\n{name}={os.environ[name]}")
+
+
+@pytest.fixture(scope="session")
+def env_file(tmp_path_factory: pytest.TempPathFactory):
+    tmp_path = tmp_path_factory.mktemp("frappe-docker")
+    file_path = tmp_path / ".env"
+    shutil.copy("example.env", file_path)
+
+    for var in ("FRAPPE_VERSION", "ERPNEXT_VERSION"):
+        _add_version_var(name=var, env_path=file_path)
+
+    yield file_path
+    os.remove(file_path)
+
+
+@pytest.fixture(scope="session")
+def compose(env_file: str):
+    return Compose(project_name="test", env_file=env_file)
+
+
+@pytest.fixture(autouse=True, scope="session")
+def frappe_setup(compose: Compose):
+    compose.stop()
+    compose("up", "-d", "--quiet-pull")
+    yield
+    compose.stop()
+
+
+@pytest.fixture(scope="session")
+def frappe_site(compose: Compose):
+    site_name = "tests"
+    compose.bench(
+        "new-site",
+        site_name,
+        "--mariadb-root-password",
+        "123",
+        "--admin-password",
+        "admin",
+    )
+    compose("restart", "backend")
+    yield site_name
+
+
+@pytest.fixture(scope="class")
+def erpnext_setup(compose: Compose):
+    compose.stop()
+
+    args = ["-f", "overrides/compose.erpnext.yaml"]
+    if CI:
+        args += ("-f", "tests/compose.ci-erpnext.yaml")
+    compose(*args, "up", "-d", "--quiet-pull")
+
+    yield
+    compose.stop()
+
+
+@pytest.fixture(scope="class")
+def erpnext_site(compose: Compose):
+    site_name = "test_erpnext_site"
+    compose.bench(
+        "new-site",
+        site_name,
+        "--mariadb-root-password",
+        "123",
+        "--admin-password",
+        "admin",
+        "--install-app",
+        "erpnext",
+    )
+    compose("restart", "backend")
+    yield site_name
+
+
+@pytest.fixture
+def postgres_setup(compose: Compose):
+    compose.stop()
+    compose("-f", "overrides/compose.postgres.yaml", "up", "-d", "--quiet-pull")
+    compose.bench("set-config", "-g", "root_login", "postgres")
+    compose.bench("set-config", "-g", "root_password", "123")
+    yield
+    compose.stop()
+
+
+@pytest.fixture
+def python_path():
+    return "/home/frappe/frappe-bench/env/bin/python"
+
+
+@dataclass
+class S3ServiceResult:
+    access_key: str
+    secret_key: str
+
+
+@pytest.fixture
+def s3_service(python_path: str, compose: Compose):
+    access_key = "AKIAIOSFODNN7EXAMPLE"
+    secret_key = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"
+    cmd = (
+        "docker",
+        "run",
+        "--name",
+        "minio",
+        "-d",
+        "-e",
+        f"MINIO_ACCESS_KEY={access_key}",
+        "-e",
+        f"MINIO_SECRET_KEY={secret_key}",
+        "--network",
+        f"{compose.project_name}_default",
+        "minio/minio",
+        "server",
+        "/data",
+    )
+    subprocess.check_call(cmd)
+
+    compose("cp", "tests/_create_bucket.py", "backend:/tmp")
+    compose.exec(
+        "-e",
+        f"S3_ACCESS_KEY={access_key}",
+        "-e",
+        f"S3_SECRET_KEY={secret_key}",
+        "backend",
+        python_path,
+        "/tmp/_create_bucket.py",
+    )
+
+    yield S3ServiceResult(access_key=access_key, secret_key=secret_key)
+    subprocess.call(("docker", "rm", "minio", "-f"))

tests/healthcheck.sh

@@ -1,28 +0,0 @@
-#!/bin/bash
-set -e
-
-get_key() {
-  jq -r ".$1" /home/frappe/frappe-bench/sites/common_site_config.json
-}
-
-get_redis_url() {
-  URL=$(get_key "$1" | sed 's|redis://||g')
-  if [[ ${URL} == *"/"* ]]; then
-    URL=$(echo "${URL}" | cut -f1 -d"/")
-  fi
-  echo "$URL"
-}
-
-check_connection() {
-  echo "Check $1"
-  wait-for-it "$1" -t 1
-}
-
-check_connection "$(get_key db_host):$(get_key db_port)"
-check_connection "$(get_redis_url redis_cache)"
-check_connection "$(get_redis_url redis_queue)"
-check_connection "$(get_redis_url redis_socketio)"
-
-if [[ "$1" = -p || "$1" = --ping-service ]]; then
-  check_connection "$2"
-fi

tests/main.py

@@ -1,517 +0,0 @@
-import os
-import shlex
-import shutil
-import ssl
-import subprocess
-import sys
-from enum import Enum
-from functools import wraps
-from textwrap import dedent
-from time import sleep
-from typing import Any, Callable, Optional
-from urllib.error import HTTPError, URLError
-from urllib.request import Request, urlopen
-
-CI = os.getenv("CI")
-SITE_NAME = "tests"
-BACKEND_SERVICES = (
-    "backend",
-    "queue-short",
-    "queue-default",
-    "queue-long",
-    "scheduler",
-)
-S3_ACCESS_KEY = "AKIAIOSFODNN7EXAMPLE"
-S3_SECRET_KEY = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"
-TTY = sys.stdout.isatty()
-
-
-def patch_print():
-    # Patch `print()` builtin to have nice logs when running GitHub Actions
-    if not CI:
-        return
-    global print
-    _old_print = print
-
-    def print(
-        *values: Any,
-        sep: Optional[str] = None,
-        end: Optional[str] = None,
-        file: Any = None,
-        flush: bool = False,
-    ):
-        return _old_print(*values, sep=sep, end=end, file=file, flush=True)
-
-
-class Color(Enum):
-    GREY = 30
-    RED = 31
-    GREEN = 32
-    YELLOW = 33
-    BLUE = 34
-    MAGENTA = 35
-    CYAN = 36
-    WHITE = 37
-
-
-def colored(text: str, color: Color):
-    return f"\033[{color.value}m{text}\033[0m"
-
-
-def log(text: str):
-    def decorator(f: Callable[..., Any]):
-        @wraps(f)
-        def wrapper(*args: Any, **kwargs: Any):
-            if CI:
-                print(f"::group::{text}")
-            else:
-                output = (
-                    f"\n{f' {text} '.center(os.get_terminal_size().columns, '=')}\n"
-                )
-                print(colored(output, Color.YELLOW))
-
-            ret = f(*args, **kwargs)
-
-            if CI:
-                print("::endgroup::")
-            return ret
-
-        return wrapper
-
-    return decorator
-
-
-def run(*cmd: str):
-    print(colored(f"> {shlex.join(cmd)}", Color.GREEN))
-    return subprocess.check_call(cmd)
-
-
-def docker_compose(*cmd: str):
-    args = [
-        "docker",
-        "compose",
-        "-p",
-        "test",
-        "--env-file",
-        "tests/.env",
-        "-f",
-        "compose.yaml",
-        "-f",
-        "overrides/compose.proxy.yaml",
-        "-f",
-        "overrides/compose.mariadb.yaml",
-        "-f",
-        "overrides/compose.redis.yaml",
-    ]
-    if CI:
-        args.extend(("-f", "tests/compose.ci.yaml"))
-    return run(*args, *cmd)
-
-
-def docker_compose_exec(*cmd: str):
-    if TTY:
-        return docker_compose("exec", *cmd)
-    else:
-        return docker_compose("exec", "-T", *cmd)
-
-
-@log("Setup .env")
-def setup_env():
-    shutil.copy("example.env", "tests/.env")
-
-    if not CI:
-        return
-
-    for var in ("FRAPPE_VERSION", "ERPNEXT_VERSION"):
-        if os.environ[var] == "develop":
-            with open(os.environ["GITHUB_ENV"], "a") as f:
-                f.write(f"\n{var}=latest")
-            os.environ[var] = "latest"
-
-    with open("tests/.env", "a") as f:
-        f.write(
-            dedent(
-                f"""
-            FRAPPE_VERSION={os.environ['FRAPPE_VERSION']}
-            ERPNEXT_VERSION={os.environ['ERPNEXT_VERSION']}
-        """
-            )
-        )
-
-    with open("tests/.env") as f:
-        print(f.read())
-
-
-@log("Print configuration")
-def print_compose_configuration():
-    docker_compose("config")
-
-
-@log("Create containers")
-def create_containers():
-    docker_compose("up", "-d", "--quiet-pull")
-
-
-@log("Check if Python services have connections")
-def ping_links_in_backends():
-    for service in BACKEND_SERVICES:
-        docker_compose("cp", "tests/healthcheck.sh", f"{service}:/tmp/")
-        for _ in range(10):
-            try:
-                docker_compose_exec(service, "bash", "/tmp/healthcheck.sh")
-                break
-            except subprocess.CalledProcessError:
-                sleep(1)
-        else:
-            raise RuntimeError(f"Connections healthcheck failed for service {service}")
-
-
-@log("Create site")
-def create_site():
-    docker_compose_exec(
-        "backend",
-        "bench",
-        "new-site",
-        SITE_NAME,
-        "--mariadb-root-password",
-        "123",
-        "--admin-password",
-        "admin",
-    )
-    docker_compose("restart", "backend")
-
-
-# This is needed to check https override
-_ssl_ctx = ssl.create_default_context()
-_ssl_ctx.check_hostname = False
-_ssl_ctx.verify_mode = ssl.CERT_NONE
-
-
-def ping_and_check_content(url: str, callback: Callable[[str], Optional[str]]):
-    request = Request(url, headers={"Host": SITE_NAME})
-    print(f"Checking {url}")
-
-    for _ in range(100):
-        try:
-            response = urlopen(request, context=_ssl_ctx)
-
-        except HTTPError as exc:
-            if exc.code not in (404, 502):
-                raise
-
-        except URLError:
-            pass
-
-        else:
-            text: str = response.read().decode()
-            ret = callback(text)
-            if ret:
-                print(ret)
-                return
-
-        sleep(0.1)
-
-    raise RuntimeError(f"Couldn't ping {url}")
-
-
-def check_index_callback(text: str):
-    if "404 page not found" not in text:
-        return text[:200]
-
-
-@log("Check /")
-def check_index():
-    ping_and_check_content("http://127.0.0.1", check_index_callback)
-
-
-@log("Check /api/method/version")
-def check_api():
-    ping_and_check_content(
-        "http://127.0.0.1/api/method/version",
-        lambda text: text if '"message"' in text else None,
-    )
-
-
-@log("Check if Frappe can connect to services in Python services")
-def ping_frappe_connections_in_backends():
-    for service in BACKEND_SERVICES:
-        docker_compose("cp", "tests/_ping_frappe_connections.py", f"{service}:/tmp/")
-        docker_compose_exec(
-            service,
-            "/home/frappe/frappe-bench/env/bin/python",
-            f"/tmp/_ping_frappe_connections.py",
-        )
-
-
-@log("Check /assets")
-def check_assets():
-    ping_and_check_content(
-        "http://127.0.0.1/assets/frappe/images/frappe-framework-logo.svg",
-        lambda text: text[:200] if text else None,
-    )
-
-
-@log("Check /files")
-def check_files():
-    file_name = "testfile.txt"
-    docker_compose(
-        "cp",
-        f"tests/{file_name}",
-        f"backend:/home/frappe/frappe-bench/sites/{SITE_NAME}/public/files/",
-    )
-    ping_and_check_content(
-        f"http://127.0.0.1/files/{file_name}",
-        lambda text: text if text == "lalala\n" else None,
-    )
-
-
-@log("Prepare S3 server")
-def prepare_s3_server():
-    run(
-        "docker",
-        "run",
-        "--name",
-        "minio",
-        "-d",
-        "-e",
-        f"MINIO_ACCESS_KEY={S3_ACCESS_KEY}",
-        "-e",
-        f"MINIO_SECRET_KEY={S3_SECRET_KEY}",
-        "--network",
-        "test_default",
-        "minio/minio",
-        "server",
-        "/data",
-    )
-    docker_compose("cp", "tests/_create_bucket.py", "backend:/tmp")
-    docker_compose_exec(
-        "-e",
-        f"S3_ACCESS_KEY={S3_ACCESS_KEY}",
-        "-e",
-        f"S3_SECRET_KEY={S3_SECRET_KEY}",
-        "backend",
-        "/home/frappe/frappe-bench/env/bin/python",
-        "/tmp/_create_bucket.py",
-    )
-
-
-@log("Push backup to S3")
-def push_backup_to_s3():
-    docker_compose_exec(
-        "backend", "bench", "--site", SITE_NAME, "backup", "--with-files"
-    )
-    docker_compose_exec(
-        "backend",
-        "push-backup",
-        "--site",
-        SITE_NAME,
-        "--bucket",
-        "frappe",
-        "--region-name",
-        "us-east-1",
-        "--endpoint-url",
-        "http://minio:9000",
-        "--aws-access-key-id",
-        S3_ACCESS_KEY,
-        "--aws-secret-access-key",
-        S3_SECRET_KEY,
-    )
-
-
-@log("Check backup files in S3")
-def check_backup_in_s3():
-    docker_compose("cp", "tests/_check_backup_files.py", "backend:/tmp")
-    docker_compose_exec(
-        "-e",
-        f"S3_ACCESS_KEY={S3_ACCESS_KEY}",
-        "-e",
-        f"S3_SECRET_KEY={S3_SECRET_KEY}",
-        "-e",
-        f"SITE_NAME={SITE_NAME}",
-        "backend",
-        "/home/frappe/frappe-bench/env/bin/python",
-        "/tmp/_check_backup_files.py",
-    )
-
-
-@log("Stop S3 container")
-def stop_s3_container():
-    run("docker", "rm", "minio", "-f")
-
-
-@log("Check Website Theme creation")
-def check_website_theme_creation():
-    docker_compose("cp", "tests/_check_website_theme.py", "backend:/tmp")
-    docker_compose_exec(
-        "backend",
-        "/home/frappe/frappe-bench/env/bin/python",
-        "/tmp/_check_website_theme.py",
-    )
-
-
-@log("Recreate with HTTPS override")
-def recreate_with_https_override():
-    docker_compose("-f", "overrides/compose.https.yaml", "up", "-d")
-
-
-@log("Check / (HTTPS)")
-def check_index_https():
-    ping_and_check_content("https://127.0.0.1", check_index_callback)
-
-
-@log("Stop containers")
-def stop_containers():
-    docker_compose("down", "-v", "--remove-orphans")
-
-
-@log("Recreate with ERPNext override")
-def create_containers_with_erpnext_override():
-    args = ["-f", "overrides/compose.erpnext.yaml"]
-    if CI:
-        args += ("-f", "tests/compose.ci-erpnext.yaml")
-
-    docker_compose(*args, "up", "-d", "--quiet-pull")
-
-
-@log("Create ERPNext site")
-def create_erpnext_site():
-    docker_compose_exec(
-        "backend",
-        "bench",
-        "new-site",
-        SITE_NAME,
-        "--mariadb-root-password",
-        "123",
-        "--admin-password",
-        "admin",
-        "--install-app",
-        "erpnext",
-    )
-    docker_compose("restart", "backend")
-
-
-@log("Check /api/method")
-def check_erpnext_api():
-    ping_and_check_content(
-        "http://127.0.0.1/api/method/erpnext.templates.pages.product_search.get_product_list",
-        lambda text: text if '"message"' in text else None,
-    )
-
-
-@log("Check /assets")
-def check_erpnext_assets():
-    ping_and_check_content(
-        "http://127.0.0.1/assets/erpnext/js/setup_wizard.js",
-        lambda text: text[:200] if text else None,
-    )
-
-
-@log("Create containers with Postgres override")
-def create_containers_with_postgres_override():
-    docker_compose("-f", "overrides/compose.postgres.yaml", "up", "-d", "--quiet-pull")
-
-
-@log("Create Postgres site")
-def create_postgres_site():
-    docker_compose_exec(
-        "backend", "bench", "set-config", "-g", "root_login", "postgres"
-    )
-    docker_compose_exec("backend", "bench", "set-config", "-g", "root_password", "123")
-    docker_compose_exec(
-        "backend",
-        "bench",
-        "new-site",
-        SITE_NAME,
-        "--db-type",
-        "postgres",
-        "--admin-password",
-        "admin",
-    )
-    docker_compose("restart", "backend")
-
-
-@log("Delete .env")
-def delete_env():
-    os.remove("tests/.env")
-
-
-@log("Show logs")
-def show_docker_compose_logs():
-    docker_compose("logs")
-
-
-def start():
-    setup_env()
-    print_compose_configuration()
-    create_containers()
-    ping_links_in_backends()
-
-
-def create_frappe_site_and_check_availability():
-    create_site()
-    check_index()
-    check_api()
-    ping_frappe_connections_in_backends()
-    check_assets()
-    check_files()
-
-
-def check_s3():
-    prepare_s3_server()
-
-    try:
-        push_backup_to_s3()
-        check_backup_in_s3()
-    finally:
-        stop_s3_container()
-
-
-def check_website_theme():
-    check_website_theme_creation()
-
-
-def check_https():
-    print_compose_configuration()
-    recreate_with_https_override()
-    check_index_https()
-    stop_containers()
-
-
-def check_erpnext():
-    print_compose_configuration()
-    create_containers_with_erpnext_override()
-    create_erpnext_site()
-    check_erpnext_api()
-    check_erpnext_assets()
-    stop_containers()
-
-
-def check_postgres():
-    print_compose_configuration()
-    create_containers_with_postgres_override()
-    create_postgres_site()
-    ping_links_in_backends()
-
-
-def main() -> int:
-    try:
-        patch_print()
-        start()
-        create_frappe_site_and_check_availability()
-        check_s3()
-        check_website_theme()
-        check_https()
-        check_erpnext()
-        check_postgres()
-
-    finally:
-        delete_env()
-        show_docker_compose_logs()
-        stop_containers()
-
-    print(colored("\nTests successfully passed!", Color.YELLOW))
-    return 0
-
-
-if __name__ == "__main__":
-    raise SystemExit(main())

tests/test_frappe_docker.py

@@ -0,0 +1,158 @@
+from pathlib import Path
+from typing import Any
+
+import pytest
+
+from tests.conftest import S3ServiceResult
+from tests.utils import Compose, check_url_content
+
+BACKEND_SERVICES = (
+    "backend",
+    "queue-short",
+    "queue-default",
+    "queue-long",
+    "scheduler",
+)
+
+
+@pytest.mark.parametrize("service", BACKEND_SERVICES)
+def test_links_in_backends(service: str, compose: Compose, python_path: str):
+    filename = "_check_connections.py"
+    compose("cp", f"tests/{filename}", f"{service}:/tmp/")
+    compose.exec(service, python_path, f"/tmp/{filename}")
+
+
+def index_cb(text: str):
+    if "404 page not found" not in text:
+        return text[:200]
+
+
+def api_cb(text: str):
+    if '"message"' in text:
+        return text
+
+
+def assets_cb(text: str):
+    if text:
+        return text[:200]
+
+
+@pytest.mark.parametrize(
+    ("url", "callback"),
+    (
+        ("/", index_cb),
+        ("/api/method/version", api_cb),
+        ("/assets/frappe/images/frappe-framework-logo.svg", assets_cb),
+    ),
+)
+def test_endpoints(url: str, callback: Any, frappe_site: str):
+    check_url_content(
+        url=f"http://127.0.0.1{url}", callback=callback, site_name=frappe_site
+    )
+
+
+def test_files_reachable(frappe_site: str, tmp_path: Path, compose: Compose):
+    content = "lalala\n"
+    file_path = tmp_path / "testfile.txt"
+
+    with file_path.open("w") as f:
+        f.write(content)
+
+    compose(
+        "cp",
+        str(file_path),
+        f"backend:/home/frappe/frappe-bench/sites/{frappe_site}/public/files/",
+    )
+
+    def callback(text: str):
+        if text == content:
+            return text
+
+    check_url_content(
+        url=f"http://127.0.0.1/files/{file_path.name}",
+        callback=callback,
+        site_name=frappe_site,
+    )
+
+
+@pytest.mark.parametrize("service", BACKEND_SERVICES)
+@pytest.mark.usefixtures("frappe_site")
+def test_frappe_connections_in_backends(
+    service: str, python_path: str, compose: Compose
+):
+    filename = "_ping_frappe_connections.py"
+    compose("cp", f"tests/{filename}", f"{service}:/tmp/")
+    compose.exec(service, python_path, f"/tmp/{filename}")
+
+
+def test_push_backup(
+    python_path: str,
+    frappe_site: str,
+    s3_service: S3ServiceResult,
+    compose: Compose,
+):
+    compose.bench("--site", frappe_site, "backup", "--with-files")
+    compose.exec(
+        "backend",
+        "push-backup",
+        "--site",
+        frappe_site,
+        "--bucket",
+        "frappe",
+        "--region-name",
+        "us-east-1",
+        "--endpoint-url",
+        "http://minio:9000",
+        "--aws-access-key-id",
+        s3_service.access_key,
+        "--aws-secret-access-key",
+        s3_service.secret_key,
+    )
+    compose("cp", "tests/_check_backup_files.py", "backend:/tmp")
+    compose.exec(
+        "-e",
+        f"S3_ACCESS_KEY={s3_service.access_key}",
+        "-e",
+        f"S3_SECRET_KEY={s3_service.secret_key}",
+        "-e",
+        f"SITE_NAME={frappe_site}",
+        "backend",
+        python_path,
+        "/tmp/_check_backup_files.py",
+    )
+
+
+def test_https(frappe_site: str, compose: Compose):
+    compose("-f", "overrides/compose.https.yaml", "up", "-d")
+    check_url_content(url="https://127.0.0.1", callback=index_cb, site_name=frappe_site)
+
+
+@pytest.mark.usefixtures("erpnext_setup")
+class TestErpnext:
+    @pytest.mark.parametrize(
+        ("url", "callback"),
+        (
+            (
+                "/api/method/erpnext.templates.pages.product_search.get_product_list",
+                api_cb,
+            ),
+            ("/assets/erpnext/js/setup_wizard.js", assets_cb),
+        ),
+    )
+    def test_endpoints(self, url: str, callback: Any, erpnext_site: str):
+        check_url_content(
+            url=f"http://127.0.0.1{url}", callback=callback, site_name=erpnext_site
+        )
+
+
+@pytest.mark.usefixtures("postgres_setup")
+class TestPostgres:
+    def test_site_creation(self, compose: Compose):
+        compose.bench(
+            "new-site",
+            "test_pg_site",
+            "--db-type",
+            "postgres",
+            "--admin-password",
+            "admin",
+        )

tests/testfile.txt

@@ -1 +0,0 @@
-lalala

tests/utils.py

@@ -0,0 +1,85 @@
+import os
+import ssl
+import subprocess
+import sys
+import time
+from typing import Callable, Optional
+from urllib.error import HTTPError, URLError
+from urllib.request import Request, urlopen
+
+CI = os.getenv("CI")
+
+
+class Compose:
+    def __init__(self, project_name: str, env_file: str):
+        self.project_name = project_name
+        self.base_cmd = (
+            "docker",
+            "compose",
+            "-p",
+            project_name,
+            "--env-file",
+            env_file,
+        )
+
+    def __call__(self, *cmd: str) -> None:
+        file_args = [
+            "-f",
+            "compose.yaml",
+            "-f",
+            "overrides/compose.proxy.yaml",
+            "-f",
+            "overrides/compose.mariadb.yaml",
+            "-f",
+            "overrides/compose.redis.yaml",
+        ]
+        if CI:
+            file_args += ("-f", "tests/compose.ci.yaml")
+
+        args = self.base_cmd + tuple(file_args) + cmd
+        subprocess.check_call(args)
+
+    def exec(self, *cmd: str) -> None:
+        if sys.stdout.isatty():
+            self("exec", *cmd)
+        else:
+            self("exec", "-T", *cmd)
+
+    def stop(self) -> None:
+        subprocess.check_call(self.base_cmd + ("down", "-v", "--remove-orphans"))
+
+    def bench(self, *cmd: str) -> None:
+        self.exec("backend", "bench", *cmd)
+
+
+def check_url_content(
+    url: str, callback: Callable[[str], Optional[str]], site_name: str
+):
+    request = Request(url, headers={"Host": site_name})
+
+    # This is needed to check https override
+    ctx = ssl.create_default_context()
+    ctx.check_hostname = False
+    ctx.verify_mode = ssl.CERT_NONE
+
+    for _ in range(100):
+        try:
+            response = urlopen(request, context=ctx)
+
+        except HTTPError as exc:
+            if exc.code not in (404, 502):
+                raise
+
+        except URLError:
+            pass
+
+        else:
+            text: str = response.read().decode()
+            ret = callback(text)
+            if ret:
+                print(ret)
+                return
+
+        time.sleep(0.1)
+
+    raise RuntimeError(f"Couldn't ping {url}")