2
0
mirror of https://github.com/frappe/frappe_docker.git synced 2024-11-08 06:15:26 +00:00

Use pytest (#705)

* Use pytest for tests

* Pin black

* Update CI

* Rename test_main to test_frappe_docker

* Force project name "test"
This commit is contained in:
Lev 2022-03-24 10:40:56 +03:00 committed by GitHub
parent 1d5a0859a8
commit 41ba718b21
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 464 additions and 555 deletions

View File

@ -34,19 +34,11 @@ jobs:
- name: Checkout - name: Checkout
uses: actions/checkout@v3 uses: actions/checkout@v3
- name: Setup Python
uses: actions/setup-python@v3
with:
python-version: 3.9
- name: Setup Buildx - name: Setup Buildx
uses: docker/setup-buildx-action@v1 uses: docker/setup-buildx-action@v1
with: with:
driver-opts: network=host driver-opts: network=host
- name: Install Docker Compose v2
uses: ndeloof/install-compose-action@4a33bc31f327b8231c4f343f6fba704fedc0fa23
- name: Get latest versions - name: Get latest versions
run: python3 ./.github/scripts/get_latest_tags.py --repo ${{ inputs.repo }} --version ${{ inputs.version }} run: python3 ./.github/scripts/get_latest_tags.py --repo ${{ inputs.repo }} --version ${{ inputs.version }}
@ -57,8 +49,21 @@ jobs:
env: env:
REGISTRY_USER: localhost:5000/frappe REGISTRY_USER: localhost:5000/frappe
- name: Setup Python
uses: actions/setup-python@v3
with:
python-version: 3.9
- name: Install Docker Compose v2
uses: ndeloof/install-compose-action@4a33bc31f327b8231c4f343f6fba704fedc0fa23
- name: Install dependencies
run: |
python -m venv venv
venv/bin/pip install -r requirements-test.txt
- name: Test - name: Test
run: python3 tests/main.py run: pytest
- name: Login - name: Login
if: ${{ inputs.push }} if: ${{ inputs.push }}

View File

@ -1,2 +1,3 @@
frappe @ git+git://github.com/frappe/frappe.git frappe @ git+git://github.com/frappe/frappe.git
boto3-stubs[s3] boto3-stubs[s3]
black==22.1.0

1
requirements-test.txt Normal file
View File

@ -0,0 +1 @@
pytest==7.1.0

View File

@ -7,3 +7,6 @@ known_third_party = frappe
[codespell] [codespell]
skip = images/bench/Dockerfile skip = images/bench/Dockerfile
[tool:pytest]
addopts = -s --exitfirst

0
tests/__init__.py Normal file
View File

View File

@ -0,0 +1,50 @@
import asyncio
import json
import socket
from typing import Any, Iterable
Address = tuple[str, int]
async def wait_for_port(address: Address) -> None:
# From https://github.com/clarketm/wait-for-it
while True:
try:
_, writer = await asyncio.open_connection(*address)
writer.close()
await writer.wait_closed()
break
except (socket.gaierror, ConnectionError, OSError, TypeError):
pass
await asyncio.sleep(0.1)
def get_redis_url(addr: str) -> Address:
result = addr.replace("redis://", "")
result = result.split("/")[0]
parts = result.split(":")
assert len(parts) == 2
return parts[0], int(parts[1])
def get_addresses(config: dict[str, Any]) -> Iterable[Address]:
yield (config["db_host"], config["db_port"])
for key in ("redis_cache", "redis_queue", "redis_socketio"):
yield get_redis_url(config[key])
async def async_main(addresses: set[Address]) -> None:
tasks = [asyncio.wait_for(wait_for_port(addr), timeout=5) for addr in addresses]
await asyncio.gather(*tasks)
def main() -> int:
with open("/home/frappe/frappe-bench/sites/common_site_config.json") as f:
config = json.load(f)
addresses = set(get_addresses(config))
asyncio.run(async_main(addresses))
return 0
if __name__ == "__main__":
raise SystemExit(main())

152
tests/conftest.py Normal file
View File

@ -0,0 +1,152 @@
import os
import shutil
import subprocess
from dataclasses import dataclass
from pathlib import Path
import pytest
from tests.utils import CI, Compose
def _add_version_var(name: str, env_path: Path):
if not os.getenv(name):
return
if (gh_env := os.getenv("GITHUB_ENV")) and os.environ[name] == "develop":
with open(gh_env, "a") as f:
f.write(f"\n{name}=latest")
os.environ[name] = "latest"
with open(env_path, "a") as f:
f.write(f"\n{name}={os.environ[name]}")
@pytest.fixture(scope="session")
def env_file(tmp_path_factory: pytest.TempPathFactory):
tmp_path = tmp_path_factory.mktemp("frappe-docker")
file_path = tmp_path / ".env"
shutil.copy("example.env", file_path)
for var in ("FRAPPE_VERSION", "ERPNEXT_VERSION"):
_add_version_var(name=var, env_path=file_path)
yield file_path
os.remove(file_path)
@pytest.fixture(scope="session")
def compose(env_file: str):
return Compose(project_name="test", env_file=env_file)
@pytest.fixture(autouse=True, scope="session")
def frappe_setup(compose: Compose):
compose.stop()
compose("up", "-d", "--quiet-pull")
yield
compose.stop()
@pytest.fixture(scope="session")
def frappe_site(compose: Compose):
site_name = "tests"
compose.bench(
"new-site",
site_name,
"--mariadb-root-password",
"123",
"--admin-password",
"admin",
)
compose("restart", "backend")
yield site_name
@pytest.fixture(scope="class")
def erpnext_setup(compose: Compose):
compose.stop()
args = ["-f", "overrides/compose.erpnext.yaml"]
if CI:
args += ("-f", "tests/compose.ci-erpnext.yaml")
compose(*args, "up", "-d", "--quiet-pull")
yield
compose.stop()
@pytest.fixture(scope="class")
def erpnext_site(compose: Compose):
site_name = "test_erpnext_site"
compose.bench(
"new-site",
site_name,
"--mariadb-root-password",
"123",
"--admin-password",
"admin",
"--install-app",
"erpnext",
)
compose("restart", "backend")
yield site_name
@pytest.fixture
def postgres_setup(compose: Compose):
compose.stop()
compose("-f", "overrides/compose.postgres.yaml", "up", "-d", "--quiet-pull")
compose.bench("set-config", "-g", "root_login", "postgres")
compose.bench("set-config", "-g", "root_password", "123")
yield
compose.stop()
@pytest.fixture
def python_path():
return "/home/frappe/frappe-bench/env/bin/python"
@dataclass
class S3ServiceResult:
access_key: str
secret_key: str
@pytest.fixture
def s3_service(python_path: str, compose: Compose):
access_key = "AKIAIOSFODNN7EXAMPLE"
secret_key = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"
cmd = (
"docker",
"run",
"--name",
"minio",
"-d",
"-e",
f"MINIO_ACCESS_KEY={access_key}",
"-e",
f"MINIO_SECRET_KEY={secret_key}",
"--network",
f"{compose.project_name}_default",
"minio/minio",
"server",
"/data",
)
subprocess.check_call(cmd)
compose("cp", "tests/_create_bucket.py", "backend:/tmp")
compose.exec(
"-e",
f"S3_ACCESS_KEY={access_key}",
"-e",
f"S3_SECRET_KEY={secret_key}",
"backend",
python_path,
"/tmp/_create_bucket.py",
)
yield S3ServiceResult(access_key=access_key, secret_key=secret_key)
subprocess.call(("docker", "rm", "minio", "-f"))

View File

@ -1,28 +0,0 @@
#!/bin/bash
set -e
get_key() {
jq -r ".$1" /home/frappe/frappe-bench/sites/common_site_config.json
}
get_redis_url() {
URL=$(get_key "$1" | sed 's|redis://||g')
if [[ ${URL} == *"/"* ]]; then
URL=$(echo "${URL}" | cut -f1 -d"/")
fi
echo "$URL"
}
check_connection() {
echo "Check $1"
wait-for-it "$1" -t 1
}
check_connection "$(get_key db_host):$(get_key db_port)"
check_connection "$(get_redis_url redis_cache)"
check_connection "$(get_redis_url redis_queue)"
check_connection "$(get_redis_url redis_socketio)"
if [[ "$1" = -p || "$1" = --ping-service ]]; then
check_connection "$2"
fi

View File

@ -1,517 +0,0 @@
import os
import shlex
import shutil
import ssl
import subprocess
import sys
from enum import Enum
from functools import wraps
from textwrap import dedent
from time import sleep
from typing import Any, Callable, Optional
from urllib.error import HTTPError, URLError
from urllib.request import Request, urlopen
CI = os.getenv("CI")
SITE_NAME = "tests"
BACKEND_SERVICES = (
"backend",
"queue-short",
"queue-default",
"queue-long",
"scheduler",
)
S3_ACCESS_KEY = "AKIAIOSFODNN7EXAMPLE"
S3_SECRET_KEY = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"
TTY = sys.stdout.isatty()
def patch_print():
# Patch `print()` builtin to have nice logs when running GitHub Actions
if not CI:
return
global print
_old_print = print
def print(
*values: Any,
sep: Optional[str] = None,
end: Optional[str] = None,
file: Any = None,
flush: bool = False,
):
return _old_print(*values, sep=sep, end=end, file=file, flush=True)
class Color(Enum):
GREY = 30
RED = 31
GREEN = 32
YELLOW = 33
BLUE = 34
MAGENTA = 35
CYAN = 36
WHITE = 37
def colored(text: str, color: Color):
return f"\033[{color.value}m{text}\033[0m"
def log(text: str):
def decorator(f: Callable[..., Any]):
@wraps(f)
def wrapper(*args: Any, **kwargs: Any):
if CI:
print(f"::group::{text}")
else:
output = (
f"\n{f' {text} '.center(os.get_terminal_size().columns, '=')}\n"
)
print(colored(output, Color.YELLOW))
ret = f(*args, **kwargs)
if CI:
print("::endgroup::")
return ret
return wrapper
return decorator
def run(*cmd: str):
print(colored(f"> {shlex.join(cmd)}", Color.GREEN))
return subprocess.check_call(cmd)
def docker_compose(*cmd: str):
args = [
"docker",
"compose",
"-p",
"test",
"--env-file",
"tests/.env",
"-f",
"compose.yaml",
"-f",
"overrides/compose.proxy.yaml",
"-f",
"overrides/compose.mariadb.yaml",
"-f",
"overrides/compose.redis.yaml",
]
if CI:
args.extend(("-f", "tests/compose.ci.yaml"))
return run(*args, *cmd)
def docker_compose_exec(*cmd: str):
if TTY:
return docker_compose("exec", *cmd)
else:
return docker_compose("exec", "-T", *cmd)
@log("Setup .env")
def setup_env():
shutil.copy("example.env", "tests/.env")
if not CI:
return
for var in ("FRAPPE_VERSION", "ERPNEXT_VERSION"):
if os.environ[var] == "develop":
with open(os.environ["GITHUB_ENV"], "a") as f:
f.write(f"\n{var}=latest")
os.environ[var] = "latest"
with open("tests/.env", "a") as f:
f.write(
dedent(
f"""
FRAPPE_VERSION={os.environ['FRAPPE_VERSION']}
ERPNEXT_VERSION={os.environ['ERPNEXT_VERSION']}
"""
)
)
with open("tests/.env") as f:
print(f.read())
@log("Print configuration")
def print_compose_configuration():
docker_compose("config")
@log("Create containers")
def create_containers():
docker_compose("up", "-d", "--quiet-pull")
@log("Check if Python services have connections")
def ping_links_in_backends():
for service in BACKEND_SERVICES:
docker_compose("cp", "tests/healthcheck.sh", f"{service}:/tmp/")
for _ in range(10):
try:
docker_compose_exec(service, "bash", "/tmp/healthcheck.sh")
break
except subprocess.CalledProcessError:
sleep(1)
else:
raise RuntimeError(f"Connections healthcheck failed for service {service}")
@log("Create site")
def create_site():
docker_compose_exec(
"backend",
"bench",
"new-site",
SITE_NAME,
"--mariadb-root-password",
"123",
"--admin-password",
"admin",
)
docker_compose("restart", "backend")
# This is needed to check https override
_ssl_ctx = ssl.create_default_context()
_ssl_ctx.check_hostname = False
_ssl_ctx.verify_mode = ssl.CERT_NONE
def ping_and_check_content(url: str, callback: Callable[[str], Optional[str]]):
request = Request(url, headers={"Host": SITE_NAME})
print(f"Checking {url}")
for _ in range(100):
try:
response = urlopen(request, context=_ssl_ctx)
except HTTPError as exc:
if exc.code not in (404, 502):
raise
except URLError:
pass
else:
text: str = response.read().decode()
ret = callback(text)
if ret:
print(ret)
return
sleep(0.1)
raise RuntimeError(f"Couldn't ping {url}")
def check_index_callback(text: str):
if "404 page not found" not in text:
return text[:200]
@log("Check /")
def check_index():
ping_and_check_content("http://127.0.0.1", check_index_callback)
@log("Check /api/method/version")
def check_api():
ping_and_check_content(
"http://127.0.0.1/api/method/version",
lambda text: text if '"message"' in text else None,
)
@log("Check if Frappe can connect to services in Python services")
def ping_frappe_connections_in_backends():
for service in BACKEND_SERVICES:
docker_compose("cp", "tests/_ping_frappe_connections.py", f"{service}:/tmp/")
docker_compose_exec(
service,
"/home/frappe/frappe-bench/env/bin/python",
f"/tmp/_ping_frappe_connections.py",
)
@log("Check /assets")
def check_assets():
ping_and_check_content(
"http://127.0.0.1/assets/frappe/images/frappe-framework-logo.svg",
lambda text: text[:200] if text else None,
)
@log("Check /files")
def check_files():
file_name = "testfile.txt"
docker_compose(
"cp",
f"tests/{file_name}",
f"backend:/home/frappe/frappe-bench/sites/{SITE_NAME}/public/files/",
)
ping_and_check_content(
f"http://127.0.0.1/files/{file_name}",
lambda text: text if text == "lalala\n" else None,
)
@log("Prepare S3 server")
def prepare_s3_server():
run(
"docker",
"run",
"--name",
"minio",
"-d",
"-e",
f"MINIO_ACCESS_KEY={S3_ACCESS_KEY}",
"-e",
f"MINIO_SECRET_KEY={S3_SECRET_KEY}",
"--network",
"test_default",
"minio/minio",
"server",
"/data",
)
docker_compose("cp", "tests/_create_bucket.py", "backend:/tmp")
docker_compose_exec(
"-e",
f"S3_ACCESS_KEY={S3_ACCESS_KEY}",
"-e",
f"S3_SECRET_KEY={S3_SECRET_KEY}",
"backend",
"/home/frappe/frappe-bench/env/bin/python",
"/tmp/_create_bucket.py",
)
@log("Push backup to S3")
def push_backup_to_s3():
docker_compose_exec(
"backend", "bench", "--site", SITE_NAME, "backup", "--with-files"
)
docker_compose_exec(
"backend",
"push-backup",
"--site",
SITE_NAME,
"--bucket",
"frappe",
"--region-name",
"us-east-1",
"--endpoint-url",
"http://minio:9000",
"--aws-access-key-id",
S3_ACCESS_KEY,
"--aws-secret-access-key",
S3_SECRET_KEY,
)
@log("Check backup files in S3")
def check_backup_in_s3():
docker_compose("cp", "tests/_check_backup_files.py", "backend:/tmp")
docker_compose_exec(
"-e",
f"S3_ACCESS_KEY={S3_ACCESS_KEY}",
"-e",
f"S3_SECRET_KEY={S3_SECRET_KEY}",
"-e",
f"SITE_NAME={SITE_NAME}",
"backend",
"/home/frappe/frappe-bench/env/bin/python",
"/tmp/_check_backup_files.py",
)
@log("Stop S3 container")
def stop_s3_container():
run("docker", "rm", "minio", "-f")
@log("Check Website Theme creation")
def check_website_theme_creation():
docker_compose("cp", "tests/_check_website_theme.py", "backend:/tmp")
docker_compose_exec(
"backend",
"/home/frappe/frappe-bench/env/bin/python",
"/tmp/_check_website_theme.py",
)
@log("Recreate with HTTPS override")
def recreate_with_https_override():
docker_compose("-f", "overrides/compose.https.yaml", "up", "-d")
@log("Check / (HTTPS)")
def check_index_https():
ping_and_check_content("https://127.0.0.1", check_index_callback)
@log("Stop containers")
def stop_containers():
docker_compose("down", "-v", "--remove-orphans")
@log("Recreate with ERPNext override")
def create_containers_with_erpnext_override():
args = ["-f", "overrides/compose.erpnext.yaml"]
if CI:
args += ("-f", "tests/compose.ci-erpnext.yaml")
docker_compose(*args, "up", "-d", "--quiet-pull")
@log("Create ERPNext site")
def create_erpnext_site():
docker_compose_exec(
"backend",
"bench",
"new-site",
SITE_NAME,
"--mariadb-root-password",
"123",
"--admin-password",
"admin",
"--install-app",
"erpnext",
)
docker_compose("restart", "backend")
@log("Check /api/method")
def check_erpnext_api():
ping_and_check_content(
"http://127.0.0.1/api/method/erpnext.templates.pages.product_search.get_product_list",
lambda text: text if '"message"' in text else None,
)
@log("Check /assets")
def check_erpnext_assets():
ping_and_check_content(
"http://127.0.0.1/assets/erpnext/js/setup_wizard.js",
lambda text: text[:200] if text else None,
)
@log("Create containers with Postgres override")
def create_containers_with_postgres_override():
docker_compose("-f", "overrides/compose.postgres.yaml", "up", "-d", "--quiet-pull")
@log("Create Postgres site")
def create_postgres_site():
docker_compose_exec(
"backend", "bench", "set-config", "-g", "root_login", "postgres"
)
docker_compose_exec("backend", "bench", "set-config", "-g", "root_password", "123")
docker_compose_exec(
"backend",
"bench",
"new-site",
SITE_NAME,
"--db-type",
"postgres",
"--admin-password",
"admin",
)
docker_compose("restart", "backend")
@log("Delete .env")
def delete_env():
os.remove("tests/.env")
@log("Show logs")
def show_docker_compose_logs():
docker_compose("logs")
def start():
setup_env()
print_compose_configuration()
create_containers()
ping_links_in_backends()
def create_frappe_site_and_check_availability():
create_site()
check_index()
check_api()
ping_frappe_connections_in_backends()
check_assets()
check_files()
def check_s3():
prepare_s3_server()
try:
push_backup_to_s3()
check_backup_in_s3()
finally:
stop_s3_container()
def check_website_theme():
check_website_theme_creation()
def check_https():
print_compose_configuration()
recreate_with_https_override()
check_index_https()
stop_containers()
def check_erpnext():
print_compose_configuration()
create_containers_with_erpnext_override()
create_erpnext_site()
check_erpnext_api()
check_erpnext_assets()
stop_containers()
def check_postgres():
print_compose_configuration()
create_containers_with_postgres_override()
create_postgres_site()
ping_links_in_backends()
def main() -> int:
try:
patch_print()
start()
create_frappe_site_and_check_availability()
check_s3()
check_website_theme()
check_https()
check_erpnext()
check_postgres()
finally:
delete_env()
show_docker_compose_logs()
stop_containers()
print(colored("\nTests successfully passed!", Color.YELLOW))
return 0
if __name__ == "__main__":
raise SystemExit(main())

158
tests/test_frappe_docker.py Normal file
View File

@ -0,0 +1,158 @@
from pathlib import Path
from typing import Any
import pytest
from tests.conftest import S3ServiceResult
from tests.utils import Compose, check_url_content
BACKEND_SERVICES = (
"backend",
"queue-short",
"queue-default",
"queue-long",
"scheduler",
)
@pytest.mark.parametrize("service", BACKEND_SERVICES)
def test_links_in_backends(service: str, compose: Compose, python_path: str):
filename = "_check_connections.py"
compose("cp", f"tests/{filename}", f"{service}:/tmp/")
compose.exec(service, python_path, f"/tmp/{filename}")
def index_cb(text: str):
if "404 page not found" not in text:
return text[:200]
def api_cb(text: str):
if '"message"' in text:
return text
def assets_cb(text: str):
if text:
return text[:200]
@pytest.mark.parametrize(
("url", "callback"),
(
("/", index_cb),
("/api/method/version", api_cb),
("/assets/frappe/images/frappe-framework-logo.svg", assets_cb),
),
)
def test_endpoints(url: str, callback: Any, frappe_site: str):
check_url_content(
url=f"http://127.0.0.1{url}", callback=callback, site_name=frappe_site
)
def test_files_reachable(frappe_site: str, tmp_path: Path, compose: Compose):
content = "lalala\n"
file_path = tmp_path / "testfile.txt"
with file_path.open("w") as f:
f.write(content)
compose(
"cp",
str(file_path),
f"backend:/home/frappe/frappe-bench/sites/{frappe_site}/public/files/",
)
def callback(text: str):
if text == content:
return text
check_url_content(
url=f"http://127.0.0.1/files/{file_path.name}",
callback=callback,
site_name=frappe_site,
)
@pytest.mark.parametrize("service", BACKEND_SERVICES)
@pytest.mark.usefixtures("frappe_site")
def test_frappe_connections_in_backends(
service: str, python_path: str, compose: Compose
):
filename = "_ping_frappe_connections.py"
compose("cp", f"tests/{filename}", f"{service}:/tmp/")
compose.exec(service, python_path, f"/tmp/{filename}")
def test_push_backup(
python_path: str,
frappe_site: str,
s3_service: S3ServiceResult,
compose: Compose,
):
compose.bench("--site", frappe_site, "backup", "--with-files")
compose.exec(
"backend",
"push-backup",
"--site",
frappe_site,
"--bucket",
"frappe",
"--region-name",
"us-east-1",
"--endpoint-url",
"http://minio:9000",
"--aws-access-key-id",
s3_service.access_key,
"--aws-secret-access-key",
s3_service.secret_key,
)
compose("cp", "tests/_check_backup_files.py", "backend:/tmp")
compose.exec(
"-e",
f"S3_ACCESS_KEY={s3_service.access_key}",
"-e",
f"S3_SECRET_KEY={s3_service.secret_key}",
"-e",
f"SITE_NAME={frappe_site}",
"backend",
python_path,
"/tmp/_check_backup_files.py",
)
def test_https(frappe_site: str, compose: Compose):
compose("-f", "overrides/compose.https.yaml", "up", "-d")
check_url_content(url="https://127.0.0.1", callback=index_cb, site_name=frappe_site)
@pytest.mark.usefixtures("erpnext_setup")
class TestErpnext:
@pytest.mark.parametrize(
("url", "callback"),
(
(
"/api/method/erpnext.templates.pages.product_search.get_product_list",
api_cb,
),
("/assets/erpnext/js/setup_wizard.js", assets_cb),
),
)
def test_endpoints(self, url: str, callback: Any, erpnext_site: str):
check_url_content(
url=f"http://127.0.0.1{url}", callback=callback, site_name=erpnext_site
)
@pytest.mark.usefixtures("postgres_setup")
class TestPostgres:
def test_site_creation(self, compose: Compose):
compose.bench(
"new-site",
"test_pg_site",
"--db-type",
"postgres",
"--admin-password",
"admin",
)

View File

@ -1 +0,0 @@
lalala

85
tests/utils.py Normal file
View File

@ -0,0 +1,85 @@
import os
import ssl
import subprocess
import sys
import time
from typing import Callable, Optional
from urllib.error import HTTPError, URLError
from urllib.request import Request, urlopen
CI = os.getenv("CI")
class Compose:
def __init__(self, project_name: str, env_file: str):
self.project_name = project_name
self.base_cmd = (
"docker",
"compose",
"-p",
project_name,
"--env-file",
env_file,
)
def __call__(self, *cmd: str) -> None:
file_args = [
"-f",
"compose.yaml",
"-f",
"overrides/compose.proxy.yaml",
"-f",
"overrides/compose.mariadb.yaml",
"-f",
"overrides/compose.redis.yaml",
]
if CI:
file_args += ("-f", "tests/compose.ci.yaml")
args = self.base_cmd + tuple(file_args) + cmd
subprocess.check_call(args)
def exec(self, *cmd: str) -> None:
if sys.stdout.isatty():
self("exec", *cmd)
else:
self("exec", "-T", *cmd)
def stop(self) -> None:
subprocess.check_call(self.base_cmd + ("down", "-v", "--remove-orphans"))
def bench(self, *cmd: str) -> None:
self.exec("backend", "bench", *cmd)
def check_url_content(
url: str, callback: Callable[[str], Optional[str]], site_name: str
):
request = Request(url, headers={"Host": site_name})
# This is needed to check https override
ctx = ssl.create_default_context()
ctx.check_hostname = False
ctx.verify_mode = ssl.CERT_NONE
for _ in range(100):
try:
response = urlopen(request, context=ctx)
except HTTPError as exc:
if exc.code not in (404, 502):
raise
except URLError:
pass
else:
text: str = response.read().decode()
ret = callback(text)
if ret:
print(ret)
return
time.sleep(0.1)
raise RuntimeError(f"Couldn't ping {url}")