2021-04-06 10:09:00 +00:00
|
|
|
from typing import Iterator, List, Tuple
|
2021-02-25 08:09:14 +00:00
|
|
|
|
2019-01-22 20:25:04 +00:00
|
|
|
import click
|
|
|
|
|
2019-06-03 22:44:12 +00:00
|
|
|
from .. import config as tutor_config
|
2019-05-11 19:20:09 +00:00
|
|
|
from .. import env as tutor_env
|
2021-04-06 10:09:00 +00:00
|
|
|
from .. import exceptions
|
2019-07-02 20:16:44 +00:00
|
|
|
from .. import images
|
|
|
|
from .. import plugins
|
2021-04-06 10:09:00 +00:00
|
|
|
from ..types import Config
|
2021-02-25 08:09:14 +00:00
|
|
|
from .context import Context
|
2019-01-22 20:25:04 +00:00
|
|
|
|
feat: run all services as unprivileged containers
With this change, containers are no longer run as "root" but as unprivileged
users. This is necessary in some environments, notably some Kubernetes
clusters.
To make this possible, we need to manually fix bind-mounted volumes in
docker-compose. This is pretty much equivalent to the behaviour in Kubernetes,
where permissions are fixed at runtime if the volume owner is incorrect. Thus,
we have a consistent behaviour between docker-compose and Kubernetes.
We achieve this by bind-mounting some repos inside "*-permissions" services.
These services run as root user on docker-compose and will fix the required
permissions, as per build/permissions/setowner.sh These services simply do not
run on Kubernetes, where we don't rely on bind-mounted volumes. There, we make
use of Kubernete's built-in volume ownership feature.
With this change, we get rid of the "openedx-dev" Docker image, in the sense
that it no longer has its own Dockerfile. Instead, the dev image is now simply
a different target in the multi-layer openedx Docker image. This makes it much
faster to build the openedx-dev image.
Because we declare the APP_USER_ID in the dev/docker-compose.yml file, we need
to pass the user ID from the host there. The only way to achieve that is with a
tutor config variable. The downside of this approach is that the
dev/docker-compose.yml file is no longer portable from one machine to the next.
We consider that this is not such a big issue, as it affects the development
environment only.
We take this opportunity to replace the base image of the "forum" image. There
is now no need to re-install ruby inside the image. The total image size is
only decreased by 10%, but re-building the image is faster.
In order to run the smtp service as non-root, we switch from namshi/smtp to
devture/exim-relay. This change should be backward-compatible.
Note that the nginx container remains privileged. We could switch to
nginxinc/nginx-unprivileged, but it's probably not worth the effort, as we are
considering to get rid of the nginx container altogether.
Close #323.
2021-09-23 10:04:19 +00:00
|
|
|
BASE_IMAGE_NAMES = ["openedx", "forum", "permissions"]
|
2020-09-17 10:53:14 +00:00
|
|
|
VENDOR_IMAGES = [
|
|
|
|
"caddy",
|
|
|
|
"elasticsearch",
|
|
|
|
"mongodb",
|
|
|
|
"mysql",
|
|
|
|
"redis",
|
|
|
|
"smtp",
|
|
|
|
]
|
2019-07-03 14:09:33 +00:00
|
|
|
|
2019-04-23 07:57:55 +00:00
|
|
|
|
2019-05-29 08:30:30 +00:00
|
|
|
@click.group(name="images", short_help="Manage docker images")
|
2021-02-25 08:09:14 +00:00
|
|
|
def images_command() -> None:
|
2019-01-22 20:25:04 +00:00
|
|
|
pass
|
|
|
|
|
2019-04-23 07:57:55 +00:00
|
|
|
|
2019-01-22 20:25:04 +00:00
|
|
|
@click.command(
|
|
|
|
short_help="Build docker images",
|
2019-05-05 09:45:24 +00:00
|
|
|
help="Build the docker images necessary for an Open edX platform.",
|
2019-02-13 19:18:47 +00:00
|
|
|
)
|
2020-10-01 22:25:03 +00:00
|
|
|
@click.argument("image_names", metavar="image", nargs=-1)
|
2019-05-22 17:17:54 +00:00
|
|
|
@click.option(
|
|
|
|
"--no-cache", is_flag=True, help="Do not use cache when building the image"
|
|
|
|
)
|
2019-01-22 20:25:04 +00:00
|
|
|
@click.option(
|
2019-05-05 09:45:24 +00:00
|
|
|
"-a",
|
|
|
|
"--build-arg",
|
2020-09-04 10:35:44 +00:00
|
|
|
"build_args",
|
2019-05-05 09:45:24 +00:00
|
|
|
multiple=True,
|
|
|
|
help="Set build-time docker ARGS in the form 'myarg=value'. This option may be specified multiple times.",
|
2019-01-22 20:25:04 +00:00
|
|
|
)
|
2020-09-04 10:35:44 +00:00
|
|
|
@click.option(
|
|
|
|
"--add-host",
|
|
|
|
"add_hosts",
|
|
|
|
multiple=True,
|
|
|
|
help="Set a custom host-to-IP mapping (host:ip).",
|
|
|
|
)
|
2021-01-19 07:48:21 +00:00
|
|
|
@click.option(
|
|
|
|
"--target",
|
|
|
|
help="Set the target build stage to build.",
|
|
|
|
)
|
2021-09-27 08:35:48 +00:00
|
|
|
@click.option(
|
|
|
|
"-d",
|
|
|
|
"--docker-arg",
|
|
|
|
"docker_args",
|
|
|
|
multiple=True,
|
|
|
|
help="Set extra options for docker build command.",
|
|
|
|
)
|
2019-12-12 16:05:56 +00:00
|
|
|
@click.pass_obj
|
2021-02-25 08:09:14 +00:00
|
|
|
def build(
|
|
|
|
context: Context,
|
|
|
|
image_names: List[str],
|
|
|
|
no_cache: bool,
|
|
|
|
build_args: List[str],
|
|
|
|
add_hosts: List[str],
|
|
|
|
target: str,
|
2021-09-27 08:35:48 +00:00
|
|
|
docker_args: List[str],
|
2021-02-25 08:09:14 +00:00
|
|
|
) -> None:
|
2019-12-12 16:05:56 +00:00
|
|
|
config = tutor_config.load(context.root)
|
2020-09-04 10:35:44 +00:00
|
|
|
command_args = []
|
|
|
|
if no_cache:
|
|
|
|
command_args.append("--no-cache")
|
|
|
|
for build_arg in build_args:
|
|
|
|
command_args += ["--build-arg", build_arg]
|
|
|
|
for add_host in add_hosts:
|
|
|
|
command_args += ["--add-host", add_host]
|
2021-01-19 07:48:21 +00:00
|
|
|
if target:
|
|
|
|
command_args += ["--target", target]
|
2021-09-27 08:35:48 +00:00
|
|
|
if docker_args:
|
|
|
|
command_args += docker_args
|
2020-10-01 22:25:03 +00:00
|
|
|
for image in image_names:
|
|
|
|
build_image(context.root, config, image, *command_args)
|
2019-10-22 14:13:50 +00:00
|
|
|
|
2019-07-02 20:16:44 +00:00
|
|
|
|
2019-03-18 21:39:35 +00:00
|
|
|
@click.command(short_help="Pull images from the Docker registry")
|
2020-10-01 22:25:03 +00:00
|
|
|
@click.argument("image_names", metavar="image", nargs=-1)
|
2019-12-12 16:05:56 +00:00
|
|
|
@click.pass_obj
|
2021-02-25 08:09:14 +00:00
|
|
|
def pull(context: Context, image_names: List[str]) -> None:
|
2019-12-12 16:05:56 +00:00
|
|
|
config = tutor_config.load(context.root)
|
2020-10-01 22:25:03 +00:00
|
|
|
for image in image_names:
|
2020-10-02 11:02:10 +00:00
|
|
|
pull_image(config, image)
|
2019-10-22 14:13:50 +00:00
|
|
|
|
|
|
|
|
2019-03-18 20:53:18 +00:00
|
|
|
@click.command(short_help="Push images to the Docker registry")
|
2020-10-01 22:25:03 +00:00
|
|
|
@click.argument("image_names", metavar="image", nargs=-1)
|
2019-12-12 16:05:56 +00:00
|
|
|
@click.pass_obj
|
2021-02-25 08:09:14 +00:00
|
|
|
def push(context: Context, image_names: List[str]) -> None:
|
2019-12-12 16:05:56 +00:00
|
|
|
config = tutor_config.load(context.root)
|
2020-10-01 22:25:03 +00:00
|
|
|
for image in image_names:
|
|
|
|
push_image(config, image)
|
|
|
|
|
|
|
|
|
|
|
|
@click.command(short_help="Print tag associated to a Docker image")
|
|
|
|
@click.argument("image_names", metavar="image", nargs=-1)
|
|
|
|
@click.pass_obj
|
2021-02-25 08:09:14 +00:00
|
|
|
def printtag(context: Context, image_names: List[str]) -> None:
|
2020-10-01 22:25:03 +00:00
|
|
|
config = tutor_config.load(context.root)
|
|
|
|
for image in image_names:
|
2021-11-25 08:53:49 +00:00
|
|
|
to_print = []
|
2020-10-01 22:25:03 +00:00
|
|
|
for _img, tag in iter_images(config, image, BASE_IMAGE_NAMES):
|
2021-11-25 08:53:49 +00:00
|
|
|
to_print.append(tag)
|
2021-02-25 08:09:14 +00:00
|
|
|
for _plugin, _img, tag in iter_plugin_images(config, image, "build-image"):
|
2021-11-25 08:53:49 +00:00
|
|
|
to_print.append(tag)
|
|
|
|
|
|
|
|
if not to_print:
|
|
|
|
raise ImageNotFoundError(image)
|
|
|
|
|
|
|
|
for tag in to_print:
|
2020-10-01 22:25:03 +00:00
|
|
|
print(tag)
|
2019-10-24 19:34:14 +00:00
|
|
|
|
|
|
|
|
2021-04-06 10:09:00 +00:00
|
|
|
def build_image(root: str, config: Config, image: str, *args: str) -> None:
|
2021-11-25 08:53:49 +00:00
|
|
|
to_build = []
|
|
|
|
|
2020-10-01 22:14:06 +00:00
|
|
|
# Build base images
|
|
|
|
for img, tag in iter_images(config, image, BASE_IMAGE_NAMES):
|
2021-11-25 08:53:49 +00:00
|
|
|
to_build.append((tutor_env.pathjoin(root, "build", img), tag, args))
|
2020-10-01 22:14:06 +00:00
|
|
|
|
|
|
|
# Build plugin images
|
|
|
|
for plugin, img, tag in iter_plugin_images(config, image, "build-image"):
|
2021-11-25 08:53:49 +00:00
|
|
|
to_build.append(
|
|
|
|
(tutor_env.pathjoin(root, "plugins", plugin, "build", img), tag, args)
|
2020-10-01 22:14:06 +00:00
|
|
|
)
|
|
|
|
|
2021-11-25 08:53:49 +00:00
|
|
|
if not to_build:
|
|
|
|
raise ImageNotFoundError(image)
|
|
|
|
|
|
|
|
for path, tag, build_args in to_build:
|
|
|
|
images.build(path, tag, *args)
|
2020-10-01 22:14:06 +00:00
|
|
|
|
|
|
|
|
2021-04-06 10:09:00 +00:00
|
|
|
def pull_image(config: Config, image: str) -> None:
|
2021-11-25 08:53:49 +00:00
|
|
|
to_pull = []
|
2020-10-02 11:02:10 +00:00
|
|
|
for _img, tag in iter_images(config, image, all_image_names(config)):
|
2021-11-25 08:53:49 +00:00
|
|
|
to_pull.append(tag)
|
2020-10-01 22:14:06 +00:00
|
|
|
for _plugin, _img, tag in iter_plugin_images(config, image, "remote-image"):
|
2021-11-25 08:53:49 +00:00
|
|
|
to_pull.append(tag)
|
|
|
|
|
|
|
|
if not to_pull:
|
|
|
|
raise ImageNotFoundError(image)
|
|
|
|
|
|
|
|
for tag in to_pull:
|
2020-10-01 22:14:06 +00:00
|
|
|
images.pull(tag)
|
|
|
|
|
|
|
|
|
2021-04-06 10:09:00 +00:00
|
|
|
def push_image(config: Config, image: str) -> None:
|
2021-11-25 08:53:49 +00:00
|
|
|
to_push = []
|
2020-10-01 22:14:06 +00:00
|
|
|
for _img, tag in iter_images(config, image, BASE_IMAGE_NAMES):
|
2021-11-25 08:53:49 +00:00
|
|
|
to_push.append(tag)
|
2020-10-01 22:14:06 +00:00
|
|
|
for _plugin, _img, tag in iter_plugin_images(config, image, "remote-image"):
|
2021-11-25 08:53:49 +00:00
|
|
|
to_push.append(tag)
|
|
|
|
|
|
|
|
if not to_push:
|
|
|
|
raise ImageNotFoundError(image)
|
|
|
|
|
|
|
|
for tag in to_push:
|
2020-10-01 22:14:06 +00:00
|
|
|
images.push(tag)
|
|
|
|
|
|
|
|
|
2021-02-25 08:09:14 +00:00
|
|
|
def iter_images(
|
2021-04-06 10:09:00 +00:00
|
|
|
config: Config, image: str, image_list: List[str]
|
2021-02-25 08:09:14 +00:00
|
|
|
) -> Iterator[Tuple[str, str]]:
|
2020-10-01 22:14:06 +00:00
|
|
|
for img in image_list:
|
2019-07-02 20:16:44 +00:00
|
|
|
if image in [img, "all"]:
|
2019-10-22 14:13:50 +00:00
|
|
|
tag = images.get_tag(config, img)
|
2020-10-01 22:14:06 +00:00
|
|
|
yield img, tag
|
|
|
|
|
2019-07-02 20:16:44 +00:00
|
|
|
|
2021-02-25 08:09:14 +00:00
|
|
|
def iter_plugin_images(
|
2021-04-06 10:09:00 +00:00
|
|
|
config: Config, image: str, hook_name: str
|
2021-02-25 08:09:14 +00:00
|
|
|
) -> Iterator[Tuple[str, str, str]]:
|
2020-10-01 22:14:06 +00:00
|
|
|
for plugin, hook in plugins.iter_hooks(config, hook_name):
|
2021-04-06 10:09:00 +00:00
|
|
|
if not isinstance(hook, dict):
|
|
|
|
raise exceptions.TutorError(
|
|
|
|
"Invalid hook '{}': expected dict, got {}".format(
|
|
|
|
hook_name, hook.__class__
|
|
|
|
)
|
|
|
|
)
|
2019-07-02 20:16:44 +00:00
|
|
|
for img, tag in hook.items():
|
|
|
|
if image in [img, "all"]:
|
2020-07-21 07:13:00 +00:00
|
|
|
tag = tutor_env.render_str(config, tag)
|
2020-10-01 22:14:06 +00:00
|
|
|
yield plugin, img, tag
|
2019-01-22 20:25:04 +00:00
|
|
|
|
2019-04-23 07:57:55 +00:00
|
|
|
|
2021-04-06 10:09:00 +00:00
|
|
|
def all_image_names(config: Config) -> List[str]:
|
2019-10-22 14:13:50 +00:00
|
|
|
return BASE_IMAGE_NAMES + vendor_image_names(config)
|
2019-07-02 20:16:44 +00:00
|
|
|
|
|
|
|
|
2021-04-06 10:09:00 +00:00
|
|
|
def vendor_image_names(config: Config) -> List[str]:
|
2020-09-17 10:53:14 +00:00
|
|
|
vendor_images = VENDOR_IMAGES[:]
|
|
|
|
for image in VENDOR_IMAGES:
|
|
|
|
if not config.get("RUN_" + image.upper(), True):
|
2019-07-03 14:09:33 +00:00
|
|
|
vendor_images.remove(image)
|
2019-07-02 20:16:44 +00:00
|
|
|
return vendor_images
|
2019-03-18 21:39:35 +00:00
|
|
|
|
2019-04-23 07:57:55 +00:00
|
|
|
|
2021-11-25 08:53:49 +00:00
|
|
|
class ImageNotFoundError(exceptions.TutorError):
|
|
|
|
def __init__(self, image_name: str):
|
|
|
|
super().__init__("Image '{}' could not be found".format(image_name))
|
|
|
|
|
|
|
|
|
2019-04-23 07:57:55 +00:00
|
|
|
images_command.add_command(build)
|
|
|
|
images_command.add_command(pull)
|
|
|
|
images_command.add_command(push)
|
2020-10-01 22:25:03 +00:00
|
|
|
images_command.add_command(printtag)
|