christianlight/tutor
6
0
Fork 0
mirror of https://github.com/ChristianLight/tutor.git synced 2024-12-13 14:43:03 +00:00
Code Issues Releases Activity

Merge remote-tracking branch 'origin/master' into nightly

Browse Source
This commit is contained in:
Overhang.IO 2022-06-03 11:31:41 +00:00
commit 9a1cca6283
6 changed files with 91 additions and 98 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
CHANGELOG.md
View File

@ -18,6 +18,9 @@ Every user-facing change should have an entry in this changelog. Please respect
## Unreleased
- [Feature] Make it possible to force the rendering of a given template, even when the template path matches an ignore pattern. (by @regisb)
- 💥[Fix] Get rid of the `tutor config render` command, which is useless now that themes can be implemented as plugins. (by @regisb)
## v13.2.3 (2022-05-30)
- [Fix] Truncate site display name to 50 characters with a warning, fixing data too long error for long site names. (by @navinkarkera)

29
tests/commands/test_config.py
View File

@ -1,9 +1,6 @@
import os
import tempfile
import unittest
from tests.helpers import temporary_root
from tutor import config as tutor_config
from .base import TestCommandMixin
@ -61,29 +58,3 @@ class ConfigTests(unittest.TestCase, TestCommandMixin):
self.assertFalse(result.exception)
self.assertEqual(0, result.exit_code)
self.assertTrue(result.output)
def test_config_render(self) -> None:
with tempfile.TemporaryDirectory() as dest:
with temporary_root() as root:
self.invoke_in_root(root, ["config", "save"])
result = self.invoke_in_root(root, ["config", "render", root, dest])
self.assertEqual(0, result.exit_code)
self.assertFalse(result.exception)
def test_config_render_with_extra_configs(self) -> None:
with tempfile.TemporaryDirectory() as dest:
with temporary_root() as root:
self.invoke_in_root(root, ["config", "save"])
result = self.invoke_in_root(
root,
[
"config",
"render",
"-x",
os.path.join(root, tutor_config.CONFIG_FILENAME),
root,
dest,
],
)
self.assertEqual(0, result.exit_code)
self.assertFalse(result.exception)

26
tests/test_env.py
View File

@ -13,22 +13,36 @@ from tutor.types import Config
class EnvTests(PluginsTestCase):
def test_walk_templates(self) -> None:
renderer = env.Renderer({}, [env.TEMPLATES_ROOT])
renderer = env.Renderer()
templates = list(renderer.walk_templates("local"))
self.assertIn("local/docker-compose.yml", templates)
def test_walk_templates_partials_are_ignored(self) -> None:
template_name = "apps/openedx/settings/partials/common_all.py"
renderer = env.Renderer({}, [env.TEMPLATES_ROOT], ignore_folders=["partials"])
renderer = env.Renderer()
templates = list(renderer.walk_templates("apps"))
self.assertIn(template_name, renderer.environment.loader.list_templates())
self.assertNotIn(template_name, templates)
def test_files_are_rendered(self) -> None:
self.assertTrue(env.is_rendered("some/file"))
self.assertFalse(env.is_rendered(".git"))
self.assertFalse(env.is_rendered(".git/subdir"))
self.assertFalse(env.is_rendered("directory/.git"))
self.assertFalse(env.is_rendered("directory/.git/somefile"))
self.assertFalse(env.is_rendered("directory/somefile.pyc"))
self.assertTrue(env.is_rendered("directory/somedir.pyc/somefile"))
self.assertFalse(env.is_rendered("directory/__pycache__"))
self.assertFalse(env.is_rendered("directory/__pycache__/somefile"))
self.assertFalse(env.is_rendered("directory/partials/extra.scss"))
self.assertFalse(env.is_rendered("directory/partials"))
self.assertFalse(env.is_rendered("partials/somefile"))
def test_is_binary_file(self) -> None:
self.assertTrue(env.is_binary_file("/home/somefile.ico"))
def test_find_os_path(self) -> None:
renderer = env.Renderer({}, [env.TEMPLATES_ROOT])
renderer = env.Renderer()
path = renderer.find_os_path("local/docker-compose.yml")
self.assertTrue(os.path.exists(path))
@ -180,14 +194,14 @@ class EnvTests(PluginsTestCase):
# Load env once
config: Config = {"PLUGINS": []}
env1 = env.Renderer.instance(config).environment
env1 = env.Renderer(config).environment
# Enable plugins
plugins.load("plugin1")
# Load env a second time
config["PLUGINS"] = ["myplugin"]
env2 = env.Renderer.instance(config).environment
env2 = env.Renderer(config).environment
self.assertNotIn("plugin1/myplugin.txt", env1.loader.list_templates())
self.assertIn("plugin1/myplugin.txt", env2.loader.list_templates())
@ -199,7 +213,7 @@ class EnvTests(PluginsTestCase):
"notsomething_test_app": 2,
"something3_test_app": 3,
}
renderer = env.Renderer.instance(config)
renderer = env.Renderer(config)
self.assertEqual([2, 3], list(renderer.iter_values_named(suffix="test_app")))
self.assertEqual([1, 3], list(renderer.iter_values_named(prefix="something")))
self.assertEqual(

24
tutor/commands/config.py
View File

@ -64,29 +64,6 @@ def save(
env.save(context.root, config)
@click.command(help="Render a template folder with eventual extra configuration files")
@click.option(
"-x",
"--extra-config",
"extra_configs",
multiple=True,
type=click.Path(exists=True, resolve_path=True, dir_okay=False),
help="Load extra configuration file (can be used multiple times)",
)
@click.argument("src", type=click.Path(exists=True, resolve_path=True))
@click.argument("dst")
@click.pass_obj
def render(context: Context, extra_configs: List[str], src: str, dst: str) -> None:
config = tutor_config.load(context.root)
for extra_config in extra_configs:
config.update(
env.render_unknown(config, tutor_config.get_yaml_file(extra_config))
)
renderer = env.Renderer(config, [src])
renderer.render_all_to(dst)
fmt.echo_info(f"Templates rendered to {dst}")
@click.command(help="Print the project root")
@click.pass_obj
def printroot(context: Context) -> None:
@ -106,6 +83,5 @@ def printvalue(context: Context, key: str) -> None:
config_command.add_command(save)
config_command.add_command(render)
config_command.add_command(printroot)
config_command.add_command(printvalue)

90
tutor/env.py
View File

@ -1,4 +1,5 @@
import os
import re
import shutil
import typing as t
from copy import deepcopy
@ -67,26 +68,13 @@ class JinjaEnvironment(jinja2.Environment):
class Renderer:
@classmethod
def instance(cls: t.Type["Renderer"], config: Config) -> "Renderer":
# Load template roots: these are required to be able to use
# {% include .. %} directives
template_roots = hooks.Filters.ENV_TEMPLATE_ROOTS.apply([TEMPLATES_ROOT])
return cls(config, template_roots, ignore_folders=["partials"])
def __init__(
self,
config: Config,
template_roots: t.List[str],
ignore_folders: t.Optional[t.List[str]] = None,
):
def __init__(self, config: t.Optional[Config] = None):
config = config or {}
self.config = deepcopy(config)
self.template_roots = template_roots
self.ignore_folders = ignore_folders or []
self.ignore_folders.append(".git")
self.template_roots = hooks.Filters.ENV_TEMPLATE_ROOTS.apply([TEMPLATES_ROOT])
# Create environment with extra filters and globals
self.environment = JinjaEnvironment(template_roots)
self.environment = JinjaEnvironment(self.template_roots)
# Filters
plugin_filters: t.Iterator[
@ -116,8 +104,11 @@ class Renderer:
full_prefix = "/".join(prefix)
env_templates: t.List[str] = self.environment.loader.list_templates()
for template in env_templates:
if template.startswith(full_prefix) and self.is_part_of_env(template):
yield template
if template.startswith(full_prefix):
# Exclude templates that match certain patterns
# Note that here we don't rely on the OS separator, as we are handling templates.
if is_rendered(template):
yield template
def iter_values_named(
self,
@ -148,23 +139,7 @@ class Renderer:
Yield:
path: template path relative to the template root
"""
yield from self.iter_templates_in(subdir + "/")
def is_part_of_env(self, path: str) -> bool:
"""
Determines whether a template should be rendered or not. Note that here we don't
rely on the OS separator, as we are handling templates
"""
parts = path.split("/")
basename = parts[-1]
is_excluded = False
is_excluded = (
is_excluded or basename.startswith(".") or basename.endswith(".pyc")
)
is_excluded = is_excluded or basename == "__pycache__"
for ignore_folder in self.ignore_folders:
is_excluded = is_excluded or ignore_folder in parts
return not is_excluded
yield from self.iter_templates_in(subdir)
def find_os_path(self, template_name: str) -> str:
path = template_name.replace("/", os.sep)
@ -238,6 +213,43 @@ class Renderer:
raise exceptions.TutorError(f"Missing configuration value: {e.args[0]}")
def is_rendered(path: str) -> bool:
"""
Return whether the template should be rendered or not.
If the path matches an include pattern, it is rendered. If not and it matches an
ignore pattern, it is not rendered. By default, all files are rendered.
"""
include_patterns: t.Iterator[str] = hooks.Filters.ENV_PATTERNS_INCLUDE.iterate()
for include_pattern in include_patterns:
if re.match(include_pattern, path):
return True
ignore_patterns: t.Iterator[str] = hooks.Filters.ENV_PATTERNS_IGNORE.iterate()
for ignore_pattern in ignore_patterns:
if re.match(ignore_pattern, path):
return False
return True
# Skip rendering some files that follow commonly-ignored patterns:
#
# .*
# *.pyc
# __pycache__
# partials
hooks.Filters.ENV_PATTERNS_IGNORE.add_items(
[
# Skip all hidden files
r"(.*/)?\.",
# Skip compiled python files
r"(.*/)?__pycache__(/.*)?$",
r".*\.pyc$",
# Skip files from "partials" folders
r"(.*/)?partials(/.*)?$",
]
)
def save(root: str, config: Config) -> None:
"""
Save the full environment, including version information.
@ -264,7 +276,7 @@ def save_all_from(prefix: str, dst: str, config: Config) -> None:
Render the templates that start with `prefix` and store them with the same
hierarchy at `dst`. Here, `prefix` can be the result of os.path.join(...).
"""
renderer = Renderer.instance(config)
renderer = Renderer(config)
renderer.render_all_to(dst, prefix.replace(os.sep, "/"))
@ -285,7 +297,7 @@ def render_file(config: Config, *path: str) -> t.Union[str, bytes]:
"""
Return the rendered contents of a template.
"""
renderer = Renderer.instance(config)
renderer = Renderer(config)
template_name = "/".join(path)
return renderer.render_template(template_name)
@ -312,7 +324,7 @@ def render_str(config: Config, text: str) -> str:
Return:
substituted (str)
"""
return Renderer.instance(config).render_str(text)
return Renderer(config).render_str(text)
def check_is_up_to_date(root: str) -> None:

17
tutor/hooks/consts.py
View File

@ -223,6 +223,23 @@ class Filters:
#: filter to modify the Tutor templates.
ENV_PATCHES = filters.get("env:patches")
#: List of template path patterns to be ignored when rendering templates to the project root. By default, we ignore:
#:
#: - hidden files (``.*``)
#: - ``__pycache__`` directories and ``*.pyc`` files
#: - "partials" directories.
#:
#: Ignored patterns are overridden by include patterns; see :py:data:`ENV_PATTERNS_INCLUDE`.
#:
#: :parameter list[str] patterns: list of regular expression patterns. E.g: ``r"(.*/)?ignored_file_name(/.*)?"``.
ENV_PATTERNS_IGNORE = filters.get("env:patterns:ignore")
#: List of template path patterns to be included when rendering templates to the project root.
#: Patterns from this list will take priority over the patterns from :py:data:`ENV_PATTERNS_IGNORE`.
#:
#: :parameter list[str] patterns: list of regular expression patterns. See :py:data:`ENV_PATTERNS_IGNORE`.
ENV_PATTERNS_INCLUDE = filters.get("env:patterns:include")
#: List of all template root folders.
#:
#: :parameter list[str] templates_root: absolute paths to folders which contain templates.