From e990291d16649854dfb1b774fc72b75bb685152b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Behmo?= Date: Wed, 2 Jun 2021 15:16:32 +0200 Subject: [PATCH] feat: upgrade pinned requirements to click 8+ We were forced to pin click to < v8 because of missing dependencies. In particular, click_repl was broken. This is no longer the case, as click_repl 0.20 was published. Also, Jinja2 now includes type annotations, which allows us to get rid of a few "# type: ignore" statements. We take the opportunity to upgrade all requirements, which allows us resolve a security issue on urllib3<1.26.0. --- requirements/base.in | 9 ++------- requirements/base.txt | 26 +++++++++++--------------- requirements/dev.txt | 39 +++++++++++++++------------------------ requirements/docs.txt | 28 +++++++++++++--------------- tests/test_env.py | 6 +++--- tutor/env.py | 17 ++++++++++------- 6 files changed, 54 insertions(+), 71 deletions(-) diff --git a/requirements/base.in b/requirements/base.in index 9a4fd12..b3a3ad9 100644 --- a/requirements/base.in +++ b/requirements/base.in @@ -1,13 +1,8 @@ appdirs -click>=7.0,<8.0 +click click_repl mypy pycryptodome -jinja2>=2.9,<3.0 +jinja2 kubernetes pyyaml>=4.2b1 - -# Add constraints until we are compatible with click 8.0 -markupsafe<2.0 -# Installing urllib3==1.26.0 causes compatibility errors with requests==2.24.0 -urllib3<1.26.0 diff --git a/requirements/base.txt b/requirements/base.txt index 05326b3..9acb263 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -8,35 +8,33 @@ appdirs==1.4.4 # via -r requirements/base.in cachetools==4.2.2 # via google-auth -certifi==2020.12.5 +certifi==2021.5.30 # via # kubernetes # requests chardet==4.0.0 # via requests -click-repl==0.1.6 +click-repl==0.2.0 # via -r requirements/base.in -click==7.1.2 +click==8.0.1 # via # -r requirements/base.in # click-repl -google-auth==1.30.0 +google-auth==1.30.1 # via kubernetes idna==2.10 # via requests -jinja2==2.11.3 +jinja2==3.0.1 # via -r requirements/base.in -kubernetes==12.0.1 +kubernetes==17.17.0 # via -r requirements/base.in -markupsafe==1.1.1 - # via - # -r requirements/base.in - # jinja2 +markupsafe==2.0.1 + # via jinja2 mypy-extensions==0.4.3 # via mypy mypy==0.812 # via -r requirements/base.in -oauthlib==3.1.0 +oauthlib==3.1.1 # via requests-oauthlib prompt-toolkit==3.0.18 # via click-repl @@ -68,19 +66,17 @@ six==1.16.0 # google-auth # kubernetes # python-dateutil - # websocket-client typed-ast==1.4.3 # via mypy typing-extensions==3.10.0.0 # via mypy -urllib3==1.25.11 +urllib3==1.26.5 # via - # -r requirements/base.in # kubernetes # requests wcwidth==0.2.5 # via prompt-toolkit -websocket-client==0.59.0 +websocket-client==1.0.1 # via kubernetes # The following packages are considered to be unsafe in a requirements file: diff --git a/requirements/dev.txt b/requirements/dev.txt index 7c5756c..1ec50fa 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -12,7 +12,7 @@ appdirs==1.4.4 # black astroid==2.5.6 # via pylint -black==21.5b1 +black==21.5b2 # via -r requirements/dev.in bleach==3.3.0 # via readme-renderer @@ -20,7 +20,7 @@ cachetools==4.2.2 # via # -r requirements/base.txt # google-auth -certifi==2020.12.5 +certifi==2021.5.30 # via # -r requirements/base.txt # kubernetes @@ -31,9 +31,9 @@ chardet==4.0.0 # via # -r requirements/base.txt # requests -click-repl==0.1.6 +click-repl==0.2.0 # via -r requirements/base.txt -click==7.1.2 +click==8.0.1 # via # -r requirements/base.txt # black @@ -45,7 +45,7 @@ cryptography==3.4.7 # via secretstorage docutils==0.17.1 # via readme-renderer -google-auth==1.30.0 +google-auth==1.30.1 # via # -r requirements/base.txt # kubernetes @@ -53,11 +53,9 @@ idna==2.10 # via # -r requirements/base.txt # requests -importlib-metadata==4.0.1 +importlib-metadata==4.4.0 # via # keyring - # pep517 - # pyinstaller # twine isort==5.8.0 # via pylint @@ -65,15 +63,15 @@ jeepney==0.6.0 # via # keyring # secretstorage -jinja2==2.11.3 +jinja2==3.0.1 # via -r requirements/base.txt keyring==23.0.1 # via twine -kubernetes==12.0.1 +kubernetes==17.17.0 # via -r requirements/base.txt lazy-object-proxy==1.6.0 # via astroid -markupsafe==1.1.1 +markupsafe==2.0.1 # via # -r requirements/base.txt # jinja2 @@ -86,7 +84,7 @@ mypy-extensions==0.4.3 # mypy mypy==0.812 # via -r requirements/base.txt -oauthlib==3.1.0 +oauthlib==3.1.1 # via # -r requirements/base.txt # requests-oauthlib @@ -123,7 +121,7 @@ pyinstaller-hooks-contrib==2021.1 # via pyinstaller pyinstaller==4.3 # via -r requirements/dev.in -pylint==2.8.2 +pylint==2.8.3 # via -r requirements/dev.in pyparsing==2.4.7 # via packaging @@ -169,29 +167,24 @@ six==1.16.0 # kubernetes # python-dateutil # readme-renderer - # websocket-client toml==0.10.2 # via # black # pep517 # pylint -tqdm==4.60.0 +tqdm==4.61.0 # via twine twine==3.4.1 # via -r requirements/dev.in typed-ast==1.4.3 # via # -r requirements/base.txt - # astroid - # black # mypy typing-extensions==3.10.0.0 # via # -r requirements/base.txt - # black - # importlib-metadata # mypy -urllib3==1.25.11 +urllib3==1.26.5 # via # -r requirements/base.txt # kubernetes @@ -202,16 +195,14 @@ wcwidth==0.2.5 # prompt-toolkit webencodings==0.5.1 # via bleach -websocket-client==0.59.0 +websocket-client==1.0.1 # via # -r requirements/base.txt # kubernetes wrapt==1.12.1 # via astroid zipp==3.4.1 - # via - # importlib-metadata - # pep517 + # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: # pip diff --git a/requirements/docs.txt b/requirements/docs.txt index 2e7aa1c..eb47ae7 100644 --- a/requirements/docs.txt +++ b/requirements/docs.txt @@ -14,7 +14,7 @@ cachetools==4.2.2 # via # -r requirements/base.txt # google-auth -certifi==2020.12.5 +certifi==2021.5.30 # via # -r requirements/base.txt # kubernetes @@ -23,9 +23,9 @@ chardet==4.0.0 # via # -r requirements/base.txt # requests -click-repl==0.1.6 +click-repl==0.2.0 # via -r requirements/base.txt -click==7.1.2 +click==8.0.1 # via # -r requirements/base.txt # click-repl @@ -33,7 +33,7 @@ docutils==0.16 # via # sphinx # sphinx-rtd-theme -google-auth==1.30.0 +google-auth==1.30.1 # via # -r requirements/base.txt # kubernetes @@ -43,24 +43,23 @@ idna==2.10 # requests imagesize==1.2.0 # via sphinx -jinja2==2.11.3 +jinja2==3.0.1 # via # -r requirements/base.txt # sphinx -kubernetes==12.0.1 +kubernetes==17.17.0 # via -r requirements/base.txt -markupsafe==1.1.1 +markupsafe==2.0.1 # via # -r requirements/base.txt # jinja2 - # sphinx mypy-extensions==0.4.3 # via # -r requirements/base.txt # mypy mypy==0.812 # via -r requirements/base.txt -oauthlib==3.1.0 +oauthlib==3.1.1 # via # -r requirements/base.txt # requests-oauthlib @@ -116,12 +115,11 @@ six==1.16.0 # google-auth # kubernetes # python-dateutil - # websocket-client snowballstemmer==2.1.0 # via sphinx sphinx-rtd-theme==0.5.2 # via -r requirements/docs.in -sphinx==4.0.1 +sphinx==4.0.2 # via # -r requirements/docs.in # sphinx-rtd-theme @@ -129,13 +127,13 @@ sphinxcontrib-applehelp==1.0.2 # via sphinx sphinxcontrib-devhelp==1.0.2 # via sphinx -sphinxcontrib-htmlhelp==1.0.3 +sphinxcontrib-htmlhelp==2.0.0 # via sphinx sphinxcontrib-jsmath==1.0.1 # via sphinx sphinxcontrib-qthelp==1.0.3 # via sphinx -sphinxcontrib-serializinghtml==1.1.4 +sphinxcontrib-serializinghtml==1.1.5 # via sphinx typed-ast==1.4.3 # via @@ -145,7 +143,7 @@ typing-extensions==3.10.0.0 # via # -r requirements/base.txt # mypy -urllib3==1.25.11 +urllib3==1.26.5 # via # -r requirements/base.txt # kubernetes @@ -154,7 +152,7 @@ wcwidth==0.2.5 # via # -r requirements/base.txt # prompt-toolkit -websocket-client==0.59.0 +websocket-client==1.0.1 # via # -r requirements/base.txt # kubernetes diff --git a/tests/test_env.py b/tests/test_env.py index e01c220..fe09c04 100644 --- a/tests/test_env.py +++ b/tests/test_env.py @@ -20,7 +20,7 @@ class EnvTests(unittest.TestCase): template_name = "apps/openedx/settings/partials/common_all.py" renderer = env.Renderer({}, [env.TEMPLATES_ROOT], ignore_folders=["partials"]) templates = list(renderer.walk_templates("apps")) - self.assertIn(template_name, renderer.environment.loader.list_templates()) # type: ignore + self.assertIn(template_name, renderer.environment.loader.list_templates()) self.assertNotIn(template_name, templates) def test_is_binary_file(self) -> None: @@ -174,5 +174,5 @@ class EnvTests(unittest.TestCase): config["PLUGINS"] = ["myplugin"] env2 = env.Renderer.instance(config).environment - self.assertNotIn("plugin1/myplugin.txt", env1.loader.list_templates()) # type: ignore - self.assertIn("plugin1/myplugin.txt", env2.loader.list_templates()) # type: ignore + self.assertNotIn("plugin1/myplugin.txt", env1.loader.list_templates()) + self.assertIn("plugin1/myplugin.txt", env2.loader.list_templates()) diff --git a/tutor/env.py b/tutor/env.py index 258a857..adb79f2 100644 --- a/tutor/env.py +++ b/tutor/env.py @@ -15,6 +15,14 @@ VERSION_FILENAME = "version" BIN_FILE_EXTENSIONS = [".ico", ".jpg", ".png", ".ttf", ".woff", ".woff2"] +class JinjaEnvironment(jinja2.Environment): + loader: jinja2.BaseLoader + + def __init__(self, template_roots: List[str]) -> None: + loader = jinja2.FileSystemLoader(template_roots) + super().__init__(loader=loader, undefined=jinja2.StrictUndefined) + + class Renderer: @classmethod def instance(cls: Type["Renderer"], config: Config) -> "Renderer": @@ -39,10 +47,7 @@ class Renderer: self.ignore_folders.append(".git") # Create environment - environment = jinja2.Environment( - loader=jinja2.FileSystemLoader(template_roots), - undefined=jinja2.StrictUndefined, - ) + environment = JinjaEnvironment(template_roots) environment.filters["common_domain"] = utils.common_domain environment.filters["encrypt"] = utils.encrypt environment.filters["list_if"] = utils.list_if @@ -61,9 +66,7 @@ class Renderer: The elements of `prefix` must contain only "/", and not os.sep. """ full_prefix = "/".join(prefix) - env_templates: List[ - str - ] = self.environment.loader.list_templates() # type:ignore[no-untyped-call] + env_templates: List[str] = self.environment.loader.list_templates() for template in env_templates: if template.startswith(full_prefix) and self.is_part_of_env(template): yield template