diff --git a/CHANGELOG.md b/CHANGELOG.md index 59fb3cf..73840d4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,23 +1,30 @@ # Changelog +This file includes a history of past releases. Changes that were not yet added to a release are in the [changelog.d/](./changelog.d) folder. + -## Unreleased + + +## v14.2.1 (2022-11-24) + +- [Improvement] Auto-completion of `plugins` and `config` arguments: `plugins enable/disable NAME`, `plugins install PATH`, `config save --set KEY=VAL`, `config save --unset KEY`, `config printvalue KEY`. (by @regisb) +- [Bugfix] Fix minimum click version (>= 8.0.0) when installing tutor from pip. - [Improvement] Enable CORS by default for both LMS and CMS by moving those settings to the `common_all` partial. (by @arbrandes) ## v14.2.0 (2022-11-21) diff --git a/Makefile b/Makefile index 6d6905d..98b36ac 100644 --- a/Makefile +++ b/Makefile @@ -83,6 +83,13 @@ coverage-browse-report: coverage-html ## Open the HTML report in the browser bundle: ## Bundle the tutor package in a single "dist/tutor" executable pyinstaller tutor.spec + +changelog-entry: ## Create a new changelog entry + scriv create + +changelog: ## Collect changelog entries in the CHANGELOG.md file + scriv collect + release: test release-unsafe ## Create a release tag and push it to origin release-unsafe: $(MAKE) release-tag release-push TAG=v$(shell make version) diff --git a/CHANGELOG-nightly.md b/changelog.d/20221124_172610_regis_nightly.md similarity index 84% rename from CHANGELOG-nightly.md rename to changelog.d/20221124_172610_regis_nightly.md index 55903cf..cb257ff 100644 --- a/CHANGELOG-nightly.md +++ b/changelog.d/20221124_172610_regis_nightly.md @@ -1,13 +1,3 @@ -# Changelog (nightly branch) - - - [Improvement] Add the `-h` help option to all commands and subcommands. Previously, we could only use `--help`, which is quite long for lazy fingers. (by @regisb) - 💥[Feature] Add an extensible `local/dev/k8s do ...` command to trigger custom job commands. These commands are used to run a series of bash scripts in designated containers. Any plugin can add custom jobs thanks to the `CLI_DO_COMMANDS` filter. This causes the following breaking changes: - The "init", "createuser", "settheme", "importdemocourse" commands were all migrated to this new interface. For instance, `tutor local init` was replaced by `tutor local do init`. @@ -23,4 +13,3 @@ facing changes. - [Bugfix] Update ``celery`` invocations for lms-worker and cms-worker to be compatible with Celery 5 CLI. - [Improvement] Point CMS at its config file using ``CMS_CFG`` environment variable instead of deprecated ``STUDIO_CFG``. - [Bugfix] Start MongoDB when running migrations, because a new data migration fails if MongoDB is not running - diff --git a/changelog.d/scriv.ini b/changelog.d/scriv.ini new file mode 100644 index 0000000..9236140 --- /dev/null +++ b/changelog.d/scriv.ini @@ -0,0 +1,7 @@ +[scriv] +version = literal: tutor/__about__.py: __version__ +categories = +format = md +md_header_level = 2 +new_fragment_template = file: scriv/new_fragment.${config:format}.j2 +entry_title_template = file: scriv/entry_title.${config:format}.j2 diff --git a/changelog.d/scriv/entry_title.md.j2 b/changelog.d/scriv/entry_title.md.j2 new file mode 100644 index 0000000..7afe894 --- /dev/null +++ b/changelog.d/scriv/entry_title.md.j2 @@ -0,0 +1 @@ +{% if version %}v{{ version }} {% endif %}({{ date.strftime('%Y-%m-%d') }}) diff --git a/changelog.d/scriv/new_fragment.md.j2 b/changelog.d/scriv/new_fragment.md.j2 new file mode 100644 index 0000000..78988a4 --- /dev/null +++ b/changelog.d/scriv/new_fragment.md.j2 @@ -0,0 +1,11 @@ + + + + diff --git a/docs/tutor.rst b/docs/tutor.rst index 04e87e3..ef9d872 100644 --- a/docs/tutor.rst +++ b/docs/tutor.rst @@ -69,9 +69,9 @@ Releasing a new version ~~~~~~~~~~~~~~~~~~~~~~~ - Bump the ``__version__`` value in ``tutor/__about__.py``. (see :ref:`versioning` below) -- Replace "Unreleased" with the version name and date in CHANGELOG.md. +- Collect changelog entries with ``make changelog``. - Create a commit with the version changelog. -- Run ``make release``: this will push to the default repo/branch for the current branch. +- Run ``make release``: this will create a tag and push to the default repo/branch for the current branch. .. _versioning: @@ -102,12 +102,7 @@ Third-party contributions to Tutor and its plugins are more than welcome! Just m - Outside of obvious bugs, contributions should be discussed first in the `official Open edX forum `__. - Once we agree on a high-level solution, you should open a pull request on the `Tutor repository `__ or the corresponding plugin. - Make sure that all tests pass by running ``make test`` (see above). -- If your PR is in the Tutor core repository, add an item to the CHANGELOG file, in the "Unreleased" section. Use the same format as the other items:: - - - [TYPE] DESCRIPTION - -Where "TYPE" is either "Bugfix", "Improvement", "Feature" or "Security". You should add an explosion emoji ("💥") before "[TYPE]" if you are making a breaking change. - +- If your PR is in the Tutor core repository, add a changelog entry by running ``make changelog-entry``. Edit the new file and follow the formatting instructions that it contains. - Write a good Git commit title and message: explain why you are making this change, what problem you are solving and which solution you adopted. Link to the relevant conversation topics in the forums and describe your use case. We *love* long, verbose descriptions :) As for the title, `conventional commits `__ are preferred. Check the repo history! Happy hacking! ☘️ diff --git a/docs/tutorials/nightly.rst b/docs/tutorials/nightly.rst index 0418a9f..6ee60f3 100644 --- a/docs/tutorials/nightly.rst +++ b/docs/tutorials/nightly.rst @@ -61,4 +61,4 @@ Making changes to Tutor Nightly In general pull requests should be open on the "master" branch of Tutor: the "master" branch is automatically merged on the "nightly" branch at every commit, such that changes made to Tutor releases find their way to Tutor Nightly as soon as they are merged. However, sometimes you want to make changes to Tutor Nightly exclusively, and not to the Tutor releases. This might be the case for instance when upgrading the running version of a third-party service (for instance: Elasticsearch, MySQL), or when the master branch requires specific changes. In that case, you should follow the instructions from the :ref:`contributing` section of the docs, with the following differences: - Open your pull request on top of the "nightly" branch instead of "master". -- Add a description of your changes to CHANGELOG-nightly.md instead of CHANGELOG.md +- Add a description of your changes by creating a changelog entry with `make changelog-entry`, as in the master branch. diff --git a/requirements/dev.in b/requirements/dev.in index 6540e89..4c1100d 100644 --- a/requirements/dev.in +++ b/requirements/dev.in @@ -1,10 +1,11 @@ -r base.txt black +coverage pip-tools pylint pyinstaller +scriv twine -coverage # doc requirement is lagging behind # https://github.com/readthedocs/sphinx_rtd_theme/issues/1323 diff --git a/requirements/dev.txt b/requirements/dev.txt index 80048a8..5d30588 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -10,6 +10,8 @@ appdirs==1.4.4 # via -r requirements/base.txt astroid==2.12.13 # via pylint +attrs==22.1.0 + # via scriv black==22.10.0 # via -r requirements/dev.in bleach==5.0.1 @@ -35,7 +37,11 @@ click==8.1.3 # via # -r requirements/base.txt # black + # click-log # pip-tools + # scriv +click-log==0.4.0 + # via scriv commonmark==0.9.1 # via rich coverage==6.5.0 @@ -69,7 +75,9 @@ jeepney==0.8.0 # keyring # secretstorage jinja2==3.1.2 - # via -r requirements/base.txt + # via + # -r requirements/base.txt + # scriv keyring==23.11.0 # via twine kubernetes==25.3.0 @@ -150,6 +158,7 @@ requests==2.28.1 # kubernetes # requests-oauthlib # requests-toolbelt + # scriv # twine requests-oauthlib==1.3.1 # via @@ -165,6 +174,8 @@ rsa==4.9 # via # -r requirements/base.txt # google-auth +scriv==0.17.0 + # via -r requirements/dev.in secretstorage==3.3.3 # via keyring six==1.16.0 diff --git a/tests/commands/test_plugins.py b/tests/commands/test_plugins.py index 81b9d3a..ccd6e08 100644 --- a/tests/commands/test_plugins.py +++ b/tests/commands/test_plugins.py @@ -2,6 +2,7 @@ import unittest from unittest.mock import Mock, patch from tutor import plugins +from tutor.commands import plugins as plugins_commands from .base import TestCommandMixin @@ -40,3 +41,17 @@ class PluginsTests(unittest.TestCase, TestCommandMixin): result = self.invoke(["plugins", "disable", "notFound"]) self.assertEqual(0, result.exit_code) self.assertFalse(result.exception) + + @patch.object( + plugins, + "iter_info", + return_value=[("aacd", None), ("abcd", None), ("abef", None), ("alba", None)], + ) + def test_plugins_name_auto_complete(self, _iter_info: Mock) -> None: + self.assertEqual([], plugins_commands.PluginName().get_names("z")) + self.assertEqual( + ["abcd", "abef"], plugins_commands.PluginName().get_names("ab") + ) + self.assertEqual( + ["all", "alba"], plugins_commands.PluginName(allow_all=True).get_names("al") + ) diff --git a/tests/test_serialize.py b/tests/test_serialize.py index 4555751..f0bd9fd 100644 --- a/tests/test_serialize.py +++ b/tests/test_serialize.py @@ -1,7 +1,5 @@ import unittest -import click - from tutor import serialize @@ -31,17 +29,15 @@ class SerializeTests(unittest.TestCase): def test_parse_empty_string(self) -> None: self.assertEqual("", serialize.parse("''")) - def test_yaml_param_type(self) -> None: - param = serialize.YamlParamType() - self.assertEqual(("name", True), param.convert("name=true", "param", {})) - self.assertEqual(("name", "abcd"), param.convert("name=abcd", "param", {})) - self.assertEqual(("name", ""), param.convert("name=", "param", {})) - with self.assertRaises(click.exceptions.BadParameter): - param.convert("name", "param", {}) - self.assertEqual(("x", "a=bcd"), param.convert("x=a=bcd", "param", {})) + def test_parse_key_value(self) -> None: + self.assertEqual(("name", True), serialize.parse_key_value("name=true")) + self.assertEqual(("name", "abcd"), serialize.parse_key_value("name=abcd")) + self.assertEqual(("name", ""), serialize.parse_key_value("name=")) + self.assertIsNone(serialize.parse_key_value("name")) + self.assertEqual(("x", "a=bcd"), serialize.parse_key_value("x=a=bcd")) self.assertEqual( ("x", {"key1": {"subkey": "value"}, "key2": {"subkey": "value"}}), - param.convert( - "x=key1:\n subkey: value\nkey2:\n subkey: value", "param", {} + serialize.parse_key_value( + "x=key1:\n subkey: value\nkey2:\n subkey: value" ), ) diff --git a/tutor/__about__.py b/tutor/__about__.py index be4248a..c05d6fe 100644 --- a/tutor/__about__.py +++ b/tutor/__about__.py @@ -2,7 +2,7 @@ import os # Increment this version number to trigger a new release. See # docs/tutor.html#versioning for information on the versioning scheme. -__version__ = "14.2.0" +__version__ = "14.2.1" # The version suffix will be appended to the actual version, separated by a # dash. Use this suffix to differentiate between the actual released version and diff --git a/tutor/commands/config.py b/tutor/commands/config.py index 83d347a..f48bf85 100644 --- a/tutor/commands/config.py +++ b/tutor/commands/config.py @@ -1,12 +1,14 @@ -from typing import List +import json +import typing as t import click +import click.shell_completion from .. import config as tutor_config from .. import env, exceptions, fmt from .. import interactive as interactive_config from .. import serialize -from ..types import Config +from ..types import Config, ConfigValue from .context import Context @@ -19,13 +21,82 @@ def config_command() -> None: pass +class ConfigKeyParamType(click.ParamType): + + name = "configkey" + + def shell_complete( + self, ctx: click.Context, param: click.Parameter, incomplete: str + ) -> t.List[click.shell_completion.CompletionItem]: + return [ + click.shell_completion.CompletionItem(key) + for key, _value in self._shell_complete_config_items(ctx, incomplete) + ] + + @staticmethod + def _shell_complete_config_items( + ctx: click.Context, incomplete: str + ) -> t.List[t.Tuple[str, ConfigValue]]: + # Here we want to auto-complete the name of the config key. For that we need to + # figure out the list of enabled plugins, and for that we need the project root. + # The project root would ordinarily be stored in ctx.obj.root, but during + # auto-completion we don't have access to our custom Tutor context. So we resort + # to a dirty hack, which is to examine the grandparent context. + root = getattr( + getattr(getattr(ctx, "parent", None), "parent", None), "params", {} + ).get("root", "") + config = tutor_config.load_full(root) + return [ + (key, value) for key, value in config.items() if key.startswith(incomplete) + ] + + +class ConfigKeyValParamType(ConfigKeyParamType): + """ + Parser for = command line arguments. + """ + + name = "configkeyval" + + def convert(self, value: str, param: t.Any, ctx: t.Any) -> t.Tuple[str, t.Any]: + result = serialize.parse_key_value(value) + if result is None: + self.fail(f"'{value}' is not of the form 'key=value'.", param, ctx) + return result + + def shell_complete( + self, ctx: click.Context, param: click.Parameter, incomplete: str + ) -> t.List[click.shell_completion.CompletionItem]: + """ + Nice and friendly = auto-completion. + """ + if "=" not in incomplete: + # Auto-complete with '='. Note the single quotes which allow users to + # further auto-complete later. + return [ + click.shell_completion.CompletionItem(f"'{key}='") + for key, value in self._shell_complete_config_items(ctx, incomplete) + ] + if incomplete.endswith("="): + # raise ValueError(f"incomplete: <{incomplete}>") + # Auto-complete with '=' + return [ + click.shell_completion.CompletionItem(f"{key}={json.dumps(value)}") + for key, value in self._shell_complete_config_items( + ctx, incomplete[:-1] + ) + ] + # Else, don't bother + return [] + + @click.command(help="Create and save configuration interactively") @click.option("-i", "--interactive", is_flag=True, help="Run interactively") @click.option( "-s", "--set", "set_vars", - type=serialize.YamlParamType(), + type=ConfigKeyValParamType(), multiple=True, metavar="KEY=VAL", help="Set a configuration value (can be used multiple times)", @@ -35,17 +106,18 @@ def config_command() -> None: "--unset", "unset_vars", multiple=True, + type=ConfigKeyParamType(), help="Remove a configuration value (can be used multiple times)", ) @click.option( - "-e", "--env-only", "env_only", is_flag=True, help="Skip updating config.yaml" + "-e", "--env-only", "env_only", is_flag=True, help="Skip updating config.yml" ) @click.pass_obj def save( context: Context, interactive: bool, set_vars: Config, - unset_vars: List[str], + unset_vars: t.List[str], env_only: bool, ) -> None: config = tutor_config.load_minimal(context.root) @@ -71,7 +143,7 @@ def printroot(context: Context) -> None: @click.command(help="Print a configuration value") -@click.argument("key") +@click.argument("key", type=ConfigKeyParamType()) @click.pass_obj def printvalue(context: Context, key: str) -> None: config = tutor_config.load(context.root) diff --git a/tutor/commands/plugins.py b/tutor/commands/plugins.py index 5438ec1..ed67149 100644 --- a/tutor/commands/plugins.py +++ b/tutor/commands/plugins.py @@ -3,6 +3,7 @@ import typing as t import urllib.request import click +import click.shell_completion from tutor import config as tutor_config from tutor import exceptions, fmt, hooks, plugins @@ -11,6 +12,31 @@ from tutor.plugins.base import PLUGINS_ROOT, PLUGINS_ROOT_ENV_VAR_NAME from .context import Context +class PluginName(click.ParamType): + """ + Convenient param type that supports plugin name autocompletion. + """ + + def __init__(self, allow_all: bool = False): + self.allow_all = allow_all + + def shell_complete( + self, ctx: click.Context, param: click.Parameter, incomplete: str + ) -> t.List[click.shell_completion.CompletionItem]: + return [ + click.shell_completion.CompletionItem(name) + for name in self.get_names(incomplete) + ] + + def get_names(self, incomplete: str) -> t.List[str]: + candidates = [] + if self.allow_all: + candidates.append("all") + candidates += [name for name, _ in plugins.iter_info()] + + return [name for name in candidates if name.startswith(incomplete)] + + @click.group( name="plugins", short_help="Manage Tutor plugins", @@ -34,12 +60,12 @@ def list_command() -> None: lines.append((plugin, status, plugin_info)) first_column_width = max([first_column_width, len(plugin) + 2]) - for line in lines: - print("{:{width}}\t{:10}\t{}".format(*line, width=first_column_width)) + for plugin, status, plugin_info in lines: + print(f"{plugin:{first_column_width}}\t{status:10}\t{plugin_info}") @click.command(help="Enable a plugin") -@click.argument("plugin_names", metavar="plugin", nargs=-1) +@click.argument("plugin_names", metavar="plugin", nargs=-1, type=PluginName()) @click.pass_obj def enable(context: Context, plugin_names: t.List[str]) -> None: config = tutor_config.load_minimal(context.root) @@ -57,7 +83,9 @@ def enable(context: Context, plugin_names: t.List[str]) -> None: short_help="Disable a plugin", help="Disable one or more plugins. Specify 'all' to disable all enabled plugins at once.", ) -@click.argument("plugin_names", metavar="plugin", nargs=-1) +@click.argument( + "plugin_names", metavar="plugin", nargs=-1, type=PluginName(allow_all=True) +) @click.pass_obj def disable(context: Context, plugin_names: t.List[str]) -> None: config = tutor_config.load_minimal(context.root) @@ -90,7 +118,7 @@ def printroot() -> None: help=f"""Install a plugin, either from a local Python/YAML file or a remote, web-hosted location. The plugin will be installed to {PLUGINS_ROOT_ENV_VAR_NAME}.""", ) -@click.argument("location") +@click.argument("location", type=click.Path(dir_okay=False)) def install(location: str) -> None: basename = os.path.basename(location) if not basename.endswith(".yml") and not basename.endswith(".py"): diff --git a/tutor/serialize.py b/tutor/serialize.py index e835c23..415f75c 100644 --- a/tutor/serialize.py +++ b/tutor/serialize.py @@ -1,32 +1,31 @@ import re -from typing import IO, Any, Iterator, Tuple, Union +import typing as t -import click import yaml from _io import TextIOWrapper from yaml.parser import ParserError from yaml.scanner import ScannerError -def load(stream: Union[str, IO[str]]) -> Any: +def load(stream: t.Union[str, t.IO[str]]) -> t.Any: return yaml.load(stream, Loader=yaml.SafeLoader) -def load_all(stream: str) -> Iterator[Any]: +def load_all(stream: str) -> t.Iterator[t.Any]: return yaml.load_all(stream, Loader=yaml.SafeLoader) -def dump(content: Any, fileobj: TextIOWrapper) -> None: +def dump(content: t.Any, fileobj: TextIOWrapper) -> None: yaml.dump(content, stream=fileobj, default_flow_style=False) -def dumps(content: Any) -> str: +def dumps(content: t.Any) -> str: result = yaml.dump(content, default_flow_style=False) assert isinstance(result, str) return result -def parse(v: Union[str, IO[str]]) -> Any: +def parse(v: t.Union[str, t.IO[str]]) -> t.Any: """ Parse a yaml-formatted string. """ @@ -37,17 +36,18 @@ def parse(v: Union[str, IO[str]]) -> Any: return v -class YamlParamType(click.ParamType): - name = "yaml" - PARAM_REGEXP = r"(?P[a-zA-Z0-9_-]+)=(?P(.|\n|\r)*)" +def parse_key_value(text: str) -> t.Optional[t.Tuple[str, t.Any]]: + """ + Parse = command line arguments. - def convert(self, value: str, param: Any, ctx: Any) -> Tuple[str, Any]: - match = re.match(self.PARAM_REGEXP, value) - if not match: - self.fail("'{}' is not of the form 'key=value'.".format(value), param, ctx) - key = match.groupdict()["key"] - value = match.groupdict()["value"] - if not value: - # Empty strings are interpreted as null values, which is incorrect. - value = "''" - return key, parse(value) + Return None if text could not be parsed. + """ + match = re.match(r"(?P[a-zA-Z0-9_-]+)=(?P(.|\n|\r)*)", text) + if not match: + return None + key = match.groupdict()["key"] + value = match.groupdict()["value"] + if not value: + # Empty strings are interpreted as null values, which is incorrect. + value = "''" + return key, parse(value)