feat: select tasks by marker metadata (#977)

Author: tobiasraabe

CHANGELOG.md

@@ -7,6 +7,8 @@ releases are available on [PyPI](https://pypi.org/project/pytask) and
 
 ## Unreleased
 
+- [#977](https://github.com/pytask-dev/pytask/pull/977) allows marker expressions
+  passed to `-m` to select tasks by marker keyword arguments.
 - [#976](https://github.com/pytask-dev/pytask/pull/976) makes invalid marker and
   keyword expressions raise the standard `SyntaxError` instead of a custom parser
   exception.

docs/source/tutorials/selecting_tasks.md

@@ -31,6 +31,26 @@ analysis.
 $ pytask -m "(data_management and not plots) or (analysis and plots)"
 ```
 
+Markers can also store metadata in keyword arguments. For example, the following marker
+records the data and model used by a prediction task.
+
+```python
+@pytask.mark.prediction(data="customer_churn", model="random_forest")
+def task_create_predictions():
+    pass
+```
+
+Pass keyword arguments in a marker expression to select tasks by this metadata.
+
+```console
+$ pytask -m 'prediction(data="customer_churn", model="random_forest")'
+```
+
+A task matches if one marker with the requested name contains all keyword arguments in
+the expression. The marker can contain additional keyword arguments. Marker expressions
+support unescaped strings, positive and negative integers, `True`, `False`, and `None`.
+Keyword arguments are only supported in marker expressions passed to `-m`.
+
 If you create your markers, use the
 [`pytask markers`](../reference_guides/commands.md#pytask-markers) command to register
 and document them.

src/_pytask/lock.py

@@ -80,6 +80,9 @@ def _expression_filter(
             f"at column {e.offset}: {e.msg}"
         )
         raise ValueError(msg) from None
+    if option == "-k" and compiled.has_keyword_arguments():
+        msg = "Keyword expressions do not support call parameters."
+        raise ValueError(msg)
 
     return {
         task.signature for task in tasks if compiled.evaluate(matcher_from_task(task))

src/_pytask/mark/__init__.py

@@ -151,11 +151,11 @@ def from_task(cls, task: PTask) -> KeywordMatcher:
 
         return cls(mapped_names)
 
-    def __call__(self, subname: str) -> bool:
+    def __call__(self, subname: str, /, **kwargs: str | int | bool | None) -> bool:
         subname = subname.lower()
         names = (name.lower() for name in self._names)
 
-        return any(subname in name for name in names)
+        return not kwargs and any(subname in name for name in names)
 
 
 def select_by_keyword(session: Session, dag: DAG) -> set[str] | None:
@@ -171,6 +171,9 @@ def select_by_keyword(session: Session, dag: DAG) -> set[str] | None:
             f"Wrong expression passed to '-k': {e.text}: at column {e.offset}: {e.msg}"
         )
         raise ValueError(msg) from None
+    if expression.has_keyword_arguments():
+        msg = "Keyword expressions do not support call parameters."
+        raise ValueError(msg)
 
     remaining: set[str] = set()
     for task in session.tasks:
@@ -190,6 +193,9 @@ def select_by_after_keyword(session: Session, after: str) -> set[str]:
             f"at column {e.offset}: {e.msg}"
         )
         raise ValueError(msg) from None
+    if expression.has_keyword_arguments():
+        msg = "Keyword expressions do not support call parameters."
+        raise ValueError(msg)
 
     ancestors: set[str] = set()
     for task in session.tasks:
@@ -199,6 +205,9 @@ def select_by_after_keyword(session: Session, after: str) -> set[str]:
     return ancestors
 
 
+_NOT_SET = object()
+
+
 @dataclass(slots=True)
 class MarkMatcher:
     """A matcher for markers which are present.
@@ -207,15 +216,22 @@ class MarkMatcher:
 
     """
 
-    own_mark_names: set[str]
+    own_mark_name_mapping: dict[str, list[Mark]]
 
     @classmethod
     def from_task(cls, task: PTask) -> MarkMatcher:
-        mark_names = {mark.name for mark in task.markers}
-        return cls(mark_names)
-
-    def __call__(self, name: str) -> bool:
-        return name in self.own_mark_names
+        mark_name_mapping: dict[str, list[Mark]] = {}
+        for mark in task.markers:
+            mark_name_mapping.setdefault(mark.name, []).append(mark)
+        return cls(mark_name_mapping)
+
+    def __call__(self, name: str, /, **kwargs: str | int | bool | None) -> bool:
+        for mark in self.own_mark_name_mapping.get(name, []):
+            if all(
+                mark.kwargs.get(key, _NOT_SET) == value for key, value in kwargs.items()
+            ):
+                return True
+        return False
 
 
 def select_by_mark(session: Session, dag: DAG) -> set[str] | None:

src/_pytask/mark/expression.py

@@ -2,22 +2,22 @@
 
 The grammar is:
 
-+------------+--------------------------------------------+
-| expression | expr? EOF                                  |
-+------------+--------------------------------------------+
-| expr       | and_expr ('or' and_expr)*                  |
-+------------+--------------------------------------------+
-| and_expr   | not_expr ('and' not_expr)*                 |
-+------------+--------------------------------------------+
-| not_expr   | ``'not' not_expr | '(' expr ')' | ident``  |
-+------------+--------------------------------------------+
-| ident      | ``(\w|:|\+|-|\.|\[|\]|\\)+``               |
-+------------+--------------------------------------------+
+expression: expr? EOF
+expr:       and_expr ('or' and_expr)*
+and_expr:   not_expr ('and' not_expr)*
+not_expr:   'not' not_expr | '(' expr ')' | ident kwargs?
+
+ident:      (\w|:|\+|-|\.|\[|\]|\\|/)+
+kwargs:     ('(' name '=' value (', ' name '=' value)* ')')
+name:       a valid identifier that is not a reserved keyword
+value:      unescaped string literal | (-)?[0-9]+ | 'False' | 'True' | 'None'
 
 The semantics are:
 
 - Empty expression evaluates to False.
-- ident evaluates to True of False according to a provided matcher function.
+- ident evaluates to True or False according to a provided matcher function.
+- ident with keyword arguments evaluates to True or False according to a provided
+  matcher function.
 - or/and/not evaluate according to the usual boolean semantics.
 
 This module is adapted from pytest's ``_pytest.mark.expression`` module:
@@ -29,20 +29,21 @@
 
 import ast
 import enum
+import keyword
 import re
-from collections.abc import Callable
 from collections.abc import Iterator
 from collections.abc import Mapping
 from collections.abc import Sequence
 from dataclasses import dataclass
 from typing import TYPE_CHECKING
+from typing import Protocol
 
 if TYPE_CHECKING:
     import types
     from typing import NoReturn
 
 
-__all__ = ["Expression"]
+__all__ = ["Expression", "ExpressionMatcher"]
 
 
 FILE_NAME = "<pytask match expression>"
@@ -56,6 +57,9 @@ class TokenType(enum.Enum):
     NOT = "not"
     IDENT = "identifier"
     EOF = "end of input"
+    EQUAL = "="
+    STRING = "string literal"
+    COMMA = ","
 
 
 @dataclass(frozen=True, slots=True)
@@ -74,7 +78,7 @@ def __init__(self, input_: str) -> None:
         self.tokens = self.lex(input_)
         self.current = next(self.tokens)
 
-    def lex(self, input_: str) -> Iterator[Token]:
+    def lex(self, input_: str) -> Iterator[Token]:  # noqa: C901, PLR0912
         pos = 0
         while pos < len(input_):
             if input_[pos] in (" ", "\t"):
@@ -85,6 +89,29 @@ def lex(self, input_: str) -> Iterator[Token]:
             elif input_[pos] == ")":
                 yield Token(TokenType.RPAREN, ")", pos)
                 pos += 1
+            elif input_[pos] == "=":
+                yield Token(TokenType.EQUAL, "=", pos)
+                pos += 1
+            elif input_[pos] == ",":
+                yield Token(TokenType.COMMA, ",", pos)
+                pos += 1
+            elif (quote_char := input_[pos]) in ("'", '"'):
+                end_quote_pos = input_.find(quote_char, pos + 1)
+                if end_quote_pos == -1:
+                    msg = f'closing quote "{quote_char}" is missing'
+                    raise SyntaxError(
+                        msg,
+                        (FILE_NAME, 1, pos + 1, input_),
+                    )
+                value = input_[pos : end_quote_pos + 1]
+                if (backslash_pos := value.find("\\")) != -1:
+                    msg = r'escaping with "\" not supported in marker expression'
+                    raise SyntaxError(
+                        msg,
+                        (FILE_NAME, 1, pos + backslash_pos + 1, input_),
+                    )
+                yield Token(TokenType.STRING, value, pos)
+                pos += len(value)
             else:
                 match = re.match(r"(:?\w|:|\+|-|\.|\[|\]|/|\\)+", input_[pos:])
                 if match:
@@ -168,19 +195,95 @@ def not_expr(s: Scanner) -> ast.expr:
     ident = s.accept(TokenType.IDENT)
     if ident:
         s.idents.add(ident.value)
-        return ast.Name(IDENT_PREFIX + ident.value, ast.Load())
+        name = ast.Name(IDENT_PREFIX + ident.value, ast.Load())
+        if s.accept(TokenType.LPAREN):
+            ret = ast.Call(func=name, args=[], keywords=all_kwargs(s))
+            s.accept(TokenType.RPAREN, reject=True)
+        else:
+            ret = name
+        return ret
     s.reject((TokenType.NOT, TokenType.LPAREN, TokenType.IDENT))
     return None  # ty: ignore[invalid-return-type]  # Unreachable: reject() raises
 
 
-class MatcherAdapter(Mapping[str, bool]):
+BUILTIN_MATCHERS = {"True": True, "False": False, "None": None}
+
+
+def single_kwarg(s: Scanner) -> ast.keyword:
+    """Parse one keyword argument."""
+    keyword_name = s.accept(TokenType.IDENT, reject=True)
+    assert keyword_name is not None
+    if not keyword_name.value.isidentifier():
+        msg = f"not a valid python identifier {keyword_name.value}"
+        raise SyntaxError(
+            msg,
+            (FILE_NAME, 1, keyword_name.pos + 1, s.input),
+        )
+    if keyword.iskeyword(keyword_name.value):
+        msg = f"unexpected reserved python keyword `{keyword_name.value}`"
+        raise SyntaxError(
+            msg,
+            (FILE_NAME, 1, keyword_name.pos + 1, s.input),
+        )
+    s.accept(TokenType.EQUAL, reject=True)
+
+    if value_token := s.accept(TokenType.STRING):
+        value: str | int | bool | None = value_token.value[1:-1]
+    else:
+        value_token = s.accept(TokenType.IDENT, reject=True)
+        assert value_token is not None
+        if (number := value_token.value).isdigit() or (
+            number.startswith("-") and number[1:].isdigit()
+        ):
+            value = int(number)
+        elif value_token.value in BUILTIN_MATCHERS:
+            value = BUILTIN_MATCHERS[value_token.value]
+        else:
+            msg = f'unexpected character/s "{value_token.value}"'
+            raise SyntaxError(
+                msg,
+                (FILE_NAME, 1, value_token.pos + 1, s.input),
+            )
+
+    return ast.keyword(keyword_name.value, ast.Constant(value))
+
+
+def all_kwargs(s: Scanner) -> list[ast.keyword]:
+    """Parse all keyword arguments."""
+    kwargs = [single_kwarg(s)]
+    while s.accept(TokenType.COMMA):
+        kwargs.append(single_kwarg(s))
+    return kwargs
+
+
+class ExpressionMatcher(Protocol):
+    """Match an identifier and optional keyword arguments."""
+
+    def __call__(self, name: str, /, **kwargs: str | int | bool | None) -> bool: ...
+
+
+@dataclass
+class MatcherNameAdapter:
+    """Adapt one matcher name to boolean and callable expression forms."""
+
+    matcher: ExpressionMatcher
+    name: str
+
+    def __bool__(self) -> bool:
+        return self.matcher(self.name)
+
+    def __call__(self, **kwargs: str | int | bool | None) -> bool:
+        return self.matcher(self.name, **kwargs)
+
+
+class MatcherAdapter(Mapping[str, MatcherNameAdapter]):
     """Adapts a matcher function to a locals mapping as required by eval()."""
 
-    def __init__(self, matcher: Callable[[str], bool]) -> None:
+    def __init__(self, matcher: ExpressionMatcher) -> None:
         self.matcher = matcher
 
-    def __getitem__(self, key: str) -> bool:
-        return self.matcher(key[len(IDENT_PREFIX) :])
+    def __getitem__(self, key: str) -> MatcherNameAdapter:
+        return MatcherNameAdapter(self.matcher, key[len(IDENT_PREFIX) :])
 
     def __iter__(self) -> Iterator[str]:  # pragma: no cover
         raise NotImplementedError
@@ -196,11 +299,17 @@ class Expression:
 
     """
 
-    __slots__ = ("_idents", "code")
+    __slots__ = ("_has_keyword_arguments", "_idents", "code")
 
-    def __init__(self, code: types.CodeType, idents: frozenset[str]) -> None:
+    def __init__(
+        self,
+        code: types.CodeType,
+        idents: frozenset[str],
+        has_keyword_arguments: bool,
+    ) -> None:
         self.code = code
         self._idents = idents
+        self._has_keyword_arguments = has_keyword_arguments
 
     @classmethod
     def compile_(cls, input_: str) -> Expression:
@@ -218,13 +327,20 @@ def compile_(cls, input_: str) -> Expression:
             filename="<pytask match expression>",
             mode="eval",
         )
-        return cls(code, idents)
+        has_keyword_arguments = any(
+            isinstance(node, ast.Call) for node in ast.walk(astexpr)
+        )
+        return cls(code, idents, has_keyword_arguments)
 
     def idents(self) -> frozenset[str]:
         """Return all identifiers which appear in the expression."""
         return self._idents
 
-    def evaluate(self, matcher: Callable[[str], bool]) -> bool:
+    def has_keyword_arguments(self) -> bool:
+        """Return whether the expression contains marker keyword arguments."""
+        return self._has_keyword_arguments
+
+    def evaluate(self, matcher: ExpressionMatcher) -> bool:
         """Evaluate the match expression.
 
         Parameters
@@ -239,7 +355,8 @@ def evaluate(self, matcher: Callable[[str], bool]) -> bool:
             Whether the expression matches or not.
 
         """
-        ret: bool = eval(  # noqa: S307
-            self.code, {"__builtins__": {}}, MatcherAdapter(matcher)
+        return bool(
+            eval(  # noqa: S307
+                self.code, {"__builtins__": {}}, MatcherAdapter(matcher)
+            )
         )
-        return ret