Coverage for python / lsst / pipe / base / _task_metadata.py: 17%
215 statements
« prev ^ index » next coverage.py v7.13.2, created at 2026-02-03 01:30 -0800
1# This file is part of pipe_base.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This software is dual licensed under the GNU General Public License and also
10# under a 3-clause BSD license. Recipients may choose which of these licenses
11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt,
12# respectively. If you choose the GPL option then the following text applies
13# (but note that there is still no warranty even if you opt for BSD instead):
14#
15# This program is free software: you can redistribute it and/or modify
16# it under the terms of the GNU General Public License as published by
17# the Free Software Foundation, either version 3 of the License, or
18# (at your option) any later version.
19#
20# This program is distributed in the hope that it will be useful,
21# but WITHOUT ANY WARRANTY; without even the implied warranty of
22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
23# GNU General Public License for more details.
24#
25# You should have received a copy of the GNU General Public License
26# along with this program. If not, see <https://www.gnu.org/licenses/>.
28__all__ = [
29 "GetDictMetadata",
30 "GetSetDictMetadata",
31 "NestedMetadataDict",
32 "SetDictMetadata",
33 "TaskMetadata",
34]
36import itertools
37import numbers
38import sys
39from collections.abc import Collection, Iterator, Mapping, Sequence
40from typing import Any, Protocol
42from pydantic import BaseModel, ConfigDict, Field, StrictBool, StrictFloat, StrictInt, StrictStr
44# The types allowed in a Task metadata field are restricted
45# to allow predictable serialization.
46_ALLOWED_PRIMITIVE_TYPES = (str, float, int, bool)
48# Note that '|' syntax for unions doesn't work when we have to use a string
49# literal (and we do since it's recursive and not an annotation).
50type NestedMetadataDict = Mapping[str, str | float | int | bool | "NestedMetadataDict"]
53class PropertySetLike(Protocol):
54 """Protocol that looks like a ``lsst.daf.base.PropertySet``.
56 Enough of the API is specified to support conversion of a
57 ``PropertySet`` to a `TaskMetadata`.
58 """
60 def paramNames(self, topLevelOnly: bool = True) -> Collection[str]: ... 60 ↛ exitline 60 didn't return from function 'paramNames' because
62 def getArray(self, name: str) -> Any: ... 62 ↛ exitline 62 didn't return from function 'getArray' because
65def _isListLike(v: Any) -> bool:
66 return isinstance(v, Sequence) and not isinstance(v, str)
69class SetDictMetadata(Protocol):
70 """Protocol for objects that can be assigned a possibly-nested `dict` of
71 primitives.
73 This protocol is satisfied by `TaskMetadata`, `lsst.daf.base.PropertySet`,
74 and `lsst.daf.base.PropertyList`, providing a consistent way to insert a
75 dictionary into these objects that avoids their historical idiosyncrasies.
77 The form in which these entries appear in the object's native keys and
78 values is implementation-defined. *Empty nested dictionaries may be
79 dropped, and if the top-level dictionary is empty this method may do
80 nothing.*
82 Neither the top-level key nor nested keys may contain ``.`` (period)
83 characters.
84 """
86 def set_dict(self, key: str, nested: NestedMetadataDict) -> None: ... 86 ↛ exitline 86 didn't return from function 'set_dict' because
89class GetDictMetadata(Protocol):
90 """Protocol for objects that can extract a possibly-nested mapping of
91 primitives.
93 This protocol is satisfied by `TaskMetadata`, `lsst.daf.base.PropertySet`,
94 and `lsst.daf.base.PropertyList`, providing a consistent way to extract a
95 dictionary from these objects that avoids their historical idiosyncrasies.
97 This is guaranteed to work for mappings inserted by
98 `~SetMapping.set_dict`. It should not be expected to work for values
99 inserted in other ways. If a value was never inserted with the given key
100 at all, *an empty `dict` will be returned* (this is a concession to
101 implementation constraints in `~lsst.daf.base.PropertyList`.
102 """
104 def get_dict(self, key: str) -> NestedMetadataDict: ... 104 ↛ exitline 104 didn't return from function 'get_dict' because
107class GetSetDictMetadata(SetDictMetadata, GetDictMetadata, Protocol):
108 """Protocol for objects that can assign and extract a possibly-nested
109 mapping of primitives.
110 """
113class TaskMetadata(BaseModel):
114 """Dict-like object for storing task metadata.
116 Metadata can be stored at two levels: single task or task plus subtasks.
117 The later is called full metadata of a task and has a form
119 topLevelTaskName:subtaskName:subsubtaskName.itemName
121 Metadata item key of a task (`itemName` above) must not contain `.`,
122 which serves as a separator in full metadata keys and turns
123 the value into sub-dictionary. Arbitrary hierarchies are supported.
124 """
126 # Pipelines regularly generate NaN and Inf so these need to be
127 # supported even though that's a JSON extension.
128 model_config = ConfigDict(ser_json_inf_nan="constants")
130 scalars: dict[str, StrictFloat | StrictInt | StrictBool | StrictStr] = Field(default_factory=dict)
131 arrays: dict[str, list[StrictFloat] | list[StrictInt] | list[StrictBool] | list[StrictStr]] = Field(
132 default_factory=dict
133 )
134 metadata: dict[str, "TaskMetadata"] = Field(default_factory=dict)
136 @classmethod
137 def from_dict(cls, d: Mapping[str, Any]) -> "TaskMetadata":
138 """Create a TaskMetadata from a dictionary.
140 Parameters
141 ----------
142 d : `~collections.abc.Mapping`
143 Mapping to convert. Can be hierarchical. Any dictionaries
144 in the hierarchy are converted to `TaskMetadata`.
146 Returns
147 -------
148 meta : `TaskMetadata`
149 Newly-constructed metadata.
150 """
151 metadata = cls()
152 for k, v in d.items():
153 metadata[k] = v
154 return metadata
156 @classmethod
157 def from_metadata(cls, ps: PropertySetLike) -> "TaskMetadata":
158 """Create a TaskMetadata from a PropertySet-like object.
160 Parameters
161 ----------
162 ps : `PropertySetLike` or `TaskMetadata`
163 A ``PropertySet``-like object to be transformed to a
164 `TaskMetadata`. A `TaskMetadata` can be copied using this
165 class method.
167 Returns
168 -------
169 tm : `TaskMetadata`
170 Newly-constructed metadata.
172 Notes
173 -----
174 Items stored in single-element arrays in the supplied object
175 will be converted to scalars in the newly-created object.
176 """
177 # Use hierarchical names to assign values from input to output.
178 # This API exists for both PropertySet and TaskMetadata.
179 # from_dict() does not work because PropertySet is not declared
180 # to be a Mapping.
181 # PropertySet.toDict() is not present in TaskMetadata so is best
182 # avoided.
183 metadata = cls()
184 for key in sorted(ps.paramNames(topLevelOnly=False)):
185 value = ps.getArray(key)
186 if len(value) == 1:
187 value = value[0]
188 metadata[key] = value
189 return metadata
191 def to_dict(self) -> dict[str, Any]:
192 """Convert the class to a simple dictionary.
194 Returns
195 -------
196 d : `dict`
197 Simple dictionary that can contain scalar values, array values
198 or other dictionary values.
200 Notes
201 -----
202 Unlike `dict()`, this method hides the model layout and combines
203 scalars, arrays, and other metadata in the same dictionary. Can be
204 used when a simple dictionary is needed. Use
205 `TaskMetadata.from_dict()` to convert it back.
206 """
207 d: dict[str, Any] = {}
208 d.update(self.scalars)
209 d.update(self.arrays)
210 for k, v in self.metadata.items():
211 d[k] = v.to_dict()
212 return d
214 def add(self, name: str, value: Any) -> None:
215 """Store a new value, adding to a list if one already exists.
217 Parameters
218 ----------
219 name : `str`
220 Name of the metadata property.
221 value : `~typing.Any`
222 Metadata property value.
223 """
224 keys = self._getKeys(name)
225 key0 = keys.pop(0)
226 if len(keys) == 0:
227 # If add() is being used, always store the value in the arrays
228 # property as a list. It's likely there will be another call.
229 slot_type, value = self._validate_value(value)
230 if slot_type == "array":
231 pass
232 elif slot_type == "scalar":
233 value = [value]
234 else:
235 raise ValueError("add() can only be used for primitive types or sequences of those types.")
237 if key0 in self.metadata:
238 raise ValueError(f"Can not add() to key '{name}' since that is a TaskMetadata")
240 if key0 in self.scalars:
241 # Convert scalar to array.
242 # MyPy should be able to figure out that List[Union[T1, T2]] is
243 # compatible with Union[List[T1], List[T2]] if the list has
244 # only one element, but it can't.
245 self.arrays[key0] = [self.scalars.pop(key0)] # type: ignore
247 if key0 in self.arrays:
248 # Check that the type is not changing.
249 if (curtype := type(self.arrays[key0][0])) is not (newtype := type(value[0])):
250 raise ValueError(f"Type mismatch in add() -- currently {curtype} but adding {newtype}")
251 self.arrays[key0].extend(value)
252 else:
253 self.arrays[key0] = value
255 return
257 self.metadata[key0].add(".".join(keys), value)
259 def getScalar(self, key: str) -> str | int | float | bool:
260 """Retrieve a scalar item even if the item is a list.
262 Parameters
263 ----------
264 key : `str`
265 Item to retrieve.
267 Returns
268 -------
269 value : `str`, `int`, `float`, or `bool`
270 Either the value associated with the key or, if the key
271 corresponds to a list, the last item in the list.
273 Raises
274 ------
275 KeyError
276 Raised if the item is not found.
277 """
278 # Used in pipe_tasks.
279 # getScalar() is the default behavior for __getitem__.
280 return self[key]
282 def getArray(self, key: str) -> list[Any]:
283 """Retrieve an item as a list even if it is a scalar.
285 Parameters
286 ----------
287 key : `str`
288 Item to retrieve.
290 Returns
291 -------
292 values : `list` of any
293 A list containing the value or values associated with this item.
295 Raises
296 ------
297 KeyError
298 Raised if the item is not found.
299 """
300 keys = self._getKeys(key)
301 key0 = keys.pop(0)
302 if len(keys) == 0:
303 if key0 in self.arrays:
304 return self.arrays[key0]
305 elif key0 in self.scalars:
306 return [self.scalars[key0]]
307 elif key0 in self.metadata:
308 return [self.metadata[key0]]
309 raise KeyError(f"'{key}' not found")
311 try:
312 return self.metadata[key0].getArray(".".join(keys))
313 except KeyError:
314 # Report the correct key.
315 raise KeyError(f"'{key}' not found") from None
317 def names(self) -> set[str]:
318 """Return the hierarchical keys from the metadata.
320 Returns
321 -------
322 names : `collections.abc.Set`
323 A set of all keys, including those from the hierarchy and the
324 top-level hierarchy.
325 """
326 names = set()
327 for k, v in self.items():
328 names.add(k) # Always include the current level
329 if isinstance(v, TaskMetadata):
330 names.update({k + "." + item for item in v.names()})
331 return names
333 def paramNames(self, topLevelOnly: bool) -> set[str]:
334 """Return hierarchical names.
336 Parameters
337 ----------
338 topLevelOnly : `bool`
339 Control whether only top-level items are returned or items
340 from the hierarchy.
342 Returns
343 -------
344 paramNames : `set` of `str`
345 If ``topLevelOnly`` is `True`, returns any keys that are not
346 part of a hierarchy. If `False` also returns fully-qualified
347 names from the hierarchy. Keys associated with the top
348 of a hierarchy are never returned.
349 """
350 # Currently used by the verify package.
351 paramNames = set()
352 for k, v in self.items():
353 if isinstance(v, TaskMetadata):
354 if not topLevelOnly:
355 paramNames.update({k + "." + item for item in v.paramNames(topLevelOnly=topLevelOnly)})
356 else:
357 paramNames.add(k)
358 return paramNames
360 @staticmethod
361 def _getKeys(key: str) -> list[str]:
362 """Return the key hierarchy.
364 Parameters
365 ----------
366 key : `str`
367 The key to analyze. Can be dot-separated.
369 Returns
370 -------
371 keys : `list` of `str`
372 The key hierarchy that has been split on ``.``.
374 Raises
375 ------
376 KeyError
377 Raised if the key is not a string.
378 """
379 try:
380 keys = key.split(".")
381 except Exception:
382 raise KeyError(f"Invalid key '{key}': only string keys are allowed") from None
383 return keys
385 def keys(self) -> tuple[str, ...]:
386 """Return the top-level keys."""
387 return tuple(k for k in self)
389 def items(self) -> Iterator[tuple[str, Any]]:
390 """Yield the top-level keys and values."""
391 yield from itertools.chain(self.scalars.items(), self.arrays.items(), self.metadata.items())
393 def __len__(self) -> int:
394 """Return the number of items."""
395 return len(self.scalars) + len(self.arrays) + len(self.metadata)
397 # This is actually a Liskov substitution violation, because
398 # pydantic.BaseModel says __iter__ should return something else. But the
399 # pydantic docs say to do exactly this to in order to make a mapping-like
400 # BaseModel, so that's what we do.
401 def __iter__(self) -> Iterator[str]: # type: ignore
402 """Return an iterator over each key."""
403 # The order of keys is not preserved since items can move
404 # from scalar to array.
405 return itertools.chain(iter(self.scalars), iter(self.arrays), iter(self.metadata))
407 def __getitem__(self, key: str) -> Any:
408 """Retrieve the item associated with the key.
410 Parameters
411 ----------
412 key : `str`
413 The key to retrieve. Can be dot-separated hierarchical.
415 Returns
416 -------
417 value : `TaskMetadata`, `float`, `int`, `bool`, `str`
418 A scalar value. For compatibility with ``PropertySet``, if the key
419 refers to an array, the final element is returned and not the
420 array itself.
422 Raises
423 ------
424 KeyError
425 Raised if the item is not found.
426 """
427 keys = self._getKeys(key)
428 key0 = keys.pop(0)
429 if len(keys) == 0:
430 if key0 in self.scalars:
431 return self.scalars[key0]
432 if key0 in self.metadata:
433 return self.metadata[key0]
434 if key0 in self.arrays:
435 arr = self.arrays[key0]
436 if not arr:
437 # If there are no elements then returning a scalar
438 # is an error.
439 raise KeyError(f"'{key}' not found")
440 return arr[-1]
441 raise KeyError(f"'{key}' not found")
442 # Hierarchical lookup so the top key can only be in the metadata
443 # property. Trap KeyError and reraise so that the correct key
444 # in the hierarchy is reported.
445 try:
446 # And forward request to that metadata.
447 return self.metadata[key0][".".join(keys)]
448 except KeyError:
449 raise KeyError(f"'{key}' not found") from None
451 def get(self, key: str, default: Any = None) -> Any:
452 """Retrieve the item associated with the key or a default.
454 Parameters
455 ----------
456 key : `str`
457 The key to retrieve. Can be dot-separated hierarchical.
458 default : `~typing.Any`
459 The value to return if the key does not exist.
461 Returns
462 -------
463 value : `TaskMetadata`, `float`, `int`, `bool`, `str`
464 A scalar value. If the key refers to an array, the final element
465 is returned and not the array itself; this is consistent with
466 `__getitem__` and `PropertySet.get`, but not ``to_dict().get``.
467 """
468 try:
469 return self[key]
470 except KeyError:
471 return default
473 def __setitem__(self, key: str, item: Any) -> None:
474 """Store the given item."""
475 keys = self._getKeys(key)
476 key0 = keys.pop(0)
477 if len(keys) == 0:
478 slots: dict[str, dict[str, Any]] = {
479 "array": self.arrays,
480 "scalar": self.scalars,
481 "metadata": self.metadata,
482 }
483 primary: dict[str, Any] | None = None
484 slot_type, item = self._validate_value(item)
485 primary = slots.pop(slot_type, None)
486 if primary is None:
487 raise AssertionError(f"Unknown slot type returned from validator: {slot_type}")
489 # Assign the value to the right place.
490 primary[key0] = item
491 for property in slots.values():
492 # Remove any other entries.
493 property.pop(key0, None)
494 return
496 # This must be hierarchical so forward to the child TaskMetadata.
497 if key0 not in self.metadata:
498 self.metadata[key0] = TaskMetadata()
499 self.metadata[key0][".".join(keys)] = item
501 # Ensure we have cleared out anything with the same name elsewhere.
502 self.scalars.pop(key0, None)
503 self.arrays.pop(key0, None)
505 def __contains__(self, key: str) -> bool:
506 """Determine if the key exists."""
507 keys = self._getKeys(key)
508 key0 = keys.pop(0)
509 if len(keys) == 0:
510 return key0 in self.scalars or key0 in self.arrays or key0 in self.metadata
512 if key0 in self.metadata:
513 return ".".join(keys) in self.metadata[key0]
514 return False
516 def __delitem__(self, key: str) -> None:
517 """Remove the specified item.
519 Raises
520 ------
521 KeyError
522 Raised if the item is not present.
523 """
524 keys = self._getKeys(key)
525 key0 = keys.pop(0)
526 if len(keys) == 0:
527 # MyPy can't figure out that this way to combine the types in the
528 # tuple is the one that matters, and annotating a local variable
529 # helps it out.
530 properties: tuple[dict[str, Any], ...] = (self.scalars, self.arrays, self.metadata)
531 for property in properties:
532 if key0 in property:
533 del property[key0]
534 return
535 raise KeyError(f"'{key}' not found'")
537 try:
538 del self.metadata[key0][".".join(keys)]
539 except KeyError:
540 # Report the correct key.
541 raise KeyError(f"'{key}' not found'") from None
543 def get_dict(self, key: str) -> NestedMetadataDict:
544 """Return a possibly-hierarchical nested `dict`.
546 This implements the `GetDictMetadata` protocol for consistency with
547 `lsst.daf.base.PropertySet` and `lsst.daf.base.PropertyList`. The
548 returned `dict` is guaranteed to be a deep copy, not a view.
550 Parameters
551 ----------
552 key : `str`
553 String key associated with the mapping. May not have a ``.``
554 character.
556 Returns
557 -------
558 value : `~collections.abc.Mapping`
559 Possibly-nested mapping, with `str` keys and values that are `int`,
560 `float`, `str`, `bool`, or another `dict` with the same key and
561 value types. Will be empty if ``key`` does not exist.
562 """
563 if value := self.get(key):
564 return value.to_dict()
565 else:
566 return {}
568 def set_dict(self, key: str, value: NestedMetadataDict) -> None:
569 """Assign a possibly-hierarchical nested `dict`.
571 This implements the `SetDictMetadata` protocol for consistency with
572 `lsst.daf.base.PropertySet` and `lsst.daf.base.PropertyList`.
574 Parameters
575 ----------
576 key : `str`
577 String key associated with the mapping. May not have a ``.``
578 character.
579 value : `~collections.abc.Mapping`
580 Possibly-nested mapping, with `str` keys and values that are `int`,
581 `float`, `str`, `bool`, or another `dict` with the same key and
582 value types. Nested keys may not have a ``.`` character.
583 """
584 self[key] = value
586 def _validate_value(self, value: Any) -> tuple[str, Any]:
587 """Validate the given value.
589 Parameters
590 ----------
591 value : Any
592 Value to check.
594 Returns
595 -------
596 slot_type : `str`
597 The type of value given. Options are "scalar", "array", "metadata".
598 item : Any
599 The item that was given but possibly modified to conform to
600 the slot type.
602 Raises
603 ------
604 ValueError
605 Raised if the value is not a recognized type.
606 """
607 # Test the simplest option first.
608 value_type = type(value)
609 if value_type in _ALLOWED_PRIMITIVE_TYPES:
610 return "scalar", value
612 if isinstance(value, TaskMetadata):
613 return "metadata", value
614 if isinstance(value, Mapping):
615 return "metadata", self.from_dict(value)
617 if _isListLike(value):
618 # For model consistency, need to check that every item in the
619 # list has the same type.
620 value = list(value)
621 if not value:
622 return "array", value
624 type0 = type(value[0])
625 for i in value:
626 if type(i) is not type0:
627 raise ValueError(
628 "Type mismatch in supplied list. TaskMetadata requires all"
629 f" elements have same type but see {type(i)} and {type0}."
630 )
632 if type0 not in _ALLOWED_PRIMITIVE_TYPES:
633 # Must check to see if we got numpy floats or something.
634 type_cast: type
635 if isinstance(value[0], numbers.Integral):
636 type_cast = int
637 elif isinstance(value[0], numbers.Real):
638 type_cast = float
639 else:
640 raise ValueError(
641 f"Supplied list has element of type '{type0}'. "
642 "TaskMetadata can only accept primitive types in lists."
643 )
645 value = [type_cast(v) for v in value]
647 return "array", value
649 # Sometimes a numpy number is given.
650 if isinstance(value, numbers.Integral):
651 value = int(value)
652 return "scalar", value
653 if isinstance(value, numbers.Real):
654 value = float(value)
655 return "scalar", value
657 raise ValueError(f"TaskMetadata does not support values of type {value!r}.")
659 # Work around the fact that Sphinx chokes on Pydantic docstring formatting,
660 # when we inherit those docstrings in our public classes.
661 if "sphinx" in sys.modules:
663 def copy(self, *args: Any, **kwargs: Any) -> Any:
664 """See `pydantic.BaseModel.copy`."""
665 return super().copy(*args, **kwargs)
667 def model_dump(self, *args: Any, **kwargs: Any) -> Any:
668 """See `pydantic.BaseModel.model_dump`."""
669 return super().model_dump(*args, **kwargs)
671 def model_dump_json(self, *args: Any, **kwargs: Any) -> Any:
672 """See `pydantic.BaseModel.model_dump_json`."""
673 return super().model_dump(*args, **kwargs)
675 def model_copy(self, *args: Any, **kwargs: Any) -> Any:
676 """See `pydantic.BaseModel.model_copy`."""
677 return super().model_copy(*args, **kwargs)
679 @classmethod
680 def model_construct(cls, *args: Any, **kwargs: Any) -> Any: # type: ignore[misc, override]
681 """See `pydantic.BaseModel.model_construct`."""
682 return super().model_construct(*args, **kwargs)
684 @classmethod
685 def model_json_schema(cls, *args: Any, **kwargs: Any) -> Any:
686 """See `pydantic.BaseModel.model_json_schema`."""
687 return super().model_json_schema(*args, **kwargs)
689 @classmethod
690 def model_validate(cls, *args: Any, **kwargs: Any) -> Any:
691 """See `pydantic.BaseModel.model_validate`."""
692 return super().model_validate(*args, **kwargs)
694 @classmethod
695 def model_validate_json(cls, *args: Any, **kwargs: Any) -> Any:
696 """See `pydantic.BaseModel.model_validate_json`."""
697 return super().model_validate_json(*args, **kwargs)
699 @classmethod
700 def model_validate_strings(cls, *args: Any, **kwargs: Any) -> Any:
701 """See `pydantic.BaseModel.model_validate_strings`."""
702 return super().model_validate_strings(*args, **kwargs)
705# Needed because a TaskMetadata can contain a TaskMetadata.
706TaskMetadata.model_rebuild()