Coverage for python/lsst/daf/base/propertyContainer/propertyContainerContinued.py: 92%

2# LSST Data Management System

6# This product includes software developed by the

7# LSST Project (http://www.lsst.org/).

9# This program is free software: you can redistribute it and/or modify

10# it under the terms of the GNU General Public License as published by

11# the Free Software Foundation, either version 3 of the License, or

12# (at your option) any later version.

13#

14# This program is distributed in the hope that it will be useful,

15# but WITHOUT ANY WARRANTY; without even the implied warranty of

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the

17# GNU General Public License for more details.

18#

19# You should have received a copy of the LSST License Statement and

20# the GNU General Public License along with this program. If not,

21# see <http://www.lsstcorp.org/LegalNotices/>.

22#

24__all__ = ["getPropertySetState", "getPropertyListState", "setPropertySetState", "setPropertyListState"]

26import enum

27import math

28import numbers

29import warnings

30import dataclasses

31from collections.abc import Mapping, KeysView, ValuesView, ItemsView

32from typing import TypeAlias, Union

34import astropy.io.fits

35import astropy.io.fits.verify

37# Ensure that C++ exceptions are properly translated to Python

38import lsst.pex.exceptions # noqa: F401

39from lsst.utils import continueClass

41from .._dafBaseLib import PropertySet, PropertyList

42from ..dateTime import DateTime

45# Note that '|' syntax for unions doesn't work when we have to use a string

46# literal (and we do since it's recursive and not an annotation).

47NestedMetadataDict: TypeAlias = Mapping[str, Union[str, float, int, bool, "NestedMetadataDict"]]

50# Map the type names to the internal type representation.

51_TYPE_MAP = {}

52for checkType in ("Bool", "Short", "Int", "Long", "LongLong", "UnsignedLongLong",

53 "Float", "Double", "String", "DateTime",

54 "PropertySet", "Undef"):

55 type_obj = getattr(PropertySet, "TYPE_" + checkType)

56 _TYPE_MAP[type_obj] = checkType

57 # Store both directions.

58 _TYPE_MAP[checkType] = type_obj

61def getPropertySetState(container, asLists=False):

62 """Get the state of a PropertySet in a form that can be pickled.

64 Parameters

65 ----------

66 container : `PropertySet`

67 The property container.

68 asLists : `bool`, optional

69 If False, the default, `tuple` will be used for the contents. If true

70 a `list` will be used.

72 Returns

73 -------

74 state : `list` of `tuple` or `list` of `list`

75 The state, as a list of tuples (or lists), each of which contains

76 the following 3 items:

78 name (a `str`)

79 the name of the item

80 elementTypeName (a `str`)

81 the suffix of a ``setX`` method name

82 which is appropriate for the data type. For example integer

83 data has ``elementTypeName="Int"` which corresponds to

84 the ``setInt`` method.

85 value

86 the data for the item, in a form compatible

87 with the set method named by ``elementTypeName``

88 """

89 names = container.names(topLevelOnly=True)

90 sequence = list if asLists else tuple

91 return [sequence((name, _propertyContainerElementTypeName(container, name),

92 _propertyContainerGet(container, name, returnStyle=ReturnStyle.AUTO)))

93 for name in names]

96def getPropertyListState(container, asLists=False):

97 """Get the state of a PropertyList in a form that can be pickled.

99 Parameters

100 ----------

101 container : `PropertyList`

102 The property container.

103 asLists : `bool`, optional

104 If False, the default, `tuple` will be used for the contents. If true

105 a `list` will be used.

106

107 Returns

108 -------

109 state : `list` of `tuple` or `list` of `list`

110 The state, as a list of tuples (or lists), each of which contains

111 the following 4 items:

112

113 name (a `str`):

114 the name of the item

115 elementTypeName (a `str`):

116 the suffix of a ``setX`` method name

117 which is appropriate for the data type. For example integer

118 data has ``elementTypeName="Int"` which corresponds to

119 the ``setInt`` method.

120 value

121 the data for the item, in a form compatible

122 with the set method named by ``elementTypeName``

123 comment (a `str`): the comment. This item is only present

124 if ``container`` is a PropertyList.

125 """

126 sequence = list if asLists else tuple

127 return [sequence((name, _propertyContainerElementTypeName(container, name),

128 _propertyContainerGet(container, name, returnStyle=ReturnStyle.AUTO),

129 container.getComment(name)))

130 for name in container.getOrderedNames()]

131

132

133def setPropertySetState(container, state):

134 """Restore the state of a PropertySet, in place.

135

136 Parameters

137 ----------

138 container : `PropertySet`

139 The property container whose state is to be restored.

140 It should be empty to start with and is updated in place.

141 state : `list`

142 The state, as returned by `getPropertySetState`

143 """

144 for name, elemType, value in state:

145 if elemType is not None: 145 ↛ 148line 145 didn't jump to line 148 because the condition on line 145 was always true

146 getattr(container, "set" + elemType)(name, value)

147 else:

148 raise ValueError(f"Unrecognized values for state restoration: ({name}, {elemType}, {value})")

149

150

151def setPropertyListState(container, state):

152 """Restore the state of a PropertyList, in place.

153

154 Parameters

155 ----------

156 container : `PropertyList`

157 The property container whose state is to be restored.

158 It should be empty to start with and is updated in place.

159 state : `list`

160 The state, as returned by ``getPropertyListState``

161 """

162 for name, elemType, value, comment in state:

163 getattr(container, "set" + elemType)(name, value, comment)

164

165

166class ReturnStyle(enum.Enum):

167 ARRAY = enum.auto()

168 SCALAR = enum.auto()

169 AUTO = enum.auto()

170

171

172def _propertyContainerElementTypeName(container, name):

173 """Return name of the type of a particular element

174

175 Parameters

176 ----------

177 container : `lsst.daf.base.PropertySet` or `lsst.daf.base.PropertyList`

178 Container including the element

179 name : `str`

180 Name of element

181 """

182 try:

183 t = container.typeOf(name)

184 except LookupError as e:

185 # KeyError is more commonly expected when asking for an element

186 # from a mapping.

187 raise KeyError(str(e)) from None

188

189 return _TYPE_MAP.get(t, None)

190

191

192def _propertyContainerGet(container, name, returnStyle):

193 """Get a value of unknown type as a scalar or array

194

195 Parameters

196 ----------

197 container : `lsst.daf.base.PropertySet` or `lsst.daf.base.PropertyList`

198 Container from which to get the value

199 name : `str`

200 Name of item

201 returnStyle : `ReturnStyle`

202 Control whether numeric or string data is returned as an array

203 or scalar (the other types, ``PropertyList``, ``PropertySet``

204 and ``PersistablePtr``, are always returned as a scalar):

205 - ReturnStyle.ARRAY: return numeric or string data types

206 as an array of values.

207 - ReturnStyle.SCALAR: return numeric or string data types

208 as a single value; if the item has multiple values then

209 return the last value.

210 - ReturnStyle.AUTO: (deprecated) return numeric or string data

211 as a scalar if there is just one item, or as an array

212 otherwise.

213

214 Raises

215 ------

216 KeyError

217 Raised if the specified key does not exist in the container.

218 TypeError

219 Raised if the value retrieved is of an unexpected type.

220 ValueError

221 Raised if the value for ``returnStyle`` is not correct.

222 """

223 if not container.exists(name):

224 raise KeyError(name + " not found")

225 if returnStyle not in ReturnStyle: 225 ↛ 226line 225 didn't jump to line 226 because the condition on line 225 was never true

226 raise ValueError("returnStyle {} must be a ReturnStyle".format(returnStyle))

227

228 elemType = _propertyContainerElementTypeName(container, name)

229 if elemType and elemType != "PropertySet":

230 value = getattr(container, "getArray" + elemType)(name)

231 if returnStyle == ReturnStyle.ARRAY or (returnStyle == ReturnStyle.AUTO and len(value) > 1):

232 return value

233 return value[-1]

234

235 if container.isPropertySetPtr(name): 235 ↛ 240line 235 didn't jump to line 240 because the condition on line 235 was always true

236 try:

237 return container.getAsPropertyListPtr(name)

238 except Exception:

239 return container.getAsPropertySetPtr(name)

240 try:

241 return container.getAsPersistablePtr(name)

242 except Exception:

243 pass

244 raise TypeError('Unknown PropertySet value type for ' + name)

245

246

247def _iterable(a):

248 """Make input iterable.

249

250 Takes whatever is given to it and yields it back one element at a time.

251 If it is not an iterable or it is a string or PropertySet/List,

252 yields itself.

253 """

254 if isinstance(a, (str, PropertyList, PropertySet)):

255 yield a

256 return

257 try:

258 yield from a

259 except Exception:

260 yield a

261

262

263def _guessIntegerType(container, name, value):

264 """Given an existing container and name, determine the type

265 that should be used for the supplied value. The supplied value

266 is assumed to be a scalar.

267

268 On Python 3 all ints are LongLong but we need to be able to store them

269 in Int containers if that is what is being used (testing for truncation).

270 Int is assumed to mean 32bit integer (2147483647 to -2147483648).

271

272 If there is no pre-existing value we have to decide what to do. For now

273 we pick Int if the value is less than maxsize.

274

275 Parameters

276 ----------

277 container : `lsst.daf.base.PropertySet` or `lsst.daf.base.PropertyList`

278 Container from which to get the value

279

280 name : `str`

281 Name of item

282

283 value : `object`

284 Value to be assigned a type. Can be an iterable.

285

286 Returns

287 -------

288 useType : `str` or none

289 Type to use for the supplied value. `None` if the input is

290 `bool` or a non-integral value.

291 """

292 maxInt = 2147483647

293 minInt = -2147483648

294 maxLongLong = 2**63 - 1

295 minLongLong = -2**63

296 maxU64 = 2**64 - 1

297 minU64 = 0

298

299 # Go through the values to find the range of supplied integers,

300 # stopping early if we don't have an integer.

301 min = None

302 max = None

303 for v in _iterable(value):

304 # Do not try to convert a bool to an integer

305 if not isinstance(v, numbers.Integral) or isinstance(v, bool):

306 return None

307

308 if min is None:

309 min = v

310 max = v

311 elif v < min:

312 min = v

313 elif v > max:

314 max = v

315

316 # Safety net

317 if min is None or max is None: 317 ↛ 318line 317 didn't jump to line 318 because the condition on line 317 was never true

318 raise RuntimeError(f"Internal logic failure calculating integer range of {value}")

319

320 def _choose_int_from_range(int_value, current_type):

321 # If this is changing type from non-integer the current type

322 # does not matter.

323 if current_type not in {"Int", "LongLong", "UnsignedLongLong"}:

324 current_type = None

325

326 if int_value <= maxInt and int_value >= minInt and current_type in (None, "Int"):

327 # Use Int only if in range and either no current type or the

328 # current type is an Int.

329 use_type = "Int"

330 elif int_value >= minLongLong and int_value < 0:

331 # All large negatives must be LongLong if they did not fit

332 # in Int clause above.

333 use_type = "LongLong"

334 elif int_value >= 0 and int_value <= maxLongLong and current_type in (None, "Int", "LongLong"):

335 # Larger than Int or already a LongLong

336 use_type = "LongLong"

337 elif int_value <= maxU64 and int_value >= minU64:

338 use_type = "UnsignedLongLong"

339 else:

340 raise RuntimeError("Unable to guess integer type for storing out of "

341 f"range value: {int_value}")

342 return use_type

343

344 if container.exists(name):

345 containerType = _propertyContainerElementTypeName(container, name)

346 else:

347 containerType = None

348

349 useTypeMin = _choose_int_from_range(min, containerType)

350 useTypeMax = _choose_int_from_range(max, containerType)

351

352 if useTypeMin == useTypeMax:

353 return useTypeMin

354

355 # When different the combinations are:

356 # Int + LongLong

357 # Int + UnsignedLongLong

358 # LongLong + UnsignedLongLong

359

360 choices = {useTypeMin, useTypeMax}

361 if choices == {"Int", "LongLong"}:

362 return "LongLong"

363

364 # If UnsignedLongLong is required things will break if the min

365 # is negative. They will break whatever we choose if that is the case

366 # but we have no choice but to return the UnsignedLongLong regardless.

367 if "UnsignedLongLong" in choices: 367 ↛ 370line 367 didn't jump to line 370 because the condition on line 367 was always true

368 return "UnsignedLongLong"

369

370 raise RuntimeError(f"Logic error in guessing integer type from {min} and {max}")

371

372

373def _propertyContainerSet(container, name, value, typeMenu, *args):

374 """Set a single Python value of unknown type

375 """

376 try:

377 exemplar = next(_iterable(value))

378 except StopIteration:

379 # Do nothing if nothing provided. This matches the behavior

380 # of the explicit setX() methods.

381 return

382 t = type(exemplar)

383 setType = _guessIntegerType(container, name, value)

384

385 if setType is not None or t in typeMenu:

386 if setType is None:

387 setType = typeMenu[t]

388 return getattr(container, "set" + setType)(name, value, *args)

389 # Allow for subclasses

390 for checkType in typeMenu: 390 ↛ 394line 390 didn't jump to line 394 because the loop on line 390 didn't complete

391 if (checkType is None and exemplar is None) or \

392 (checkType is not None and isinstance(exemplar, checkType)):

393 return getattr(container, "set" + typeMenu[checkType])(name, value, *args)

394 raise TypeError("Unknown value type for key '%s': %s" % (name, t))

395

396

397def _propertyContainerAdd(container, name, value, typeMenu, *args):

398 """Add a single Python value of unknown type

399 """

400 try:

401 exemplar = next(_iterable(value))

402 except StopIteration:

403 # Adding an empty iterable to an existing entry is a no-op

404 # since there is nothing to add.

405 return

406 t = type(exemplar)

407 addType = _guessIntegerType(container, name, exemplar)

408

409 if addType is not None or t in typeMenu:

410 if addType is None:

411 addType = typeMenu[t]

412 return getattr(container, "add" + addType)(name, value, *args)

413 # Allow for subclasses

414 for checkType in typeMenu: 414 ↛ 418line 414 didn't jump to line 418 because the loop on line 414 didn't complete

415 if (checkType is None and exemplar is None) or \

416 (checkType is not None and isinstance(exemplar, checkType)):

417 return getattr(container, "add" + typeMenu[checkType])(name, value, *args)

418 raise TypeError("Unknown value type for key '%s': %s" % (name, t))

419

420

421def _makePropertySet(state):

422 """Make a `PropertySet` from the state returned by `getPropertySetState`

423

424 Parameters

425 ----------

426 state : `list`

427 The data returned by `getPropertySetState`.

428 """

429 ps = PropertySet()

430 setPropertySetState(ps, state)

431 return ps

432

433

434def _makePropertyList(state):

435 """Make a `PropertyList` from the state returned by

436 `getPropertyListState`

437

438 Parameters

439 ----------

440 state : `list`

441 The data returned by `getPropertySetState`.

442 """

443 pl = PropertyList()

444 setPropertyListState(pl, state)

445 return pl

446

447

448@continueClass

449class PropertySet:

450 # Mapping of type to method names;

451 # int types are omitted due to use of _guessIntegerType

452 _typeMenu = {bool: "Bool",

453 float: "Double",

454 str: "String",

455 DateTime: "DateTime",

456 PropertySet: "PropertySet",

457 PropertyList: "PropertySet",

458 None: "Undef",

459 }

460

461 @classmethod

462 def from_mapping(cls, metadata):

463 """Create a `PropertySet` from a mapping or dict-like object.

464

465 Parameters

466 ----------

467 metadata : `collections.abc.Mapping`

468 Metadata from which to create the `PropertySet`.

469 Can be a mapping, a `~dataclasses.dataclass` or anything that

470 supports ``toDict()``, ``to_dict()`` or ``dict()`` method.

471 It is assumed that the dictionary is expanded recursively by these

472 methods or that the Python type can be understood by `PropertySet`.

473

474 Returns

475 -------

476 ps : `PropertySet`

477 The new `PropertySet`.

478 """

479 ps = cls()

480 d = None

481 if isinstance(metadata, Mapping):

482 d = metadata

483 elif dataclasses.is_dataclass(metadata):

484 d = dataclasses.asdict(metadata)

485 else:

486 for attr in ("to_dict", "toDict", "dict"):

487 if hasattr(metadata, attr):

488 d = getattr(metadata, attr)()

489 break

490 if d is None:

491 raise ValueError("Unable to extract mappings from the supplied metadata of type"

492 f" {type(metadata)}")

493 ps.update(d)

494 return ps

495

496 def get(self, name, default=None):

497 """Return an item as a scalar, else default.

498

499 Identical to `getScalar` except that a default value is returned

500 if the requested key is not present. If an array item is requested

501 the final value in the array will be returned.

502

503 Parameters

504 ----------

505 name : `str`

506 Name of item

507 default : `object`, optional

508 Default value to use if the named item is not present.

509

510 Returns

511 -------

512 value : any type supported by container

513 Single value of any type supported by the container, else the

514 default value if the requested item is not present in the

515 container. For array items the most recently added value is

516 returned.

517 """

518 try:

519 return _propertyContainerGet(self, name, returnStyle=ReturnStyle.SCALAR)

520 except KeyError:

521 return default

522

523 def getArray(self, name):

524 """Return an item as an array if the item is numeric or string

525

526 If the item is a `PropertySet`, `PropertyList` or

527 `lsst.daf.base.PersistablePtr` then return the item as a scalar.

528

529 Parameters

530 ----------

531 name : `str`

532 Name of item

533

534 Returns

535 -------

536 values : `list` of any type supported by container

537 The contents of the item, guaranteed to be returned as a `list.`

538

539 Raises

540 ------

541 KeyError

542 Raised if the item does not exist.

543 """

544 return _propertyContainerGet(self, name, returnStyle=ReturnStyle.ARRAY)

545

546 def getScalar(self, name):

547 """Return an item as a scalar

548

549 If the item has more than one value then the last value is returned.

550

551 Parameters

552 ----------

553 name : `str`

554 Name of item

555

556 Returns

557 -------

558 value : scalar item

559 Value stored in the item. If the item refers to an array the

560 most recently added value is returned.

561

562 Raises

563 ------

564 KeyError

565 Raised if the item does not exist.

566 """

567 return _propertyContainerGet(self, name, returnStyle=ReturnStyle.SCALAR)

568

569 def set(self, name, value):

570 """Set the value of an item

571

572 If the item already exists it is silently replaced; the types

573 need not match.

574

575 Parameters

576 ----------

577 name : `str`

578 Name of item

579 value : any supported type

580 Value of item; may be a scalar or array

581 """

582 return _propertyContainerSet(self, name, value, self._typeMenu)

583

584 def add(self, name, value):

585 """Append one or more values to a given item, which need not exist

586

587 If the item exists then the new value(s) are appended;

588 otherwise it is like calling `set`

589

590 Parameters

591 ----------

592 name : `str`

593 Name of item

594 value : any supported type

595 Value of item; may be a scalar or array

596

597 Notes

598 -----

599 If ``value`` is an `lsst.daf.base.PropertySet` or

600 `lsst.daf.base.PropertyList` then ``value`` replaces

601 the existing value. Also the item is added as a live

602 reference, so updating ``value`` will update this container

603 and vice-versa.

604

605 Raises

606 ------

607 lsst::pex::exceptions::TypeError

608 Raised if the type of `value` is incompatible with the existing

609 value of the item.

610 """

611 return _propertyContainerAdd(self, name, value, self._typeMenu)

612

613 def update(self, addition):

614 """Update the current container with the supplied additions.

615

616 Parameters

617 ----------

618 addition : `collections.abc.Mapping` or `PropertySet`

619 The content to merge into the current container.

620

621 Notes

622 -----

623 This is not the same as calling `PropertySet.combine` since the

624 behavior differs when both mappings contain the same key. This

625 method updates by overwriting existing values completely with

626 the new value.

627 """

628 if isinstance(addition, PropertySet):

629 # To support array values we can not use the dict interface

630 # and instead use the copy() method which overwrites

631 for k in addition:

632 self.copy(k, addition, k)

633 else:

634 for k, v in addition.items():

635 self[k] = v

636

637 def toDict(self):

638 """Returns a (possibly nested) dictionary with all properties.

639

640 Returns

641 -------

642 d : `dict`

643 Dictionary with all names and values (no comments).

644 """

645

646 d = {}

647 for name in self.names():

648 v = _propertyContainerGet(self, name, returnStyle=ReturnStyle.AUTO)

649

650 if isinstance(v, PropertySet):

651 d[name] = PropertySet.toDict(v)

652 else:

653 d[name] = v

654 return d

655

656 def __eq__(self, other):

657 if type(self) is not type(other): 657 ↛ 658line 657 didn't jump to line 658 because the condition on line 657 was never true

658 return NotImplemented

659

660 if len(self) != len(other): 660 ↛ 661line 660 didn't jump to line 661 because the condition on line 660 was never true

661 return False

662

663 for name in self:

664 if (self_typeOf := self.typeOf(name)) != other.typeOf(name): 664 ↛ 665line 664 didn't jump to line 665 because the condition on line 664 was never true

665 return False

666

667 if (v1 := _propertyContainerGet(self, name, returnStyle=ReturnStyle.AUTO)) != \

668 (v2 := _propertyContainerGet(other, name, returnStyle=ReturnStyle.AUTO)):

669 # It is possible that we have floats that are NaN. When

670 # equating two PropertySets if there are fields with NaN

671 # these should equate equal.

672 if self_typeOf in (_TYPE_MAP["Float"], _TYPE_MAP["Double"]) \ 672 ↛ 674line 672 didn't jump to line 674 because the condition on line 672 was never true

673 and math.isnan(v1) and math.isnan(v2):

674 pass

675 else:

676 return False

677

678 return True

679

680 def __copy__(self):

681 # Copy without having to go through pickle state

682 ps = PropertySet()

683 for itemName in self:

684 ps.copy(itemName, self, itemName)

685 return ps

686

687 def __deepcopy__(self, memo):

688 result = self.deepCopy()

689 memo[id(self)] = result

690 return result

691

692 def __contains__(self, name):

693 """Determines if the name is found at the top level hierarchy

694 of the container.

695

696 Notes

697 ------

698 Does not use `PropertySet.exists()`` because that includes support

699 for "."-delimited names. This method is consistent with the

700 items returned from ``__iter__``.

701 """

702 return name in self.names(topLevelOnly=True)

703

704 def __setitem__(self, name, value):

705 """Assigns the supplied value to the container.

706

707 Parameters

708 ----------

709 name : `str`

710 Name of item to update.

711 value : Value to assign

712 Can be any value supported by the container's ``set()``

713 method. `~collections.abc.Mapping` are converted to

714 `PropertySet` before assignment.

715

716 Notes

717 -----

718 Uses `PropertySet.set`, overwriting any previous value.

719 """

720 if isinstance(value, Mapping):

721 # Create a property set instead

722 ps = PropertySet()

723 for k, v in value.items():

724 ps[k] = v

725 value = ps

726 self.set(name, value)

727

728 def __getitem__(self, name):

729 """Returns a scalar item from the container.

730

731 Notes

732 -----

733 Uses `PropertySet.getScalar` to guarantee that a single value

734 will be returned.

735 """

736 return self.getScalar(name)

737

738 def __delitem__(self, name):

739 if self.exists(name): 739 ↛ 743line 739 didn't jump to line 743 because the condition on line 739 was always true

740 # dot-delimited names should work so cannot use "in".

741 self.remove(name)

742 else:

743 raise KeyError(f"{name} not present in dict")

744

745 def __str__(self):

746 return self.toString()

747

748 def __len__(self):

749 return self.nameCount(topLevelOnly=True)

750

751 def __iter__(self):

752 for n in self.names(topLevelOnly=True):

753 yield n

754

755 def keys(self):

756 return KeysView(self)

757

758 def items(self):

759 return ItemsView(self)

760

761 def values(self):

762 return ValuesView(self)

763

764 def pop(self, name, default=None):

765 """Remove the named key and return its value.

766

767 Parameters

768 ----------

769 name : `str`

770 Name of the key to remove. Can be hierarchical.

771 default : Any, optional

772 Value to return if the key is not present.

773

774 Returns

775 -------

776 value : Any

777 The value of the item as would be returned using `getScalar()`.

778

779 Raises

780 ------

781 KeyError

782 Raised if no default is given and the key is missing.

783 """

784 if self.exists(name):

785 value = self[name]

786 self.remove(name)

787 else:

788 if default is None:

789 raise KeyError(name)

790 value = default

791 return value

792

793 def __reduce__(self):

794 # It would be a bit simpler to use __setstate__ and __getstate__.

795 # However, implementing __setstate__ in Python causes segfaults

796 # because pickle creates a new instance by calling

797 # object.__new__(PropertyList, *args) which bypasses

798 # the pybind11 memory allocation step.

799 return (_makePropertySet, (getPropertySetState(self),))

800

801 def get_dict(self, key: str) -> NestedMetadataDict:

802 """Return a possibly-hierarchical nested `dict`.

803

804 This implements the `lsst.pipe.base.GetDictMetadata` protocol for

805 consistency with `lsst.pipe.base.TaskMetadata` and `PropertyList`.

806

807 Parameters

808 ----------

809 key : `str`

810 String key associated with the mapping. May not have a ``.``

811 character.

812

813 Returns

814 -------

815 value : `~collections.abc.Mapping`

816 Possibly-nested mapping, with `str` keys and values that are `int`,

817 `float`, `str`, `bool`, or another `dict` with the same key and

818 value types. Will be empty if ``key`` does not exist.

819

820 Raises

821 ------

822 TypeError

823 Raised if the value associated with this key is not a nested

824 dictionary, but does exist. Note that this behavior is not

825 consistent with `PropertyList` (which returns an empty `dict`).

826 """

827 if self.exists(key):

828 return self.getScalar(key).toDict()

829 else:

830 return {}

831

832 def set_dict(self, key: str, value: NestedMetadataDict) -> None:

833 """Assign a possibly-hierarchical nested `dict`.

834

835 This implements the `lsst.pipe.base.SetDictMetadata` protocol for

836 consistency with `lsst.pipe.base.TaskMetadata` and `PropertyList`.

837

838 Parameters

839 ----------

840 key : `str`

841 String key associated with the mapping. May not have a ``.``

842 character.

843 value : `~collections.abc.Mapping`

844 Possibly-nested mapping, with `str` keys and values that are `int`,

845 `float`, `str`, `bool`, or another `dict` with the same key and

846 value types. May not have a ``.``

847 character.

848 """

849 self.set(key, PropertySet.from_mapping(value))

850

851

852@continueClass

853class PropertyList:

854 # Mapping of type to method names

855 _typeMenu = {bool: "Bool",

856 int: "Int",

857 float: "Double",

858 str: "String",

859 DateTime: "DateTime",

860 PropertySet: "PropertySet",

861 PropertyList: "PropertySet",

862 None: "Undef",

863 }

864

865 COMMENTSUFFIX = "#COMMENT"

866 """Special suffix used to indicate that a named item being assigned

867 using dict syntax is referring to a comment, not value."""

868

869 def get(self, name, default=None):

870 """Return an item as a scalar, else default.

871

872 Identical to `getScalar` except that a default value is returned

873 if the requested key is not present. If an array item is requested

874 the final value in the array will be returned.

875

876 Parameters

877 ----------

878 name : ``str``

879 Name of item

880 default : `object`, optional

881 Default value to use if the named item is not present.

882

883 Returns

884 -------

885 value : any type supported by container

886 Single value of any type supported by the container, else the

887 default value if the requested item is not present in the

888 container. For array items the most recently added value is

889 returned.

890 """

891 try:

892 return _propertyContainerGet(self, name, returnStyle=ReturnStyle.SCALAR)

893 except KeyError:

894 return default

895

896 def getArray(self, name):

897 """Return an item as a list.

898

899 Parameters

900 ----------

901 name : `str`

902 Name of item

903

904 Returns

905 -------

906 values : `list` of values

907 The contents of the item, guaranteed to be returned as a `list.`

908

909 Raises

910 ------

911 KeyError

912 Raised if the item does not exist.

913 """

914 return _propertyContainerGet(self, name, returnStyle=ReturnStyle.ARRAY)

915

916 def getScalar(self, name):

917 """Return an item as a scalar

918

919 If the item has more than one value then the last value is returned.

920

921 Parameters

922 ----------

923 name : `str`

924 Name of item.

925

926 Returns

927 -------

928 value : scalar item

929 Value stored in the item. If the item refers to an array the

930 most recently added value is returned.

931

932 Raises

933 ------

934 KeyError

935 Raised if the item does not exist.

936 """

937 return _propertyContainerGet(self, name, returnStyle=ReturnStyle.SCALAR)

938

939 def set(self, name, value, comment=None):

940 """Set the value of an item

941

942 If the item already exists it is silently replaced; the types

943 need not match.

944

945 Parameters

946 ----------

947 name : `str`

948 Name of item

949 value : any supported type

950 Value of item; may be a scalar or array

951 """

952 args = []

953 if comment is not None:

954 args.append(comment)

955 return _propertyContainerSet(self, name, value, self._typeMenu, *args)

956

957 def add(self, name, value, comment=None):

958 """Append one or more values to a given item, which need not exist

959

960 If the item exists then the new value(s) are appended;

961 otherwise it is like calling `set`

962

963 Parameters

964 ----------

965 name : `str`

966 Name of item

967 value : any supported type

968 Value of item; may be a scalar or array

969

970 Notes

971 -----

972 If `value` is an `lsst.daf.base.PropertySet` items are added

973 using dotted names (e.g. if name="a" and value contains

974 an item "b" which is another PropertySet and contains an

975 item "c" which is numeric or string, then the value of "c"

976 is added as "a.b.c", appended to the existing values of

977 "a.b.c" if any (in which case the types must be compatible).

978

979 Raises

980 ------

981 lsst::pex::exceptions::TypeError

982 Raise if the type of ``value`` is incompatible with the existing

983 value of the item.

984 """

985 args = []

986 if comment is not None:

987 args.append(comment)

988 return _propertyContainerAdd(self, name, value, self._typeMenu, *args)

989

990 def setComment(self, name, comment):

991 """Set the comment for an existing entry.

992

993 Parameters

994 ----------

995 name : `str`

996 Name of the key to receive updated comment.

997 comment : `comment`

998 New comment string.

999 """

1000 # The only way to do this is to replace the existing entry with

1001 # one that has the new comment

1002 containerType = _propertyContainerElementTypeName(self, name)

1003 if self.isArray(name): 1003 ↛ 1004line 1003 didn't jump to line 1004 because the condition on line 1003 was never true

1004 value = self.getArray(name)

1005 else:

1006 value = self.getScalar(name)

1007 getattr(self, f"set{containerType}")(name, value, comment)

1008

1009 @classmethod

1010 def from_astropy_header(cls, header: astropy.io.fits.Header) -> "PropertyList":

1011 """Convert an `astropy.io.fits.Header` into a `PropertyList`.

1012

1013 Parameters

1014 ----------

1015 header : `astropy.io.fits.Header`

1016 FITS header to convert.

1017

1018 Returns

1019 -------

1020 propertyList : `PropertyList`

1021 Equivalent legacy metadata. Blank (keyword-less) cards and

1022 cards whose values Astropy cannot parse (e.g. from unterminated

1023 quoted strings in real raw headers) are skipped. Cards with

1024 `astropy.io.fits.card.Undefined` values (e.g. ``SEEING =``) are

1025 stored as undefined properties (``TYPE_Undef``), preserving any

1026 comment text. COMMENT and HISTORY cards hold their text as

1027 string values, as they do both in FITS convention and when read

1028 back by `lsst.afw.fits.readMetadata`.

1029

1030 Notes

1031 -----

1032 Comments on cards are preserved via `PropertyList.add`, and so are

1033 keyed by name: cards that repeat a keyword share that keyword's

1034 comment slot.

1035 """

1036 result = cls()

1037 for card in header.cards:

1038 if not card.keyword: 1038 ↛ 1040line 1038 didn't jump to line 1040 because the condition on line 1038 was never true

1039 # Truly blank cards have no name to key on.

1040 continue

1041 try:

1042 value = card.value

1043 except astropy.io.fits.verify.VerifyError:

1044 # Values astropy cannot parse (e.g. from unterminated quoted

1045 # strings) have no representation in legacy metadata.

1046 continue

1047 comment = card.comment or None

1048 if isinstance(value, astropy.io.fits.card.Undefined):

1049 value = None

1050 result.add(card.keyword, value, comment)

1051 return result

1052

1053 def to_astropy_header(self) -> astropy.io.fits.Header:

1054 """Convert this `PropertyList` into an `astropy.io.fits.Header`.

1055

1056 Returns

1057 -------

1058 header : `astropy.io.fits.Header`

1059 FITS header with one card per stored value, in insertion order.

1060 Undefined properties become cards with no value

1061 (`astropy.io.fits.card.Undefined`); string values stored under

1062 the COMMENT and HISTORY keywords render as bare comment cards

1063 (``COMMENT text``) rather than ``COMMENT = 'text'``. Comments

1064 set via `PropertyList.add` or `setComment` are restored.

1065 """

1066 header = astropy.io.fits.Header()

1067 with warnings.catch_warnings():

1068 # Silence warnings about long keys becoming HIERARCH and

1069 # comments being truncated; both are unavoidable here.

1070 warnings.simplefilter("ignore", category=astropy.io.fits.verify.VerifyWarning)

1071 for name in self.getOrderedNames():

1072 values = self.getArray(name)

1073 comment = self.getComment(name) or None

1074 for value in values:

1075 header.append(astropy.io.fits.Card(name, value, comment), end=True)

1076 return header

1077

1078 def toList(self):

1079 """Return a list of tuples of name, value, comment for each property

1080 in the order that they were inserted.

1081

1082 Returns

1083 -------

1084 ret : `list` of `tuple`

1085 Tuples of name, value, comment for each property in the order

1086 in which they were inserted.

1087 """

1088 orderedNames = self.getOrderedNames()

1089 ret = []

1090 for name in orderedNames:

1091 if self.isArray(name):

1092 values = _propertyContainerGet(self, name, returnStyle=ReturnStyle.AUTO)

1093 for v in values:

1094 ret.append((name, v, self.getComment(name)))

1095 else:

1096 ret.append((name, _propertyContainerGet(self, name, returnStyle=ReturnStyle.AUTO),

1097 self.getComment(name)))

1098 return ret

1099

1100 def toOrderedDict(self):

1101 """Return an ordered dictionary with all properties in the order that

1102 they were inserted.

1103

1104 Returns

1105 -------

1106 d : `dict`

1107 Ordered dictionary with all properties in the order that they

1108 were inserted. Comments are not included.

1109

1110 Notes

1111 -----

1112 As of Python 3.6 dicts retain their insertion order.

1113 """

1114 d = {}

1115 for name in self:

1116 d[name] = _propertyContainerGet(self, name, returnStyle=ReturnStyle.AUTO)

1117 return d

1118

1119 # For PropertyList the two are equivalent

1120 toDict = toOrderedDict

1121

1122 def __eq__(self, other):

1123 # super() doesn't seem to work properly in @continueClass;

1124 # note that super with arguments seems to work at first, but actually

1125 # doesn't either.

1126 if not PropertySet.__eq__(self, other): 1126 ↛ 1127line 1126 didn't jump to line 1127 because the condition on line 1126 was never true

1127 return False

1128

1129 for name in self:

1130 if self.getComment(name) != other.getComment(name): 1130 ↛ 1131line 1130 didn't jump to line 1131 because the condition on line 1130 was never true

1131 return False

1132

1133 return True

1134

1135 def __copy__(self):

1136 # Copy without having to go through pickle state

1137 pl = PropertyList()

1138 for itemName in self:

1139 pl.copy(itemName, self, itemName)

1140 return pl

1141

1142 def __deepcopy__(self, memo):

1143 result = self.deepCopy()

1144 memo[id(self)] = result

1145 return result

1146

1147 def __iter__(self):

1148 for n in self.getOrderedNames():

1149 yield n

1150

1151 def __setitem__(self, name, value):

1152 """Assigns the supplied value to the container.

1153

1154 Parameters

1155 ----------

1156 name : `str`

1157 Name of item to update. If the name ends with

1158 `PropertyList.COMMENTSUFFIX`, the comment is updated rather

1159 than the value.

1160 value : Value to assign

1161 Can be any value supported by the container's ``set()``

1162 method. `~collections.abc.Mapping` are converted to

1163 `PropertySet` before assignment.

1164

1165 Notes

1166 -----

1167 Uses `PropertySet.set`, overwriting any previous value.

1168 """

1169 if name.endswith(self.COMMENTSUFFIX):

1170 name = name[:-len(self.COMMENTSUFFIX)]

1171 self.setComment(name, value)

1172 return

1173 if isinstance(value, Mapping):

1174 # Create a property set instead

1175 ps = PropertySet()

1176 for k, v in value.items():

1177 ps[k] = v

1178 value = ps

1179 self.set(name, value)

1180

1181 def __reduce__(self):

1182 # It would be a bit simpler to use __setstate__ and __getstate__.

1183 # However, implementing __setstate__ in Python causes segfaults

1184 # because pickle creates a new instance by calling

1185 # object.__new__(PropertyList, *args) which bypasses

1186 # the pybind11 memory allocation step.

1187 return (_makePropertyList, (getPropertyListState(self),))

1188

1189 def get_dict(self, key: str) -> NestedMetadataDict:

1190 """Return a possibly-hierarchical nested `dict`.

1191

1192 This implements the `lsst.pipe.base.GetDictMetadata` protocol for

1193 consistency with `lsst.pipe.base.TaskMetadata` and `PropertySet`.

1194

1195 Parameters

1196 ----------

1197 key : `str`

1198 String key associated with the mapping. May not have a ``.``

1199 character.

1200

1201 Returns

1202 -------

1203 value : `~collections.abc.Mapping`

1204 Possibly-nested mapping, with `str` keys and values that are `int`,

1205 `float`, `str`, `bool`, or another `dict` with the same key and

1206 value types. Will be empty if ``key`` does not exist.

1207 """

1208 result: NestedMetadataDict = {}

1209 name: str

1210 for name in self.getOrderedNames():

1211 levels = name.split(".")

1212 if levels[0] == key:

1213 nested = result

1214 for level_key in levels[1:-1]:

1215 nested = result.setdefault(level_key, {})

1216 nested[levels[-1]] = self[name]

1217 return result

1218

1219 def set_dict(self, key: str, value: NestedMetadataDict) -> None:

1220 """Assign a possibly-hierarchical nested `dict`.

1221

1222 This implements the `lsst.pipe.base.SetDictMetadata` protocol for

1223 consistency with `lsst.pipe.base.TaskMetadata` and `PropertySet`.

1224

1225 Parameters

1226 ----------

1227 key : `str`

1228 String key associated with the mapping. May not have a ``.``

1229 character.

1230 value : `~collections.abc.Mapping`

1231 Possibly-nested mapping, with `str` keys and values that are `int`,

1232 `float`, `str`, `bool`, or another `dict` with the same key and

1233 value types. Nested keys may not have a ``.`` character.

1234 """

1235 self.set(key, PropertySet.from_mapping(value))