Coverage for python/lsst/obs/base/defineVisits.py: 31%

344 statements  

« prev     ^ index     » next       coverage.py v6.4.4, created at 2022-09-07 04:12 -0700

1# This file is part of obs_base. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22from __future__ import annotations 

23 

24__all__ = [ 

25 "DefineVisitsConfig", 

26 "DefineVisitsTask", 

27 "GroupExposuresConfig", 

28 "GroupExposuresTask", 

29 "VisitDefinitionData", 

30 "VisitSystem", 

31] 

32 

33import cmath 

34import dataclasses 

35import enum 

36import math 

37import operator 

38from abc import ABCMeta, abstractmethod 

39from collections import defaultdict 

40from typing import ( 

41 Any, 

42 Callable, 

43 ClassVar, 

44 Dict, 

45 FrozenSet, 

46 Iterable, 

47 List, 

48 Optional, 

49 Set, 

50 Tuple, 

51 Type, 

52 TypeVar, 

53) 

54 

55import lsst.geom 

56from lsst.afw.cameraGeom import FOCAL_PLANE, PIXELS 

57from lsst.daf.butler import ( 

58 Butler, 

59 DataCoordinate, 

60 DataId, 

61 DimensionGraph, 

62 DimensionRecord, 

63 Progress, 

64 Timespan, 

65) 

66from lsst.geom import Box2D 

67from lsst.pex.config import Config, Field, makeRegistry, registerConfigurable 

68from lsst.pipe.base import Instrument, Task 

69from lsst.sphgeom import ConvexPolygon, Region, UnitVector3d 

70from lsst.utils.introspection import get_full_type_name 

71 

72from ._instrument import loadCamera 

73 

74 

75class VisitSystem(enum.Enum): 

76 """Enumeration used to label different visit systems.""" 

77 

78 ONE_TO_ONE = 0 

79 """Each exposure is assigned to its own visit.""" 

80 

81 BY_GROUP_METADATA = 1 

82 """Visit membership is defined by the value of the exposure.group_id.""" 

83 

84 BY_SEQ_START_END = 2 

85 """Visit membership is defined by the values of the ``exposure.day_obs``, 

86 ``exposure.seq_start``, and ``exposure.seq_end`` values. 

87 """ 

88 

89 @classmethod 

90 def all(cls) -> FrozenSet[VisitSystem]: 

91 """Return a `frozenset` containing all members.""" 

92 return frozenset(cls.__members__.values()) 

93 

94 @classmethod 

95 def from_name(cls, external_name: str) -> VisitSystem: 

96 """Construct the enumeration from given name.""" 

97 name = external_name.upper() 

98 name = name.replace("-", "_") 

99 try: 

100 return cls.__members__[name] 

101 except KeyError: 

102 raise KeyError(f"Visit system named '{external_name}' not known.") from None 

103 

104 @classmethod 

105 def from_names(cls, names: Optional[Iterable[str]]) -> FrozenSet[VisitSystem]: 

106 """Return a `frozenset` of all the visit systems matching the supplied 

107 names. 

108 

109 Parameters 

110 ---------- 

111 names : iterable of `str`, or `None` 

112 Names of visit systems. Case insensitive. If `None` or empty, all 

113 the visit systems are returned. 

114 

115 Returns 

116 ------- 

117 systems : `frozenset` of `VisitSystem` 

118 The matching visit systems. 

119 """ 

120 if not names: 

121 return cls.all() 

122 

123 return frozenset({cls.from_name(name) for name in names}) 

124 

125 def __str__(self) -> str: 

126 name = self.name.lower() 

127 name = name.replace("_", "-") 

128 return name 

129 

130 

131@dataclasses.dataclass 

132class VisitDefinitionData: 

133 """Struct representing a group of exposures that will be used to define a 

134 visit. 

135 """ 

136 

137 instrument: str 

138 """Name of the instrument this visit will be associated with. 

139 """ 

140 

141 id: int 

142 """Integer ID of the visit. 

143 

144 This must be unique across all visit systems for the instrument. 

145 """ 

146 

147 name: str 

148 """String name for the visit. 

149 

150 This must be unique across all visit systems for the instrument. 

151 """ 

152 

153 visit_systems: Set[VisitSystem] 

154 """All the visit systems associated with this visit.""" 

155 

156 exposures: List[DimensionRecord] = dataclasses.field(default_factory=list) 

157 """Dimension records for the exposures that are part of this visit. 

158 """ 

159 

160 

161@dataclasses.dataclass 

162class _VisitRecords: 

163 """Struct containing the dimension records associated with a visit.""" 

164 

165 visit: DimensionRecord 

166 """Record for the 'visit' dimension itself. 

167 """ 

168 

169 visit_definition: List[DimensionRecord] 

170 """Records for 'visit_definition', which relates 'visit' to 'exposure'. 

171 """ 

172 

173 visit_detector_region: List[DimensionRecord] 

174 """Records for 'visit_detector_region', which associates the combination 

175 of a 'visit' and a 'detector' with a region on the sky. 

176 """ 

177 

178 visit_system_membership: List[DimensionRecord] 

179 """Records relating visits to an associated visit system.""" 

180 

181 

182class GroupExposuresConfig(Config): 

183 pass 

184 

185 

186class GroupExposuresTask(Task, metaclass=ABCMeta): 

187 """Abstract base class for the subtask of `DefineVisitsTask` that is 

188 responsible for grouping exposures into visits. 

189 

190 Subclasses should be registered with `GroupExposuresTask.registry` to 

191 enable use by `DefineVisitsTask`, and should generally correspond to a 

192 particular 'visit_system' dimension value. They are also responsible for 

193 defining visit IDs and names that are unique across all visit systems in 

194 use by an instrument. 

195 

196 Parameters 

197 ---------- 

198 config : `GroupExposuresConfig` 

199 Configuration information. 

200 **kwargs 

201 Additional keyword arguments forwarded to the `Task` constructor. 

202 """ 

203 

204 def __init__(self, config: GroupExposuresConfig, **kwargs: Any): 

205 Task.__init__(self, config=config, **kwargs) 

206 

207 ConfigClass = GroupExposuresConfig 

208 

209 _DefaultName = "groupExposures" 

210 

211 registry = makeRegistry( 

212 doc="Registry of algorithms for grouping exposures into visits.", 

213 configBaseType=GroupExposuresConfig, 

214 ) 

215 

216 @abstractmethod 

217 def group(self, exposures: List[DimensionRecord]) -> Iterable[VisitDefinitionData]: 

218 """Group the given exposures into visits. 

219 

220 Parameters 

221 ---------- 

222 exposures : `list` [ `DimensionRecord` ] 

223 DimensionRecords (for the 'exposure' dimension) describing the 

224 exposures to group. 

225 

226 Returns 

227 ------- 

228 visits : `Iterable` [ `VisitDefinitionData` ] 

229 Structs identifying the visits and the exposures associated with 

230 them. This may be an iterator or a container. 

231 """ 

232 raise NotImplementedError() 

233 

234 def getVisitSystems(self) -> Set[VisitSystem]: 

235 """Return identifiers for the 'visit_system' dimension this 

236 algorithm implements. 

237 

238 Returns 

239 ------- 

240 visit_systems : `Set` [`VisitSystem`] 

241 The visit systems used by this algorithm. 

242 """ 

243 raise NotImplementedError() 

244 

245 

246class ComputeVisitRegionsConfig(Config): 

247 padding: Field[int] = Field( 

248 dtype=int, 

249 default=250, 

250 doc=( 

251 "Pad raw image bounding boxes with specified number of pixels " 

252 "when calculating their (conservatively large) region on the " 

253 "sky. Note that the config value for pixelMargin of the " 

254 "reference object loaders in meas_algorithms should be <= " 

255 "the value set here." 

256 ), 

257 ) 

258 

259 

260class ComputeVisitRegionsTask(Task, metaclass=ABCMeta): 

261 """Abstract base class for the subtask of `DefineVisitsTask` that is 

262 responsible for extracting spatial regions for visits and visit+detector 

263 combinations. 

264 

265 Subclasses should be registered with `ComputeVisitRegionsTask.registry` to 

266 enable use by `DefineVisitsTask`. 

267 

268 Parameters 

269 ---------- 

270 config : `ComputeVisitRegionsConfig` 

271 Configuration information. 

272 butler : `lsst.daf.butler.Butler` 

273 The butler to use. 

274 **kwargs 

275 Additional keyword arguments forwarded to the `Task` constructor. 

276 """ 

277 

278 def __init__(self, config: ComputeVisitRegionsConfig, *, butler: Butler, **kwargs: Any): 

279 Task.__init__(self, config=config, **kwargs) 

280 self.butler = butler 

281 self.instrumentMap: Dict[str, Instrument] = {} 

282 

283 ConfigClass = ComputeVisitRegionsConfig 

284 

285 _DefaultName = "computeVisitRegions" 

286 

287 registry = makeRegistry( 

288 doc=( 

289 "Registry of algorithms for computing on-sky regions for visits " 

290 "and visit+detector combinations." 

291 ), 

292 configBaseType=ComputeVisitRegionsConfig, 

293 ) 

294 

295 def getInstrument(self, instrumentName: str) -> Instrument: 

296 """Retrieve an `~lsst.obs.base.Instrument` associated with this 

297 instrument name. 

298 

299 Parameters 

300 ---------- 

301 instrumentName : `str` 

302 The name of the instrument. 

303 

304 Returns 

305 ------- 

306 instrument : `~lsst.obs.base.Instrument` 

307 The associated instrument object. 

308 

309 Notes 

310 ----- 

311 The result is cached. 

312 """ 

313 instrument = self.instrumentMap.get(instrumentName) 

314 if instrument is None: 

315 instrument = Instrument.fromName(instrumentName, self.butler.registry) 

316 self.instrumentMap[instrumentName] = instrument 

317 return instrument 

318 

319 @abstractmethod 

320 def compute( 

321 self, visit: VisitDefinitionData, *, collections: Any = None 

322 ) -> Tuple[Region, Dict[int, Region]]: 

323 """Compute regions for the given visit and all detectors in that visit. 

324 

325 Parameters 

326 ---------- 

327 visit : `VisitDefinitionData` 

328 Struct describing the visit and the exposures associated with it. 

329 collections : Any, optional 

330 Collections to be searched for raws and camera geometry, overriding 

331 ``self.butler.collections``. 

332 Can be any of the types supported by the ``collections`` argument 

333 to butler construction. 

334 

335 Returns 

336 ------- 

337 visitRegion : `lsst.sphgeom.Region` 

338 Region for the full visit. 

339 visitDetectorRegions : `dict` [ `int`, `lsst.sphgeom.Region` ] 

340 Dictionary mapping detector ID to the region for that detector. 

341 Should include all detectors in the visit. 

342 """ 

343 raise NotImplementedError() 

344 

345 

346class DefineVisitsConfig(Config): 

347 groupExposures = GroupExposuresTask.registry.makeField( 

348 doc="Algorithm for grouping exposures into visits.", 

349 default="one-to-one-and-by-counter", 

350 ) 

351 computeVisitRegions = ComputeVisitRegionsTask.registry.makeField( 

352 doc="Algorithm from computing visit and visit+detector regions.", 

353 default="single-raw-wcs", 

354 ) 

355 ignoreNonScienceExposures: Field[bool] = Field( 

356 doc=( 

357 "If True, silently ignore input exposures that do not have " 

358 "observation_type=SCIENCE. If False, raise an exception if one " 

359 "encountered." 

360 ), 

361 dtype=bool, 

362 optional=False, 

363 default=True, 

364 ) 

365 

366 

367class DefineVisitsTask(Task): 

368 """Driver Task for defining visits (and their spatial regions) in Gen3 

369 Butler repositories. 

370 

371 Parameters 

372 ---------- 

373 config : `DefineVisitsConfig` 

374 Configuration for the task. 

375 butler : `~lsst.daf.butler.Butler` 

376 Writeable butler instance. Will be used to read `raw.wcs` and `camera` 

377 datasets and insert/sync dimension data. 

378 **kwargs 

379 Additional keyword arguments are forwarded to the `lsst.pipe.base.Task` 

380 constructor. 

381 

382 Notes 

383 ----- 

384 Each instance of `DefineVisitsTask` reads from / writes to the same Butler. 

385 Each invocation of `DefineVisitsTask.run` processes an independent group of 

386 exposures into one or more new vists, all belonging to the same visit 

387 system and instrument. 

388 

389 The actual work of grouping exposures and computing regions is delegated 

390 to pluggable subtasks (`GroupExposuresTask` and `ComputeVisitRegionsTask`), 

391 respectively. The defaults are to create one visit for every exposure, 

392 and to use exactly one (arbitrary) detector-level raw dataset's WCS along 

393 with camera geometry to compute regions for all detectors. Other 

394 implementations can be created and configured for instruments for which 

395 these choices are unsuitable (e.g. because visits and exposures are not 

396 one-to-one, or because ``raw.wcs`` datasets for different detectors may not 

397 be consistent with camera geomery). 

398 

399 It is not necessary in general to ingest all raws for an exposure before 

400 defining a visit that includes the exposure; this depends entirely on the 

401 `ComputeVisitRegionTask` subclass used. For the default configuration, 

402 a single raw for each exposure is sufficient. 

403 

404 Defining the same visit the same way multiple times (e.g. via multiple 

405 invocations of this task on the same exposures, with the same 

406 configuration) is safe, but it may be inefficient, as most of the work must 

407 be done before new visits can be compared to existing visits. 

408 """ 

409 

410 def __init__(self, config: DefineVisitsConfig, *, butler: Butler, **kwargs: Any): 

411 config.validate() # Not a CmdlineTask nor PipelineTask, so have to validate the config here. 

412 super().__init__(config, **kwargs) 

413 self.butler = butler 

414 self.universe = self.butler.registry.dimensions 

415 self.progress = Progress("obs.base.DefineVisitsTask") 

416 self.makeSubtask("groupExposures") 

417 self.makeSubtask("computeVisitRegions", butler=self.butler) 

418 

419 def _reduce_kwargs(self) -> dict: 

420 # Add extra parameters to pickle 

421 return dict(**super()._reduce_kwargs(), butler=self.butler) 

422 

423 ConfigClass: ClassVar[Type[Config]] = DefineVisitsConfig 

424 

425 _DefaultName: ClassVar[str] = "defineVisits" 

426 

427 config: DefineVisitsConfig 

428 groupExposures: GroupExposuresTask 

429 computeVisitRegions: ComputeVisitRegionsTask 

430 

431 def _buildVisitRecords( 

432 self, definition: VisitDefinitionData, *, collections: Any = None 

433 ) -> _VisitRecords: 

434 """Build the DimensionRecords associated with a visit. 

435 

436 Parameters 

437 ---------- 

438 definition : `VisitDefinitionData` 

439 Struct with identifiers for the visit and records for its 

440 constituent exposures. 

441 collections : Any, optional 

442 Collections to be searched for raws and camera geometry, overriding 

443 ``self.butler.collections``. 

444 Can be any of the types supported by the ``collections`` argument 

445 to butler construction. 

446 

447 Results 

448 ------- 

449 records : `_VisitRecords` 

450 Struct containing DimensionRecords for the visit, including 

451 associated dimension elements. 

452 """ 

453 dimension = self.universe["visit"] 

454 

455 # Some registries support additional items. 

456 supported = {meta.name for meta in dimension.metadata} 

457 

458 # Compute all regions. 

459 visitRegion, visitDetectorRegions = self.computeVisitRegions.compute( 

460 definition, collections=collections 

461 ) 

462 # Aggregate other exposure quantities. 

463 timespan = Timespan( 

464 begin=_reduceOrNone(min, (e.timespan.begin for e in definition.exposures)), 

465 end=_reduceOrNone(max, (e.timespan.end for e in definition.exposures)), 

466 ) 

467 exposure_time = _reduceOrNone(operator.add, (e.exposure_time for e in definition.exposures)) 

468 physical_filter = _reduceOrNone(_value_if_equal, (e.physical_filter for e in definition.exposures)) 

469 target_name = _reduceOrNone(_value_if_equal, (e.target_name for e in definition.exposures)) 

470 science_program = _reduceOrNone(_value_if_equal, (e.science_program for e in definition.exposures)) 

471 

472 # observing day for a visit is defined by the earliest observation 

473 # of the visit 

474 observing_day = _reduceOrNone(min, (e.day_obs for e in definition.exposures)) 

475 observation_reason = _reduceOrNone( 

476 _value_if_equal, (e.observation_reason for e in definition.exposures) 

477 ) 

478 if observation_reason is None: 

479 # Be explicit about there being multiple reasons 

480 # MyPy can't really handle DimensionRecord fields as 

481 # DimensionRecord classes are dynamically defined; easiest to just 

482 # shush it when it complains. 

483 observation_reason = "various" # type: ignore 

484 

485 # Use the mean zenith angle as an approximation 

486 zenith_angle = _reduceOrNone(operator.add, (e.zenith_angle for e in definition.exposures)) 

487 if zenith_angle is not None: 

488 zenith_angle /= len(definition.exposures) 

489 

490 # New records that may not be supported. 

491 extras: Dict[str, Any] = {} 

492 if "seq_num" in supported: 

493 extras["seq_num"] = _reduceOrNone(min, (e.seq_num for e in definition.exposures)) 

494 if "azimuth" in supported: 

495 # Must take into account 0/360 problem. 

496 extras["azimuth"] = _calc_mean_angle([e.azimuth for e in definition.exposures]) 

497 

498 # visit_system handling changed. This is the logic for visit/exposure 

499 # that has support for seq_start/seq_end. 

500 if "seq_num" in supported: 

501 # Map visit to exposure. 

502 visit_definition = [ 

503 self.universe["visit_definition"].RecordClass( 

504 instrument=definition.instrument, 

505 visit=definition.id, 

506 exposure=exposure.id, 

507 ) 

508 for exposure in definition.exposures 

509 ] 

510 

511 # Map visit to visit system. 

512 visit_system_membership = [] 

513 for visit_system in self.groupExposures.getVisitSystems(): 

514 if visit_system in definition.visit_systems: 

515 record = self.universe["visit_system_membership"].RecordClass( 

516 instrument=definition.instrument, 

517 visit=definition.id, 

518 visit_system=visit_system.value, 

519 ) 

520 visit_system_membership.append(record) 

521 

522 else: 

523 # The old approach can only handle one visit system at a time. 

524 # If we have been configured with multiple options, prefer the 

525 # one-to-one. 

526 visit_systems = self.groupExposures.getVisitSystems() 

527 if len(visit_systems) > 1: 

528 one_to_one = VisitSystem.from_name("one-to-one") 

529 if one_to_one not in visit_systems: 

530 raise ValueError( 

531 f"Multiple visit systems specified ({visit_systems}) for use with old" 

532 " dimension universe but unable to find one-to-one." 

533 ) 

534 visit_system = one_to_one 

535 else: 

536 visit_system = visit_systems.pop() 

537 

538 extras["visit_system"] = visit_system.value 

539 

540 # The old visit_definition included visit system. 

541 visit_definition = [ 

542 self.universe["visit_definition"].RecordClass( 

543 instrument=definition.instrument, 

544 visit=definition.id, 

545 exposure=exposure.id, 

546 visit_system=visit_system.value, 

547 ) 

548 for exposure in definition.exposures 

549 ] 

550 

551 # This concept does not exist in old schema. 

552 visit_system_membership = [] 

553 

554 # Construct the actual DimensionRecords. 

555 return _VisitRecords( 

556 visit=dimension.RecordClass( 

557 instrument=definition.instrument, 

558 id=definition.id, 

559 name=definition.name, 

560 physical_filter=physical_filter, 

561 target_name=target_name, 

562 science_program=science_program, 

563 observation_reason=observation_reason, 

564 day_obs=observing_day, 

565 zenith_angle=zenith_angle, 

566 exposure_time=exposure_time, 

567 timespan=timespan, 

568 region=visitRegion, 

569 # TODO: no seeing value in exposure dimension records, so we 

570 # can't set that here. But there are many other columns that 

571 # both dimensions should probably have as well. 

572 **extras, 

573 ), 

574 visit_definition=visit_definition, 

575 visit_system_membership=visit_system_membership, 

576 visit_detector_region=[ 

577 self.universe["visit_detector_region"].RecordClass( 

578 instrument=definition.instrument, 

579 visit=definition.id, 

580 detector=detectorId, 

581 region=detectorRegion, 

582 ) 

583 for detectorId, detectorRegion in visitDetectorRegions.items() 

584 ], 

585 ) 

586 

587 def run( 

588 self, 

589 dataIds: Iterable[DataId], 

590 *, 

591 collections: Optional[str] = None, 

592 update_records: bool = False, 

593 ) -> None: 

594 """Add visit definitions to the registry for the given exposures. 

595 

596 Parameters 

597 ---------- 

598 dataIds : `Iterable` [ `dict` or `DataCoordinate` ] 

599 Exposure-level data IDs. These must all correspond to the same 

600 instrument, and are expected to be on-sky science exposures. 

601 collections : Any, optional 

602 Collections to be searched for raws and camera geometry, overriding 

603 ``self.butler.collections``. 

604 Can be any of the types supported by the ``collections`` argument 

605 to butler construction. 

606 update_records : `bool`, optional 

607 If `True` (`False` is default), update existing visit records that 

608 conflict with the new ones instead of rejecting them (and when this 

609 occurs, update visit_detector_region as well). THIS IS AN ADVANCED 

610 OPTION THAT SHOULD ONLY BE USED TO FIX REGIONS AND/OR METADATA THAT 

611 ARE KNOWN TO BE BAD, AND IT CANNOT BE USED TO REMOVE EXPOSURES OR 

612 DETECTORS FROM A VISIT. 

613 

614 Raises 

615 ------ 

616 lsst.daf.butler.registry.ConflictingDefinitionError 

617 Raised if a visit ID conflict is detected and the existing visit 

618 differs from the new one. 

619 """ 

620 # Normalize, expand, and deduplicate data IDs. 

621 self.log.info("Preprocessing data IDs.") 

622 dimensions = DimensionGraph(self.universe, names=["exposure"]) 

623 data_id_set: Set[DataCoordinate] = { 

624 self.butler.registry.expandDataId(d, graph=dimensions) for d in dataIds 

625 } 

626 if not data_id_set: 

627 raise RuntimeError("No exposures given.") 

628 # Extract exposure DimensionRecords, check that there's only one 

629 # instrument in play, and check for non-science exposures. 

630 exposures = [] 

631 instruments = set() 

632 for dataId in data_id_set: 

633 record = dataId.records["exposure"] 

634 assert record is not None, "Guaranteed by expandDataIds call earlier." 

635 if record.tracking_ra is None or record.tracking_dec is None or record.sky_angle is None: 

636 if self.config.ignoreNonScienceExposures: 

637 continue 

638 else: 

639 raise RuntimeError( 

640 f"Input exposure {dataId} has observation_type " 

641 f"{record.observation_type}, but is not on sky." 

642 ) 

643 instruments.add(dataId["instrument"]) 

644 exposures.append(record) 

645 if not exposures: 

646 self.log.info("No on-sky exposures found after filtering.") 

647 return 

648 if len(instruments) > 1: 

649 raise RuntimeError( 

650 f"All data IDs passed to DefineVisitsTask.run must be " 

651 f"from the same instrument; got {instruments}." 

652 ) 

653 (instrument,) = instruments 

654 # Ensure the visit_system our grouping algorithm uses is in the 

655 # registry, if it wasn't already. 

656 visitSystems = self.groupExposures.getVisitSystems() 

657 for visitSystem in visitSystems: 

658 self.log.info("Registering visit_system %d: %s.", visitSystem.value, visitSystem) 

659 self.butler.registry.syncDimensionData( 

660 "visit_system", {"instrument": instrument, "id": visitSystem.value, "name": str(visitSystem)} 

661 ) 

662 # Group exposures into visits, delegating to subtask. 

663 self.log.info("Grouping %d exposure(s) into visits.", len(exposures)) 

664 definitions = list(self.groupExposures.group(exposures)) 

665 # Iterate over visits, compute regions, and insert dimension data, one 

666 # transaction per visit. If a visit already exists, we skip all other 

667 # inserts. 

668 self.log.info("Computing regions and other metadata for %d visit(s).", len(definitions)) 

669 for visitDefinition in self.progress.wrap( 

670 definitions, total=len(definitions), desc="Computing regions and inserting visits" 

671 ): 

672 visitRecords = self._buildVisitRecords(visitDefinition, collections=collections) 

673 with self.butler.registry.transaction(): 

674 inserted_or_updated = self.butler.registry.syncDimensionData( 

675 "visit", 

676 visitRecords.visit, 

677 update=update_records, 

678 ) 

679 if inserted_or_updated: 

680 if inserted_or_updated is True: 

681 # This is a new visit, not an update to an existing 

682 # one, so insert visit definition. 

683 # We don't allow visit definitions to change even when 

684 # asked to update, because we'd have to delete the old 

685 # visit_definitions first and also worry about what 

686 # this does to datasets that already use the visit. 

687 self.butler.registry.insertDimensionData( 

688 "visit_definition", *visitRecords.visit_definition 

689 ) 

690 if visitRecords.visit_system_membership: 

691 self.butler.registry.insertDimensionData( 

692 "visit_system_membership", *visitRecords.visit_system_membership 

693 ) 

694 # [Re]Insert visit_detector_region records for both inserts 

695 # and updates, because we do allow updating to affect the 

696 # region calculations. 

697 self.butler.registry.insertDimensionData( 

698 "visit_detector_region", *visitRecords.visit_detector_region, replace=update_records 

699 ) 

700 

701 

702_T = TypeVar("_T") 

703 

704 

705def _reduceOrNone(func: Callable[[_T, _T], Optional[_T]], iterable: Iterable[Optional[_T]]) -> Optional[_T]: 

706 """Apply a binary function to pairs of elements in an iterable until a 

707 single value is returned, but return `None` if any element is `None` or 

708 there are no elements. 

709 """ 

710 r: Optional[_T] = None 

711 for v in iterable: 

712 if v is None: 

713 return None 

714 if r is None: 

715 r = v 

716 else: 

717 r = func(r, v) 

718 return r 

719 

720 

721def _value_if_equal(a: _T, b: _T) -> Optional[_T]: 

722 """Return either argument if they are equal, or `None` if they are not.""" 

723 return a if a == b else None 

724 

725 

726def _calc_mean_angle(angles: List[float]) -> float: 

727 """Calculate the mean angle, taking into account 0/360 wrapping. 

728 

729 Parameters 

730 ---------- 

731 angles : `list` [`float`] 

732 Angles to average together, in degrees. 

733 

734 Returns 

735 ------- 

736 average : `float` 

737 Average angle in degrees. 

738 """ 

739 # Save on all the math if we only have one value. 

740 if len(angles) == 1: 

741 return angles[0] 

742 

743 # Convert polar coordinates of unit circle to complex values. 

744 # Average the complex values. 

745 # Convert back to a phase angle. 

746 return math.degrees(cmath.phase(sum(cmath.rect(1.0, math.radians(d)) for d in angles) / len(angles))) 

747 

748 

749class _GroupExposuresOneToOneConfig(GroupExposuresConfig): 

750 visitSystemId: Field[int] = Field( 

751 doc="Integer ID of the visit_system implemented by this grouping algorithm.", 

752 dtype=int, 

753 default=0, 

754 deprecated="No longer used. Replaced by enum.", 

755 ) 

756 visitSystemName: Field[str] = Field( 

757 doc="String name of the visit_system implemented by this grouping algorithm.", 

758 dtype=str, 

759 default="one-to-one", 

760 deprecated="No longer used. Replaced by enum.", 

761 ) 

762 

763 

764@registerConfigurable("one-to-one", GroupExposuresTask.registry) 

765class _GroupExposuresOneToOneTask(GroupExposuresTask, metaclass=ABCMeta): 

766 """An exposure grouping algorithm that simply defines one visit for each 

767 exposure, reusing the exposures identifiers for the visit. 

768 """ 

769 

770 ConfigClass = _GroupExposuresOneToOneConfig 

771 

772 def group(self, exposures: List[DimensionRecord]) -> Iterable[VisitDefinitionData]: 

773 # Docstring inherited from GroupExposuresTask. 

774 visit_systems = {VisitSystem.from_name("one-to-one")} 

775 for exposure in exposures: 

776 yield VisitDefinitionData( 

777 instrument=exposure.instrument, 

778 id=exposure.id, 

779 name=exposure.obs_id, 

780 exposures=[exposure], 

781 visit_systems=visit_systems, 

782 ) 

783 

784 def getVisitSystems(self) -> Set[VisitSystem]: 

785 # Docstring inherited from GroupExposuresTask. 

786 return set(VisitSystem.from_names(["one-to-one"])) 

787 

788 

789class _GroupExposuresByGroupMetadataConfig(GroupExposuresConfig): 

790 visitSystemId: Field[int] = Field( 

791 doc="Integer ID of the visit_system implemented by this grouping algorithm.", 

792 dtype=int, 

793 default=1, 

794 deprecated="No longer used. Replaced by enum.", 

795 ) 

796 visitSystemName: Field[str] = Field( 

797 doc="String name of the visit_system implemented by this grouping algorithm.", 

798 dtype=str, 

799 default="by-group-metadata", 

800 deprecated="No longer used. Replaced by enum.", 

801 ) 

802 

803 

804@registerConfigurable("by-group-metadata", GroupExposuresTask.registry) 

805class _GroupExposuresByGroupMetadataTask(GroupExposuresTask, metaclass=ABCMeta): 

806 """An exposure grouping algorithm that uses exposure.group_name and 

807 exposure.group_id. 

808 

809 This algorithm _assumes_ exposure.group_id (generally populated from 

810 `astro_metadata_translator.ObservationInfo.visit_id`) is not just unique, 

811 but disjoint from all `ObservationInfo.exposure_id` values - if it isn't, 

812 it will be impossible to ever use both this grouping algorithm and the 

813 one-to-one algorithm for a particular camera in the same data repository. 

814 """ 

815 

816 ConfigClass = _GroupExposuresByGroupMetadataConfig 

817 

818 def group(self, exposures: List[DimensionRecord]) -> Iterable[VisitDefinitionData]: 

819 # Docstring inherited from GroupExposuresTask. 

820 visit_systems = {VisitSystem.from_name("by-group-metadata")} 

821 groups = defaultdict(list) 

822 for exposure in exposures: 

823 groups[exposure.group_name].append(exposure) 

824 for visitName, exposuresInGroup in groups.items(): 

825 instrument = exposuresInGroup[0].instrument 

826 visitId = exposuresInGroup[0].group_id 

827 assert all( 

828 e.group_id == visitId for e in exposuresInGroup 

829 ), "Grouping by exposure.group_name does not yield consistent group IDs" 

830 yield VisitDefinitionData( 

831 instrument=instrument, 

832 id=visitId, 

833 name=visitName, 

834 exposures=exposuresInGroup, 

835 visit_systems=visit_systems, 

836 ) 

837 

838 def getVisitSystems(self) -> Set[VisitSystem]: 

839 # Docstring inherited from GroupExposuresTask. 

840 return set(VisitSystem.from_names(["by-group-metadata"])) 

841 

842 

843class _GroupExposuresByCounterAndExposuresConfig(GroupExposuresConfig): 

844 visitSystemId: Field[int] = Field( 

845 doc="Integer ID of the visit_system implemented by this grouping algorithm.", 

846 dtype=int, 

847 default=2, 

848 deprecated="No longer used. Replaced by enum.", 

849 ) 

850 visitSystemName: Field[str] = Field( 

851 doc="String name of the visit_system implemented by this grouping algorithm.", 

852 dtype=str, 

853 default="by-counter-and-exposures", 

854 deprecated="No longer used. Replaced by enum.", 

855 ) 

856 

857 

858@registerConfigurable("one-to-one-and-by-counter", GroupExposuresTask.registry) 

859class _GroupExposuresByCounterAndExposuresTask(GroupExposuresTask, metaclass=ABCMeta): 

860 """An exposure grouping algorithm that uses the sequence start and 

861 sequence end metadata to create multi-exposure visits, but also 

862 creates one-to-one visits. 

863 

864 This algorithm uses the exposure.seq_start and 

865 exposure.seq_end fields to collect related snaps. 

866 It also groups single exposures. 

867 """ 

868 

869 ConfigClass = _GroupExposuresByCounterAndExposuresConfig 

870 

871 def group(self, exposures: List[DimensionRecord]) -> Iterable[VisitDefinitionData]: 

872 # Docstring inherited from GroupExposuresTask. 

873 system_one_to_one = VisitSystem.from_name("one-to-one") 

874 system_seq_start_end = VisitSystem.from_name("by-seq-start-end") 

875 

876 groups = defaultdict(list) 

877 for exposure in exposures: 

878 groups[exposure.day_obs, exposure.seq_start, exposure.seq_end].append(exposure) 

879 for visit_key, exposures_in_group in groups.items(): 

880 instrument = exposures_in_group[0].instrument 

881 

882 # It is possible that the first exposure in a visit has not 

883 # been ingested. This can be determined and if that is the case 

884 # we can not reliably define the multi-exposure visit. 

885 skip_multi = False 

886 sorted_exposures = sorted(exposures_in_group, key=lambda e: e.seq_num) 

887 first = sorted_exposures.pop(0) 

888 if first.seq_num != first.seq_start: 

889 # Special case seq_num == 0 since that implies that the 

890 # instrument has no counters and therefore no multi-exposure 

891 # visits. 

892 if first.seq_num == 0: 

893 self.log.warning( 

894 "First exposure for visit %s is not present. Skipping the multi-snap definition.", 

895 visit_key, 

896 ) 

897 skip_multi = True 

898 

899 # Define the one-to-one visits. 

900 num_exposures = len(exposures_in_group) 

901 for exposure in exposures_in_group: 

902 # Default is to use the exposure ID and name unless 

903 # this is the first exposure in a multi-exposure visit. 

904 visit_name = exposure.obs_id 

905 visit_id = exposure.id 

906 visit_systems = {system_one_to_one} 

907 

908 if num_exposures == 1: 

909 # This is also a by-counter visit. 

910 # It will use the same visit_name and visit_id. 

911 visit_systems.add(system_seq_start_end) 

912 

913 elif num_exposures > 1 and not skip_multi and exposure == first: 

914 # This is the first legitimate exposure in a multi-exposure 

915 # visit. It therefore needs a modified visit name and ID 

916 # so it does not clash with the multi-exposure visit 

917 # definition. 

918 visit_name = f"{visit_name}_first" 

919 visit_id = int(f"9{visit_id}") 

920 

921 yield VisitDefinitionData( 

922 instrument=instrument, 

923 id=visit_id, 

924 name=visit_name, 

925 exposures=[exposure], 

926 visit_systems=visit_systems, 

927 ) 

928 

929 # Multi-exposure visit. 

930 if not skip_multi and num_exposures > 1: 

931 # Define the visit using the first exposure 

932 visit_name = first.obs_id 

933 visit_id = first.id 

934 

935 yield VisitDefinitionData( 

936 instrument=instrument, 

937 id=visit_id, 

938 name=visit_name, 

939 exposures=exposures_in_group, 

940 visit_systems={system_seq_start_end}, 

941 ) 

942 

943 def getVisitSystems(self) -> Set[VisitSystem]: 

944 # Docstring inherited from GroupExposuresTask. 

945 # Using a Config for this is difficult because what this grouping 

946 # algorithm is doing is using two visit systems. 

947 # One is using metadata (but not by-group) and the other is the 

948 # one-to-one. For now hard-code in class. 

949 return set(VisitSystem.from_names(["one-to-one", "by-seq-start-end"])) 

950 

951 

952class _ComputeVisitRegionsFromSingleRawWcsConfig(ComputeVisitRegionsConfig): 

953 mergeExposures: Field[bool] = Field( 

954 doc=( 

955 "If True, merge per-detector regions over all exposures in a " 

956 "visit (via convex hull) instead of using the first exposure and " 

957 "assuming its regions are valid for all others." 

958 ), 

959 dtype=bool, 

960 default=False, 

961 ) 

962 detectorId: Field[Optional[int]] = Field( 

963 doc=( 

964 "Load the WCS for the detector with this ID. If None, use an " 

965 "arbitrary detector (the first found in a query of the data " 

966 "repository for each exposure (or all exposures, if " 

967 "mergeExposures is True)." 

968 ), 

969 dtype=int, 

970 optional=True, 

971 default=None, 

972 ) 

973 requireVersionedCamera: Field[bool] = Field( 

974 doc=( 

975 "If True, raise LookupError if version camera geometry cannot be " 

976 "loaded for an exposure. If False, use the nominal camera from " 

977 "the Instrument class instead." 

978 ), 

979 dtype=bool, 

980 optional=False, 

981 default=False, 

982 ) 

983 

984 

985@registerConfigurable("single-raw-wcs", ComputeVisitRegionsTask.registry) 

986class _ComputeVisitRegionsFromSingleRawWcsTask(ComputeVisitRegionsTask): 

987 """A visit region calculator that uses a single raw WCS and a camera to 

988 project the bounding boxes of all detectors onto the sky, relating 

989 different detectors by their positions in focal plane coordinates. 

990 

991 Notes 

992 ----- 

993 Most instruments should have their raw WCSs determined from a combination 

994 of boresight angle, rotator angle, and camera geometry, and hence this 

995 algorithm should produce stable results regardless of which detector the 

996 raw corresponds to. If this is not the case (e.g. because a per-file FITS 

997 WCS is used instead), either the ID of the detector should be fixed (see 

998 the ``detectorId`` config parameter) or a different algorithm used. 

999 """ 

1000 

1001 ConfigClass = _ComputeVisitRegionsFromSingleRawWcsConfig 

1002 config: _ComputeVisitRegionsFromSingleRawWcsConfig 

1003 

1004 def computeExposureBounds( 

1005 self, exposure: DimensionRecord, *, collections: Any = None 

1006 ) -> Dict[int, List[UnitVector3d]]: 

1007 """Compute the lists of unit vectors on the sphere that correspond to 

1008 the sky positions of detector corners. 

1009 

1010 Parameters 

1011 ---------- 

1012 exposure : `DimensionRecord` 

1013 Dimension record for the exposure. 

1014 collections : Any, optional 

1015 Collections to be searched for raws and camera geometry, overriding 

1016 ``self.butler.collections``. 

1017 Can be any of the types supported by the ``collections`` argument 

1018 to butler construction. 

1019 

1020 Returns 

1021 ------- 

1022 bounds : `dict` 

1023 Dictionary mapping detector ID to a list of unit vectors on the 

1024 sphere representing that detector's corners projected onto the sky. 

1025 """ 

1026 if collections is None: 

1027 collections = self.butler.collections 

1028 camera, versioned = loadCamera(self.butler, exposure.dataId, collections=collections) 

1029 if not versioned and self.config.requireVersionedCamera: 

1030 raise LookupError(f"No versioned camera found for exposure {exposure.dataId}.") 

1031 

1032 # Derive WCS from boresight information -- if available in registry 

1033 use_registry = True 

1034 try: 

1035 orientation = lsst.geom.Angle(exposure.sky_angle, lsst.geom.degrees) 

1036 radec = lsst.geom.SpherePoint( 

1037 lsst.geom.Angle(exposure.tracking_ra, lsst.geom.degrees), 

1038 lsst.geom.Angle(exposure.tracking_dec, lsst.geom.degrees), 

1039 ) 

1040 except AttributeError: 

1041 use_registry = False 

1042 

1043 if use_registry: 

1044 if self.config.detectorId is None: 

1045 detectorId = next(camera.getIdIter()) 

1046 else: 

1047 detectorId = self.config.detectorId 

1048 wcsDetector = camera[detectorId] 

1049 

1050 # Ask the raw formatter to create the relevant WCS 

1051 # This allows flips to be taken into account 

1052 instrument = self.getInstrument(exposure.instrument) 

1053 rawFormatter = instrument.getRawFormatter({"detector": detectorId}) 

1054 

1055 try: 

1056 wcs = rawFormatter.makeRawSkyWcsFromBoresight(radec, orientation, wcsDetector) # type: ignore 

1057 except AttributeError: 

1058 raise TypeError( 

1059 f"Raw formatter is {get_full_type_name(rawFormatter)} but visit" 

1060 " definition requires it to support 'makeRawSkyWcsFromBoresight'" 

1061 ) from None 

1062 else: 

1063 if self.config.detectorId is None: 

1064 wcsRefsIter = self.butler.registry.queryDatasets( 

1065 "raw.wcs", dataId=exposure.dataId, collections=collections 

1066 ) 

1067 if not wcsRefsIter: 

1068 raise LookupError( 

1069 f"No raw.wcs datasets found for data ID {exposure.dataId} " 

1070 f"in collections {collections}." 

1071 ) 

1072 wcsRef = next(iter(wcsRefsIter)) 

1073 wcsDetector = camera[wcsRef.dataId["detector"]] 

1074 wcs = self.butler.getDirect(wcsRef) 

1075 else: 

1076 wcsDetector = camera[self.config.detectorId] 

1077 wcs = self.butler.get( 

1078 "raw.wcs", 

1079 dataId=exposure.dataId, 

1080 detector=self.config.detectorId, 

1081 collections=collections, 

1082 ) 

1083 fpToSky = wcsDetector.getTransform(FOCAL_PLANE, PIXELS).then(wcs.getTransform()) 

1084 bounds = {} 

1085 for detector in camera: 

1086 pixelsToSky = detector.getTransform(PIXELS, FOCAL_PLANE).then(fpToSky) 

1087 pixCorners = Box2D(detector.getBBox().dilatedBy(self.config.padding)).getCorners() 

1088 bounds[detector.getId()] = [ 

1089 skyCorner.getVector() for skyCorner in pixelsToSky.applyForward(pixCorners) 

1090 ] 

1091 return bounds 

1092 

1093 def compute( 

1094 self, visit: VisitDefinitionData, *, collections: Any = None 

1095 ) -> Tuple[Region, Dict[int, Region]]: 

1096 # Docstring inherited from ComputeVisitRegionsTask. 

1097 if self.config.mergeExposures: 

1098 detectorBounds: Dict[int, List[UnitVector3d]] = defaultdict(list) 

1099 for exposure in visit.exposures: 

1100 exposureDetectorBounds = self.computeExposureBounds(exposure, collections=collections) 

1101 for detectorId, bounds in exposureDetectorBounds.items(): 

1102 detectorBounds[detectorId].extend(bounds) 

1103 else: 

1104 detectorBounds = self.computeExposureBounds(visit.exposures[0], collections=collections) 

1105 visitBounds = [] 

1106 detectorRegions = {} 

1107 for detectorId, bounds in detectorBounds.items(): 

1108 detectorRegions[detectorId] = ConvexPolygon.convexHull(bounds) 

1109 visitBounds.extend(bounds) 

1110 return ConvexPolygon.convexHull(visitBounds), detectorRegions