Coverage for python/lsst/obs/base/defineVisits.py: 27%

353 statements  

« prev     ^ index     » next       coverage.py v7.2.4, created at 2023-04-29 03:23 -0700

1# This file is part of obs_base. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22from __future__ import annotations 

23 

24__all__ = [ 

25 "DefineVisitsConfig", 

26 "DefineVisitsTask", 

27 "GroupExposuresConfig", 

28 "GroupExposuresTask", 

29 "VisitDefinitionData", 

30 "VisitSystem", 

31] 

32 

33import cmath 

34import dataclasses 

35import enum 

36import math 

37import operator 

38from abc import ABCMeta, abstractmethod 

39from collections import defaultdict 

40from typing import ( 

41 Any, 

42 Callable, 

43 ClassVar, 

44 Dict, 

45 FrozenSet, 

46 Iterable, 

47 List, 

48 Optional, 

49 Set, 

50 Tuple, 

51 Type, 

52 TypeVar, 

53 cast, 

54) 

55 

56import lsst.geom 

57from lsst.afw.cameraGeom import FOCAL_PLANE, PIXELS 

58from lsst.daf.butler import ( 

59 Butler, 

60 DataCoordinate, 

61 DataId, 

62 DimensionGraph, 

63 DimensionRecord, 

64 Progress, 

65 Timespan, 

66) 

67from lsst.geom import Box2D 

68from lsst.pex.config import Config, Field, makeRegistry, registerConfigurable 

69from lsst.pipe.base import Instrument, Task 

70from lsst.sphgeom import ConvexPolygon, Region, UnitVector3d 

71from lsst.utils.introspection import get_full_type_name 

72 

73from ._instrument import loadCamera 

74 

75 

76class VisitSystem(enum.Enum): 

77 """Enumeration used to label different visit systems.""" 

78 

79 ONE_TO_ONE = 0 

80 """Each exposure is assigned to its own visit.""" 

81 

82 BY_GROUP_METADATA = 1 

83 """Visit membership is defined by the value of the exposure.group_id.""" 

84 

85 BY_SEQ_START_END = 2 

86 """Visit membership is defined by the values of the ``exposure.day_obs``, 

87 ``exposure.seq_start``, and ``exposure.seq_end`` values. 

88 """ 

89 

90 @classmethod 

91 def all(cls) -> FrozenSet[VisitSystem]: 

92 """Return a `frozenset` containing all members.""" 

93 return frozenset(cls.__members__.values()) 

94 

95 @classmethod 

96 def from_name(cls, external_name: str) -> VisitSystem: 

97 """Construct the enumeration from given name.""" 

98 name = external_name.upper() 

99 name = name.replace("-", "_") 

100 try: 

101 return cls.__members__[name] 

102 except KeyError: 

103 raise KeyError(f"Visit system named '{external_name}' not known.") from None 

104 

105 @classmethod 

106 def from_names(cls, names: Optional[Iterable[str]]) -> FrozenSet[VisitSystem]: 

107 """Return a `frozenset` of all the visit systems matching the supplied 

108 names. 

109 

110 Parameters 

111 ---------- 

112 names : iterable of `str`, or `None` 

113 Names of visit systems. Case insensitive. If `None` or empty, all 

114 the visit systems are returned. 

115 

116 Returns 

117 ------- 

118 systems : `frozenset` of `VisitSystem` 

119 The matching visit systems. 

120 """ 

121 if not names: 

122 return cls.all() 

123 

124 return frozenset({cls.from_name(name) for name in names}) 

125 

126 def __str__(self) -> str: 

127 name = self.name.lower() 

128 name = name.replace("_", "-") 

129 return name 

130 

131 

132@dataclasses.dataclass 

133class VisitDefinitionData: 

134 """Struct representing a group of exposures that will be used to define a 

135 visit. 

136 """ 

137 

138 instrument: str 

139 """Name of the instrument this visit will be associated with. 

140 """ 

141 

142 id: int 

143 """Integer ID of the visit. 

144 

145 This must be unique across all visit systems for the instrument. 

146 """ 

147 

148 name: str 

149 """String name for the visit. 

150 

151 This must be unique across all visit systems for the instrument. 

152 """ 

153 

154 visit_systems: Set[VisitSystem] 

155 """All the visit systems associated with this visit.""" 

156 

157 exposures: List[DimensionRecord] = dataclasses.field(default_factory=list) 

158 """Dimension records for the exposures that are part of this visit. 

159 """ 

160 

161 

162@dataclasses.dataclass 

163class _VisitRecords: 

164 """Struct containing the dimension records associated with a visit.""" 

165 

166 visit: DimensionRecord 

167 """Record for the 'visit' dimension itself. 

168 """ 

169 

170 visit_definition: List[DimensionRecord] 

171 """Records for 'visit_definition', which relates 'visit' to 'exposure'. 

172 """ 

173 

174 visit_detector_region: List[DimensionRecord] 

175 """Records for 'visit_detector_region', which associates the combination 

176 of a 'visit' and a 'detector' with a region on the sky. 

177 """ 

178 

179 visit_system_membership: List[DimensionRecord] 

180 """Records relating visits to an associated visit system.""" 

181 

182 

183class GroupExposuresConfig(Config): 

184 pass 

185 

186 

187class GroupExposuresTask(Task, metaclass=ABCMeta): 

188 """Abstract base class for the subtask of `DefineVisitsTask` that is 

189 responsible for grouping exposures into visits. 

190 

191 Subclasses should be registered with `GroupExposuresTask.registry` to 

192 enable use by `DefineVisitsTask`, and should generally correspond to a 

193 particular 'visit_system' dimension value. They are also responsible for 

194 defining visit IDs and names that are unique across all visit systems in 

195 use by an instrument. 

196 

197 Parameters 

198 ---------- 

199 config : `GroupExposuresConfig` 

200 Configuration information. 

201 **kwargs 

202 Additional keyword arguments forwarded to the `Task` constructor. 

203 """ 

204 

205 def __init__(self, config: GroupExposuresConfig, **kwargs: Any): 

206 Task.__init__(self, config=config, **kwargs) 

207 

208 ConfigClass = GroupExposuresConfig 

209 

210 _DefaultName = "groupExposures" 

211 

212 registry = makeRegistry( 

213 doc="Registry of algorithms for grouping exposures into visits.", 

214 configBaseType=GroupExposuresConfig, 

215 ) 

216 

217 @abstractmethod 

218 def group(self, exposures: List[DimensionRecord]) -> Iterable[VisitDefinitionData]: 

219 """Group the given exposures into visits. 

220 

221 Parameters 

222 ---------- 

223 exposures : `list` [ `DimensionRecord` ] 

224 DimensionRecords (for the 'exposure' dimension) describing the 

225 exposures to group. 

226 

227 Returns 

228 ------- 

229 visits : `Iterable` [ `VisitDefinitionData` ] 

230 Structs identifying the visits and the exposures associated with 

231 them. This may be an iterator or a container. 

232 """ 

233 raise NotImplementedError() 

234 

235 def getVisitSystems(self) -> Set[VisitSystem]: 

236 """Return identifiers for the 'visit_system' dimension this 

237 algorithm implements. 

238 

239 Returns 

240 ------- 

241 visit_systems : `Set` [`VisitSystem`] 

242 The visit systems used by this algorithm. 

243 """ 

244 raise NotImplementedError() 

245 

246 

247class ComputeVisitRegionsConfig(Config): 

248 padding: Field[int] = Field( 

249 dtype=int, 

250 default=250, 

251 doc=( 

252 "Pad raw image bounding boxes with specified number of pixels " 

253 "when calculating their (conservatively large) region on the " 

254 "sky. Note that the config value for pixelMargin of the " 

255 "reference object loaders in meas_algorithms should be <= " 

256 "the value set here." 

257 ), 

258 ) 

259 

260 

261class ComputeVisitRegionsTask(Task, metaclass=ABCMeta): 

262 """Abstract base class for the subtask of `DefineVisitsTask` that is 

263 responsible for extracting spatial regions for visits and visit+detector 

264 combinations. 

265 

266 Subclasses should be registered with `ComputeVisitRegionsTask.registry` to 

267 enable use by `DefineVisitsTask`. 

268 

269 Parameters 

270 ---------- 

271 config : `ComputeVisitRegionsConfig` 

272 Configuration information. 

273 butler : `lsst.daf.butler.Butler` 

274 The butler to use. 

275 **kwargs 

276 Additional keyword arguments forwarded to the `Task` constructor. 

277 """ 

278 

279 def __init__(self, config: ComputeVisitRegionsConfig, *, butler: Butler, **kwargs: Any): 

280 Task.__init__(self, config=config, **kwargs) 

281 self.butler = butler 

282 self.instrumentMap: Dict[str, Instrument] = {} 

283 

284 ConfigClass = ComputeVisitRegionsConfig 

285 

286 _DefaultName = "computeVisitRegions" 

287 

288 registry = makeRegistry( 

289 doc="Registry of algorithms for computing on-sky regions for visits and visit+detector combinations.", 

290 configBaseType=ComputeVisitRegionsConfig, 

291 ) 

292 

293 def getInstrument(self, instrumentName: str) -> Instrument: 

294 """Retrieve an `~lsst.obs.base.Instrument` associated with this 

295 instrument name. 

296 

297 Parameters 

298 ---------- 

299 instrumentName : `str` 

300 The name of the instrument. 

301 

302 Returns 

303 ------- 

304 instrument : `~lsst.obs.base.Instrument` 

305 The associated instrument object. 

306 

307 Notes 

308 ----- 

309 The result is cached. 

310 """ 

311 instrument = self.instrumentMap.get(instrumentName) 

312 if instrument is None: 

313 instrument = Instrument.fromName(instrumentName, self.butler.registry) 

314 self.instrumentMap[instrumentName] = instrument 

315 return instrument 

316 

317 @abstractmethod 

318 def compute( 

319 self, visit: VisitDefinitionData, *, collections: Any = None 

320 ) -> Tuple[Region, Dict[int, Region]]: 

321 """Compute regions for the given visit and all detectors in that visit. 

322 

323 Parameters 

324 ---------- 

325 visit : `VisitDefinitionData` 

326 Struct describing the visit and the exposures associated with it. 

327 collections : Any, optional 

328 Collections to be searched for raws and camera geometry, overriding 

329 ``self.butler.collections``. 

330 Can be any of the types supported by the ``collections`` argument 

331 to butler construction. 

332 

333 Returns 

334 ------- 

335 visitRegion : `lsst.sphgeom.Region` 

336 Region for the full visit. 

337 visitDetectorRegions : `dict` [ `int`, `lsst.sphgeom.Region` ] 

338 Dictionary mapping detector ID to the region for that detector. 

339 Should include all detectors in the visit. 

340 """ 

341 raise NotImplementedError() 

342 

343 

344class DefineVisitsConfig(Config): 

345 groupExposures = GroupExposuresTask.registry.makeField( 

346 doc="Algorithm for grouping exposures into visits.", 

347 default="one-to-one-and-by-counter", 

348 ) 

349 computeVisitRegions = ComputeVisitRegionsTask.registry.makeField( 

350 doc="Algorithm from computing visit and visit+detector regions.", 

351 default="single-raw-wcs", 

352 ) 

353 ignoreNonScienceExposures: Field[bool] = Field( 

354 doc=( 

355 "If True, silently ignore input exposures that do not have " 

356 "observation_type=SCIENCE. If False, raise an exception if one " 

357 "encountered." 

358 ), 

359 dtype=bool, 

360 optional=False, 

361 default=True, 

362 ) 

363 updateObsCoreTable: Field[bool] = Field( 

364 doc=( 

365 "If True, update exposure regions in obscore table after visits " 

366 "are defined. If False, do not update obscore table." 

367 ), 

368 dtype=bool, 

369 default=True, 

370 ) 

371 

372 

373class DefineVisitsTask(Task): 

374 """Driver Task for defining visits (and their spatial regions) in Gen3 

375 Butler repositories. 

376 

377 Parameters 

378 ---------- 

379 config : `DefineVisitsConfig` 

380 Configuration for the task. 

381 butler : `~lsst.daf.butler.Butler` 

382 Writeable butler instance. Will be used to read `raw.wcs` and `camera` 

383 datasets and insert/sync dimension data. 

384 **kwargs 

385 Additional keyword arguments are forwarded to the `lsst.pipe.base.Task` 

386 constructor. 

387 

388 Notes 

389 ----- 

390 Each instance of `DefineVisitsTask` reads from / writes to the same Butler. 

391 Each invocation of `DefineVisitsTask.run` processes an independent group of 

392 exposures into one or more new vists, all belonging to the same visit 

393 system and instrument. 

394 

395 The actual work of grouping exposures and computing regions is delegated 

396 to pluggable subtasks (`GroupExposuresTask` and `ComputeVisitRegionsTask`), 

397 respectively. The defaults are to create one visit for every exposure, 

398 and to use exactly one (arbitrary) detector-level raw dataset's WCS along 

399 with camera geometry to compute regions for all detectors. Other 

400 implementations can be created and configured for instruments for which 

401 these choices are unsuitable (e.g. because visits and exposures are not 

402 one-to-one, or because ``raw.wcs`` datasets for different detectors may not 

403 be consistent with camera geomery). 

404 

405 It is not necessary in general to ingest all raws for an exposure before 

406 defining a visit that includes the exposure; this depends entirely on the 

407 `ComputeVisitRegionTask` subclass used. For the default configuration, 

408 a single raw for each exposure is sufficient. 

409 

410 Defining the same visit the same way multiple times (e.g. via multiple 

411 invocations of this task on the same exposures, with the same 

412 configuration) is safe, but it may be inefficient, as most of the work must 

413 be done before new visits can be compared to existing visits. 

414 """ 

415 

416 def __init__(self, config: DefineVisitsConfig, *, butler: Butler, **kwargs: Any): 

417 config.validate() # Not a CmdlineTask nor PipelineTask, so have to validate the config here. 

418 super().__init__(config, **kwargs) 

419 self.butler = butler 

420 self.universe = self.butler.registry.dimensions 

421 self.progress = Progress("obs.base.DefineVisitsTask") 

422 self.makeSubtask("groupExposures") 

423 self.makeSubtask("computeVisitRegions", butler=self.butler) 

424 

425 def _reduce_kwargs(self) -> dict: 

426 # Add extra parameters to pickle 

427 return dict(**super()._reduce_kwargs(), butler=self.butler) 

428 

429 ConfigClass: ClassVar[Type[Config]] = DefineVisitsConfig 

430 

431 _DefaultName: ClassVar[str] = "defineVisits" 

432 

433 config: DefineVisitsConfig 

434 groupExposures: GroupExposuresTask 

435 computeVisitRegions: ComputeVisitRegionsTask 

436 

437 def _buildVisitRecords( 

438 self, definition: VisitDefinitionData, *, collections: Any = None 

439 ) -> _VisitRecords: 

440 """Build the DimensionRecords associated with a visit. 

441 

442 Parameters 

443 ---------- 

444 definition : `VisitDefinitionData` 

445 Struct with identifiers for the visit and records for its 

446 constituent exposures. 

447 collections : Any, optional 

448 Collections to be searched for raws and camera geometry, overriding 

449 ``self.butler.collections``. 

450 Can be any of the types supported by the ``collections`` argument 

451 to butler construction. 

452 

453 Results 

454 ------- 

455 records : `_VisitRecords` 

456 Struct containing DimensionRecords for the visit, including 

457 associated dimension elements. 

458 """ 

459 dimension = self.universe["visit"] 

460 

461 # Some registries support additional items. 

462 supported = {meta.name for meta in dimension.metadata} 

463 

464 # Compute all regions. 

465 visitRegion, visitDetectorRegions = self.computeVisitRegions.compute( 

466 definition, collections=collections 

467 ) 

468 # Aggregate other exposure quantities. 

469 timespan = Timespan( 

470 begin=_reduceOrNone(min, (e.timespan.begin for e in definition.exposures)), 

471 end=_reduceOrNone(max, (e.timespan.end for e in definition.exposures)), 

472 ) 

473 exposure_time = _reduceOrNone(operator.add, (e.exposure_time for e in definition.exposures)) 

474 physical_filter = _reduceOrNone(_value_if_equal, (e.physical_filter for e in definition.exposures)) 

475 target_name = _reduceOrNone(_value_if_equal, (e.target_name for e in definition.exposures)) 

476 science_program = _reduceOrNone(_value_if_equal, (e.science_program for e in definition.exposures)) 

477 

478 # observing day for a visit is defined by the earliest observation 

479 # of the visit 

480 observing_day = _reduceOrNone(min, (e.day_obs for e in definition.exposures)) 

481 observation_reason = _reduceOrNone( 

482 _value_if_equal, (e.observation_reason for e in definition.exposures) 

483 ) 

484 if observation_reason is None: 

485 # Be explicit about there being multiple reasons 

486 # MyPy can't really handle DimensionRecord fields as 

487 # DimensionRecord classes are dynamically defined; easiest to just 

488 # shush it when it complains. 

489 observation_reason = "various" # type: ignore 

490 

491 # Use the mean zenith angle as an approximation 

492 zenith_angle = _reduceOrNone(operator.add, (e.zenith_angle for e in definition.exposures)) 

493 if zenith_angle is not None: 

494 zenith_angle /= len(definition.exposures) 

495 

496 # New records that may not be supported. 

497 extras: Dict[str, Any] = {} 

498 if "seq_num" in supported: 

499 extras["seq_num"] = _reduceOrNone(min, (e.seq_num for e in definition.exposures)) 

500 if "azimuth" in supported: 

501 # Must take into account 0/360 problem. 

502 extras["azimuth"] = _calc_mean_angle([e.azimuth for e in definition.exposures]) 

503 

504 # visit_system handling changed. This is the logic for visit/exposure 

505 # that has support for seq_start/seq_end. 

506 if "seq_num" in supported: 

507 # Map visit to exposure. 

508 visit_definition = [ 

509 self.universe["visit_definition"].RecordClass( 

510 instrument=definition.instrument, 

511 visit=definition.id, 

512 exposure=exposure.id, 

513 ) 

514 for exposure in definition.exposures 

515 ] 

516 

517 # Map visit to visit system. 

518 visit_system_membership = [] 

519 for visit_system in self.groupExposures.getVisitSystems(): 

520 if visit_system in definition.visit_systems: 

521 record = self.universe["visit_system_membership"].RecordClass( 

522 instrument=definition.instrument, 

523 visit=definition.id, 

524 visit_system=visit_system.value, 

525 ) 

526 visit_system_membership.append(record) 

527 

528 else: 

529 # The old approach can only handle one visit system at a time. 

530 # If we have been configured with multiple options, prefer the 

531 # one-to-one. 

532 visit_systems = self.groupExposures.getVisitSystems() 

533 if len(visit_systems) > 1: 

534 one_to_one = VisitSystem.from_name("one-to-one") 

535 if one_to_one not in visit_systems: 

536 raise ValueError( 

537 f"Multiple visit systems specified ({visit_systems}) for use with old" 

538 " dimension universe but unable to find one-to-one." 

539 ) 

540 visit_system = one_to_one 

541 else: 

542 visit_system = visit_systems.pop() 

543 

544 extras["visit_system"] = visit_system.value 

545 

546 # The old visit_definition included visit system. 

547 visit_definition = [ 

548 self.universe["visit_definition"].RecordClass( 

549 instrument=definition.instrument, 

550 visit=definition.id, 

551 exposure=exposure.id, 

552 visit_system=visit_system.value, 

553 ) 

554 for exposure in definition.exposures 

555 ] 

556 

557 # This concept does not exist in old schema. 

558 visit_system_membership = [] 

559 

560 # Construct the actual DimensionRecords. 

561 return _VisitRecords( 

562 visit=dimension.RecordClass( 

563 instrument=definition.instrument, 

564 id=definition.id, 

565 name=definition.name, 

566 physical_filter=physical_filter, 

567 target_name=target_name, 

568 science_program=science_program, 

569 observation_reason=observation_reason, 

570 day_obs=observing_day, 

571 zenith_angle=zenith_angle, 

572 exposure_time=exposure_time, 

573 timespan=timespan, 

574 region=visitRegion, 

575 # TODO: no seeing value in exposure dimension records, so we 

576 # can't set that here. But there are many other columns that 

577 # both dimensions should probably have as well. 

578 **extras, 

579 ), 

580 visit_definition=visit_definition, 

581 visit_system_membership=visit_system_membership, 

582 visit_detector_region=[ 

583 self.universe["visit_detector_region"].RecordClass( 

584 instrument=definition.instrument, 

585 visit=definition.id, 

586 detector=detectorId, 

587 region=detectorRegion, 

588 ) 

589 for detectorId, detectorRegion in visitDetectorRegions.items() 

590 ], 

591 ) 

592 

593 def run( 

594 self, 

595 dataIds: Iterable[DataId], 

596 *, 

597 collections: Optional[str] = None, 

598 update_records: bool = False, 

599 ) -> None: 

600 """Add visit definitions to the registry for the given exposures. 

601 

602 Parameters 

603 ---------- 

604 dataIds : `Iterable` [ `dict` or `DataCoordinate` ] 

605 Exposure-level data IDs. These must all correspond to the same 

606 instrument, and are expected to be on-sky science exposures. 

607 collections : Any, optional 

608 Collections to be searched for raws and camera geometry, overriding 

609 ``self.butler.collections``. 

610 Can be any of the types supported by the ``collections`` argument 

611 to butler construction. 

612 update_records : `bool`, optional 

613 If `True` (`False` is default), update existing visit records that 

614 conflict with the new ones instead of rejecting them (and when this 

615 occurs, update visit_detector_region as well). THIS IS AN ADVANCED 

616 OPTION THAT SHOULD ONLY BE USED TO FIX REGIONS AND/OR METADATA THAT 

617 ARE KNOWN TO BE BAD, AND IT CANNOT BE USED TO REMOVE EXPOSURES OR 

618 DETECTORS FROM A VISIT. 

619 

620 Raises 

621 ------ 

622 lsst.daf.butler.registry.ConflictingDefinitionError 

623 Raised if a visit ID conflict is detected and the existing visit 

624 differs from the new one. 

625 """ 

626 # Normalize, expand, and deduplicate data IDs. 

627 self.log.info("Preprocessing data IDs.") 

628 dimensions = DimensionGraph(self.universe, names=["exposure"]) 

629 data_id_set: Set[DataCoordinate] = { 

630 self.butler.registry.expandDataId(d, graph=dimensions) for d in dataIds 

631 } 

632 if not data_id_set: 

633 raise RuntimeError("No exposures given.") 

634 # Extract exposure DimensionRecords, check that there's only one 

635 # instrument in play, and check for non-science exposures. 

636 exposures = [] 

637 instruments = set() 

638 for dataId in data_id_set: 

639 record = dataId.records["exposure"] 

640 assert record is not None, "Guaranteed by expandDataIds call earlier." 

641 if record.tracking_ra is None or record.tracking_dec is None or record.sky_angle is None: 

642 if self.config.ignoreNonScienceExposures: 

643 continue 

644 else: 

645 raise RuntimeError( 

646 f"Input exposure {dataId} has observation_type " 

647 f"{record.observation_type}, but is not on sky." 

648 ) 

649 instruments.add(dataId["instrument"]) 

650 exposures.append(record) 

651 if not exposures: 

652 self.log.info("No on-sky exposures found after filtering.") 

653 return 

654 if len(instruments) > 1: 

655 raise RuntimeError( 

656 "All data IDs passed to DefineVisitsTask.run must be " 

657 f"from the same instrument; got {instruments}." 

658 ) 

659 (instrument,) = instruments 

660 # Ensure the visit_system our grouping algorithm uses is in the 

661 # registry, if it wasn't already. 

662 visitSystems = self.groupExposures.getVisitSystems() 

663 for visitSystem in visitSystems: 

664 self.log.info("Registering visit_system %d: %s.", visitSystem.value, visitSystem) 

665 self.butler.registry.syncDimensionData( 

666 "visit_system", {"instrument": instrument, "id": visitSystem.value, "name": str(visitSystem)} 

667 ) 

668 # Group exposures into visits, delegating to subtask. 

669 self.log.info("Grouping %d exposure(s) into visits.", len(exposures)) 

670 definitions = list(self.groupExposures.group(exposures)) 

671 # Iterate over visits, compute regions, and insert dimension data, one 

672 # transaction per visit. If a visit already exists, we skip all other 

673 # inserts. 

674 self.log.info("Computing regions and other metadata for %d visit(s).", len(definitions)) 

675 for visitDefinition in self.progress.wrap( 

676 definitions, total=len(definitions), desc="Computing regions and inserting visits" 

677 ): 

678 visitRecords = self._buildVisitRecords(visitDefinition, collections=collections) 

679 with self.butler.registry.transaction(): 

680 inserted_or_updated = self.butler.registry.syncDimensionData( 

681 "visit", 

682 visitRecords.visit, 

683 update=update_records, 

684 ) 

685 if inserted_or_updated: 

686 if inserted_or_updated is True: 

687 # This is a new visit, not an update to an existing 

688 # one, so insert visit definition. 

689 # We don't allow visit definitions to change even when 

690 # asked to update, because we'd have to delete the old 

691 # visit_definitions first and also worry about what 

692 # this does to datasets that already use the visit. 

693 self.butler.registry.insertDimensionData( 

694 "visit_definition", *visitRecords.visit_definition 

695 ) 

696 if visitRecords.visit_system_membership: 

697 self.butler.registry.insertDimensionData( 

698 "visit_system_membership", *visitRecords.visit_system_membership 

699 ) 

700 # [Re]Insert visit_detector_region records for both inserts 

701 # and updates, because we do allow updating to affect the 

702 # region calculations. 

703 self.butler.registry.insertDimensionData( 

704 "visit_detector_region", *visitRecords.visit_detector_region, replace=update_records 

705 ) 

706 

707 # Update obscore exposure records with region information 

708 # from corresponding visits. 

709 if self.config.updateObsCoreTable: 

710 if obscore_manager := self.butler.registry.obsCoreTableManager: 

711 obscore_updates: list[tuple[int, int, Region]] = [] 

712 exposure_ids = [rec.exposure for rec in visitRecords.visit_definition] 

713 for record in visitRecords.visit_detector_region: 

714 obscore_updates += [ 

715 (exposure, record.detector, record.region) for exposure in exposure_ids 

716 ] 

717 if obscore_updates: 

718 obscore_manager.update_exposure_regions( 

719 cast(str, instrument), obscore_updates 

720 ) 

721 

722 

723_T = TypeVar("_T") 

724 

725 

726def _reduceOrNone(func: Callable[[_T, _T], Optional[_T]], iterable: Iterable[Optional[_T]]) -> Optional[_T]: 

727 """Apply a binary function to pairs of elements in an iterable until a 

728 single value is returned, but return `None` if any element is `None` or 

729 there are no elements. 

730 """ 

731 r: Optional[_T] = None 

732 for v in iterable: 

733 if v is None: 

734 return None 

735 if r is None: 

736 r = v 

737 else: 

738 r = func(r, v) 

739 return r 

740 

741 

742def _value_if_equal(a: _T, b: _T) -> Optional[_T]: 

743 """Return either argument if they are equal, or `None` if they are not.""" 

744 return a if a == b else None 

745 

746 

747def _calc_mean_angle(angles: List[float]) -> float: 

748 """Calculate the mean angle, taking into account 0/360 wrapping. 

749 

750 Parameters 

751 ---------- 

752 angles : `list` [`float`] 

753 Angles to average together, in degrees. 

754 

755 Returns 

756 ------- 

757 average : `float` 

758 Average angle in degrees. 

759 """ 

760 # Save on all the math if we only have one value. 

761 if len(angles) == 1: 

762 return angles[0] 

763 

764 # Convert polar coordinates of unit circle to complex values. 

765 # Average the complex values. 

766 # Convert back to a phase angle. 

767 return math.degrees(cmath.phase(sum(cmath.rect(1.0, math.radians(d)) for d in angles) / len(angles))) 

768 

769 

770class _GroupExposuresOneToOneConfig(GroupExposuresConfig): 

771 visitSystemId: Field[int] = Field( 

772 doc="Integer ID of the visit_system implemented by this grouping algorithm.", 

773 dtype=int, 

774 default=0, 

775 deprecated="No longer used. Replaced by enum.", 

776 ) 

777 visitSystemName: Field[str] = Field( 

778 doc="String name of the visit_system implemented by this grouping algorithm.", 

779 dtype=str, 

780 default="one-to-one", 

781 deprecated="No longer used. Replaced by enum.", 

782 ) 

783 

784 

785@registerConfigurable("one-to-one", GroupExposuresTask.registry) 

786class _GroupExposuresOneToOneTask(GroupExposuresTask, metaclass=ABCMeta): 

787 """An exposure grouping algorithm that simply defines one visit for each 

788 exposure, reusing the exposures identifiers for the visit. 

789 """ 

790 

791 ConfigClass = _GroupExposuresOneToOneConfig 

792 

793 def group(self, exposures: List[DimensionRecord]) -> Iterable[VisitDefinitionData]: 

794 # Docstring inherited from GroupExposuresTask. 

795 visit_systems = {VisitSystem.from_name("one-to-one")} 

796 for exposure in exposures: 

797 yield VisitDefinitionData( 

798 instrument=exposure.instrument, 

799 id=exposure.id, 

800 name=exposure.obs_id, 

801 exposures=[exposure], 

802 visit_systems=visit_systems, 

803 ) 

804 

805 def getVisitSystems(self) -> Set[VisitSystem]: 

806 # Docstring inherited from GroupExposuresTask. 

807 return set(VisitSystem.from_names(["one-to-one"])) 

808 

809 

810class _GroupExposuresByGroupMetadataConfig(GroupExposuresConfig): 

811 visitSystemId: Field[int] = Field( 

812 doc="Integer ID of the visit_system implemented by this grouping algorithm.", 

813 dtype=int, 

814 default=1, 

815 deprecated="No longer used. Replaced by enum.", 

816 ) 

817 visitSystemName: Field[str] = Field( 

818 doc="String name of the visit_system implemented by this grouping algorithm.", 

819 dtype=str, 

820 default="by-group-metadata", 

821 deprecated="No longer used. Replaced by enum.", 

822 ) 

823 

824 

825@registerConfigurable("by-group-metadata", GroupExposuresTask.registry) 

826class _GroupExposuresByGroupMetadataTask(GroupExposuresTask, metaclass=ABCMeta): 

827 """An exposure grouping algorithm that uses exposure.group_name and 

828 exposure.group_id. 

829 

830 This algorithm _assumes_ exposure.group_id (generally populated from 

831 `astro_metadata_translator.ObservationInfo.visit_id`) is not just unique, 

832 but disjoint from all `ObservationInfo.exposure_id` values - if it isn't, 

833 it will be impossible to ever use both this grouping algorithm and the 

834 one-to-one algorithm for a particular camera in the same data repository. 

835 """ 

836 

837 ConfigClass = _GroupExposuresByGroupMetadataConfig 

838 

839 def group(self, exposures: List[DimensionRecord]) -> Iterable[VisitDefinitionData]: 

840 # Docstring inherited from GroupExposuresTask. 

841 visit_systems = {VisitSystem.from_name("by-group-metadata")} 

842 groups = defaultdict(list) 

843 for exposure in exposures: 

844 groups[exposure.group_name].append(exposure) 

845 for visitName, exposuresInGroup in groups.items(): 

846 instrument = exposuresInGroup[0].instrument 

847 visitId = exposuresInGroup[0].group_id 

848 assert all( 

849 e.group_id == visitId for e in exposuresInGroup 

850 ), "Grouping by exposure.group_name does not yield consistent group IDs" 

851 yield VisitDefinitionData( 

852 instrument=instrument, 

853 id=visitId, 

854 name=visitName, 

855 exposures=exposuresInGroup, 

856 visit_systems=visit_systems, 

857 ) 

858 

859 def getVisitSystems(self) -> Set[VisitSystem]: 

860 # Docstring inherited from GroupExposuresTask. 

861 return set(VisitSystem.from_names(["by-group-metadata"])) 

862 

863 

864class _GroupExposuresByCounterAndExposuresConfig(GroupExposuresConfig): 

865 visitSystemId: Field[int] = Field( 

866 doc="Integer ID of the visit_system implemented by this grouping algorithm.", 

867 dtype=int, 

868 default=2, 

869 deprecated="No longer used. Replaced by enum.", 

870 ) 

871 visitSystemName: Field[str] = Field( 

872 doc="String name of the visit_system implemented by this grouping algorithm.", 

873 dtype=str, 

874 default="by-counter-and-exposures", 

875 deprecated="No longer used. Replaced by enum.", 

876 ) 

877 

878 

879@registerConfigurable("one-to-one-and-by-counter", GroupExposuresTask.registry) 

880class _GroupExposuresByCounterAndExposuresTask(GroupExposuresTask, metaclass=ABCMeta): 

881 """An exposure grouping algorithm that uses the sequence start and 

882 sequence end metadata to create multi-exposure visits, but also 

883 creates one-to-one visits. 

884 

885 This algorithm uses the exposure.seq_start and 

886 exposure.seq_end fields to collect related snaps. 

887 It also groups single exposures. 

888 """ 

889 

890 ConfigClass = _GroupExposuresByCounterAndExposuresConfig 

891 

892 def group(self, exposures: List[DimensionRecord]) -> Iterable[VisitDefinitionData]: 

893 # Docstring inherited from GroupExposuresTask. 

894 system_one_to_one = VisitSystem.from_name("one-to-one") 

895 system_seq_start_end = VisitSystem.from_name("by-seq-start-end") 

896 

897 groups = defaultdict(list) 

898 for exposure in exposures: 

899 groups[exposure.day_obs, exposure.seq_start, exposure.seq_end].append(exposure) 

900 for visit_key, exposures_in_group in groups.items(): 

901 instrument = exposures_in_group[0].instrument 

902 

903 # It is possible that the first exposure in a visit has not 

904 # been ingested. This can be determined and if that is the case 

905 # we can not reliably define the multi-exposure visit. 

906 skip_multi = False 

907 sorted_exposures = sorted(exposures_in_group, key=lambda e: e.seq_num) 

908 first = sorted_exposures.pop(0) 

909 if first.seq_num != first.seq_start: 

910 # Special case seq_num == 0 since that implies that the 

911 # instrument has no counters and therefore no multi-exposure 

912 # visits. 

913 if first.seq_num != 0: 

914 self.log.warning( 

915 "First exposure for visit %s is not present. Skipping the multi-snap definition.", 

916 visit_key, 

917 ) 

918 skip_multi = True 

919 

920 # Define the one-to-one visits. 

921 num_exposures = len(exposures_in_group) 

922 for exposure in exposures_in_group: 

923 # Default is to use the exposure ID and name unless 

924 # this is the first exposure in a multi-exposure visit. 

925 visit_name = exposure.obs_id 

926 visit_id = exposure.id 

927 visit_systems = {system_one_to_one} 

928 

929 if num_exposures == 1: 

930 # This is also a by-counter visit. 

931 # It will use the same visit_name and visit_id. 

932 visit_systems.add(system_seq_start_end) 

933 

934 elif num_exposures > 1 and not skip_multi and exposure == first: 

935 # This is the first legitimate exposure in a multi-exposure 

936 # visit. It therefore needs a modified visit name and ID 

937 # so it does not clash with the multi-exposure visit 

938 # definition. 

939 visit_name = f"{visit_name}_first" 

940 visit_id = int(f"9{visit_id}") 

941 

942 yield VisitDefinitionData( 

943 instrument=instrument, 

944 id=visit_id, 

945 name=visit_name, 

946 exposures=[exposure], 

947 visit_systems=visit_systems, 

948 ) 

949 

950 # Multi-exposure visit. 

951 if not skip_multi and num_exposures > 1: 

952 # Define the visit using the first exposure 

953 visit_name = first.obs_id 

954 visit_id = first.id 

955 

956 yield VisitDefinitionData( 

957 instrument=instrument, 

958 id=visit_id, 

959 name=visit_name, 

960 exposures=exposures_in_group, 

961 visit_systems={system_seq_start_end}, 

962 ) 

963 

964 def getVisitSystems(self) -> Set[VisitSystem]: 

965 # Docstring inherited from GroupExposuresTask. 

966 # Using a Config for this is difficult because what this grouping 

967 # algorithm is doing is using two visit systems. 

968 # One is using metadata (but not by-group) and the other is the 

969 # one-to-one. For now hard-code in class. 

970 return set(VisitSystem.from_names(["one-to-one", "by-seq-start-end"])) 

971 

972 

973class _ComputeVisitRegionsFromSingleRawWcsConfig(ComputeVisitRegionsConfig): 

974 mergeExposures: Field[bool] = Field( 

975 doc=( 

976 "If True, merge per-detector regions over all exposures in a " 

977 "visit (via convex hull) instead of using the first exposure and " 

978 "assuming its regions are valid for all others." 

979 ), 

980 dtype=bool, 

981 default=False, 

982 ) 

983 detectorId: Field[Optional[int]] = Field( 

984 doc=( 

985 "Load the WCS for the detector with this ID. If None, use an " 

986 "arbitrary detector (the first found in a query of the data " 

987 "repository for each exposure (or all exposures, if " 

988 "mergeExposures is True)." 

989 ), 

990 dtype=int, 

991 optional=True, 

992 default=None, 

993 ) 

994 requireVersionedCamera: Field[bool] = Field( 

995 doc=( 

996 "If True, raise LookupError if version camera geometry cannot be " 

997 "loaded for an exposure. If False, use the nominal camera from " 

998 "the Instrument class instead." 

999 ), 

1000 dtype=bool, 

1001 optional=False, 

1002 default=False, 

1003 ) 

1004 

1005 

1006@registerConfigurable("single-raw-wcs", ComputeVisitRegionsTask.registry) 

1007class _ComputeVisitRegionsFromSingleRawWcsTask(ComputeVisitRegionsTask): 

1008 """A visit region calculator that uses a single raw WCS and a camera to 

1009 project the bounding boxes of all detectors onto the sky, relating 

1010 different detectors by their positions in focal plane coordinates. 

1011 

1012 Notes 

1013 ----- 

1014 Most instruments should have their raw WCSs determined from a combination 

1015 of boresight angle, rotator angle, and camera geometry, and hence this 

1016 algorithm should produce stable results regardless of which detector the 

1017 raw corresponds to. If this is not the case (e.g. because a per-file FITS 

1018 WCS is used instead), either the ID of the detector should be fixed (see 

1019 the ``detectorId`` config parameter) or a different algorithm used. 

1020 """ 

1021 

1022 ConfigClass = _ComputeVisitRegionsFromSingleRawWcsConfig 

1023 config: _ComputeVisitRegionsFromSingleRawWcsConfig 

1024 

1025 def computeExposureBounds( 

1026 self, exposure: DimensionRecord, *, collections: Any = None 

1027 ) -> Dict[int, List[UnitVector3d]]: 

1028 """Compute the lists of unit vectors on the sphere that correspond to 

1029 the sky positions of detector corners. 

1030 

1031 Parameters 

1032 ---------- 

1033 exposure : `DimensionRecord` 

1034 Dimension record for the exposure. 

1035 collections : Any, optional 

1036 Collections to be searched for raws and camera geometry, overriding 

1037 ``self.butler.collections``. 

1038 Can be any of the types supported by the ``collections`` argument 

1039 to butler construction. 

1040 

1041 Returns 

1042 ------- 

1043 bounds : `dict` 

1044 Dictionary mapping detector ID to a list of unit vectors on the 

1045 sphere representing that detector's corners projected onto the sky. 

1046 """ 

1047 if collections is None: 

1048 collections = self.butler.collections 

1049 camera, versioned = loadCamera(self.butler, exposure.dataId, collections=collections) 

1050 if not versioned and self.config.requireVersionedCamera: 

1051 raise LookupError(f"No versioned camera found for exposure {exposure.dataId}.") 

1052 

1053 # Derive WCS from boresight information -- if available in registry 

1054 use_registry = True 

1055 try: 

1056 orientation = lsst.geom.Angle(exposure.sky_angle, lsst.geom.degrees) 

1057 radec = lsst.geom.SpherePoint( 

1058 lsst.geom.Angle(exposure.tracking_ra, lsst.geom.degrees), 

1059 lsst.geom.Angle(exposure.tracking_dec, lsst.geom.degrees), 

1060 ) 

1061 except AttributeError: 

1062 use_registry = False 

1063 

1064 if use_registry: 

1065 if self.config.detectorId is None: 

1066 detectorId = next(camera.getIdIter()) 

1067 else: 

1068 detectorId = self.config.detectorId 

1069 wcsDetector = camera[detectorId] 

1070 

1071 # Ask the raw formatter to create the relevant WCS 

1072 # This allows flips to be taken into account 

1073 instrument = self.getInstrument(exposure.instrument) 

1074 rawFormatter = instrument.getRawFormatter({"detector": detectorId}) 

1075 

1076 try: 

1077 wcs = rawFormatter.makeRawSkyWcsFromBoresight(radec, orientation, wcsDetector) # type: ignore 

1078 except AttributeError: 

1079 raise TypeError( 

1080 f"Raw formatter is {get_full_type_name(rawFormatter)} but visit" 

1081 " definition requires it to support 'makeRawSkyWcsFromBoresight'" 

1082 ) from None 

1083 else: 

1084 if self.config.detectorId is None: 

1085 wcsRefsIter = self.butler.registry.queryDatasets( 

1086 "raw.wcs", dataId=exposure.dataId, collections=collections 

1087 ) 

1088 if not wcsRefsIter: 

1089 raise LookupError( 

1090 f"No raw.wcs datasets found for data ID {exposure.dataId} " 

1091 f"in collections {collections}." 

1092 ) 

1093 wcsRef = next(iter(wcsRefsIter)) 

1094 wcsDetector = camera[wcsRef.dataId["detector"]] 

1095 wcs = self.butler.get(wcsRef) 

1096 else: 

1097 wcsDetector = camera[self.config.detectorId] 

1098 wcs = self.butler.get( 

1099 "raw.wcs", 

1100 dataId=exposure.dataId, 

1101 detector=self.config.detectorId, 

1102 collections=collections, 

1103 ) 

1104 fpToSky = wcsDetector.getTransform(FOCAL_PLANE, PIXELS).then(wcs.getTransform()) 

1105 bounds = {} 

1106 for detector in camera: 

1107 pixelsToSky = detector.getTransform(PIXELS, FOCAL_PLANE).then(fpToSky) 

1108 pixCorners = Box2D(detector.getBBox().dilatedBy(self.config.padding)).getCorners() 

1109 bounds[detector.getId()] = [ 

1110 skyCorner.getVector() for skyCorner in pixelsToSky.applyForward(pixCorners) 

1111 ] 

1112 return bounds 

1113 

1114 def compute( 

1115 self, visit: VisitDefinitionData, *, collections: Any = None 

1116 ) -> Tuple[Region, Dict[int, Region]]: 

1117 # Docstring inherited from ComputeVisitRegionsTask. 

1118 if self.config.mergeExposures: 

1119 detectorBounds: Dict[int, List[UnitVector3d]] = defaultdict(list) 

1120 for exposure in visit.exposures: 

1121 exposureDetectorBounds = self.computeExposureBounds(exposure, collections=collections) 

1122 for detectorId, bounds in exposureDetectorBounds.items(): 

1123 detectorBounds[detectorId].extend(bounds) 

1124 else: 

1125 detectorBounds = self.computeExposureBounds(visit.exposures[0], collections=collections) 

1126 visitBounds = [] 

1127 detectorRegions = {} 

1128 for detectorId, bounds in detectorBounds.items(): 

1129 detectorRegions[detectorId] = ConvexPolygon.convexHull(bounds) 

1130 visitBounds.extend(bounds) 

1131 return ConvexPolygon.convexHull(visitBounds), detectorRegions