Coverage for python/lsst/obs/base/defineVisits.py: 31%

342 statements  

« prev     ^ index     » next       coverage.py v6.4.1, created at 2022-06-23 02:50 -0700

1# This file is part of obs_base. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22from __future__ import annotations 

23 

24__all__ = [ 

25 "DefineVisitsConfig", 

26 "DefineVisitsTask", 

27 "GroupExposuresConfig", 

28 "GroupExposuresTask", 

29 "VisitDefinitionData", 

30 "VisitSystem", 

31] 

32 

33import cmath 

34import dataclasses 

35import enum 

36import math 

37import operator 

38from abc import ABCMeta, abstractmethod 

39from collections import defaultdict 

40from typing import Any, Callable, ClassVar, Dict, FrozenSet, Iterable, List, Optional, Set, Tuple, TypeVar 

41 

42import lsst.geom 

43from lsst.afw.cameraGeom import FOCAL_PLANE, PIXELS 

44from lsst.daf.butler import ( 

45 Butler, 

46 DataCoordinate, 

47 DataId, 

48 DimensionGraph, 

49 DimensionRecord, 

50 Progress, 

51 Timespan, 

52) 

53from lsst.geom import Box2D 

54from lsst.pex.config import Config, Field, makeRegistry, registerConfigurable 

55from lsst.pipe.base import Instrument, Task 

56from lsst.sphgeom import ConvexPolygon, Region, UnitVector3d 

57from lsst.utils.introspection import get_full_type_name 

58 

59from ._instrument import loadCamera 

60 

61 

62class VisitSystem(enum.Enum): 

63 """Enumeration used to label different visit systems.""" 

64 

65 ONE_TO_ONE = 0 

66 """Each exposure is assigned to its own visit.""" 

67 

68 BY_GROUP_METADATA = 1 

69 """Visit membership is defined by the value of the exposure.group_id.""" 

70 

71 BY_SEQ_START_END = 2 

72 """Visit membership is defined by the values of the ``exposure.day_obs``, 

73 ``exposure.seq_start``, and ``exposure.seq_end`` values. 

74 """ 

75 

76 @classmethod 

77 def all(cls) -> FrozenSet[VisitSystem]: 

78 """Return a `frozenset` containing all members.""" 

79 return frozenset(cls.__members__.values()) 

80 

81 @classmethod 

82 def from_name(cls, external_name: str) -> VisitSystem: 

83 """Construct the enumeration from given name.""" 

84 name = external_name.upper() 

85 name = name.replace("-", "_") 

86 try: 

87 return cls.__members__[name] 

88 except KeyError: 

89 raise KeyError(f"Visit system named '{external_name}' not known.") from None 

90 

91 @classmethod 

92 def from_names(cls, names: Optional[Iterable[str]]) -> FrozenSet[VisitSystem]: 

93 """Return a `frozenset` of all the visit systems matching the supplied 

94 names. 

95 

96 Parameters 

97 ---------- 

98 names : iterable of `str`, or `None` 

99 Names of visit systems. Case insensitive. If `None` or empty, all 

100 the visit systems are returned. 

101 

102 Returns 

103 ------- 

104 systems : `frozenset` of `VisitSystem` 

105 The matching visit systems. 

106 """ 

107 if not names: 

108 return cls.all() 

109 

110 return frozenset({cls.from_name(name) for name in names}) 

111 

112 def __str__(self) -> str: 

113 name = self.name.lower() 

114 name = name.replace("_", "-") 

115 return name 

116 

117 

118@dataclasses.dataclass 

119class VisitDefinitionData: 

120 """Struct representing a group of exposures that will be used to define a 

121 visit. 

122 """ 

123 

124 instrument: str 

125 """Name of the instrument this visit will be associated with. 

126 """ 

127 

128 id: int 

129 """Integer ID of the visit. 

130 

131 This must be unique across all visit systems for the instrument. 

132 """ 

133 

134 name: str 

135 """String name for the visit. 

136 

137 This must be unique across all visit systems for the instrument. 

138 """ 

139 

140 visit_systems: Set[VisitSystem] 

141 """All the visit systems associated with this visit.""" 

142 

143 exposures: List[DimensionRecord] = dataclasses.field(default_factory=list) 

144 """Dimension records for the exposures that are part of this visit. 

145 """ 

146 

147 

148@dataclasses.dataclass 

149class _VisitRecords: 

150 """Struct containing the dimension records associated with a visit.""" 

151 

152 visit: DimensionRecord 

153 """Record for the 'visit' dimension itself. 

154 """ 

155 

156 visit_definition: List[DimensionRecord] 

157 """Records for 'visit_definition', which relates 'visit' to 'exposure'. 

158 """ 

159 

160 visit_detector_region: List[DimensionRecord] 

161 """Records for 'visit_detector_region', which associates the combination 

162 of a 'visit' and a 'detector' with a region on the sky. 

163 """ 

164 

165 visit_system_membership: List[DimensionRecord] 

166 """Records relating visits to an associated visit system.""" 

167 

168 

169class GroupExposuresConfig(Config): 

170 pass 

171 

172 

173class GroupExposuresTask(Task, metaclass=ABCMeta): 

174 """Abstract base class for the subtask of `DefineVisitsTask` that is 

175 responsible for grouping exposures into visits. 

176 

177 Subclasses should be registered with `GroupExposuresTask.registry` to 

178 enable use by `DefineVisitsTask`, and should generally correspond to a 

179 particular 'visit_system' dimension value. They are also responsible for 

180 defining visit IDs and names that are unique across all visit systems in 

181 use by an instrument. 

182 

183 Parameters 

184 ---------- 

185 config : `GroupExposuresConfig` 

186 Configuration information. 

187 **kwargs 

188 Additional keyword arguments forwarded to the `Task` constructor. 

189 """ 

190 

191 def __init__(self, config: GroupExposuresConfig, **kwargs: Any): 

192 Task.__init__(self, config=config, **kwargs) 

193 

194 ConfigClass = GroupExposuresConfig 

195 

196 _DefaultName = "groupExposures" 

197 

198 registry = makeRegistry( 

199 doc="Registry of algorithms for grouping exposures into visits.", 

200 configBaseType=GroupExposuresConfig, 

201 ) 

202 

203 @abstractmethod 

204 def group(self, exposures: List[DimensionRecord]) -> Iterable[VisitDefinitionData]: 

205 """Group the given exposures into visits. 

206 

207 Parameters 

208 ---------- 

209 exposures : `list` [ `DimensionRecord` ] 

210 DimensionRecords (for the 'exposure' dimension) describing the 

211 exposures to group. 

212 

213 Returns 

214 ------- 

215 visits : `Iterable` [ `VisitDefinitionData` ] 

216 Structs identifying the visits and the exposures associated with 

217 them. This may be an iterator or a container. 

218 """ 

219 raise NotImplementedError() 

220 

221 def getVisitSystems(self) -> Set[VisitSystem]: 

222 """Return identifiers for the 'visit_system' dimension this 

223 algorithm implements. 

224 

225 Returns 

226 ------- 

227 visit_systems : `Set` [`VisitSystem`] 

228 The visit systems used by this algorithm. 

229 """ 

230 raise NotImplementedError() 

231 

232 

233class ComputeVisitRegionsConfig(Config): 

234 padding = Field( 

235 dtype=int, 

236 default=250, 

237 doc=( 

238 "Pad raw image bounding boxes with specified number of pixels " 

239 "when calculating their (conservatively large) region on the " 

240 "sky. Note that the config value for pixelMargin of the " 

241 "reference object loaders in meas_algorithms should be <= " 

242 "the value set here." 

243 ), 

244 ) 

245 

246 

247class ComputeVisitRegionsTask(Task, metaclass=ABCMeta): 

248 """Abstract base class for the subtask of `DefineVisitsTask` that is 

249 responsible for extracting spatial regions for visits and visit+detector 

250 combinations. 

251 

252 Subclasses should be registered with `ComputeVisitRegionsTask.registry` to 

253 enable use by `DefineVisitsTask`. 

254 

255 Parameters 

256 ---------- 

257 config : `ComputeVisitRegionsConfig` 

258 Configuration information. 

259 butler : `lsst.daf.butler.Butler` 

260 The butler to use. 

261 **kwargs 

262 Additional keyword arguments forwarded to the `Task` constructor. 

263 """ 

264 

265 def __init__(self, config: ComputeVisitRegionsConfig, *, butler: Butler, **kwargs: Any): 

266 Task.__init__(self, config=config, **kwargs) 

267 self.butler = butler 

268 self.instrumentMap: Dict[str, Instrument] = {} 

269 

270 ConfigClass = ComputeVisitRegionsConfig 

271 

272 _DefaultName = "computeVisitRegions" 

273 

274 registry = makeRegistry( 

275 doc=( 

276 "Registry of algorithms for computing on-sky regions for visits " 

277 "and visit+detector combinations." 

278 ), 

279 configBaseType=ComputeVisitRegionsConfig, 

280 ) 

281 

282 def getInstrument(self, instrumentName: str) -> Instrument: 

283 """Retrieve an `~lsst.obs.base.Instrument` associated with this 

284 instrument name. 

285 

286 Parameters 

287 ---------- 

288 instrumentName : `str` 

289 The name of the instrument. 

290 

291 Returns 

292 ------- 

293 instrument : `~lsst.obs.base.Instrument` 

294 The associated instrument object. 

295 

296 Notes 

297 ----- 

298 The result is cached. 

299 """ 

300 instrument = self.instrumentMap.get(instrumentName) 

301 if instrument is None: 

302 instrument = Instrument.fromName(instrumentName, self.butler.registry) 

303 self.instrumentMap[instrumentName] = instrument 

304 return instrument 

305 

306 @abstractmethod 

307 def compute( 

308 self, visit: VisitDefinitionData, *, collections: Any = None 

309 ) -> Tuple[Region, Dict[int, Region]]: 

310 """Compute regions for the given visit and all detectors in that visit. 

311 

312 Parameters 

313 ---------- 

314 visit : `VisitDefinitionData` 

315 Struct describing the visit and the exposures associated with it. 

316 collections : Any, optional 

317 Collections to be searched for raws and camera geometry, overriding 

318 ``self.butler.collections``. 

319 Can be any of the types supported by the ``collections`` argument 

320 to butler construction. 

321 

322 Returns 

323 ------- 

324 visitRegion : `lsst.sphgeom.Region` 

325 Region for the full visit. 

326 visitDetectorRegions : `dict` [ `int`, `lsst.sphgeom.Region` ] 

327 Dictionary mapping detector ID to the region for that detector. 

328 Should include all detectors in the visit. 

329 """ 

330 raise NotImplementedError() 

331 

332 

333class DefineVisitsConfig(Config): 

334 groupExposures = GroupExposuresTask.registry.makeField( 

335 doc="Algorithm for grouping exposures into visits.", 

336 default="one-to-one-and-by-counter", 

337 ) 

338 computeVisitRegions = ComputeVisitRegionsTask.registry.makeField( 

339 doc="Algorithm from computing visit and visit+detector regions.", 

340 default="single-raw-wcs", 

341 ) 

342 ignoreNonScienceExposures = Field( 

343 doc=( 

344 "If True, silently ignore input exposures that do not have " 

345 "observation_type=SCIENCE. If False, raise an exception if one " 

346 "encountered." 

347 ), 

348 dtype=bool, 

349 optional=False, 

350 default=True, 

351 ) 

352 

353 

354class DefineVisitsTask(Task): 

355 """Driver Task for defining visits (and their spatial regions) in Gen3 

356 Butler repositories. 

357 

358 Parameters 

359 ---------- 

360 config : `DefineVisitsConfig` 

361 Configuration for the task. 

362 butler : `~lsst.daf.butler.Butler` 

363 Writeable butler instance. Will be used to read `raw.wcs` and `camera` 

364 datasets and insert/sync dimension data. 

365 **kwargs 

366 Additional keyword arguments are forwarded to the `lsst.pipe.base.Task` 

367 constructor. 

368 

369 Notes 

370 ----- 

371 Each instance of `DefineVisitsTask` reads from / writes to the same Butler. 

372 Each invocation of `DefineVisitsTask.run` processes an independent group of 

373 exposures into one or more new vists, all belonging to the same visit 

374 system and instrument. 

375 

376 The actual work of grouping exposures and computing regions is delegated 

377 to pluggable subtasks (`GroupExposuresTask` and `ComputeVisitRegionsTask`), 

378 respectively. The defaults are to create one visit for every exposure, 

379 and to use exactly one (arbitrary) detector-level raw dataset's WCS along 

380 with camera geometry to compute regions for all detectors. Other 

381 implementations can be created and configured for instruments for which 

382 these choices are unsuitable (e.g. because visits and exposures are not 

383 one-to-one, or because ``raw.wcs`` datasets for different detectors may not 

384 be consistent with camera geomery). 

385 

386 It is not necessary in general to ingest all raws for an exposure before 

387 defining a visit that includes the exposure; this depends entirely on the 

388 `ComputeVisitRegionTask` subclass used. For the default configuration, 

389 a single raw for each exposure is sufficient. 

390 

391 Defining the same visit the same way multiple times (e.g. via multiple 

392 invocations of this task on the same exposures, with the same 

393 configuration) is safe, but it may be inefficient, as most of the work must 

394 be done before new visits can be compared to existing visits. 

395 """ 

396 

397 def __init__(self, config: DefineVisitsConfig, *, butler: Butler, **kwargs: Any): 

398 config.validate() # Not a CmdlineTask nor PipelineTask, so have to validate the config here. 

399 super().__init__(config, **kwargs) 

400 self.butler = butler 

401 self.universe = self.butler.registry.dimensions 

402 self.progress = Progress("obs.base.DefineVisitsTask") 

403 self.makeSubtask("groupExposures") 

404 self.makeSubtask("computeVisitRegions", butler=self.butler) 

405 

406 def _reduce_kwargs(self) -> dict: 

407 # Add extra parameters to pickle 

408 return dict(**super()._reduce_kwargs(), butler=self.butler) 

409 

410 ConfigClass: ClassVar[Config] = DefineVisitsConfig 

411 

412 _DefaultName: ClassVar[str] = "defineVisits" 

413 

414 groupExposures: GroupExposuresTask 

415 computeVisitRegions: ComputeVisitRegionsTask 

416 

417 def _buildVisitRecords( 

418 self, definition: VisitDefinitionData, *, collections: Any = None 

419 ) -> _VisitRecords: 

420 """Build the DimensionRecords associated with a visit. 

421 

422 Parameters 

423 ---------- 

424 definition : `VisitDefinitionData` 

425 Struct with identifiers for the visit and records for its 

426 constituent exposures. 

427 collections : Any, optional 

428 Collections to be searched for raws and camera geometry, overriding 

429 ``self.butler.collections``. 

430 Can be any of the types supported by the ``collections`` argument 

431 to butler construction. 

432 

433 Results 

434 ------- 

435 records : `_VisitRecords` 

436 Struct containing DimensionRecords for the visit, including 

437 associated dimension elements. 

438 """ 

439 dimension = self.universe["visit"] 

440 

441 # Some registries support additional items. 

442 supported = {meta.name for meta in dimension.metadata} 

443 

444 # Compute all regions. 

445 visitRegion, visitDetectorRegions = self.computeVisitRegions.compute( 

446 definition, collections=collections 

447 ) 

448 # Aggregate other exposure quantities. 

449 timespan = Timespan( 

450 begin=_reduceOrNone(min, (e.timespan.begin for e in definition.exposures)), 

451 end=_reduceOrNone(max, (e.timespan.end for e in definition.exposures)), 

452 ) 

453 exposure_time = _reduceOrNone(operator.add, (e.exposure_time for e in definition.exposures)) 

454 physical_filter = _reduceOrNone(_value_if_equal, (e.physical_filter for e in definition.exposures)) 

455 target_name = _reduceOrNone(_value_if_equal, (e.target_name for e in definition.exposures)) 

456 science_program = _reduceOrNone(_value_if_equal, (e.science_program for e in definition.exposures)) 

457 

458 # observing day for a visit is defined by the earliest observation 

459 # of the visit 

460 observing_day = _reduceOrNone(min, (e.day_obs for e in definition.exposures)) 

461 observation_reason = _reduceOrNone( 

462 _value_if_equal, (e.observation_reason for e in definition.exposures) 

463 ) 

464 if observation_reason is None: 

465 # Be explicit about there being multiple reasons 

466 # MyPy can't really handle DimensionRecord fields as 

467 # DimensionRecord classes are dynamically defined; easiest to just 

468 # shush it when it complains. 

469 observation_reason = "various" # type: ignore 

470 

471 # Use the mean zenith angle as an approximation 

472 zenith_angle = _reduceOrNone(operator.add, (e.zenith_angle for e in definition.exposures)) 

473 if zenith_angle is not None: 

474 zenith_angle /= len(definition.exposures) 

475 

476 # New records that may not be supported. 

477 extras: Dict[str, Any] = {} 

478 if "seq_num" in supported: 

479 extras["seq_num"] = _reduceOrNone(min, (e.seq_num for e in definition.exposures)) 

480 if "azimuth" in supported: 

481 # Must take into account 0/360 problem. 

482 extras["azimuth"] = _calc_mean_angle([e.azimuth for e in definition.exposures]) 

483 

484 # visit_system handling changed. This is the logic for visit/exposure 

485 # that has support for seq_start/seq_end. 

486 if "seq_num" in supported: 

487 # Map visit to exposure. 

488 visit_definition = [ 

489 self.universe["visit_definition"].RecordClass( 

490 instrument=definition.instrument, 

491 visit=definition.id, 

492 exposure=exposure.id, 

493 ) 

494 for exposure in definition.exposures 

495 ] 

496 

497 # Map visit to visit system. 

498 visit_system_membership = [] 

499 for visit_system in self.groupExposures.getVisitSystems(): 

500 if visit_system in definition.visit_systems: 

501 record = self.universe["visit_system_membership"].RecordClass( 

502 instrument=definition.instrument, 

503 visit=definition.id, 

504 visit_system=visit_system.value, 

505 ) 

506 visit_system_membership.append(record) 

507 

508 else: 

509 # The old approach can only handle one visit system at a time. 

510 # If we have been configured with multiple options, prefer the 

511 # one-to-one. 

512 visit_systems = self.groupExposures.getVisitSystems() 

513 if len(visit_systems) > 1: 

514 one_to_one = VisitSystem.from_name("one-to-one") 

515 if one_to_one not in visit_systems: 

516 raise ValueError( 

517 f"Multiple visit systems specified ({visit_systems}) for use with old" 

518 " dimension universe but unable to find one-to-one." 

519 ) 

520 visit_system = one_to_one 

521 else: 

522 visit_system = visit_systems.pop() 

523 

524 extras["visit_system"] = visit_system.value 

525 

526 # The old visit_definition included visit system. 

527 visit_definition = [ 

528 self.universe["visit_definition"].RecordClass( 

529 instrument=definition.instrument, 

530 visit=definition.id, 

531 exposure=exposure.id, 

532 visit_system=visit_system.value, 

533 ) 

534 for exposure in definition.exposures 

535 ] 

536 

537 # This concept does not exist in old schema. 

538 visit_system_membership = [] 

539 

540 # Construct the actual DimensionRecords. 

541 return _VisitRecords( 

542 visit=dimension.RecordClass( 

543 instrument=definition.instrument, 

544 id=definition.id, 

545 name=definition.name, 

546 physical_filter=physical_filter, 

547 target_name=target_name, 

548 science_program=science_program, 

549 observation_reason=observation_reason, 

550 day_obs=observing_day, 

551 zenith_angle=zenith_angle, 

552 exposure_time=exposure_time, 

553 timespan=timespan, 

554 region=visitRegion, 

555 # TODO: no seeing value in exposure dimension records, so we 

556 # can't set that here. But there are many other columns that 

557 # both dimensions should probably have as well. 

558 **extras, 

559 ), 

560 visit_definition=visit_definition, 

561 visit_system_membership=visit_system_membership, 

562 visit_detector_region=[ 

563 self.universe["visit_detector_region"].RecordClass( 

564 instrument=definition.instrument, 

565 visit=definition.id, 

566 detector=detectorId, 

567 region=detectorRegion, 

568 ) 

569 for detectorId, detectorRegion in visitDetectorRegions.items() 

570 ], 

571 ) 

572 

573 def run( 

574 self, 

575 dataIds: Iterable[DataId], 

576 *, 

577 collections: Optional[str] = None, 

578 update_records: bool = False, 

579 ) -> None: 

580 """Add visit definitions to the registry for the given exposures. 

581 

582 Parameters 

583 ---------- 

584 dataIds : `Iterable` [ `dict` or `DataCoordinate` ] 

585 Exposure-level data IDs. These must all correspond to the same 

586 instrument, and are expected to be on-sky science exposures. 

587 collections : Any, optional 

588 Collections to be searched for raws and camera geometry, overriding 

589 ``self.butler.collections``. 

590 Can be any of the types supported by the ``collections`` argument 

591 to butler construction. 

592 update_records : `bool`, optional 

593 If `True` (`False` is default), update existing visit records that 

594 conflict with the new ones instead of rejecting them (and when this 

595 occurs, update visit_detector_region as well). THIS IS AN ADVANCED 

596 OPTION THAT SHOULD ONLY BE USED TO FIX REGIONS AND/OR METADATA THAT 

597 ARE KNOWN TO BE BAD, AND IT CANNOT BE USED TO REMOVE EXPOSURES OR 

598 DETECTORS FROM A VISIT. 

599 

600 Raises 

601 ------ 

602 lsst.daf.butler.registry.ConflictingDefinitionError 

603 Raised if a visit ID conflict is detected and the existing visit 

604 differs from the new one. 

605 """ 

606 # Normalize, expand, and deduplicate data IDs. 

607 self.log.info("Preprocessing data IDs.") 

608 dimensions = DimensionGraph(self.universe, names=["exposure"]) 

609 data_id_set: Set[DataCoordinate] = { 

610 self.butler.registry.expandDataId(d, graph=dimensions) for d in dataIds 

611 } 

612 if not data_id_set: 

613 raise RuntimeError("No exposures given.") 

614 # Extract exposure DimensionRecords, check that there's only one 

615 # instrument in play, and check for non-science exposures. 

616 exposures = [] 

617 instruments = set() 

618 for dataId in data_id_set: 

619 record = dataId.records["exposure"] 

620 assert record is not None, "Guaranteed by expandDataIds call earlier." 

621 if record.tracking_ra is None or record.tracking_dec is None or record.sky_angle is None: 

622 if self.config.ignoreNonScienceExposures: 

623 continue 

624 else: 

625 raise RuntimeError( 

626 f"Input exposure {dataId} has observation_type " 

627 f"{record.observation_type}, but is not on sky." 

628 ) 

629 instruments.add(dataId["instrument"]) 

630 exposures.append(record) 

631 if not exposures: 

632 self.log.info("No on-sky exposures found after filtering.") 

633 return 

634 if len(instruments) > 1: 

635 raise RuntimeError( 

636 f"All data IDs passed to DefineVisitsTask.run must be " 

637 f"from the same instrument; got {instruments}." 

638 ) 

639 (instrument,) = instruments 

640 # Ensure the visit_system our grouping algorithm uses is in the 

641 # registry, if it wasn't already. 

642 visitSystems = self.groupExposures.getVisitSystems() 

643 for visitSystem in visitSystems: 

644 self.log.info("Registering visit_system %d: %s.", visitSystem.value, visitSystem) 

645 self.butler.registry.syncDimensionData( 

646 "visit_system", {"instrument": instrument, "id": visitSystem.value, "name": str(visitSystem)} 

647 ) 

648 # Group exposures into visits, delegating to subtask. 

649 self.log.info("Grouping %d exposure(s) into visits.", len(exposures)) 

650 definitions = list(self.groupExposures.group(exposures)) 

651 # Iterate over visits, compute regions, and insert dimension data, one 

652 # transaction per visit. If a visit already exists, we skip all other 

653 # inserts. 

654 self.log.info("Computing regions and other metadata for %d visit(s).", len(definitions)) 

655 for visitDefinition in self.progress.wrap( 

656 definitions, total=len(definitions), desc="Computing regions and inserting visits" 

657 ): 

658 visitRecords = self._buildVisitRecords(visitDefinition, collections=collections) 

659 with self.butler.registry.transaction(): 

660 inserted_or_updated = self.butler.registry.syncDimensionData( 

661 "visit", 

662 visitRecords.visit, 

663 update=update_records, 

664 ) 

665 if inserted_or_updated: 

666 if inserted_or_updated is True: 

667 # This is a new visit, not an update to an existing 

668 # one, so insert visit definition. 

669 # We don't allow visit definitions to change even when 

670 # asked to update, because we'd have to delete the old 

671 # visit_definitions first and also worry about what 

672 # this does to datasets that already use the visit. 

673 self.butler.registry.insertDimensionData( 

674 "visit_definition", *visitRecords.visit_definition 

675 ) 

676 if visitRecords.visit_system_membership: 

677 self.butler.registry.insertDimensionData( 

678 "visit_system_membership", *visitRecords.visit_system_membership 

679 ) 

680 # [Re]Insert visit_detector_region records for both inserts 

681 # and updates, because we do allow updating to affect the 

682 # region calculations. 

683 self.butler.registry.insertDimensionData( 

684 "visit_detector_region", *visitRecords.visit_detector_region, replace=update_records 

685 ) 

686 

687 

688_T = TypeVar("_T") 

689 

690 

691def _reduceOrNone(func: Callable[[_T, _T], Optional[_T]], iterable: Iterable[Optional[_T]]) -> Optional[_T]: 

692 """Apply a binary function to pairs of elements in an iterable until a 

693 single value is returned, but return `None` if any element is `None` or 

694 there are no elements. 

695 """ 

696 r: Optional[_T] = None 

697 for v in iterable: 

698 if v is None: 

699 return None 

700 if r is None: 

701 r = v 

702 else: 

703 r = func(r, v) 

704 return r 

705 

706 

707def _value_if_equal(a: _T, b: _T) -> Optional[_T]: 

708 """Return either argument if they are equal, or `None` if they are not.""" 

709 return a if a == b else None 

710 

711 

712def _calc_mean_angle(angles: List[float]) -> float: 

713 """Calculate the mean angle, taking into account 0/360 wrapping. 

714 

715 Parameters 

716 ---------- 

717 angles : `list` [`float`] 

718 Angles to average together, in degrees. 

719 

720 Returns 

721 ------- 

722 average : `float` 

723 Average angle in degrees. 

724 """ 

725 # Save on all the math if we only have one value. 

726 if len(angles) == 1: 

727 return angles[0] 

728 

729 # Convert polar coordinates of unit circle to complex values. 

730 # Average the complex values. 

731 # Convert back to a phase angle. 

732 return math.degrees(cmath.phase(sum(cmath.rect(1.0, math.radians(d)) for d in angles) / len(angles))) 

733 

734 

735class _GroupExposuresOneToOneConfig(GroupExposuresConfig): 

736 visitSystemId = Field( 

737 doc="Integer ID of the visit_system implemented by this grouping algorithm.", 

738 dtype=int, 

739 default=0, 

740 deprecated="No longer used. Replaced by enum.", 

741 ) 

742 visitSystemName = Field( 

743 doc="String name of the visit_system implemented by this grouping algorithm.", 

744 dtype=str, 

745 default="one-to-one", 

746 deprecated="No longer used. Replaced by enum.", 

747 ) 

748 

749 

750@registerConfigurable("one-to-one", GroupExposuresTask.registry) 

751class _GroupExposuresOneToOneTask(GroupExposuresTask, metaclass=ABCMeta): 

752 """An exposure grouping algorithm that simply defines one visit for each 

753 exposure, reusing the exposures identifiers for the visit. 

754 """ 

755 

756 ConfigClass = _GroupExposuresOneToOneConfig 

757 

758 def group(self, exposures: List[DimensionRecord]) -> Iterable[VisitDefinitionData]: 

759 # Docstring inherited from GroupExposuresTask. 

760 visit_systems = {VisitSystem.from_name("one-to-one")} 

761 for exposure in exposures: 

762 yield VisitDefinitionData( 

763 instrument=exposure.instrument, 

764 id=exposure.id, 

765 name=exposure.obs_id, 

766 exposures=[exposure], 

767 visit_systems=visit_systems, 

768 ) 

769 

770 def getVisitSystems(self) -> Set[VisitSystem]: 

771 # Docstring inherited from GroupExposuresTask. 

772 return set(VisitSystem.from_names(["one-to-one"])) 

773 

774 

775class _GroupExposuresByGroupMetadataConfig(GroupExposuresConfig): 

776 visitSystemId = Field( 

777 doc="Integer ID of the visit_system implemented by this grouping algorithm.", 

778 dtype=int, 

779 default=1, 

780 deprecated="No longer used. Replaced by enum.", 

781 ) 

782 visitSystemName = Field( 

783 doc="String name of the visit_system implemented by this grouping algorithm.", 

784 dtype=str, 

785 default="by-group-metadata", 

786 deprecated="No longer used. Replaced by enum.", 

787 ) 

788 

789 

790@registerConfigurable("by-group-metadata", GroupExposuresTask.registry) 

791class _GroupExposuresByGroupMetadataTask(GroupExposuresTask, metaclass=ABCMeta): 

792 """An exposure grouping algorithm that uses exposure.group_name and 

793 exposure.group_id. 

794 

795 This algorithm _assumes_ exposure.group_id (generally populated from 

796 `astro_metadata_translator.ObservationInfo.visit_id`) is not just unique, 

797 but disjoint from all `ObservationInfo.exposure_id` values - if it isn't, 

798 it will be impossible to ever use both this grouping algorithm and the 

799 one-to-one algorithm for a particular camera in the same data repository. 

800 """ 

801 

802 ConfigClass = _GroupExposuresByGroupMetadataConfig 

803 

804 def group(self, exposures: List[DimensionRecord]) -> Iterable[VisitDefinitionData]: 

805 # Docstring inherited from GroupExposuresTask. 

806 visit_systems = {VisitSystem.from_name("by-group-metadata")} 

807 groups = defaultdict(list) 

808 for exposure in exposures: 

809 groups[exposure.group_name].append(exposure) 

810 for visitName, exposuresInGroup in groups.items(): 

811 instrument = exposuresInGroup[0].instrument 

812 visitId = exposuresInGroup[0].group_id 

813 assert all( 

814 e.group_id == visitId for e in exposuresInGroup 

815 ), "Grouping by exposure.group_name does not yield consistent group IDs" 

816 yield VisitDefinitionData( 

817 instrument=instrument, 

818 id=visitId, 

819 name=visitName, 

820 exposures=exposuresInGroup, 

821 visit_systems=visit_systems, 

822 ) 

823 

824 def getVisitSystems(self) -> Set[VisitSystem]: 

825 # Docstring inherited from GroupExposuresTask. 

826 return set(VisitSystem.from_names(["by-group-metadata"])) 

827 

828 

829class _GroupExposuresByCounterAndExposuresConfig(GroupExposuresConfig): 

830 visitSystemId = Field( 

831 doc="Integer ID of the visit_system implemented by this grouping algorithm.", 

832 dtype=int, 

833 default=2, 

834 deprecated="No longer used. Replaced by enum.", 

835 ) 

836 visitSystemName = Field( 

837 doc="String name of the visit_system implemented by this grouping algorithm.", 

838 dtype=str, 

839 default="by-counter-and-exposures", 

840 deprecated="No longer used. Replaced by enum.", 

841 ) 

842 

843 

844@registerConfigurable("one-to-one-and-by-counter", GroupExposuresTask.registry) 

845class _GroupExposuresByCounterAndExposuresTask(GroupExposuresTask, metaclass=ABCMeta): 

846 """An exposure grouping algorithm that uses the sequence start and 

847 sequence end metadata to create multi-exposure visits, but also 

848 creates one-to-one visits. 

849 

850 This algorithm uses the exposure.seq_start and 

851 exposure.seq_end fields to collect related snaps. 

852 It also groups single exposures. 

853 """ 

854 

855 ConfigClass = _GroupExposuresByCounterAndExposuresConfig 

856 

857 def group(self, exposures: List[DimensionRecord]) -> Iterable[VisitDefinitionData]: 

858 # Docstring inherited from GroupExposuresTask. 

859 system_one_to_one = VisitSystem.from_name("one-to-one") 

860 system_seq_start_end = VisitSystem.from_name("by-seq-start-end") 

861 

862 groups = defaultdict(list) 

863 for exposure in exposures: 

864 groups[exposure.day_obs, exposure.seq_start, exposure.seq_end].append(exposure) 

865 for visit_key, exposures_in_group in groups.items(): 

866 instrument = exposures_in_group[0].instrument 

867 

868 # It is possible that the first exposure in a visit has not 

869 # been ingested. This can be determined and if that is the case 

870 # we can not reliably define the multi-exposure visit. 

871 skip_multi = False 

872 sorted_exposures = sorted(exposures_in_group, key=lambda e: e.seq_num) 

873 first = sorted_exposures.pop(0) 

874 if first.seq_num != first.seq_start: 

875 # Special case seq_num == 0 since that implies that the 

876 # instrument has no counters and therefore no multi-exposure 

877 # visits. 

878 if first.seq_num == 0: 

879 self.log.warning( 

880 "First exposure for visit %s is not present. Skipping the multi-snap definition.", 

881 visit_key, 

882 ) 

883 skip_multi = True 

884 

885 # Define the one-to-one visits. 

886 num_exposures = len(exposures_in_group) 

887 for exposure in exposures_in_group: 

888 # Default is to use the exposure ID and name unless 

889 # this is the first exposure in a multi-exposure visit. 

890 visit_name = exposure.obs_id 

891 visit_id = exposure.id 

892 visit_systems = {system_one_to_one} 

893 

894 if num_exposures == 1: 

895 # This is also a by-counter visit. 

896 # It will use the same visit_name and visit_id. 

897 visit_systems.add(system_seq_start_end) 

898 

899 elif num_exposures > 1 and not skip_multi and exposure == first: 

900 # This is the first legitimate exposure in a multi-exposure 

901 # visit. It therefore needs a modified visit name and ID 

902 # so it does not clash with the multi-exposure visit 

903 # definition. 

904 visit_name = f"{visit_name}_first" 

905 visit_id = int(f"9{visit_id}") 

906 

907 yield VisitDefinitionData( 

908 instrument=instrument, 

909 id=visit_id, 

910 name=visit_name, 

911 exposures=[exposure], 

912 visit_systems=visit_systems, 

913 ) 

914 

915 # Multi-exposure visit. 

916 if not skip_multi and num_exposures > 1: 

917 # Define the visit using the first exposure 

918 visit_name = first.obs_id 

919 visit_id = first.id 

920 

921 yield VisitDefinitionData( 

922 instrument=instrument, 

923 id=visit_id, 

924 name=visit_name, 

925 exposures=exposures_in_group, 

926 visit_systems={system_seq_start_end}, 

927 ) 

928 

929 def getVisitSystems(self) -> Set[VisitSystem]: 

930 # Docstring inherited from GroupExposuresTask. 

931 # Using a Config for this is difficult because what this grouping 

932 # algorithm is doing is using two visit systems. 

933 # One is using metadata (but not by-group) and the other is the 

934 # one-to-one. For now hard-code in class. 

935 return set(VisitSystem.from_names(["one-to-one", "by-seq-start-end"])) 

936 

937 

938class _ComputeVisitRegionsFromSingleRawWcsConfig(ComputeVisitRegionsConfig): 

939 mergeExposures = Field( 

940 doc=( 

941 "If True, merge per-detector regions over all exposures in a " 

942 "visit (via convex hull) instead of using the first exposure and " 

943 "assuming its regions are valid for all others." 

944 ), 

945 dtype=bool, 

946 default=False, 

947 ) 

948 detectorId = Field( 

949 doc=( 

950 "Load the WCS for the detector with this ID. If None, use an " 

951 "arbitrary detector (the first found in a query of the data " 

952 "repository for each exposure (or all exposures, if " 

953 "mergeExposures is True)." 

954 ), 

955 dtype=int, 

956 optional=True, 

957 default=None, 

958 ) 

959 requireVersionedCamera = Field( 

960 doc=( 

961 "If True, raise LookupError if version camera geometry cannot be " 

962 "loaded for an exposure. If False, use the nominal camera from " 

963 "the Instrument class instead." 

964 ), 

965 dtype=bool, 

966 optional=False, 

967 default=False, 

968 ) 

969 

970 

971@registerConfigurable("single-raw-wcs", ComputeVisitRegionsTask.registry) 

972class _ComputeVisitRegionsFromSingleRawWcsTask(ComputeVisitRegionsTask): 

973 """A visit region calculator that uses a single raw WCS and a camera to 

974 project the bounding boxes of all detectors onto the sky, relating 

975 different detectors by their positions in focal plane coordinates. 

976 

977 Notes 

978 ----- 

979 Most instruments should have their raw WCSs determined from a combination 

980 of boresight angle, rotator angle, and camera geometry, and hence this 

981 algorithm should produce stable results regardless of which detector the 

982 raw corresponds to. If this is not the case (e.g. because a per-file FITS 

983 WCS is used instead), either the ID of the detector should be fixed (see 

984 the ``detectorId`` config parameter) or a different algorithm used. 

985 """ 

986 

987 ConfigClass = _ComputeVisitRegionsFromSingleRawWcsConfig 

988 

989 def computeExposureBounds( 

990 self, exposure: DimensionRecord, *, collections: Any = None 

991 ) -> Dict[int, List[UnitVector3d]]: 

992 """Compute the lists of unit vectors on the sphere that correspond to 

993 the sky positions of detector corners. 

994 

995 Parameters 

996 ---------- 

997 exposure : `DimensionRecord` 

998 Dimension record for the exposure. 

999 collections : Any, optional 

1000 Collections to be searched for raws and camera geometry, overriding 

1001 ``self.butler.collections``. 

1002 Can be any of the types supported by the ``collections`` argument 

1003 to butler construction. 

1004 

1005 Returns 

1006 ------- 

1007 bounds : `dict` 

1008 Dictionary mapping detector ID to a list of unit vectors on the 

1009 sphere representing that detector's corners projected onto the sky. 

1010 """ 

1011 if collections is None: 

1012 collections = self.butler.collections 

1013 camera, versioned = loadCamera(self.butler, exposure.dataId, collections=collections) 

1014 if not versioned and self.config.requireVersionedCamera: 

1015 raise LookupError(f"No versioned camera found for exposure {exposure.dataId}.") 

1016 

1017 # Derive WCS from boresight information -- if available in registry 

1018 use_registry = True 

1019 try: 

1020 orientation = lsst.geom.Angle(exposure.sky_angle, lsst.geom.degrees) 

1021 radec = lsst.geom.SpherePoint( 

1022 lsst.geom.Angle(exposure.tracking_ra, lsst.geom.degrees), 

1023 lsst.geom.Angle(exposure.tracking_dec, lsst.geom.degrees), 

1024 ) 

1025 except AttributeError: 

1026 use_registry = False 

1027 

1028 if use_registry: 

1029 if self.config.detectorId is None: 

1030 detectorId = next(camera.getIdIter()) 

1031 else: 

1032 detectorId = self.config.detectorId 

1033 wcsDetector = camera[detectorId] 

1034 

1035 # Ask the raw formatter to create the relevant WCS 

1036 # This allows flips to be taken into account 

1037 instrument = self.getInstrument(exposure.instrument) 

1038 rawFormatter = instrument.getRawFormatter({"detector": detectorId}) 

1039 

1040 try: 

1041 wcs = rawFormatter.makeRawSkyWcsFromBoresight(radec, orientation, wcsDetector) # type: ignore 

1042 except AttributeError: 

1043 raise TypeError( 

1044 f"Raw formatter is {get_full_type_name(rawFormatter)} but visit" 

1045 " definition requires it to support 'makeRawSkyWcsFromBoresight'" 

1046 ) from None 

1047 else: 

1048 if self.config.detectorId is None: 

1049 wcsRefsIter = self.butler.registry.queryDatasets( 

1050 "raw.wcs", dataId=exposure.dataId, collections=collections 

1051 ) 

1052 if not wcsRefsIter: 

1053 raise LookupError( 

1054 f"No raw.wcs datasets found for data ID {exposure.dataId} " 

1055 f"in collections {collections}." 

1056 ) 

1057 wcsRef = next(iter(wcsRefsIter)) 

1058 wcsDetector = camera[wcsRef.dataId["detector"]] 

1059 wcs = self.butler.getDirect(wcsRef) 

1060 else: 

1061 wcsDetector = camera[self.config.detectorId] 

1062 wcs = self.butler.get( 

1063 "raw.wcs", 

1064 dataId=exposure.dataId, 

1065 detector=self.config.detectorId, 

1066 collections=collections, 

1067 ) 

1068 fpToSky = wcsDetector.getTransform(FOCAL_PLANE, PIXELS).then(wcs.getTransform()) 

1069 bounds = {} 

1070 for detector in camera: 

1071 pixelsToSky = detector.getTransform(PIXELS, FOCAL_PLANE).then(fpToSky) 

1072 pixCorners = Box2D(detector.getBBox().dilatedBy(self.config.padding)).getCorners() 

1073 bounds[detector.getId()] = [ 

1074 skyCorner.getVector() for skyCorner in pixelsToSky.applyForward(pixCorners) 

1075 ] 

1076 return bounds 

1077 

1078 def compute( 

1079 self, visit: VisitDefinitionData, *, collections: Any = None 

1080 ) -> Tuple[Region, Dict[int, Region]]: 

1081 # Docstring inherited from ComputeVisitRegionsTask. 

1082 if self.config.mergeExposures: 

1083 detectorBounds: Dict[int, List[UnitVector3d]] = defaultdict(list) 

1084 for exposure in visit.exposures: 

1085 exposureDetectorBounds = self.computeExposureBounds(exposure, collections=collections) 

1086 for detectorId, bounds in exposureDetectorBounds.items(): 

1087 detectorBounds[detectorId].extend(bounds) 

1088 else: 

1089 detectorBounds = self.computeExposureBounds(visit.exposures[0], collections=collections) 

1090 visitBounds = [] 

1091 detectorRegions = {} 

1092 for detectorId, bounds in detectorBounds.items(): 

1093 detectorRegions[detectorId] = ConvexPolygon.convexHull(bounds) 

1094 visitBounds.extend(bounds) 

1095 return ConvexPolygon.convexHull(visitBounds), detectorRegions