Coverage for python/lsst/obs/base/defineVisits.py: 28%

344 statements  

« prev     ^ index     » next       coverage.py v6.5.0, created at 2023-03-16 02:10 -0700

1# This file is part of obs_base. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22from __future__ import annotations 

23 

24__all__ = [ 

25 "DefineVisitsConfig", 

26 "DefineVisitsTask", 

27 "GroupExposuresConfig", 

28 "GroupExposuresTask", 

29 "VisitDefinitionData", 

30 "VisitSystem", 

31] 

32 

33import cmath 

34import dataclasses 

35import enum 

36import math 

37import operator 

38from abc import ABCMeta, abstractmethod 

39from collections import defaultdict 

40from typing import ( 

41 Any, 

42 Callable, 

43 ClassVar, 

44 Dict, 

45 FrozenSet, 

46 Iterable, 

47 List, 

48 Optional, 

49 Set, 

50 Tuple, 

51 Type, 

52 TypeVar, 

53) 

54 

55import lsst.geom 

56from lsst.afw.cameraGeom import FOCAL_PLANE, PIXELS 

57from lsst.daf.butler import ( 

58 Butler, 

59 DataCoordinate, 

60 DataId, 

61 DimensionGraph, 

62 DimensionRecord, 

63 Progress, 

64 Timespan, 

65) 

66from lsst.geom import Box2D 

67from lsst.pex.config import Config, Field, makeRegistry, registerConfigurable 

68from lsst.pipe.base import Instrument, Task 

69from lsst.sphgeom import ConvexPolygon, Region, UnitVector3d 

70from lsst.utils.introspection import get_full_type_name 

71 

72from ._instrument import loadCamera 

73 

74 

75class VisitSystem(enum.Enum): 

76 """Enumeration used to label different visit systems.""" 

77 

78 ONE_TO_ONE = 0 

79 """Each exposure is assigned to its own visit.""" 

80 

81 BY_GROUP_METADATA = 1 

82 """Visit membership is defined by the value of the exposure.group_id.""" 

83 

84 BY_SEQ_START_END = 2 

85 """Visit membership is defined by the values of the ``exposure.day_obs``, 

86 ``exposure.seq_start``, and ``exposure.seq_end`` values. 

87 """ 

88 

89 @classmethod 

90 def all(cls) -> FrozenSet[VisitSystem]: 

91 """Return a `frozenset` containing all members.""" 

92 return frozenset(cls.__members__.values()) 

93 

94 @classmethod 

95 def from_name(cls, external_name: str) -> VisitSystem: 

96 """Construct the enumeration from given name.""" 

97 name = external_name.upper() 

98 name = name.replace("-", "_") 

99 try: 

100 return cls.__members__[name] 

101 except KeyError: 

102 raise KeyError(f"Visit system named '{external_name}' not known.") from None 

103 

104 @classmethod 

105 def from_names(cls, names: Optional[Iterable[str]]) -> FrozenSet[VisitSystem]: 

106 """Return a `frozenset` of all the visit systems matching the supplied 

107 names. 

108 

109 Parameters 

110 ---------- 

111 names : iterable of `str`, or `None` 

112 Names of visit systems. Case insensitive. If `None` or empty, all 

113 the visit systems are returned. 

114 

115 Returns 

116 ------- 

117 systems : `frozenset` of `VisitSystem` 

118 The matching visit systems. 

119 """ 

120 if not names: 

121 return cls.all() 

122 

123 return frozenset({cls.from_name(name) for name in names}) 

124 

125 def __str__(self) -> str: 

126 name = self.name.lower() 

127 name = name.replace("_", "-") 

128 return name 

129 

130 

131@dataclasses.dataclass 

132class VisitDefinitionData: 

133 """Struct representing a group of exposures that will be used to define a 

134 visit. 

135 """ 

136 

137 instrument: str 

138 """Name of the instrument this visit will be associated with. 

139 """ 

140 

141 id: int 

142 """Integer ID of the visit. 

143 

144 This must be unique across all visit systems for the instrument. 

145 """ 

146 

147 name: str 

148 """String name for the visit. 

149 

150 This must be unique across all visit systems for the instrument. 

151 """ 

152 

153 visit_systems: Set[VisitSystem] 

154 """All the visit systems associated with this visit.""" 

155 

156 exposures: List[DimensionRecord] = dataclasses.field(default_factory=list) 

157 """Dimension records for the exposures that are part of this visit. 

158 """ 

159 

160 

161@dataclasses.dataclass 

162class _VisitRecords: 

163 """Struct containing the dimension records associated with a visit.""" 

164 

165 visit: DimensionRecord 

166 """Record for the 'visit' dimension itself. 

167 """ 

168 

169 visit_definition: List[DimensionRecord] 

170 """Records for 'visit_definition', which relates 'visit' to 'exposure'. 

171 """ 

172 

173 visit_detector_region: List[DimensionRecord] 

174 """Records for 'visit_detector_region', which associates the combination 

175 of a 'visit' and a 'detector' with a region on the sky. 

176 """ 

177 

178 visit_system_membership: List[DimensionRecord] 

179 """Records relating visits to an associated visit system.""" 

180 

181 

182class GroupExposuresConfig(Config): 

183 pass 

184 

185 

186class GroupExposuresTask(Task, metaclass=ABCMeta): 

187 """Abstract base class for the subtask of `DefineVisitsTask` that is 

188 responsible for grouping exposures into visits. 

189 

190 Subclasses should be registered with `GroupExposuresTask.registry` to 

191 enable use by `DefineVisitsTask`, and should generally correspond to a 

192 particular 'visit_system' dimension value. They are also responsible for 

193 defining visit IDs and names that are unique across all visit systems in 

194 use by an instrument. 

195 

196 Parameters 

197 ---------- 

198 config : `GroupExposuresConfig` 

199 Configuration information. 

200 **kwargs 

201 Additional keyword arguments forwarded to the `Task` constructor. 

202 """ 

203 

204 def __init__(self, config: GroupExposuresConfig, **kwargs: Any): 

205 Task.__init__(self, config=config, **kwargs) 

206 

207 ConfigClass = GroupExposuresConfig 

208 

209 _DefaultName = "groupExposures" 

210 

211 registry = makeRegistry( 

212 doc="Registry of algorithms for grouping exposures into visits.", 

213 configBaseType=GroupExposuresConfig, 

214 ) 

215 

216 @abstractmethod 

217 def group(self, exposures: List[DimensionRecord]) -> Iterable[VisitDefinitionData]: 

218 """Group the given exposures into visits. 

219 

220 Parameters 

221 ---------- 

222 exposures : `list` [ `DimensionRecord` ] 

223 DimensionRecords (for the 'exposure' dimension) describing the 

224 exposures to group. 

225 

226 Returns 

227 ------- 

228 visits : `Iterable` [ `VisitDefinitionData` ] 

229 Structs identifying the visits and the exposures associated with 

230 them. This may be an iterator or a container. 

231 """ 

232 raise NotImplementedError() 

233 

234 def getVisitSystems(self) -> Set[VisitSystem]: 

235 """Return identifiers for the 'visit_system' dimension this 

236 algorithm implements. 

237 

238 Returns 

239 ------- 

240 visit_systems : `Set` [`VisitSystem`] 

241 The visit systems used by this algorithm. 

242 """ 

243 raise NotImplementedError() 

244 

245 

246class ComputeVisitRegionsConfig(Config): 

247 padding: Field[int] = Field( 

248 dtype=int, 

249 default=250, 

250 doc=( 

251 "Pad raw image bounding boxes with specified number of pixels " 

252 "when calculating their (conservatively large) region on the " 

253 "sky. Note that the config value for pixelMargin of the " 

254 "reference object loaders in meas_algorithms should be <= " 

255 "the value set here." 

256 ), 

257 ) 

258 

259 

260class ComputeVisitRegionsTask(Task, metaclass=ABCMeta): 

261 """Abstract base class for the subtask of `DefineVisitsTask` that is 

262 responsible for extracting spatial regions for visits and visit+detector 

263 combinations. 

264 

265 Subclasses should be registered with `ComputeVisitRegionsTask.registry` to 

266 enable use by `DefineVisitsTask`. 

267 

268 Parameters 

269 ---------- 

270 config : `ComputeVisitRegionsConfig` 

271 Configuration information. 

272 butler : `lsst.daf.butler.Butler` 

273 The butler to use. 

274 **kwargs 

275 Additional keyword arguments forwarded to the `Task` constructor. 

276 """ 

277 

278 def __init__(self, config: ComputeVisitRegionsConfig, *, butler: Butler, **kwargs: Any): 

279 Task.__init__(self, config=config, **kwargs) 

280 self.butler = butler 

281 self.instrumentMap: Dict[str, Instrument] = {} 

282 

283 ConfigClass = ComputeVisitRegionsConfig 

284 

285 _DefaultName = "computeVisitRegions" 

286 

287 registry = makeRegistry( 

288 doc="Registry of algorithms for computing on-sky regions for visits and visit+detector combinations.", 

289 configBaseType=ComputeVisitRegionsConfig, 

290 ) 

291 

292 def getInstrument(self, instrumentName: str) -> Instrument: 

293 """Retrieve an `~lsst.obs.base.Instrument` associated with this 

294 instrument name. 

295 

296 Parameters 

297 ---------- 

298 instrumentName : `str` 

299 The name of the instrument. 

300 

301 Returns 

302 ------- 

303 instrument : `~lsst.obs.base.Instrument` 

304 The associated instrument object. 

305 

306 Notes 

307 ----- 

308 The result is cached. 

309 """ 

310 instrument = self.instrumentMap.get(instrumentName) 

311 if instrument is None: 

312 instrument = Instrument.fromName(instrumentName, self.butler.registry) 

313 self.instrumentMap[instrumentName] = instrument 

314 return instrument 

315 

316 @abstractmethod 

317 def compute( 

318 self, visit: VisitDefinitionData, *, collections: Any = None 

319 ) -> Tuple[Region, Dict[int, Region]]: 

320 """Compute regions for the given visit and all detectors in that visit. 

321 

322 Parameters 

323 ---------- 

324 visit : `VisitDefinitionData` 

325 Struct describing the visit and the exposures associated with it. 

326 collections : Any, optional 

327 Collections to be searched for raws and camera geometry, overriding 

328 ``self.butler.collections``. 

329 Can be any of the types supported by the ``collections`` argument 

330 to butler construction. 

331 

332 Returns 

333 ------- 

334 visitRegion : `lsst.sphgeom.Region` 

335 Region for the full visit. 

336 visitDetectorRegions : `dict` [ `int`, `lsst.sphgeom.Region` ] 

337 Dictionary mapping detector ID to the region for that detector. 

338 Should include all detectors in the visit. 

339 """ 

340 raise NotImplementedError() 

341 

342 

343class DefineVisitsConfig(Config): 

344 groupExposures = GroupExposuresTask.registry.makeField( 

345 doc="Algorithm for grouping exposures into visits.", 

346 default="one-to-one-and-by-counter", 

347 ) 

348 computeVisitRegions = ComputeVisitRegionsTask.registry.makeField( 

349 doc="Algorithm from computing visit and visit+detector regions.", 

350 default="single-raw-wcs", 

351 ) 

352 ignoreNonScienceExposures: Field[bool] = Field( 

353 doc=( 

354 "If True, silently ignore input exposures that do not have " 

355 "observation_type=SCIENCE. If False, raise an exception if one " 

356 "encountered." 

357 ), 

358 dtype=bool, 

359 optional=False, 

360 default=True, 

361 ) 

362 

363 

364class DefineVisitsTask(Task): 

365 """Driver Task for defining visits (and their spatial regions) in Gen3 

366 Butler repositories. 

367 

368 Parameters 

369 ---------- 

370 config : `DefineVisitsConfig` 

371 Configuration for the task. 

372 butler : `~lsst.daf.butler.Butler` 

373 Writeable butler instance. Will be used to read `raw.wcs` and `camera` 

374 datasets and insert/sync dimension data. 

375 **kwargs 

376 Additional keyword arguments are forwarded to the `lsst.pipe.base.Task` 

377 constructor. 

378 

379 Notes 

380 ----- 

381 Each instance of `DefineVisitsTask` reads from / writes to the same Butler. 

382 Each invocation of `DefineVisitsTask.run` processes an independent group of 

383 exposures into one or more new vists, all belonging to the same visit 

384 system and instrument. 

385 

386 The actual work of grouping exposures and computing regions is delegated 

387 to pluggable subtasks (`GroupExposuresTask` and `ComputeVisitRegionsTask`), 

388 respectively. The defaults are to create one visit for every exposure, 

389 and to use exactly one (arbitrary) detector-level raw dataset's WCS along 

390 with camera geometry to compute regions for all detectors. Other 

391 implementations can be created and configured for instruments for which 

392 these choices are unsuitable (e.g. because visits and exposures are not 

393 one-to-one, or because ``raw.wcs`` datasets for different detectors may not 

394 be consistent with camera geomery). 

395 

396 It is not necessary in general to ingest all raws for an exposure before 

397 defining a visit that includes the exposure; this depends entirely on the 

398 `ComputeVisitRegionTask` subclass used. For the default configuration, 

399 a single raw for each exposure is sufficient. 

400 

401 Defining the same visit the same way multiple times (e.g. via multiple 

402 invocations of this task on the same exposures, with the same 

403 configuration) is safe, but it may be inefficient, as most of the work must 

404 be done before new visits can be compared to existing visits. 

405 """ 

406 

407 def __init__(self, config: DefineVisitsConfig, *, butler: Butler, **kwargs: Any): 

408 config.validate() # Not a CmdlineTask nor PipelineTask, so have to validate the config here. 

409 super().__init__(config, **kwargs) 

410 self.butler = butler 

411 self.universe = self.butler.registry.dimensions 

412 self.progress = Progress("obs.base.DefineVisitsTask") 

413 self.makeSubtask("groupExposures") 

414 self.makeSubtask("computeVisitRegions", butler=self.butler) 

415 

416 def _reduce_kwargs(self) -> dict: 

417 # Add extra parameters to pickle 

418 return dict(**super()._reduce_kwargs(), butler=self.butler) 

419 

420 ConfigClass: ClassVar[Type[Config]] = DefineVisitsConfig 

421 

422 _DefaultName: ClassVar[str] = "defineVisits" 

423 

424 config: DefineVisitsConfig 

425 groupExposures: GroupExposuresTask 

426 computeVisitRegions: ComputeVisitRegionsTask 

427 

428 def _buildVisitRecords( 

429 self, definition: VisitDefinitionData, *, collections: Any = None 

430 ) -> _VisitRecords: 

431 """Build the DimensionRecords associated with a visit. 

432 

433 Parameters 

434 ---------- 

435 definition : `VisitDefinitionData` 

436 Struct with identifiers for the visit and records for its 

437 constituent exposures. 

438 collections : Any, optional 

439 Collections to be searched for raws and camera geometry, overriding 

440 ``self.butler.collections``. 

441 Can be any of the types supported by the ``collections`` argument 

442 to butler construction. 

443 

444 Results 

445 ------- 

446 records : `_VisitRecords` 

447 Struct containing DimensionRecords for the visit, including 

448 associated dimension elements. 

449 """ 

450 dimension = self.universe["visit"] 

451 

452 # Some registries support additional items. 

453 supported = {meta.name for meta in dimension.metadata} 

454 

455 # Compute all regions. 

456 visitRegion, visitDetectorRegions = self.computeVisitRegions.compute( 

457 definition, collections=collections 

458 ) 

459 # Aggregate other exposure quantities. 

460 timespan = Timespan( 

461 begin=_reduceOrNone(min, (e.timespan.begin for e in definition.exposures)), 

462 end=_reduceOrNone(max, (e.timespan.end for e in definition.exposures)), 

463 ) 

464 exposure_time = _reduceOrNone(operator.add, (e.exposure_time for e in definition.exposures)) 

465 physical_filter = _reduceOrNone(_value_if_equal, (e.physical_filter for e in definition.exposures)) 

466 target_name = _reduceOrNone(_value_if_equal, (e.target_name for e in definition.exposures)) 

467 science_program = _reduceOrNone(_value_if_equal, (e.science_program for e in definition.exposures)) 

468 

469 # observing day for a visit is defined by the earliest observation 

470 # of the visit 

471 observing_day = _reduceOrNone(min, (e.day_obs for e in definition.exposures)) 

472 observation_reason = _reduceOrNone( 

473 _value_if_equal, (e.observation_reason for e in definition.exposures) 

474 ) 

475 if observation_reason is None: 

476 # Be explicit about there being multiple reasons 

477 # MyPy can't really handle DimensionRecord fields as 

478 # DimensionRecord classes are dynamically defined; easiest to just 

479 # shush it when it complains. 

480 observation_reason = "various" # type: ignore 

481 

482 # Use the mean zenith angle as an approximation 

483 zenith_angle = _reduceOrNone(operator.add, (e.zenith_angle for e in definition.exposures)) 

484 if zenith_angle is not None: 

485 zenith_angle /= len(definition.exposures) 

486 

487 # New records that may not be supported. 

488 extras: Dict[str, Any] = {} 

489 if "seq_num" in supported: 

490 extras["seq_num"] = _reduceOrNone(min, (e.seq_num for e in definition.exposures)) 

491 if "azimuth" in supported: 

492 # Must take into account 0/360 problem. 

493 extras["azimuth"] = _calc_mean_angle([e.azimuth for e in definition.exposures]) 

494 

495 # visit_system handling changed. This is the logic for visit/exposure 

496 # that has support for seq_start/seq_end. 

497 if "seq_num" in supported: 

498 # Map visit to exposure. 

499 visit_definition = [ 

500 self.universe["visit_definition"].RecordClass( 

501 instrument=definition.instrument, 

502 visit=definition.id, 

503 exposure=exposure.id, 

504 ) 

505 for exposure in definition.exposures 

506 ] 

507 

508 # Map visit to visit system. 

509 visit_system_membership = [] 

510 for visit_system in self.groupExposures.getVisitSystems(): 

511 if visit_system in definition.visit_systems: 

512 record = self.universe["visit_system_membership"].RecordClass( 

513 instrument=definition.instrument, 

514 visit=definition.id, 

515 visit_system=visit_system.value, 

516 ) 

517 visit_system_membership.append(record) 

518 

519 else: 

520 # The old approach can only handle one visit system at a time. 

521 # If we have been configured with multiple options, prefer the 

522 # one-to-one. 

523 visit_systems = self.groupExposures.getVisitSystems() 

524 if len(visit_systems) > 1: 

525 one_to_one = VisitSystem.from_name("one-to-one") 

526 if one_to_one not in visit_systems: 

527 raise ValueError( 

528 f"Multiple visit systems specified ({visit_systems}) for use with old" 

529 " dimension universe but unable to find one-to-one." 

530 ) 

531 visit_system = one_to_one 

532 else: 

533 visit_system = visit_systems.pop() 

534 

535 extras["visit_system"] = visit_system.value 

536 

537 # The old visit_definition included visit system. 

538 visit_definition = [ 

539 self.universe["visit_definition"].RecordClass( 

540 instrument=definition.instrument, 

541 visit=definition.id, 

542 exposure=exposure.id, 

543 visit_system=visit_system.value, 

544 ) 

545 for exposure in definition.exposures 

546 ] 

547 

548 # This concept does not exist in old schema. 

549 visit_system_membership = [] 

550 

551 # Construct the actual DimensionRecords. 

552 return _VisitRecords( 

553 visit=dimension.RecordClass( 

554 instrument=definition.instrument, 

555 id=definition.id, 

556 name=definition.name, 

557 physical_filter=physical_filter, 

558 target_name=target_name, 

559 science_program=science_program, 

560 observation_reason=observation_reason, 

561 day_obs=observing_day, 

562 zenith_angle=zenith_angle, 

563 exposure_time=exposure_time, 

564 timespan=timespan, 

565 region=visitRegion, 

566 # TODO: no seeing value in exposure dimension records, so we 

567 # can't set that here. But there are many other columns that 

568 # both dimensions should probably have as well. 

569 **extras, 

570 ), 

571 visit_definition=visit_definition, 

572 visit_system_membership=visit_system_membership, 

573 visit_detector_region=[ 

574 self.universe["visit_detector_region"].RecordClass( 

575 instrument=definition.instrument, 

576 visit=definition.id, 

577 detector=detectorId, 

578 region=detectorRegion, 

579 ) 

580 for detectorId, detectorRegion in visitDetectorRegions.items() 

581 ], 

582 ) 

583 

584 def run( 

585 self, 

586 dataIds: Iterable[DataId], 

587 *, 

588 collections: Optional[str] = None, 

589 update_records: bool = False, 

590 ) -> None: 

591 """Add visit definitions to the registry for the given exposures. 

592 

593 Parameters 

594 ---------- 

595 dataIds : `Iterable` [ `dict` or `DataCoordinate` ] 

596 Exposure-level data IDs. These must all correspond to the same 

597 instrument, and are expected to be on-sky science exposures. 

598 collections : Any, optional 

599 Collections to be searched for raws and camera geometry, overriding 

600 ``self.butler.collections``. 

601 Can be any of the types supported by the ``collections`` argument 

602 to butler construction. 

603 update_records : `bool`, optional 

604 If `True` (`False` is default), update existing visit records that 

605 conflict with the new ones instead of rejecting them (and when this 

606 occurs, update visit_detector_region as well). THIS IS AN ADVANCED 

607 OPTION THAT SHOULD ONLY BE USED TO FIX REGIONS AND/OR METADATA THAT 

608 ARE KNOWN TO BE BAD, AND IT CANNOT BE USED TO REMOVE EXPOSURES OR 

609 DETECTORS FROM A VISIT. 

610 

611 Raises 

612 ------ 

613 lsst.daf.butler.registry.ConflictingDefinitionError 

614 Raised if a visit ID conflict is detected and the existing visit 

615 differs from the new one. 

616 """ 

617 # Normalize, expand, and deduplicate data IDs. 

618 self.log.info("Preprocessing data IDs.") 

619 dimensions = DimensionGraph(self.universe, names=["exposure"]) 

620 data_id_set: Set[DataCoordinate] = { 

621 self.butler.registry.expandDataId(d, graph=dimensions) for d in dataIds 

622 } 

623 if not data_id_set: 

624 raise RuntimeError("No exposures given.") 

625 # Extract exposure DimensionRecords, check that there's only one 

626 # instrument in play, and check for non-science exposures. 

627 exposures = [] 

628 instruments = set() 

629 for dataId in data_id_set: 

630 record = dataId.records["exposure"] 

631 assert record is not None, "Guaranteed by expandDataIds call earlier." 

632 if record.tracking_ra is None or record.tracking_dec is None or record.sky_angle is None: 

633 if self.config.ignoreNonScienceExposures: 

634 continue 

635 else: 

636 raise RuntimeError( 

637 f"Input exposure {dataId} has observation_type " 

638 f"{record.observation_type}, but is not on sky." 

639 ) 

640 instruments.add(dataId["instrument"]) 

641 exposures.append(record) 

642 if not exposures: 

643 self.log.info("No on-sky exposures found after filtering.") 

644 return 

645 if len(instruments) > 1: 

646 raise RuntimeError( 

647 "All data IDs passed to DefineVisitsTask.run must be " 

648 f"from the same instrument; got {instruments}." 

649 ) 

650 (instrument,) = instruments 

651 # Ensure the visit_system our grouping algorithm uses is in the 

652 # registry, if it wasn't already. 

653 visitSystems = self.groupExposures.getVisitSystems() 

654 for visitSystem in visitSystems: 

655 self.log.info("Registering visit_system %d: %s.", visitSystem.value, visitSystem) 

656 self.butler.registry.syncDimensionData( 

657 "visit_system", {"instrument": instrument, "id": visitSystem.value, "name": str(visitSystem)} 

658 ) 

659 # Group exposures into visits, delegating to subtask. 

660 self.log.info("Grouping %d exposure(s) into visits.", len(exposures)) 

661 definitions = list(self.groupExposures.group(exposures)) 

662 # Iterate over visits, compute regions, and insert dimension data, one 

663 # transaction per visit. If a visit already exists, we skip all other 

664 # inserts. 

665 self.log.info("Computing regions and other metadata for %d visit(s).", len(definitions)) 

666 for visitDefinition in self.progress.wrap( 

667 definitions, total=len(definitions), desc="Computing regions and inserting visits" 

668 ): 

669 visitRecords = self._buildVisitRecords(visitDefinition, collections=collections) 

670 with self.butler.registry.transaction(): 

671 inserted_or_updated = self.butler.registry.syncDimensionData( 

672 "visit", 

673 visitRecords.visit, 

674 update=update_records, 

675 ) 

676 if inserted_or_updated: 

677 if inserted_or_updated is True: 

678 # This is a new visit, not an update to an existing 

679 # one, so insert visit definition. 

680 # We don't allow visit definitions to change even when 

681 # asked to update, because we'd have to delete the old 

682 # visit_definitions first and also worry about what 

683 # this does to datasets that already use the visit. 

684 self.butler.registry.insertDimensionData( 

685 "visit_definition", *visitRecords.visit_definition 

686 ) 

687 if visitRecords.visit_system_membership: 

688 self.butler.registry.insertDimensionData( 

689 "visit_system_membership", *visitRecords.visit_system_membership 

690 ) 

691 # [Re]Insert visit_detector_region records for both inserts 

692 # and updates, because we do allow updating to affect the 

693 # region calculations. 

694 self.butler.registry.insertDimensionData( 

695 "visit_detector_region", *visitRecords.visit_detector_region, replace=update_records 

696 ) 

697 

698 

699_T = TypeVar("_T") 

700 

701 

702def _reduceOrNone(func: Callable[[_T, _T], Optional[_T]], iterable: Iterable[Optional[_T]]) -> Optional[_T]: 

703 """Apply a binary function to pairs of elements in an iterable until a 

704 single value is returned, but return `None` if any element is `None` or 

705 there are no elements. 

706 """ 

707 r: Optional[_T] = None 

708 for v in iterable: 

709 if v is None: 

710 return None 

711 if r is None: 

712 r = v 

713 else: 

714 r = func(r, v) 

715 return r 

716 

717 

718def _value_if_equal(a: _T, b: _T) -> Optional[_T]: 

719 """Return either argument if they are equal, or `None` if they are not.""" 

720 return a if a == b else None 

721 

722 

723def _calc_mean_angle(angles: List[float]) -> float: 

724 """Calculate the mean angle, taking into account 0/360 wrapping. 

725 

726 Parameters 

727 ---------- 

728 angles : `list` [`float`] 

729 Angles to average together, in degrees. 

730 

731 Returns 

732 ------- 

733 average : `float` 

734 Average angle in degrees. 

735 """ 

736 # Save on all the math if we only have one value. 

737 if len(angles) == 1: 

738 return angles[0] 

739 

740 # Convert polar coordinates of unit circle to complex values. 

741 # Average the complex values. 

742 # Convert back to a phase angle. 

743 return math.degrees(cmath.phase(sum(cmath.rect(1.0, math.radians(d)) for d in angles) / len(angles))) 

744 

745 

746class _GroupExposuresOneToOneConfig(GroupExposuresConfig): 

747 visitSystemId: Field[int] = Field( 

748 doc="Integer ID of the visit_system implemented by this grouping algorithm.", 

749 dtype=int, 

750 default=0, 

751 deprecated="No longer used. Replaced by enum.", 

752 ) 

753 visitSystemName: Field[str] = Field( 

754 doc="String name of the visit_system implemented by this grouping algorithm.", 

755 dtype=str, 

756 default="one-to-one", 

757 deprecated="No longer used. Replaced by enum.", 

758 ) 

759 

760 

761@registerConfigurable("one-to-one", GroupExposuresTask.registry) 

762class _GroupExposuresOneToOneTask(GroupExposuresTask, metaclass=ABCMeta): 

763 """An exposure grouping algorithm that simply defines one visit for each 

764 exposure, reusing the exposures identifiers for the visit. 

765 """ 

766 

767 ConfigClass = _GroupExposuresOneToOneConfig 

768 

769 def group(self, exposures: List[DimensionRecord]) -> Iterable[VisitDefinitionData]: 

770 # Docstring inherited from GroupExposuresTask. 

771 visit_systems = {VisitSystem.from_name("one-to-one")} 

772 for exposure in exposures: 

773 yield VisitDefinitionData( 

774 instrument=exposure.instrument, 

775 id=exposure.id, 

776 name=exposure.obs_id, 

777 exposures=[exposure], 

778 visit_systems=visit_systems, 

779 ) 

780 

781 def getVisitSystems(self) -> Set[VisitSystem]: 

782 # Docstring inherited from GroupExposuresTask. 

783 return set(VisitSystem.from_names(["one-to-one"])) 

784 

785 

786class _GroupExposuresByGroupMetadataConfig(GroupExposuresConfig): 

787 visitSystemId: Field[int] = Field( 

788 doc="Integer ID of the visit_system implemented by this grouping algorithm.", 

789 dtype=int, 

790 default=1, 

791 deprecated="No longer used. Replaced by enum.", 

792 ) 

793 visitSystemName: Field[str] = Field( 

794 doc="String name of the visit_system implemented by this grouping algorithm.", 

795 dtype=str, 

796 default="by-group-metadata", 

797 deprecated="No longer used. Replaced by enum.", 

798 ) 

799 

800 

801@registerConfigurable("by-group-metadata", GroupExposuresTask.registry) 

802class _GroupExposuresByGroupMetadataTask(GroupExposuresTask, metaclass=ABCMeta): 

803 """An exposure grouping algorithm that uses exposure.group_name and 

804 exposure.group_id. 

805 

806 This algorithm _assumes_ exposure.group_id (generally populated from 

807 `astro_metadata_translator.ObservationInfo.visit_id`) is not just unique, 

808 but disjoint from all `ObservationInfo.exposure_id` values - if it isn't, 

809 it will be impossible to ever use both this grouping algorithm and the 

810 one-to-one algorithm for a particular camera in the same data repository. 

811 """ 

812 

813 ConfigClass = _GroupExposuresByGroupMetadataConfig 

814 

815 def group(self, exposures: List[DimensionRecord]) -> Iterable[VisitDefinitionData]: 

816 # Docstring inherited from GroupExposuresTask. 

817 visit_systems = {VisitSystem.from_name("by-group-metadata")} 

818 groups = defaultdict(list) 

819 for exposure in exposures: 

820 groups[exposure.group_name].append(exposure) 

821 for visitName, exposuresInGroup in groups.items(): 

822 instrument = exposuresInGroup[0].instrument 

823 visitId = exposuresInGroup[0].group_id 

824 assert all( 

825 e.group_id == visitId for e in exposuresInGroup 

826 ), "Grouping by exposure.group_name does not yield consistent group IDs" 

827 yield VisitDefinitionData( 

828 instrument=instrument, 

829 id=visitId, 

830 name=visitName, 

831 exposures=exposuresInGroup, 

832 visit_systems=visit_systems, 

833 ) 

834 

835 def getVisitSystems(self) -> Set[VisitSystem]: 

836 # Docstring inherited from GroupExposuresTask. 

837 return set(VisitSystem.from_names(["by-group-metadata"])) 

838 

839 

840class _GroupExposuresByCounterAndExposuresConfig(GroupExposuresConfig): 

841 visitSystemId: Field[int] = Field( 

842 doc="Integer ID of the visit_system implemented by this grouping algorithm.", 

843 dtype=int, 

844 default=2, 

845 deprecated="No longer used. Replaced by enum.", 

846 ) 

847 visitSystemName: Field[str] = Field( 

848 doc="String name of the visit_system implemented by this grouping algorithm.", 

849 dtype=str, 

850 default="by-counter-and-exposures", 

851 deprecated="No longer used. Replaced by enum.", 

852 ) 

853 

854 

855@registerConfigurable("one-to-one-and-by-counter", GroupExposuresTask.registry) 

856class _GroupExposuresByCounterAndExposuresTask(GroupExposuresTask, metaclass=ABCMeta): 

857 """An exposure grouping algorithm that uses the sequence start and 

858 sequence end metadata to create multi-exposure visits, but also 

859 creates one-to-one visits. 

860 

861 This algorithm uses the exposure.seq_start and 

862 exposure.seq_end fields to collect related snaps. 

863 It also groups single exposures. 

864 """ 

865 

866 ConfigClass = _GroupExposuresByCounterAndExposuresConfig 

867 

868 def group(self, exposures: List[DimensionRecord]) -> Iterable[VisitDefinitionData]: 

869 # Docstring inherited from GroupExposuresTask. 

870 system_one_to_one = VisitSystem.from_name("one-to-one") 

871 system_seq_start_end = VisitSystem.from_name("by-seq-start-end") 

872 

873 groups = defaultdict(list) 

874 for exposure in exposures: 

875 groups[exposure.day_obs, exposure.seq_start, exposure.seq_end].append(exposure) 

876 for visit_key, exposures_in_group in groups.items(): 

877 instrument = exposures_in_group[0].instrument 

878 

879 # It is possible that the first exposure in a visit has not 

880 # been ingested. This can be determined and if that is the case 

881 # we can not reliably define the multi-exposure visit. 

882 skip_multi = False 

883 sorted_exposures = sorted(exposures_in_group, key=lambda e: e.seq_num) 

884 first = sorted_exposures.pop(0) 

885 if first.seq_num != first.seq_start: 

886 # Special case seq_num == 0 since that implies that the 

887 # instrument has no counters and therefore no multi-exposure 

888 # visits. 

889 if first.seq_num != 0: 

890 self.log.warning( 

891 "First exposure for visit %s is not present. Skipping the multi-snap definition.", 

892 visit_key, 

893 ) 

894 skip_multi = True 

895 

896 # Define the one-to-one visits. 

897 num_exposures = len(exposures_in_group) 

898 for exposure in exposures_in_group: 

899 # Default is to use the exposure ID and name unless 

900 # this is the first exposure in a multi-exposure visit. 

901 visit_name = exposure.obs_id 

902 visit_id = exposure.id 

903 visit_systems = {system_one_to_one} 

904 

905 if num_exposures == 1: 

906 # This is also a by-counter visit. 

907 # It will use the same visit_name and visit_id. 

908 visit_systems.add(system_seq_start_end) 

909 

910 elif num_exposures > 1 and not skip_multi and exposure == first: 

911 # This is the first legitimate exposure in a multi-exposure 

912 # visit. It therefore needs a modified visit name and ID 

913 # so it does not clash with the multi-exposure visit 

914 # definition. 

915 visit_name = f"{visit_name}_first" 

916 visit_id = int(f"9{visit_id}") 

917 

918 yield VisitDefinitionData( 

919 instrument=instrument, 

920 id=visit_id, 

921 name=visit_name, 

922 exposures=[exposure], 

923 visit_systems=visit_systems, 

924 ) 

925 

926 # Multi-exposure visit. 

927 if not skip_multi and num_exposures > 1: 

928 # Define the visit using the first exposure 

929 visit_name = first.obs_id 

930 visit_id = first.id 

931 

932 yield VisitDefinitionData( 

933 instrument=instrument, 

934 id=visit_id, 

935 name=visit_name, 

936 exposures=exposures_in_group, 

937 visit_systems={system_seq_start_end}, 

938 ) 

939 

940 def getVisitSystems(self) -> Set[VisitSystem]: 

941 # Docstring inherited from GroupExposuresTask. 

942 # Using a Config for this is difficult because what this grouping 

943 # algorithm is doing is using two visit systems. 

944 # One is using metadata (but not by-group) and the other is the 

945 # one-to-one. For now hard-code in class. 

946 return set(VisitSystem.from_names(["one-to-one", "by-seq-start-end"])) 

947 

948 

949class _ComputeVisitRegionsFromSingleRawWcsConfig(ComputeVisitRegionsConfig): 

950 mergeExposures: Field[bool] = Field( 

951 doc=( 

952 "If True, merge per-detector regions over all exposures in a " 

953 "visit (via convex hull) instead of using the first exposure and " 

954 "assuming its regions are valid for all others." 

955 ), 

956 dtype=bool, 

957 default=False, 

958 ) 

959 detectorId: Field[Optional[int]] = Field( 

960 doc=( 

961 "Load the WCS for the detector with this ID. If None, use an " 

962 "arbitrary detector (the first found in a query of the data " 

963 "repository for each exposure (or all exposures, if " 

964 "mergeExposures is True)." 

965 ), 

966 dtype=int, 

967 optional=True, 

968 default=None, 

969 ) 

970 requireVersionedCamera: Field[bool] = Field( 

971 doc=( 

972 "If True, raise LookupError if version camera geometry cannot be " 

973 "loaded for an exposure. If False, use the nominal camera from " 

974 "the Instrument class instead." 

975 ), 

976 dtype=bool, 

977 optional=False, 

978 default=False, 

979 ) 

980 

981 

982@registerConfigurable("single-raw-wcs", ComputeVisitRegionsTask.registry) 

983class _ComputeVisitRegionsFromSingleRawWcsTask(ComputeVisitRegionsTask): 

984 """A visit region calculator that uses a single raw WCS and a camera to 

985 project the bounding boxes of all detectors onto the sky, relating 

986 different detectors by their positions in focal plane coordinates. 

987 

988 Notes 

989 ----- 

990 Most instruments should have their raw WCSs determined from a combination 

991 of boresight angle, rotator angle, and camera geometry, and hence this 

992 algorithm should produce stable results regardless of which detector the 

993 raw corresponds to. If this is not the case (e.g. because a per-file FITS 

994 WCS is used instead), either the ID of the detector should be fixed (see 

995 the ``detectorId`` config parameter) or a different algorithm used. 

996 """ 

997 

998 ConfigClass = _ComputeVisitRegionsFromSingleRawWcsConfig 

999 config: _ComputeVisitRegionsFromSingleRawWcsConfig 

1000 

1001 def computeExposureBounds( 

1002 self, exposure: DimensionRecord, *, collections: Any = None 

1003 ) -> Dict[int, List[UnitVector3d]]: 

1004 """Compute the lists of unit vectors on the sphere that correspond to 

1005 the sky positions of detector corners. 

1006 

1007 Parameters 

1008 ---------- 

1009 exposure : `DimensionRecord` 

1010 Dimension record for the exposure. 

1011 collections : Any, optional 

1012 Collections to be searched for raws and camera geometry, overriding 

1013 ``self.butler.collections``. 

1014 Can be any of the types supported by the ``collections`` argument 

1015 to butler construction. 

1016 

1017 Returns 

1018 ------- 

1019 bounds : `dict` 

1020 Dictionary mapping detector ID to a list of unit vectors on the 

1021 sphere representing that detector's corners projected onto the sky. 

1022 """ 

1023 if collections is None: 

1024 collections = self.butler.collections 

1025 camera, versioned = loadCamera(self.butler, exposure.dataId, collections=collections) 

1026 if not versioned and self.config.requireVersionedCamera: 

1027 raise LookupError(f"No versioned camera found for exposure {exposure.dataId}.") 

1028 

1029 # Derive WCS from boresight information -- if available in registry 

1030 use_registry = True 

1031 try: 

1032 orientation = lsst.geom.Angle(exposure.sky_angle, lsst.geom.degrees) 

1033 radec = lsst.geom.SpherePoint( 

1034 lsst.geom.Angle(exposure.tracking_ra, lsst.geom.degrees), 

1035 lsst.geom.Angle(exposure.tracking_dec, lsst.geom.degrees), 

1036 ) 

1037 except AttributeError: 

1038 use_registry = False 

1039 

1040 if use_registry: 

1041 if self.config.detectorId is None: 

1042 detectorId = next(camera.getIdIter()) 

1043 else: 

1044 detectorId = self.config.detectorId 

1045 wcsDetector = camera[detectorId] 

1046 

1047 # Ask the raw formatter to create the relevant WCS 

1048 # This allows flips to be taken into account 

1049 instrument = self.getInstrument(exposure.instrument) 

1050 rawFormatter = instrument.getRawFormatter({"detector": detectorId}) 

1051 

1052 try: 

1053 wcs = rawFormatter.makeRawSkyWcsFromBoresight(radec, orientation, wcsDetector) # type: ignore 

1054 except AttributeError: 

1055 raise TypeError( 

1056 f"Raw formatter is {get_full_type_name(rawFormatter)} but visit" 

1057 " definition requires it to support 'makeRawSkyWcsFromBoresight'" 

1058 ) from None 

1059 else: 

1060 if self.config.detectorId is None: 

1061 wcsRefsIter = self.butler.registry.queryDatasets( 

1062 "raw.wcs", dataId=exposure.dataId, collections=collections 

1063 ) 

1064 if not wcsRefsIter: 

1065 raise LookupError( 

1066 f"No raw.wcs datasets found for data ID {exposure.dataId} " 

1067 f"in collections {collections}." 

1068 ) 

1069 wcsRef = next(iter(wcsRefsIter)) 

1070 wcsDetector = camera[wcsRef.dataId["detector"]] 

1071 wcs = self.butler.getDirect(wcsRef) 

1072 else: 

1073 wcsDetector = camera[self.config.detectorId] 

1074 wcs = self.butler.get( 

1075 "raw.wcs", 

1076 dataId=exposure.dataId, 

1077 detector=self.config.detectorId, 

1078 collections=collections, 

1079 ) 

1080 fpToSky = wcsDetector.getTransform(FOCAL_PLANE, PIXELS).then(wcs.getTransform()) 

1081 bounds = {} 

1082 for detector in camera: 

1083 pixelsToSky = detector.getTransform(PIXELS, FOCAL_PLANE).then(fpToSky) 

1084 pixCorners = Box2D(detector.getBBox().dilatedBy(self.config.padding)).getCorners() 

1085 bounds[detector.getId()] = [ 

1086 skyCorner.getVector() for skyCorner in pixelsToSky.applyForward(pixCorners) 

1087 ] 

1088 return bounds 

1089 

1090 def compute( 

1091 self, visit: VisitDefinitionData, *, collections: Any = None 

1092 ) -> Tuple[Region, Dict[int, Region]]: 

1093 # Docstring inherited from ComputeVisitRegionsTask. 

1094 if self.config.mergeExposures: 

1095 detectorBounds: Dict[int, List[UnitVector3d]] = defaultdict(list) 

1096 for exposure in visit.exposures: 

1097 exposureDetectorBounds = self.computeExposureBounds(exposure, collections=collections) 

1098 for detectorId, bounds in exposureDetectorBounds.items(): 

1099 detectorBounds[detectorId].extend(bounds) 

1100 else: 

1101 detectorBounds = self.computeExposureBounds(visit.exposures[0], collections=collections) 

1102 visitBounds = [] 

1103 detectorRegions = {} 

1104 for detectorId, bounds in detectorBounds.items(): 

1105 detectorRegions[detectorId] = ConvexPolygon.convexHull(bounds) 

1106 visitBounds.extend(bounds) 

1107 return ConvexPolygon.convexHull(visitBounds), detectorRegions