Coverage for python/lsst/meas/base/forcedPhotCcd.py: 22%

296 statements  

« prev     ^ index     » next       coverage.py v7.3.2, created at 2023-11-18 11:04 +0000

1# This file is part of meas_base. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22import warnings 

23 

24import pandas as pd 

25import numpy as np 

26 

27import lsst.pex.config 

28import lsst.pex.exceptions 

29import lsst.pipe.base 

30import lsst.geom 

31import lsst.afw.detection 

32import lsst.afw.geom 

33import lsst.afw.image 

34import lsst.afw.table 

35import lsst.sphgeom 

36 

37from lsst.utils.introspection import find_outside_stacklevel 

38from lsst.pipe.base import PipelineTaskConnections 

39import lsst.pipe.base.connectionTypes as cT 

40 

41import lsst.pipe.base as pipeBase 

42from lsst.skymap import BaseSkyMap 

43 

44from .forcedMeasurement import ForcedMeasurementTask 

45from .applyApCorr import ApplyApCorrTask 

46from .catalogCalculation import CatalogCalculationTask 

47from ._id_generator import DetectorVisitIdGeneratorConfig 

48 

49__all__ = ("ForcedPhotCcdConfig", "ForcedPhotCcdTask", 

50 "ForcedPhotCcdFromDataFrameTask", "ForcedPhotCcdFromDataFrameConfig") 

51 

52 

53class ForcedPhotCcdConnections(PipelineTaskConnections, 

54 dimensions=("instrument", "visit", "detector", "skymap", "tract"), 

55 defaultTemplates={"inputCoaddName": "deep", 

56 "inputName": "calexp", 

57 "skyWcsName": "gbdesAstrometricFit", 

58 "photoCalibName": "fgcm"}, 

59 # TODO: remove on DM-39854 

60 deprecatedTemplates={"skyWcsName": "Deprecated; will be removed after v26.", 

61 "photoCalibName": "Deprecated; will be removed after v26." 

62 }): 

63 inputSchema = cT.InitInput( 

64 doc="Schema for the input measurement catalogs.", 

65 name="{inputCoaddName}Coadd_ref_schema", 

66 storageClass="SourceCatalog", 

67 ) 

68 outputSchema = cT.InitOutput( 

69 doc="Schema for the output forced measurement catalogs.", 

70 name="forced_src_schema", 

71 storageClass="SourceCatalog", 

72 ) 

73 exposure = cT.Input( 

74 doc="Input exposure to perform photometry on.", 

75 name="{inputName}", 

76 storageClass="ExposureF", 

77 dimensions=["instrument", "visit", "detector"], 

78 ) 

79 refCat = cT.Input( 

80 doc="Catalog of shapes and positions at which to force photometry.", 

81 name="{inputCoaddName}Coadd_ref", 

82 storageClass="SourceCatalog", 

83 dimensions=["skymap", "tract", "patch"], 

84 multiple=True, 

85 deferLoad=True, 

86 ) 

87 skyMap = cT.Input( 

88 doc="SkyMap dataset that defines the coordinate system of the reference catalog.", 

89 name=BaseSkyMap.SKYMAP_DATASET_TYPE_NAME, 

90 storageClass="SkyMap", 

91 dimensions=["skymap"], 

92 ) 

93 skyCorr = cT.Input( 

94 doc="Input Sky Correction to be subtracted from the calexp if doApplySkyCorr=True", 

95 name="skyCorr", 

96 storageClass="Background", 

97 dimensions=("instrument", "visit", "detector"), 

98 ) 

99 visitSummary = cT.Input( 

100 doc="Input visit-summary catalog with updated calibration objects.", 

101 name="finalVisitSummary", 

102 storageClass="ExposureCatalog", 

103 dimensions=("instrument", "visit"), 

104 ) 

105 externalSkyWcsTractCatalog = cT.Input( 

106 doc=("Per-tract, per-visit wcs calibrations. These catalogs use the detector " 

107 "id for the catalog id, sorted on id for fast lookup."), 

108 name="{skyWcsName}SkyWcsCatalog", 

109 storageClass="ExposureCatalog", 

110 dimensions=["instrument", "visit", "tract"], 

111 # TODO: remove on DM-39854 

112 deprecated="Deprecated in favor of 'visitSummary'; will be removed after v26." 

113 ) 

114 externalSkyWcsGlobalCatalog = cT.Input( 

115 doc=("Per-visit wcs calibrations computed globally (with no tract information). " 

116 "These catalogs use the detector id for the catalog id, sorted on id for " 

117 "fast lookup."), 

118 name="finalVisitSummary", 

119 storageClass="ExposureCatalog", 

120 dimensions=["instrument", "visit"], 

121 # TODO: remove on DM-39854 

122 deprecated="Deprecated in favor of 'visitSummary'; will be removed after v26." 

123 ) 

124 externalPhotoCalibTractCatalog = cT.Input( 

125 doc=("Per-tract, per-visit photometric calibrations. These catalogs use the " 

126 "detector id for the catalog id, sorted on id for fast lookup."), 

127 name="{photoCalibName}PhotoCalibCatalog", 

128 storageClass="ExposureCatalog", 

129 dimensions=["instrument", "visit", "tract"], 

130 # TODO: remove on DM-39854 

131 deprecated="Deprecated in favor of 'visitSummary'; will be removed after v26." 

132 ) 

133 externalPhotoCalibGlobalCatalog = cT.Input( 

134 doc=("Per-visit photometric calibrations computed globally (with no tract " 

135 "information). These catalogs use the detector id for the catalog id, " 

136 "sorted on id for fast lookup."), 

137 name="finalVisitSummary", 

138 storageClass="ExposureCatalog", 

139 dimensions=["instrument", "visit"], 

140 # TODO: remove on DM-39854 

141 deprecated="Deprecated in favor of 'visitSummary'; will be removed after v26." 

142 ) 

143 finalizedPsfApCorrCatalog = cT.Input( 

144 doc=("Per-visit finalized psf models and aperture correction maps. " 

145 "These catalogs use the detector id for the catalog id, " 

146 "sorted on id for fast lookup."), 

147 name="finalized_psf_ap_corr_catalog", 

148 storageClass="ExposureCatalog", 

149 dimensions=["instrument", "visit"], 

150 # TODO: remove on DM-39854 

151 deprecated="Deprecated in favor of 'visitSummary'; will be removed after v26." 

152 ) 

153 measCat = cT.Output( 

154 doc="Output forced photometry catalog.", 

155 name="forced_src", 

156 storageClass="SourceCatalog", 

157 dimensions=["instrument", "visit", "detector", "skymap", "tract"], 

158 ) 

159 

160 def __init__(self, *, config=None): 

161 super().__init__(config=config) 

162 if not config.doApplySkyCorr: 

163 self.inputs.remove("skyCorr") 

164 if config.doApplyExternalSkyWcs: 

165 if config.useGlobalExternalSkyWcs: 

166 self.inputs.remove("externalSkyWcsTractCatalog") 

167 else: 

168 self.inputs.remove("externalSkyWcsGlobalCatalog") 

169 else: 

170 self.inputs.remove("externalSkyWcsTractCatalog") 

171 self.inputs.remove("externalSkyWcsGlobalCatalog") 

172 if config.doApplyExternalPhotoCalib: 

173 if config.useGlobalExternalPhotoCalib: 

174 self.inputs.remove("externalPhotoCalibTractCatalog") 

175 else: 

176 self.inputs.remove("externalPhotoCalibGlobalCatalog") 

177 else: 

178 self.inputs.remove("externalPhotoCalibTractCatalog") 

179 self.inputs.remove("externalPhotoCalibGlobalCatalog") 

180 if not config.doApplyFinalizedPsf: 

181 self.inputs.remove("finalizedPsfApCorrCatalog") 

182 

183 

184class ForcedPhotCcdConfig(pipeBase.PipelineTaskConfig, 

185 pipelineConnections=ForcedPhotCcdConnections): 

186 """Config class for forced measurement driver task.""" 

187 measurement = lsst.pex.config.ConfigurableField( 

188 target=ForcedMeasurementTask, 

189 doc="subtask to do forced measurement" 

190 ) 

191 coaddName = lsst.pex.config.Field( 

192 doc="coadd name: typically one of deep or goodSeeing", 

193 dtype=str, 

194 default="deep", 

195 ) 

196 doApCorr = lsst.pex.config.Field( 

197 dtype=bool, 

198 default=True, 

199 doc="Run subtask to apply aperture corrections" 

200 ) 

201 applyApCorr = lsst.pex.config.ConfigurableField( 

202 target=ApplyApCorrTask, 

203 doc="Subtask to apply aperture corrections" 

204 ) 

205 catalogCalculation = lsst.pex.config.ConfigurableField( 

206 target=CatalogCalculationTask, 

207 doc="Subtask to run catalogCalculation plugins on catalog" 

208 ) 

209 doApplyExternalPhotoCalib = lsst.pex.config.Field( 

210 dtype=bool, 

211 default=False, 

212 doc=("Whether to apply external photometric calibration via an " 

213 "`lsst.afw.image.PhotoCalib` object."), 

214 # TODO: remove on DM-39854 

215 deprecated="Removed in favor of the 'visitSummary' connection; will be removed after v26.", 

216 ) 

217 useGlobalExternalPhotoCalib = lsst.pex.config.Field( 

218 dtype=bool, 

219 default=False, 

220 doc=("When using doApplyExternalPhotoCalib, use 'global' calibrations " 

221 "that are not run per-tract. When False, use per-tract photometric " 

222 "calibration files."), 

223 # TODO: remove on DM-39854 

224 deprecated="Removed in favor of the 'visitSummary' connection; will be removed after v26.", 

225 ) 

226 doApplyExternalSkyWcs = lsst.pex.config.Field( 

227 dtype=bool, 

228 default=False, 

229 doc=("Whether to apply external astrometric calibration via an " 

230 "`lsst.afw.geom.SkyWcs` object."), 

231 # TODO: remove on DM-39854 

232 deprecated="Removed in favor of the 'visitSummary' connection; will be removed after v26.", 

233 ) 

234 useGlobalExternalSkyWcs = lsst.pex.config.Field( 

235 dtype=bool, 

236 default=False, 

237 doc=("When using doApplyExternalSkyWcs, use 'global' calibrations " 

238 "that are not run per-tract. When False, use per-tract wcs " 

239 "files."), 

240 # TODO: remove on DM-39854 

241 deprecated="Removed in favor of the 'visitSummary' connection; will be removed after v26.", 

242 ) 

243 doApplyFinalizedPsf = lsst.pex.config.Field( 

244 dtype=bool, 

245 default=False, 

246 doc="Whether to apply finalized psf models and aperture correction map.", 

247 # TODO: remove on DM-39854 

248 deprecated="Removed in favor of the 'visitSummary' connection; will be removed after v26.", 

249 ) 

250 doApplySkyCorr = lsst.pex.config.Field( 

251 dtype=bool, 

252 default=False, 

253 doc="Apply sky correction?", 

254 ) 

255 includePhotoCalibVar = lsst.pex.config.Field( 

256 dtype=bool, 

257 default=False, 

258 doc="Add photometric calibration variance to warp variance plane?", 

259 ) 

260 footprintSource = lsst.pex.config.ChoiceField( 

261 dtype=str, 

262 doc="Where to obtain footprints to install in the measurement catalog, prior to measurement.", 

263 allowed={ 

264 "transformed": "Transform footprints from the reference catalog (downgrades HeavyFootprints).", 

265 "psf": ("Use the scaled shape of the PSF at the position of each source (does not generate " 

266 "HeavyFootprints)."), 

267 }, 

268 optional=True, 

269 default="transformed", 

270 ) 

271 psfFootprintScaling = lsst.pex.config.Field( 

272 dtype=float, 

273 doc="Scaling factor to apply to the PSF shape when footprintSource='psf' (ignored otherwise).", 

274 default=3.0, 

275 ) 

276 idGenerator = DetectorVisitIdGeneratorConfig.make_field() 

277 

278 def setDefaults(self): 

279 # Docstring inherited. 

280 super().setDefaults() 

281 # Footprints here will not be entirely correct, so don't try to make 

282 # a biased correction for blended neighbors. 

283 self.measurement.doReplaceWithNoise = False 

284 # Only run a minimal set of plugins, as these measurements are only 

285 # needed for PSF-like sources. 

286 self.measurement.plugins.names = ["base_PixelFlags", 

287 "base_TransformedCentroid", 

288 "base_PsfFlux", 

289 "base_LocalBackground", 

290 "base_LocalPhotoCalib", 

291 "base_LocalWcs", 

292 ] 

293 self.measurement.slots.shape = None 

294 # Keep track of which footprints contain streaks 

295 self.measurement.plugins['base_PixelFlags'].masksFpAnywhere = ['STREAK'] 

296 self.measurement.plugins['base_PixelFlags'].masksFpCenter = ['STREAK'] 

297 # Make catalogCalculation a no-op by default as no modelFlux is setup 

298 # by default in ForcedMeasurementTask. 

299 self.catalogCalculation.plugins.names = [] 

300 

301 

302class ForcedPhotCcdTask(pipeBase.PipelineTask): 

303 """A pipeline task for performing forced measurement on CCD images. 

304 

305 Parameters 

306 ---------- 

307 refSchema : `lsst.afw.table.Schema`, optional 

308 The schema of the reference catalog, passed to the constructor of the 

309 references subtask. Optional, but must be specified if ``initInputs`` 

310 is not; if both are specified, ``initInputs`` takes precedence. 

311 initInputs : `dict` 

312 Dictionary that can contain a key ``inputSchema`` containing the 

313 schema. If present will override the value of ``refSchema``. 

314 **kwargs 

315 Keyword arguments are passed to the supertask constructor. 

316 """ 

317 

318 ConfigClass = ForcedPhotCcdConfig 

319 _DefaultName = "forcedPhotCcd" 

320 dataPrefix = "" 

321 

322 def __init__(self, refSchema=None, initInputs=None, **kwargs): 

323 super().__init__(**kwargs) 

324 

325 if initInputs is not None: 

326 refSchema = initInputs['inputSchema'].schema 

327 

328 if refSchema is None: 

329 raise ValueError("No reference schema provided.") 

330 

331 self.makeSubtask("measurement", refSchema=refSchema) 

332 # It is necessary to get the schema internal to the forced measurement 

333 # task until such a time that the schema is not owned by the 

334 # measurement task, but is passed in by an external caller. 

335 if self.config.doApCorr: 

336 self.makeSubtask("applyApCorr", schema=self.measurement.schema) 

337 self.makeSubtask('catalogCalculation', schema=self.measurement.schema) 

338 self.outputSchema = lsst.afw.table.SourceCatalog(self.measurement.schema) 

339 

340 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

341 inputs = butlerQC.get(inputRefs) 

342 

343 tract = butlerQC.quantum.dataId['tract'] 

344 skyMap = inputs.pop('skyMap') 

345 inputs['refWcs'] = skyMap[tract].getWcs() 

346 

347 # Connections only exist if they are configured to be used. 

348 skyCorr = inputs.pop('skyCorr', None) 

349 if self.config.useGlobalExternalSkyWcs: 

350 externalSkyWcsCatalog = inputs.pop('externalSkyWcsGlobalCatalog', None) 

351 else: 

352 externalSkyWcsCatalog = inputs.pop('externalSkyWcsTractCatalog', None) 

353 if self.config.useGlobalExternalPhotoCalib: 

354 externalPhotoCalibCatalog = inputs.pop('externalPhotoCalibGlobalCatalog', None) 

355 else: 

356 externalPhotoCalibCatalog = inputs.pop('externalPhotoCalibTractCatalog', None) 

357 finalizedPsfApCorrCatalog = inputs.pop('finalizedPsfApCorrCatalog', None) 

358 

359 inputs['exposure'] = self.prepareCalibratedExposure( 

360 inputs['exposure'], 

361 skyCorr=skyCorr, 

362 externalSkyWcsCatalog=externalSkyWcsCatalog, 

363 externalPhotoCalibCatalog=externalPhotoCalibCatalog, 

364 finalizedPsfApCorrCatalog=finalizedPsfApCorrCatalog, 

365 visitSummary=inputs.pop("visitSummary"), 

366 ) 

367 

368 inputs['refCat'] = self.mergeAndFilterReferences(inputs['exposure'], inputs['refCat'], 

369 inputs['refWcs']) 

370 

371 if inputs['refCat'] is None: 

372 self.log.info("No WCS for exposure %s. No %s catalog will be written.", 

373 butlerQC.quantum.dataId, outputRefs.measCat.datasetType.name) 

374 else: 

375 inputs['measCat'], inputs['exposureId'] = self.generateMeasCat(inputRefs.exposure.dataId, 

376 inputs['exposure'], 

377 inputs['refCat'], inputs['refWcs']) 

378 self.attachFootprints(inputs['measCat'], inputs['refCat'], inputs['exposure'], inputs['refWcs']) 

379 outputs = self.run(**inputs) 

380 butlerQC.put(outputs, outputRefs) 

381 

382 def prepareCalibratedExposure(self, exposure, skyCorr=None, externalSkyWcsCatalog=None, 

383 externalPhotoCalibCatalog=None, finalizedPsfApCorrCatalog=None, 

384 visitSummary=None): 

385 """Prepare a calibrated exposure and apply external calibrations 

386 and sky corrections if so configured. 

387 

388 Parameters 

389 ---------- 

390 exposure : `lsst.afw.image.exposure.Exposure` 

391 Input exposure to adjust calibrations. 

392 skyCorr : `lsst.afw.math.backgroundList`, optional 

393 Sky correction frame to apply if doApplySkyCorr=True. 

394 externalSkyWcsCatalog : `lsst.afw.table.ExposureCatalog`, optional 

395 Exposure catalog with external skyWcs to be applied if 

396 config.doApplyExternalSkyWcs=True. Catalog uses the detector id 

397 for the catalog id, sorted on id for fast lookup. Deprecated in 

398 favor of ``visitSummary`` and will be removed after v26. 

399 externalPhotoCalibCatalog : `lsst.afw.table.ExposureCatalog`, optional 

400 Exposure catalog with external photoCalib to be applied if 

401 config.doApplyExternalPhotoCalib=True. Catalog uses the detector 

402 id for the catalog id, sorted on id for fast lookup. Deprecated in 

403 favor of ``visitSummary`` and will be removed after v26. 

404 finalizedPsfApCorrCatalog : `lsst.afw.table.ExposureCatalog`, optional 

405 Exposure catalog with finalized psf models and aperture correction 

406 maps to be applied if config.doApplyFinalizedPsf=True. Catalog 

407 uses the detector id for the catalog id, sorted on id for fast 

408 lookup. Deprecated in favor of ``visitSummary`` and will be removed 

409 after v26. 

410 visitSummary : `lsst.afw.table.ExposureCatalog`, optional 

411 Exposure catalog with update calibrations; any not-None calibration 

412 objects attached will be used. These are applied first and may be 

413 overridden by other arguments. 

414 

415 Returns 

416 ------- 

417 exposure : `lsst.afw.image.exposure.Exposure` 

418 Exposure with adjusted calibrations. 

419 """ 

420 detectorId = exposure.getInfo().getDetector().getId() 

421 

422 if visitSummary is not None: 

423 row = visitSummary.find(detectorId) 

424 if row is None: 

425 raise RuntimeError(f"Detector id {detectorId} not found in visitSummary.") 

426 if (photoCalib := row.getPhotoCalib()) is not None: 

427 exposure.setPhotoCalib(photoCalib) 

428 if (skyWcs := row.getWcs()) is not None: 

429 exposure.setWcs(skyWcs) 

430 if (psf := row.getPsf()) is not None: 

431 exposure.setPsf(psf) 

432 if (apCorrMap := row.getApCorrMap()) is not None: 

433 exposure.info.setApCorrMap(apCorrMap) 

434 

435 if externalPhotoCalibCatalog is not None: 

436 # TODO: remove on DM-39854 

437 warnings.warn( 

438 "The 'externalPhotoCalibCatalog' argument is deprecated in favor of 'visitSummary' and will " 

439 "be removed after v26.", 

440 FutureWarning, 

441 stacklevel=find_outside_stacklevel("lsst.meas.base"), 

442 ) 

443 row = externalPhotoCalibCatalog.find(detectorId) 

444 if row is None: 

445 self.log.warning("Detector id %s not found in externalPhotoCalibCatalog; " 

446 "Using original photoCalib.", detectorId) 

447 else: 

448 photoCalib = row.getPhotoCalib() 

449 if photoCalib is None: 

450 self.log.warning("Detector id %s has None for photoCalib in externalPhotoCalibCatalog; " 

451 "Using original photoCalib.", detectorId) 

452 else: 

453 exposure.setPhotoCalib(photoCalib) 

454 

455 if externalSkyWcsCatalog is not None: 

456 # TODO: remove on DM-39854 

457 warnings.warn( 

458 "The 'externalSkyWcsCatalog' argument is deprecated in favor of 'visitSummary' and will " 

459 "be removed after v26.", 

460 FutureWarning, 

461 stacklevel=find_outside_stacklevel("lsst.meas.base"), 

462 ) 

463 row = externalSkyWcsCatalog.find(detectorId) 

464 if row is None: 

465 self.log.warning("Detector id %s not found in externalSkyWcsCatalog; " 

466 "Using original skyWcs.", detectorId) 

467 else: 

468 skyWcs = row.getWcs() 

469 if skyWcs is None: 

470 self.log.warning("Detector id %s has None for skyWcs in externalSkyWcsCatalog; " 

471 "Using original skyWcs.", detectorId) 

472 else: 

473 exposure.setWcs(skyWcs) 

474 

475 if finalizedPsfApCorrCatalog is not None: 

476 # TODO: remove on DM-39854 

477 warnings.warn( 

478 "The 'finalizedPsfApCorrCatalog' argument is deprecated in favor of 'visitSummary' and will " 

479 "be removed after v26.", 

480 FutureWarning, 

481 stacklevel=find_outside_stacklevel("lsst.meas.base"), 

482 ) 

483 row = finalizedPsfApCorrCatalog.find(detectorId) 

484 if row is None: 

485 self.log.warning("Detector id %s not found in finalizedPsfApCorrCatalog; " 

486 "Using original psf.", detectorId) 

487 else: 

488 psf = row.getPsf() 

489 apCorrMap = row.getApCorrMap() 

490 if psf is None or apCorrMap is None: 

491 self.log.warning("Detector id %s has None for psf/apCorrMap in " 

492 "finalizedPsfApCorrCatalog; Using original psf.", detectorId) 

493 else: 

494 exposure.setPsf(psf) 

495 exposure.setApCorrMap(apCorrMap) 

496 

497 if skyCorr is not None: 

498 exposure.maskedImage -= skyCorr.getImage() 

499 

500 return exposure 

501 

502 def mergeAndFilterReferences(self, exposure, refCats, refWcs): 

503 """Filter reference catalog so that all sources are within the 

504 boundaries of the exposure. 

505 

506 Parameters 

507 ---------- 

508 exposure : `lsst.afw.image.exposure.Exposure` 

509 Exposure to generate the catalog for. 

510 refCats : sequence of `lsst.daf.butler.DeferredDatasetHandle` 

511 Handles for catalogs of shapes and positions at which to force 

512 photometry. 

513 refWcs : `lsst.afw.image.SkyWcs` 

514 Reference world coordinate system. 

515 

516 Returns 

517 ------- 

518 refSources : `lsst.afw.table.SourceCatalog` 

519 Filtered catalog of forced sources to measure. 

520 

521 Notes 

522 ----- 

523 The majority of this code is based on the methods of 

524 lsst.meas.algorithms.loadReferenceObjects.ReferenceObjectLoader 

525 

526 """ 

527 mergedRefCat = None 

528 

529 # Step 1: Determine bounds of the exposure photometry will 

530 # be performed on. 

531 expWcs = exposure.getWcs() 

532 if expWcs is None: 

533 self.log.info("Exposure has no WCS. Returning None for mergedRefCat.") 

534 else: 

535 expRegion = exposure.getBBox(lsst.afw.image.PARENT) 

536 expBBox = lsst.geom.Box2D(expRegion) 

537 expBoxCorners = expBBox.getCorners() 

538 expSkyCorners = [expWcs.pixelToSky(corner).getVector() for 

539 corner in expBoxCorners] 

540 expPolygon = lsst.sphgeom.ConvexPolygon(expSkyCorners) 

541 

542 # Step 2: Filter out reference catalog sources that are 

543 # not contained within the exposure boundaries, or whose 

544 # parents are not within the exposure boundaries. Note 

545 # that within a single input refCat, the parents always 

546 # appear before the children. 

547 for refCat in refCats: 

548 refCat = refCat.get() 

549 if mergedRefCat is None: 

550 mergedRefCat = lsst.afw.table.SourceCatalog(refCat.table) 

551 containedIds = {0} # zero as a parent ID means "this is a parent" 

552 for record in refCat: 

553 if (expPolygon.contains(record.getCoord().getVector()) and record.getParent() 

554 in containedIds): 

555 record.setFootprint(record.getFootprint()) 

556 mergedRefCat.append(record) 

557 containedIds.add(record.getId()) 

558 if mergedRefCat is None: 

559 raise RuntimeError("No reference objects for forced photometry.") 

560 mergedRefCat.sort(lsst.afw.table.SourceTable.getParentKey()) 

561 return mergedRefCat 

562 

563 def generateMeasCat(self, dataId, exposure, refCat, refWcs): 

564 """Generate a measurement catalog. 

565 

566 Parameters 

567 ---------- 

568 dataId : `lsst.daf.butler.DataCoordinate` 

569 Butler data ID for this image, with ``{visit, detector}`` keys. 

570 exposure : `lsst.afw.image.exposure.Exposure` 

571 Exposure to generate the catalog for. 

572 refCat : `lsst.afw.table.SourceCatalog` 

573 Catalog of shapes and positions at which to force photometry. 

574 refWcs : `lsst.afw.image.SkyWcs` 

575 Reference world coordinate system. 

576 This parameter is not currently used. 

577 

578 Returns 

579 ------- 

580 measCat : `lsst.afw.table.SourceCatalog` 

581 Catalog of forced sources to measure. 

582 expId : `int` 

583 Unique binary id associated with the input exposure 

584 """ 

585 id_generator = self.config.idGenerator.apply(dataId) 

586 measCat = self.measurement.generateMeasCat(exposure, refCat, refWcs, 

587 idFactory=id_generator.make_table_id_factory()) 

588 return measCat, id_generator.catalog_id 

589 

590 def run(self, measCat, exposure, refCat, refWcs, exposureId=None): 

591 """Perform forced measurement on a single exposure. 

592 

593 Parameters 

594 ---------- 

595 measCat : `lsst.afw.table.SourceCatalog` 

596 The measurement catalog, based on the sources listed in the 

597 reference catalog. 

598 exposure : `lsst.afw.image.Exposure` 

599 The measurement image upon which to perform forced detection. 

600 refCat : `lsst.afw.table.SourceCatalog` 

601 The reference catalog of sources to measure. 

602 refWcs : `lsst.afw.image.SkyWcs` 

603 The WCS for the references. 

604 exposureId : `int` 

605 Optional unique exposureId used for random seed in measurement 

606 task. 

607 

608 Returns 

609 ------- 

610 result : `lsst.pipe.base.Struct` 

611 Structure with fields: 

612 

613 ``measCat`` 

614 Catalog of forced measurement results 

615 (`lsst.afw.table.SourceCatalog`). 

616 """ 

617 self.measurement.run(measCat, exposure, refCat, refWcs, exposureId=exposureId) 

618 if self.config.doApCorr: 

619 apCorrMap = exposure.getInfo().getApCorrMap() 

620 if apCorrMap is None: 

621 self.log.warning("Forced exposure image does not have valid aperture correction; skipping.") 

622 else: 

623 self.applyApCorr.run( 

624 catalog=measCat, 

625 apCorrMap=apCorrMap, 

626 ) 

627 self.catalogCalculation.run(measCat) 

628 

629 return pipeBase.Struct(measCat=measCat) 

630 

631 def attachFootprints(self, sources, refCat, exposure, refWcs): 

632 """Attach footprints to blank sources prior to measurements. 

633 

634 Notes 

635 ----- 

636 `~lsst.afw.detection.Footprint` objects for forced photometry must 

637 be in the pixel coordinate system of the image being measured, while 

638 the actual detections may start out in a different coordinate system. 

639 

640 Subclasses of this class may implement this method to define how 

641 those `~lsst.afw.detection.Footprint` objects should be generated. 

642 

643 This default implementation transforms depends on the 

644 ``footprintSource`` configuration parameter. 

645 """ 

646 if self.config.footprintSource == "transformed": 

647 return self.measurement.attachTransformedFootprints(sources, refCat, exposure, refWcs) 

648 elif self.config.footprintSource == "psf": 

649 return self.measurement.attachPsfShapeFootprints(sources, exposure, 

650 scaling=self.config.psfFootprintScaling) 

651 

652 

653class ForcedPhotCcdFromDataFrameConnections(PipelineTaskConnections, 

654 dimensions=("instrument", "visit", "detector", "skymap", "tract"), 

655 defaultTemplates={"inputCoaddName": "goodSeeing", 

656 "inputName": "calexp", 

657 "skyWcsName": "gbdesAstrometricFit", 

658 "photoCalibName": "fgcm"}, 

659 # TODO: remove on DM-39854 

660 deprecatedTemplates={ 

661 "skyWcsName": "Deprecated; will be removed after v26.", 

662 "photoCalibName": "Deprecated; will be removed after v26." 

663 }): 

664 refCat = cT.Input( 

665 doc="Catalog of positions at which to force photometry.", 

666 name="{inputCoaddName}Diff_fullDiaObjTable", 

667 storageClass="DataFrame", 

668 dimensions=["skymap", "tract", "patch"], 

669 multiple=True, 

670 deferLoad=True, 

671 ) 

672 exposure = cT.Input( 

673 doc="Input exposure to perform photometry on.", 

674 name="{inputName}", 

675 storageClass="ExposureF", 

676 dimensions=["instrument", "visit", "detector"], 

677 ) 

678 skyCorr = cT.Input( 

679 doc="Input Sky Correction to be subtracted from the calexp if doApplySkyCorr=True", 

680 name="skyCorr", 

681 storageClass="Background", 

682 dimensions=("instrument", "visit", "detector"), 

683 ) 

684 visitSummary = cT.Input( 

685 doc="Input visit-summary catalog with updated calibration objects.", 

686 name="finalVisitSummary", 

687 storageClass="ExposureCatalog", 

688 dimensions=("instrument", "visit"), 

689 ) 

690 externalSkyWcsTractCatalog = cT.Input( 

691 doc=("Per-tract, per-visit wcs calibrations. These catalogs use the detector " 

692 "id for the catalog id, sorted on id for fast lookup."), 

693 name="{skyWcsName}SkyWcsCatalog", 

694 storageClass="ExposureCatalog", 

695 dimensions=["instrument", "visit", "tract"], 

696 # TODO: remove on DM-39854 

697 deprecated="Deprecated in favor of 'visitSummary'; will be removed after v26." 

698 ) 

699 externalSkyWcsGlobalCatalog = cT.Input( 

700 doc=("Per-visit wcs calibrations computed globally (with no tract information). " 

701 "These catalogs use the detector id for the catalog id, sorted on id for " 

702 "fast lookup."), 

703 name="{skyWcsName}SkyWcsCatalog", 

704 storageClass="ExposureCatalog", 

705 dimensions=["instrument", "visit"], 

706 # TODO: remove on DM-39854 

707 deprecated="Deprecated in favor of 'visitSummary'; will be removed after v26." 

708 ) 

709 externalPhotoCalibTractCatalog = cT.Input( 

710 doc=("Per-tract, per-visit photometric calibrations. These catalogs use the " 

711 "detector id for the catalog id, sorted on id for fast lookup."), 

712 name="{photoCalibName}PhotoCalibCatalog", 

713 storageClass="ExposureCatalog", 

714 dimensions=["instrument", "visit", "tract"], 

715 # TODO: remove on DM-39854 

716 deprecated="Deprecated in favor of 'visitSummary'; will be removed after v26." 

717 ) 

718 externalPhotoCalibGlobalCatalog = cT.Input( 

719 doc=("Per-visit photometric calibrations computed globally (with no tract " 

720 "information). These catalogs use the detector id for the catalog id, " 

721 "sorted on id for fast lookup."), 

722 name="{photoCalibName}PhotoCalibCatalog", 

723 storageClass="ExposureCatalog", 

724 dimensions=["instrument", "visit"], 

725 # TODO: remove on DM-39854 

726 deprecated="Deprecated in favor of 'visitSummary'; will be removed after v26." 

727 ) 

728 finalizedPsfApCorrCatalog = cT.Input( 

729 doc=("Per-visit finalized psf models and aperture correction maps. " 

730 "These catalogs use the detector id for the catalog id, " 

731 "sorted on id for fast lookup."), 

732 name="finalized_psf_ap_corr_catalog", 

733 storageClass="ExposureCatalog", 

734 dimensions=["instrument", "visit"], 

735 # TODO: remove on DM-39854 

736 deprecated="Deprecated in favor of 'visitSummary'; will be removed after v26." 

737 ) 

738 measCat = cT.Output( 

739 doc="Output forced photometry catalog.", 

740 name="forced_src_diaObject", 

741 storageClass="SourceCatalog", 

742 dimensions=["instrument", "visit", "detector", "skymap", "tract"], 

743 ) 

744 outputSchema = cT.InitOutput( 

745 doc="Schema for the output forced measurement catalogs.", 

746 name="forced_src_diaObject_schema", 

747 storageClass="SourceCatalog", 

748 ) 

749 

750 def __init__(self, *, config=None): 

751 super().__init__(config=config) 

752 if not config.doApplySkyCorr: 

753 self.inputs.remove("skyCorr") 

754 if config.doApplyExternalSkyWcs: 

755 if config.useGlobalExternalSkyWcs: 

756 self.inputs.remove("externalSkyWcsTractCatalog") 

757 else: 

758 self.inputs.remove("externalSkyWcsGlobalCatalog") 

759 else: 

760 self.inputs.remove("externalSkyWcsTractCatalog") 

761 self.inputs.remove("externalSkyWcsGlobalCatalog") 

762 if config.doApplyExternalPhotoCalib: 

763 if config.useGlobalExternalPhotoCalib: 

764 self.inputs.remove("externalPhotoCalibTractCatalog") 

765 else: 

766 self.inputs.remove("externalPhotoCalibGlobalCatalog") 

767 else: 

768 self.inputs.remove("externalPhotoCalibTractCatalog") 

769 self.inputs.remove("externalPhotoCalibGlobalCatalog") 

770 if not config.doApplyFinalizedPsf: 

771 self.inputs.remove("finalizedPsfApCorrCatalog") 

772 

773 

774class ForcedPhotCcdFromDataFrameConfig(ForcedPhotCcdConfig, 

775 pipelineConnections=ForcedPhotCcdFromDataFrameConnections): 

776 def setDefaults(self): 

777 super().setDefaults() 

778 self.footprintSource = "psf" 

779 self.measurement.doReplaceWithNoise = False 

780 # Only run a minimal set of plugins, as these measurements are only 

781 # needed for PSF-like sources. 

782 self.measurement.plugins.names = ["base_PixelFlags", 

783 "base_TransformedCentroidFromCoord", 

784 "base_PsfFlux", 

785 "base_LocalBackground", 

786 "base_LocalPhotoCalib", 

787 "base_LocalWcs", 

788 ] 

789 self.measurement.slots.shape = None 

790 # Keep track of which footprints contain streaks 

791 self.measurement.plugins['base_PixelFlags'].masksFpAnywhere = ['STREAK'] 

792 self.measurement.plugins['base_PixelFlags'].masksFpCenter = ['STREAK'] 

793 # Make catalogCalculation a no-op by default as no modelFlux is setup 

794 # by default in ForcedMeasurementTask. 

795 self.catalogCalculation.plugins.names = [] 

796 

797 self.measurement.copyColumns = {'id': 'diaObjectId', 'coord_ra': 'coord_ra', 'coord_dec': 'coord_dec'} 

798 self.measurement.slots.centroid = "base_TransformedCentroidFromCoord" 

799 self.measurement.slots.psfFlux = "base_PsfFlux" 

800 

801 def validate(self): 

802 super().validate() 

803 if self.footprintSource == "transformed": 

804 raise ValueError("Cannot transform footprints from reference catalog, " 

805 "because DataFrames can't hold footprints.") 

806 

807 

808class ForcedPhotCcdFromDataFrameTask(ForcedPhotCcdTask): 

809 """Force Photometry on a per-detector exposure with coords from a DataFrame 

810 

811 Uses input from a DataFrame instead of SourceCatalog 

812 like the base class ForcedPhotCcd does. 

813 Writes out a SourceCatalog so that the downstream 

814 WriteForcedSourceTableTask can be reused with output from this Task. 

815 """ 

816 _DefaultName = "forcedPhotCcdFromDataFrame" 

817 ConfigClass = ForcedPhotCcdFromDataFrameConfig 

818 

819 def __init__(self, refSchema=None, initInputs=None, **kwargs): 

820 # Parent's init assumes that we have a reference schema; Cannot reuse 

821 pipeBase.PipelineTask.__init__(self, **kwargs) 

822 

823 self.makeSubtask("measurement", refSchema=lsst.afw.table.SourceTable.makeMinimalSchema()) 

824 

825 if self.config.doApCorr: 

826 self.makeSubtask("applyApCorr", schema=self.measurement.schema) 

827 self.makeSubtask('catalogCalculation', schema=self.measurement.schema) 

828 self.outputSchema = lsst.afw.table.SourceCatalog(self.measurement.schema) 

829 

830 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

831 inputs = butlerQC.get(inputRefs) 

832 

833 # When run with dataframes, we do not need a reference wcs. 

834 inputs['refWcs'] = None 

835 

836 # Connections only exist if they are configured to be used. 

837 skyCorr = inputs.pop('skyCorr', None) 

838 if self.config.useGlobalExternalSkyWcs: 

839 externalSkyWcsCatalog = inputs.pop('externalSkyWcsGlobalCatalog', None) 

840 else: 

841 externalSkyWcsCatalog = inputs.pop('externalSkyWcsTractCatalog', None) 

842 if self.config.useGlobalExternalPhotoCalib: 

843 externalPhotoCalibCatalog = inputs.pop('externalPhotoCalibGlobalCatalog', None) 

844 else: 

845 externalPhotoCalibCatalog = inputs.pop('externalPhotoCalibTractCatalog', None) 

846 finalizedPsfApCorrCatalog = inputs.pop('finalizedPsfApCorrCatalog', None) 

847 

848 inputs['exposure'] = self.prepareCalibratedExposure( 

849 inputs['exposure'], 

850 skyCorr=skyCorr, 

851 externalSkyWcsCatalog=externalSkyWcsCatalog, 

852 externalPhotoCalibCatalog=externalPhotoCalibCatalog, 

853 finalizedPsfApCorrCatalog=finalizedPsfApCorrCatalog, 

854 visitSummary=inputs.pop("visitSummary"), 

855 ) 

856 

857 self.log.info("Filtering ref cats: %s", ','.join([str(i.dataId) for i in inputs['refCat']])) 

858 if inputs["exposure"].getWcs() is not None: 

859 refCat = self.df2RefCat([i.get(parameters={"columns": ['diaObjectId', 'ra', 'dec']}) 

860 for i in inputs['refCat']], 

861 inputs['exposure'].getBBox(), inputs['exposure'].getWcs()) 

862 inputs['refCat'] = refCat 

863 # generateMeasCat does not use the refWcs. 

864 inputs['measCat'], inputs['exposureId'] = self.generateMeasCat( 

865 inputRefs.exposure.dataId, inputs['exposure'], inputs['refCat'], inputs['refWcs'] 

866 ) 

867 # attachFootprints only uses refWcs in ``transformed`` mode, which is not 

868 # supported in the DataFrame-backed task. 

869 self.attachFootprints(inputs["measCat"], inputs["refCat"], inputs["exposure"], inputs["refWcs"]) 

870 outputs = self.run(**inputs) 

871 

872 butlerQC.put(outputs, outputRefs) 

873 else: 

874 self.log.info("No WCS for %s. Skipping and no %s catalog will be written.", 

875 butlerQC.quantum.dataId, outputRefs.measCat.datasetType.name) 

876 

877 def df2RefCat(self, dfList, exposureBBox, exposureWcs): 

878 """Convert list of DataFrames to reference catalog 

879 

880 Concatenate list of DataFrames presumably from multiple patches and 

881 downselect rows that overlap the exposureBBox using the exposureWcs. 

882 

883 Parameters 

884 ---------- 

885 dfList : `list` of `pandas.DataFrame` 

886 Each element containst diaObjects with ra/dec position in degrees 

887 Columns 'diaObjectId', 'ra', 'dec' are expected 

888 exposureBBox : `lsst.geom.Box2I` 

889 Bounding box on which to select rows that overlap 

890 exposureWcs : `lsst.afw.geom.SkyWcs` 

891 World coordinate system to convert sky coords in ref cat to 

892 pixel coords with which to compare with exposureBBox 

893 

894 Returns 

895 ------- 

896 refCat : `lsst.afw.table.SourceTable` 

897 Source Catalog with minimal schema that overlaps exposureBBox 

898 """ 

899 df = pd.concat(dfList) 

900 # translate ra/dec coords in dataframe to detector pixel coords 

901 # to down select rows that overlap the detector bbox 

902 mapping = exposureWcs.getTransform().getMapping() 

903 x, y = mapping.applyInverse(np.array(df[['ra', 'dec']].values*2*np.pi/360).T) 

904 inBBox = lsst.geom.Box2D(exposureBBox).contains(x, y) 

905 refCat = self.df2SourceCat(df[inBBox]) 

906 return refCat 

907 

908 def df2SourceCat(self, df): 

909 """Create minimal schema SourceCatalog from a pandas DataFrame. 

910 

911 The forced measurement subtask expects this as input. 

912 

913 Parameters 

914 ---------- 

915 df : `pandas.DataFrame` 

916 DiaObjects with locations and ids. 

917 

918 Returns 

919 ------- 

920 outputCatalog : `lsst.afw.table.SourceTable` 

921 Output catalog with minimal schema. 

922 """ 

923 schema = lsst.afw.table.SourceTable.makeMinimalSchema() 

924 outputCatalog = lsst.afw.table.SourceCatalog(schema) 

925 outputCatalog.reserve(len(df)) 

926 

927 for diaObjectId, ra, dec in df[['ra', 'dec']].itertuples(): 

928 outputRecord = outputCatalog.addNew() 

929 outputRecord.setId(diaObjectId) 

930 outputRecord.setCoord(lsst.geom.SpherePoint(ra, dec, lsst.geom.degrees)) 

931 return outputCatalog