Coverage for python/lsst/meas/base/forcedPhotCcd.py: 23%

271 statements  

« prev     ^ index     » next       coverage.py v7.2.7, created at 2023-06-24 09:54 +0000

1# This file is part of meas_base. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22import pandas as pd 

23import numpy as np 

24 

25import lsst.pex.config 

26import lsst.pex.exceptions 

27import lsst.pipe.base 

28import lsst.geom 

29import lsst.afw.detection 

30import lsst.afw.geom 

31import lsst.afw.image 

32import lsst.afw.table 

33import lsst.sphgeom 

34 

35from lsst.pipe.base import PipelineTaskConnections 

36import lsst.pipe.base.connectionTypes as cT 

37 

38import lsst.pipe.base as pipeBase 

39from lsst.skymap import BaseSkyMap 

40 

41from .forcedMeasurement import ForcedMeasurementTask 

42from .applyApCorr import ApplyApCorrTask 

43from .catalogCalculation import CatalogCalculationTask 

44from ._id_generator import DetectorVisitIdGeneratorConfig 

45 

46__all__ = ("ForcedPhotCcdConfig", "ForcedPhotCcdTask", 

47 "ForcedPhotCcdFromDataFrameTask", "ForcedPhotCcdFromDataFrameConfig") 

48 

49 

50class ForcedPhotCcdConnections(PipelineTaskConnections, 

51 dimensions=("instrument", "visit", "detector", "skymap", "tract"), 

52 defaultTemplates={"inputCoaddName": "deep", 

53 "inputName": "calexp", 

54 "skyWcsName": "gbdesAstrometricFit", 

55 "photoCalibName": "fgcm"}): 

56 inputSchema = cT.InitInput( 

57 doc="Schema for the input measurement catalogs.", 

58 name="{inputCoaddName}Coadd_ref_schema", 

59 storageClass="SourceCatalog", 

60 ) 

61 outputSchema = cT.InitOutput( 

62 doc="Schema for the output forced measurement catalogs.", 

63 name="forced_src_schema", 

64 storageClass="SourceCatalog", 

65 ) 

66 exposure = cT.Input( 

67 doc="Input exposure to perform photometry on.", 

68 name="{inputName}", 

69 storageClass="ExposureF", 

70 dimensions=["instrument", "visit", "detector"], 

71 ) 

72 refCat = cT.Input( 

73 doc="Catalog of shapes and positions at which to force photometry.", 

74 name="{inputCoaddName}Coadd_ref", 

75 storageClass="SourceCatalog", 

76 dimensions=["skymap", "tract", "patch"], 

77 multiple=True, 

78 deferLoad=True, 

79 ) 

80 skyMap = cT.Input( 

81 doc="SkyMap dataset that defines the coordinate system of the reference catalog.", 

82 name=BaseSkyMap.SKYMAP_DATASET_TYPE_NAME, 

83 storageClass="SkyMap", 

84 dimensions=["skymap"], 

85 ) 

86 skyCorr = cT.Input( 

87 doc="Input Sky Correction to be subtracted from the calexp if doApplySkyCorr=True", 

88 name="skyCorr", 

89 storageClass="Background", 

90 dimensions=("instrument", "visit", "detector"), 

91 ) 

92 externalSkyWcsTractCatalog = cT.Input( 

93 doc=("Per-tract, per-visit wcs calibrations. These catalogs use the detector " 

94 "id for the catalog id, sorted on id for fast lookup."), 

95 name="{skyWcsName}SkyWcsCatalog", 

96 storageClass="ExposureCatalog", 

97 dimensions=["instrument", "visit", "tract"], 

98 ) 

99 externalSkyWcsGlobalCatalog = cT.Input( 

100 doc=("Per-visit wcs calibrations computed globally (with no tract information). " 

101 "These catalogs use the detector id for the catalog id, sorted on id for " 

102 "fast lookup."), 

103 name="finalVisitSummary", 

104 storageClass="ExposureCatalog", 

105 dimensions=["instrument", "visit"], 

106 ) 

107 externalPhotoCalibTractCatalog = cT.Input( 

108 doc=("Per-tract, per-visit photometric calibrations. These catalogs use the " 

109 "detector id for the catalog id, sorted on id for fast lookup."), 

110 name="{photoCalibName}PhotoCalibCatalog", 

111 storageClass="ExposureCatalog", 

112 dimensions=["instrument", "visit", "tract"], 

113 ) 

114 externalPhotoCalibGlobalCatalog = cT.Input( 

115 doc=("Per-visit photometric calibrations computed globally (with no tract " 

116 "information). These catalogs use the detector id for the catalog id, " 

117 "sorted on id for fast lookup."), 

118 name="finalVisitSummary", 

119 storageClass="ExposureCatalog", 

120 dimensions=["instrument", "visit"], 

121 ) 

122 finalizedPsfApCorrCatalog = cT.Input( 

123 doc=("Per-visit finalized psf models and aperture correction maps. " 

124 "These catalogs use the detector id for the catalog id, " 

125 "sorted on id for fast lookup."), 

126 name="finalized_psf_ap_corr_catalog", 

127 storageClass="ExposureCatalog", 

128 dimensions=["instrument", "visit"], 

129 ) 

130 measCat = cT.Output( 

131 doc="Output forced photometry catalog.", 

132 name="forced_src", 

133 storageClass="SourceCatalog", 

134 dimensions=["instrument", "visit", "detector", "skymap", "tract"], 

135 ) 

136 

137 def __init__(self, *, config=None): 

138 super().__init__(config=config) 

139 if not config.doApplySkyCorr: 

140 self.inputs.remove("skyCorr") 

141 if config.doApplyExternalSkyWcs: 

142 if config.useGlobalExternalSkyWcs: 

143 self.inputs.remove("externalSkyWcsTractCatalog") 

144 else: 

145 self.inputs.remove("externalSkyWcsGlobalCatalog") 

146 else: 

147 self.inputs.remove("externalSkyWcsTractCatalog") 

148 self.inputs.remove("externalSkyWcsGlobalCatalog") 

149 if config.doApplyExternalPhotoCalib: 

150 if config.useGlobalExternalPhotoCalib: 

151 self.inputs.remove("externalPhotoCalibTractCatalog") 

152 else: 

153 self.inputs.remove("externalPhotoCalibGlobalCatalog") 

154 else: 

155 self.inputs.remove("externalPhotoCalibTractCatalog") 

156 self.inputs.remove("externalPhotoCalibGlobalCatalog") 

157 if not config.doApplyFinalizedPsf: 

158 self.inputs.remove("finalizedPsfApCorrCatalog") 

159 

160 

161class ForcedPhotCcdConfig(pipeBase.PipelineTaskConfig, 

162 pipelineConnections=ForcedPhotCcdConnections): 

163 """Config class for forced measurement driver task.""" 

164 measurement = lsst.pex.config.ConfigurableField( 

165 target=ForcedMeasurementTask, 

166 doc="subtask to do forced measurement" 

167 ) 

168 coaddName = lsst.pex.config.Field( 

169 doc="coadd name: typically one of deep or goodSeeing", 

170 dtype=str, 

171 default="deep", 

172 ) 

173 doApCorr = lsst.pex.config.Field( 

174 dtype=bool, 

175 default=True, 

176 doc="Run subtask to apply aperture corrections" 

177 ) 

178 applyApCorr = lsst.pex.config.ConfigurableField( 

179 target=ApplyApCorrTask, 

180 doc="Subtask to apply aperture corrections" 

181 ) 

182 catalogCalculation = lsst.pex.config.ConfigurableField( 

183 target=CatalogCalculationTask, 

184 doc="Subtask to run catalogCalculation plugins on catalog" 

185 ) 

186 doApplyUberCal = lsst.pex.config.Field( 

187 dtype=bool, 

188 doc="Apply meas_mosaic ubercal results to input calexps?", 

189 default=False, 

190 deprecated="Deprecated by DM-23352; use doApplyExternalPhotoCalib and doApplyExternalSkyWcs instead", 

191 ) 

192 doApplyExternalPhotoCalib = lsst.pex.config.Field( 

193 dtype=bool, 

194 default=False, 

195 doc=("Whether to apply external photometric calibration via an " 

196 "`lsst.afw.image.PhotoCalib` object."), 

197 ) 

198 useGlobalExternalPhotoCalib = lsst.pex.config.Field( 

199 dtype=bool, 

200 default=True, 

201 doc=("When using doApplyExternalPhotoCalib, use 'global' calibrations " 

202 "that are not run per-tract. When False, use per-tract photometric " 

203 "calibration files.") 

204 ) 

205 doApplyExternalSkyWcs = lsst.pex.config.Field( 

206 dtype=bool, 

207 default=False, 

208 doc=("Whether to apply external astrometric calibration via an " 

209 "`lsst.afw.geom.SkyWcs` object."), 

210 ) 

211 useGlobalExternalSkyWcs = lsst.pex.config.Field( 

212 dtype=bool, 

213 default=True, 

214 doc=("When using doApplyExternalSkyWcs, use 'global' calibrations " 

215 "that are not run per-tract. When False, use per-tract wcs " 

216 "files.") 

217 ) 

218 doApplyFinalizedPsf = lsst.pex.config.Field( 

219 dtype=bool, 

220 default=False, 

221 doc="Whether to apply finalized psf models and aperture correction map.", 

222 ) 

223 doApplySkyCorr = lsst.pex.config.Field( 

224 dtype=bool, 

225 default=False, 

226 doc="Apply sky correction?", 

227 ) 

228 includePhotoCalibVar = lsst.pex.config.Field( 

229 dtype=bool, 

230 default=False, 

231 doc="Add photometric calibration variance to warp variance plane?", 

232 ) 

233 footprintSource = lsst.pex.config.ChoiceField( 

234 dtype=str, 

235 doc="Where to obtain footprints to install in the measurement catalog, prior to measurement.", 

236 allowed={ 

237 "transformed": "Transform footprints from the reference catalog (downgrades HeavyFootprints).", 

238 "psf": ("Use the scaled shape of the PSF at the position of each source (does not generate " 

239 "HeavyFootprints)."), 

240 }, 

241 optional=True, 

242 default="transformed", 

243 ) 

244 psfFootprintScaling = lsst.pex.config.Field( 

245 dtype=float, 

246 doc="Scaling factor to apply to the PSF shape when footprintSource='psf' (ignored otherwise).", 

247 default=3.0, 

248 ) 

249 idGenerator = DetectorVisitIdGeneratorConfig.make_field() 

250 

251 def setDefaults(self): 

252 # Docstring inherited. 

253 super().setDefaults() 

254 # Footprints here will not be entirely correct, so don't try to make 

255 # a biased correction for blended neighbors. 

256 self.measurement.doReplaceWithNoise = False 

257 # Only run a minimal set of plugins, as these measurements are only 

258 # needed for PSF-like sources. 

259 self.measurement.plugins.names = ["base_PixelFlags", 

260 "base_TransformedCentroid", 

261 "base_PsfFlux", 

262 "base_LocalBackground", 

263 "base_LocalPhotoCalib", 

264 "base_LocalWcs", 

265 ] 

266 self.measurement.slots.shape = None 

267 # Make catalogCalculation a no-op by default as no modelFlux is setup 

268 # by default in ForcedMeasurementTask. 

269 self.catalogCalculation.plugins.names = [] 

270 

271 

272class ForcedPhotCcdTask(pipeBase.PipelineTask): 

273 """A pipeline task for performing forced measurement on CCD images. 

274 

275 Parameters 

276 ---------- 

277 refSchema : `lsst.afw.table.Schema`, optional 

278 The schema of the reference catalog, passed to the constructor of the 

279 references subtask. Optional, but must be specified if ``initInputs`` 

280 is not; if both are specified, ``initInputs`` takes precedence. 

281 initInputs : `dict` 

282 Dictionary that can contain a key ``inputSchema`` containing the 

283 schema. If present will override the value of ``refSchema``. 

284 **kwargs 

285 Keyword arguments are passed to the supertask constructor. 

286 """ 

287 

288 ConfigClass = ForcedPhotCcdConfig 

289 _DefaultName = "forcedPhotCcd" 

290 dataPrefix = "" 

291 

292 def __init__(self, refSchema=None, initInputs=None, **kwargs): 

293 super().__init__(**kwargs) 

294 

295 if initInputs is not None: 

296 refSchema = initInputs['inputSchema'].schema 

297 

298 if refSchema is None: 

299 raise ValueError("No reference schema provided.") 

300 

301 self.makeSubtask("measurement", refSchema=refSchema) 

302 # It is necessary to get the schema internal to the forced measurement 

303 # task until such a time that the schema is not owned by the 

304 # measurement task, but is passed in by an external caller. 

305 if self.config.doApCorr: 

306 self.makeSubtask("applyApCorr", schema=self.measurement.schema) 

307 self.makeSubtask('catalogCalculation', schema=self.measurement.schema) 

308 self.outputSchema = lsst.afw.table.SourceCatalog(self.measurement.schema) 

309 

310 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

311 inputs = butlerQC.get(inputRefs) 

312 

313 tract = butlerQC.quantum.dataId['tract'] 

314 skyMap = inputs.pop('skyMap') 

315 inputs['refWcs'] = skyMap[tract].getWcs() 

316 

317 # Connections only exist if they are configured to be used. 

318 skyCorr = inputs.pop('skyCorr', None) 

319 if self.config.useGlobalExternalSkyWcs: 

320 externalSkyWcsCatalog = inputs.pop('externalSkyWcsGlobalCatalog', None) 

321 else: 

322 externalSkyWcsCatalog = inputs.pop('externalSkyWcsTractCatalog', None) 

323 if self.config.useGlobalExternalPhotoCalib: 

324 externalPhotoCalibCatalog = inputs.pop('externalPhotoCalibGlobalCatalog', None) 

325 else: 

326 externalPhotoCalibCatalog = inputs.pop('externalPhotoCalibTractCatalog', None) 

327 finalizedPsfApCorrCatalog = inputs.pop('finalizedPsfApCorrCatalog', None) 

328 

329 inputs['exposure'] = self.prepareCalibratedExposure( 

330 inputs['exposure'], 

331 skyCorr=skyCorr, 

332 externalSkyWcsCatalog=externalSkyWcsCatalog, 

333 externalPhotoCalibCatalog=externalPhotoCalibCatalog, 

334 finalizedPsfApCorrCatalog=finalizedPsfApCorrCatalog 

335 ) 

336 

337 inputs['refCat'] = self.mergeAndFilterReferences(inputs['exposure'], inputs['refCat'], 

338 inputs['refWcs']) 

339 

340 if inputs['refCat'] is None: 

341 self.log.info("No WCS for exposure %s. No %s catalog will be written.", 

342 butlerQC.quantum.dataId, outputRefs.measCat.datasetType.name) 

343 else: 

344 inputs['measCat'], inputs['exposureId'] = self.generateMeasCat(inputRefs.exposure.dataId, 

345 inputs['exposure'], 

346 inputs['refCat'], inputs['refWcs']) 

347 self.attachFootprints(inputs['measCat'], inputs['refCat'], inputs['exposure'], inputs['refWcs']) 

348 outputs = self.run(**inputs) 

349 butlerQC.put(outputs, outputRefs) 

350 

351 def prepareCalibratedExposure(self, exposure, skyCorr=None, externalSkyWcsCatalog=None, 

352 externalPhotoCalibCatalog=None, finalizedPsfApCorrCatalog=None): 

353 """Prepare a calibrated exposure and apply external calibrations 

354 and sky corrections if so configured. 

355 

356 Parameters 

357 ---------- 

358 exposure : `lsst.afw.image.exposure.Exposure` 

359 Input exposure to adjust calibrations. 

360 skyCorr : `lsst.afw.math.backgroundList`, optional 

361 Sky correction frame to apply if doApplySkyCorr=True. 

362 externalSkyWcsCatalog : `lsst.afw.table.ExposureCatalog`, optional 

363 Exposure catalog with external skyWcs to be applied 

364 if config.doApplyExternalSkyWcs=True. Catalog uses the detector id 

365 for the catalog id, sorted on id for fast lookup. 

366 externalPhotoCalibCatalog : `lsst.afw.table.ExposureCatalog`, optional 

367 Exposure catalog with external photoCalib to be applied 

368 if config.doApplyExternalPhotoCalib=True. Catalog uses the detector 

369 id for the catalog id, sorted on id for fast lookup. 

370 finalizedPsfApCorrCatalog : `lsst.afw.table.ExposureCatalog`, optional 

371 Exposure catalog with finalized psf models and aperture correction 

372 maps to be applied if config.doApplyFinalizedPsf=True. Catalog uses 

373 the detector id for the catalog id, sorted on id for fast lookup. 

374 

375 Returns 

376 ------- 

377 exposure : `lsst.afw.image.exposure.Exposure` 

378 Exposure with adjusted calibrations. 

379 """ 

380 detectorId = exposure.getInfo().getDetector().getId() 

381 

382 if externalPhotoCalibCatalog is not None: 

383 row = externalPhotoCalibCatalog.find(detectorId) 

384 if row is None: 

385 self.log.warning("Detector id %s not found in externalPhotoCalibCatalog; " 

386 "Using original photoCalib.", detectorId) 

387 else: 

388 photoCalib = row.getPhotoCalib() 

389 if photoCalib is None: 

390 self.log.warning("Detector id %s has None for photoCalib in externalPhotoCalibCatalog; " 

391 "Using original photoCalib.", detectorId) 

392 else: 

393 exposure.setPhotoCalib(photoCalib) 

394 

395 if externalSkyWcsCatalog is not None: 

396 row = externalSkyWcsCatalog.find(detectorId) 

397 if row is None: 

398 self.log.warning("Detector id %s not found in externalSkyWcsCatalog; " 

399 "Using original skyWcs.", detectorId) 

400 else: 

401 skyWcs = row.getWcs() 

402 if skyWcs is None: 

403 self.log.warning("Detector id %s has None for skyWcs in externalSkyWcsCatalog; " 

404 "Using original skyWcs.", detectorId) 

405 else: 

406 exposure.setWcs(skyWcs) 

407 

408 if finalizedPsfApCorrCatalog is not None: 

409 row = finalizedPsfApCorrCatalog.find(detectorId) 

410 if row is None: 

411 self.log.warning("Detector id %s not found in finalizedPsfApCorrCatalog; " 

412 "Using original psf.", detectorId) 

413 else: 

414 psf = row.getPsf() 

415 apCorrMap = row.getApCorrMap() 

416 if psf is None or apCorrMap is None: 

417 self.log.warning("Detector id %s has None for psf/apCorrMap in " 

418 "finalizedPsfApCorrCatalog; Using original psf.", detectorId) 

419 else: 

420 exposure.setPsf(psf) 

421 exposure.setApCorrMap(apCorrMap) 

422 

423 if skyCorr is not None: 

424 exposure.maskedImage -= skyCorr.getImage() 

425 

426 return exposure 

427 

428 def mergeAndFilterReferences(self, exposure, refCats, refWcs): 

429 """Filter reference catalog so that all sources are within the 

430 boundaries of the exposure. 

431 

432 Parameters 

433 ---------- 

434 exposure : `lsst.afw.image.exposure.Exposure` 

435 Exposure to generate the catalog for. 

436 refCats : sequence of `lsst.daf.butler.DeferredDatasetHandle` 

437 Handles for catalogs of shapes and positions at which to force 

438 photometry. 

439 refWcs : `lsst.afw.image.SkyWcs` 

440 Reference world coordinate system. 

441 

442 Returns 

443 ------- 

444 refSources : `lsst.afw.table.SourceCatalog` 

445 Filtered catalog of forced sources to measure. 

446 

447 Notes 

448 ----- 

449 The majority of this code is based on the methods of 

450 lsst.meas.algorithms.loadReferenceObjects.ReferenceObjectLoader 

451 

452 """ 

453 mergedRefCat = None 

454 

455 # Step 1: Determine bounds of the exposure photometry will 

456 # be performed on. 

457 expWcs = exposure.getWcs() 

458 if expWcs is None: 

459 self.log.info("Exposure has no WCS. Returning None for mergedRefCat.") 

460 else: 

461 expRegion = exposure.getBBox(lsst.afw.image.PARENT) 

462 expBBox = lsst.geom.Box2D(expRegion) 

463 expBoxCorners = expBBox.getCorners() 

464 expSkyCorners = [expWcs.pixelToSky(corner).getVector() for 

465 corner in expBoxCorners] 

466 expPolygon = lsst.sphgeom.ConvexPolygon(expSkyCorners) 

467 

468 # Step 2: Filter out reference catalog sources that are 

469 # not contained within the exposure boundaries, or whose 

470 # parents are not within the exposure boundaries. Note 

471 # that within a single input refCat, the parents always 

472 # appear before the children. 

473 for refCat in refCats: 

474 refCat = refCat.get() 

475 if mergedRefCat is None: 

476 mergedRefCat = lsst.afw.table.SourceCatalog(refCat.table) 

477 containedIds = {0} # zero as a parent ID means "this is a parent" 

478 for record in refCat: 

479 if (expPolygon.contains(record.getCoord().getVector()) and record.getParent() 

480 in containedIds): 

481 record.setFootprint(record.getFootprint()) 

482 mergedRefCat.append(record) 

483 containedIds.add(record.getId()) 

484 if mergedRefCat is None: 

485 raise RuntimeError("No reference objects for forced photometry.") 

486 mergedRefCat.sort(lsst.afw.table.SourceTable.getParentKey()) 

487 return mergedRefCat 

488 

489 def generateMeasCat(self, dataId, exposure, refCat, refWcs): 

490 """Generate a measurement catalog. 

491 

492 Parameters 

493 ---------- 

494 dataId : `lsst.daf.butler.DataCoordinate` 

495 Butler data ID for this image, with ``{visit, detector}`` keys. 

496 exposure : `lsst.afw.image.exposure.Exposure` 

497 Exposure to generate the catalog for. 

498 refCat : `lsst.afw.table.SourceCatalog` 

499 Catalog of shapes and positions at which to force photometry. 

500 refWcs : `lsst.afw.image.SkyWcs` 

501 Reference world coordinate system. 

502 This parameter is not currently used. 

503 

504 Returns 

505 ------- 

506 measCat : `lsst.afw.table.SourceCatalog` 

507 Catalog of forced sources to measure. 

508 expId : `int` 

509 Unique binary id associated with the input exposure 

510 """ 

511 id_generator = self.config.idGenerator.apply(dataId) 

512 measCat = self.measurement.generateMeasCat(exposure, refCat, refWcs, 

513 idFactory=id_generator.make_table_id_factory()) 

514 return measCat, id_generator.catalog_id 

515 

516 def run(self, measCat, exposure, refCat, refWcs, exposureId=None): 

517 """Perform forced measurement on a single exposure. 

518 

519 Parameters 

520 ---------- 

521 measCat : `lsst.afw.table.SourceCatalog` 

522 The measurement catalog, based on the sources listed in the 

523 reference catalog. 

524 exposure : `lsst.afw.image.Exposure` 

525 The measurement image upon which to perform forced detection. 

526 refCat : `lsst.afw.table.SourceCatalog` 

527 The reference catalog of sources to measure. 

528 refWcs : `lsst.afw.image.SkyWcs` 

529 The WCS for the references. 

530 exposureId : `int` 

531 Optional unique exposureId used for random seed in measurement 

532 task. 

533 

534 Returns 

535 ------- 

536 result : `lsst.pipe.base.Struct` 

537 Structure with fields: 

538 

539 ``measCat`` 

540 Catalog of forced measurement results 

541 (`lsst.afw.table.SourceCatalog`). 

542 """ 

543 self.measurement.run(measCat, exposure, refCat, refWcs, exposureId=exposureId) 

544 if self.config.doApCorr: 

545 self.applyApCorr.run( 

546 catalog=measCat, 

547 apCorrMap=exposure.getInfo().getApCorrMap() 

548 ) 

549 self.catalogCalculation.run(measCat) 

550 

551 return pipeBase.Struct(measCat=measCat) 

552 

553 def attachFootprints(self, sources, refCat, exposure, refWcs): 

554 """Attach footprints to blank sources prior to measurements. 

555 

556 Notes 

557 ----- 

558 `~lsst.afw.detection.Footprint` objects for forced photometry must 

559 be in the pixel coordinate system of the image being measured, while 

560 the actual detections may start out in a different coordinate system. 

561 

562 Subclasses of this class may implement this method to define how 

563 those `~lsst.afw.detection.Footprint` objects should be generated. 

564 

565 This default implementation transforms depends on the 

566 ``footprintSource`` configuration parameter. 

567 """ 

568 if self.config.footprintSource == "transformed": 

569 return self.measurement.attachTransformedFootprints(sources, refCat, exposure, refWcs) 

570 elif self.config.footprintSource == "psf": 

571 return self.measurement.attachPsfShapeFootprints(sources, exposure, 

572 scaling=self.config.psfFootprintScaling) 

573 

574 

575class ForcedPhotCcdFromDataFrameConnections(PipelineTaskConnections, 

576 dimensions=("instrument", "visit", "detector", "skymap", "tract"), 

577 defaultTemplates={"inputCoaddName": "goodSeeing", 

578 "inputName": "calexp", 

579 "skyWcsName": "gbdesAstrometricFit", 

580 "photoCalibName": "fgcm"}): 

581 refCat = cT.Input( 

582 doc="Catalog of positions at which to force photometry.", 

583 name="{inputCoaddName}Diff_fullDiaObjTable", 

584 storageClass="DataFrame", 

585 dimensions=["skymap", "tract", "patch"], 

586 multiple=True, 

587 deferLoad=True, 

588 ) 

589 exposure = cT.Input( 

590 doc="Input exposure to perform photometry on.", 

591 name="{inputName}", 

592 storageClass="ExposureF", 

593 dimensions=["instrument", "visit", "detector"], 

594 ) 

595 skyCorr = cT.Input( 

596 doc="Input Sky Correction to be subtracted from the calexp if doApplySkyCorr=True", 

597 name="skyCorr", 

598 storageClass="Background", 

599 dimensions=("instrument", "visit", "detector"), 

600 ) 

601 externalSkyWcsTractCatalog = cT.Input( 

602 doc=("Per-tract, per-visit wcs calibrations. These catalogs use the detector " 

603 "id for the catalog id, sorted on id for fast lookup."), 

604 name="{skyWcsName}SkyWcsCatalog", 

605 storageClass="ExposureCatalog", 

606 dimensions=["instrument", "visit", "tract"], 

607 ) 

608 externalSkyWcsGlobalCatalog = cT.Input( 

609 doc=("Per-visit wcs calibrations computed globally (with no tract information). " 

610 "These catalogs use the detector id for the catalog id, sorted on id for " 

611 "fast lookup."), 

612 name="{skyWcsName}SkyWcsCatalog", 

613 storageClass="ExposureCatalog", 

614 dimensions=["instrument", "visit"], 

615 ) 

616 externalPhotoCalibTractCatalog = cT.Input( 

617 doc=("Per-tract, per-visit photometric calibrations. These catalogs use the " 

618 "detector id for the catalog id, sorted on id for fast lookup."), 

619 name="{photoCalibName}PhotoCalibCatalog", 

620 storageClass="ExposureCatalog", 

621 dimensions=["instrument", "visit", "tract"], 

622 ) 

623 externalPhotoCalibGlobalCatalog = cT.Input( 

624 doc=("Per-visit photometric calibrations computed globally (with no tract " 

625 "information). These catalogs use the detector id for the catalog id, " 

626 "sorted on id for fast lookup."), 

627 name="{photoCalibName}PhotoCalibCatalog", 

628 storageClass="ExposureCatalog", 

629 dimensions=["instrument", "visit"], 

630 ) 

631 finalizedPsfApCorrCatalog = cT.Input( 

632 doc=("Per-visit finalized psf models and aperture correction maps. " 

633 "These catalogs use the detector id for the catalog id, " 

634 "sorted on id for fast lookup."), 

635 name="finalized_psf_ap_corr_catalog", 

636 storageClass="ExposureCatalog", 

637 dimensions=["instrument", "visit"], 

638 ) 

639 measCat = cT.Output( 

640 doc="Output forced photometry catalog.", 

641 name="forced_src_diaObject", 

642 storageClass="SourceCatalog", 

643 dimensions=["instrument", "visit", "detector", "skymap", "tract"], 

644 ) 

645 outputSchema = cT.InitOutput( 

646 doc="Schema for the output forced measurement catalogs.", 

647 name="forced_src_diaObject_schema", 

648 storageClass="SourceCatalog", 

649 ) 

650 

651 def __init__(self, *, config=None): 

652 super().__init__(config=config) 

653 if not config.doApplySkyCorr: 

654 self.inputs.remove("skyCorr") 

655 if config.doApplyExternalSkyWcs: 

656 if config.useGlobalExternalSkyWcs: 

657 self.inputs.remove("externalSkyWcsTractCatalog") 

658 else: 

659 self.inputs.remove("externalSkyWcsGlobalCatalog") 

660 else: 

661 self.inputs.remove("externalSkyWcsTractCatalog") 

662 self.inputs.remove("externalSkyWcsGlobalCatalog") 

663 if config.doApplyExternalPhotoCalib: 

664 if config.useGlobalExternalPhotoCalib: 

665 self.inputs.remove("externalPhotoCalibTractCatalog") 

666 else: 

667 self.inputs.remove("externalPhotoCalibGlobalCatalog") 

668 else: 

669 self.inputs.remove("externalPhotoCalibTractCatalog") 

670 self.inputs.remove("externalPhotoCalibGlobalCatalog") 

671 if not config.doApplyFinalizedPsf: 

672 self.inputs.remove("finalizedPsfApCorrCatalog") 

673 

674 

675class ForcedPhotCcdFromDataFrameConfig(ForcedPhotCcdConfig, 

676 pipelineConnections=ForcedPhotCcdFromDataFrameConnections): 

677 def setDefaults(self): 

678 super().setDefaults() 

679 self.footprintSource = "psf" 

680 self.measurement.doReplaceWithNoise = False 

681 # Only run a minimal set of plugins, as these measurements are only 

682 # needed for PSF-like sources. 

683 self.measurement.plugins.names = ["base_PixelFlags", 

684 "base_TransformedCentroidFromCoord", 

685 "base_PsfFlux", 

686 "base_LocalBackground", 

687 "base_LocalPhotoCalib", 

688 "base_LocalWcs", 

689 ] 

690 self.measurement.slots.shape = None 

691 # Make catalogCalculation a no-op by default as no modelFlux is setup 

692 # by default in ForcedMeasurementTask. 

693 self.catalogCalculation.plugins.names = [] 

694 

695 self.measurement.copyColumns = {'id': 'diaObjectId', 'coord_ra': 'coord_ra', 'coord_dec': 'coord_dec'} 

696 self.measurement.slots.centroid = "base_TransformedCentroidFromCoord" 

697 self.measurement.slots.psfFlux = "base_PsfFlux" 

698 

699 def validate(self): 

700 super().validate() 

701 if self.footprintSource == "transformed": 

702 raise ValueError("Cannot transform footprints from reference catalog, " 

703 "because DataFrames can't hold footprints.") 

704 

705 

706class ForcedPhotCcdFromDataFrameTask(ForcedPhotCcdTask): 

707 """Force Photometry on a per-detector exposure with coords from a DataFrame 

708 

709 Uses input from a DataFrame instead of SourceCatalog 

710 like the base class ForcedPhotCcd does. 

711 Writes out a SourceCatalog so that the downstream 

712 WriteForcedSourceTableTask can be reused with output from this Task. 

713 """ 

714 _DefaultName = "forcedPhotCcdFromDataFrame" 

715 ConfigClass = ForcedPhotCcdFromDataFrameConfig 

716 

717 def __init__(self, refSchema=None, initInputs=None, **kwargs): 

718 # Parent's init assumes that we have a reference schema; Cannot reuse 

719 pipeBase.PipelineTask.__init__(self, **kwargs) 

720 

721 self.makeSubtask("measurement", refSchema=lsst.afw.table.SourceTable.makeMinimalSchema()) 

722 

723 if self.config.doApCorr: 

724 self.makeSubtask("applyApCorr", schema=self.measurement.schema) 

725 self.makeSubtask('catalogCalculation', schema=self.measurement.schema) 

726 self.outputSchema = lsst.afw.table.SourceCatalog(self.measurement.schema) 

727 

728 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

729 inputs = butlerQC.get(inputRefs) 

730 

731 # When run with dataframes, we do not need a reference wcs. 

732 inputs['refWcs'] = None 

733 

734 # Connections only exist if they are configured to be used. 

735 skyCorr = inputs.pop('skyCorr', None) 

736 if self.config.useGlobalExternalSkyWcs: 

737 externalSkyWcsCatalog = inputs.pop('externalSkyWcsGlobalCatalog', None) 

738 else: 

739 externalSkyWcsCatalog = inputs.pop('externalSkyWcsTractCatalog', None) 

740 if self.config.useGlobalExternalPhotoCalib: 

741 externalPhotoCalibCatalog = inputs.pop('externalPhotoCalibGlobalCatalog', None) 

742 else: 

743 externalPhotoCalibCatalog = inputs.pop('externalPhotoCalibTractCatalog', None) 

744 finalizedPsfApCorrCatalog = inputs.pop('finalizedPsfApCorrCatalog', None) 

745 

746 inputs['exposure'] = self.prepareCalibratedExposure( 

747 inputs['exposure'], 

748 skyCorr=skyCorr, 

749 externalSkyWcsCatalog=externalSkyWcsCatalog, 

750 externalPhotoCalibCatalog=externalPhotoCalibCatalog, 

751 finalizedPsfApCorrCatalog=finalizedPsfApCorrCatalog 

752 ) 

753 

754 self.log.info("Filtering ref cats: %s", ','.join([str(i.dataId) for i in inputs['refCat']])) 

755 if inputs["exposure"].getWcs() is not None: 

756 refCat = self.df2RefCat([i.get(parameters={"columns": ['diaObjectId', 'ra', 'dec']}) 

757 for i in inputs['refCat']], 

758 inputs['exposure'].getBBox(), inputs['exposure'].getWcs()) 

759 inputs['refCat'] = refCat 

760 # generateMeasCat does not use the refWcs. 

761 inputs['measCat'], inputs['exposureId'] = self.generateMeasCat( 

762 inputRefs.exposure.dataId, inputs['exposure'], inputs['refCat'], inputs['refWcs'] 

763 ) 

764 # attachFootprints only uses refWcs in ``transformed`` mode, which is not 

765 # supported in the DataFrame-backed task. 

766 self.attachFootprints(inputs["measCat"], inputs["refCat"], inputs["exposure"], inputs["refWcs"]) 

767 outputs = self.run(**inputs) 

768 

769 butlerQC.put(outputs, outputRefs) 

770 else: 

771 self.log.info("No WCS for %s. Skipping and no %s catalog will be written.", 

772 butlerQC.quantum.dataId, outputRefs.measCat.datasetType.name) 

773 

774 def df2RefCat(self, dfList, exposureBBox, exposureWcs): 

775 """Convert list of DataFrames to reference catalog 

776 

777 Concatenate list of DataFrames presumably from multiple patches and 

778 downselect rows that overlap the exposureBBox using the exposureWcs. 

779 

780 Parameters 

781 ---------- 

782 dfList : `list` of `pandas.DataFrame` 

783 Each element containst diaObjects with ra/dec position in degrees 

784 Columns 'diaObjectId', 'ra', 'dec' are expected 

785 exposureBBox : `lsst.geom.Box2I` 

786 Bounding box on which to select rows that overlap 

787 exposureWcs : `lsst.afw.geom.SkyWcs` 

788 World coordinate system to convert sky coords in ref cat to 

789 pixel coords with which to compare with exposureBBox 

790 

791 Returns 

792 ------- 

793 refCat : `lsst.afw.table.SourceTable` 

794 Source Catalog with minimal schema that overlaps exposureBBox 

795 """ 

796 df = pd.concat(dfList) 

797 # translate ra/dec coords in dataframe to detector pixel coords 

798 # to down select rows that overlap the detector bbox 

799 mapping = exposureWcs.getTransform().getMapping() 

800 x, y = mapping.applyInverse(np.array(df[['ra', 'dec']].values*2*np.pi/360).T) 

801 inBBox = lsst.geom.Box2D(exposureBBox).contains(x, y) 

802 refCat = self.df2SourceCat(df[inBBox]) 

803 return refCat 

804 

805 def df2SourceCat(self, df): 

806 """Create minimal schema SourceCatalog from a pandas DataFrame. 

807 

808 The forced measurement subtask expects this as input. 

809 

810 Parameters 

811 ---------- 

812 df : `pandas.DataFrame` 

813 DiaObjects with locations and ids. 

814 

815 Returns 

816 ------- 

817 outputCatalog : `lsst.afw.table.SourceTable` 

818 Output catalog with minimal schema. 

819 """ 

820 schema = lsst.afw.table.SourceTable.makeMinimalSchema() 

821 outputCatalog = lsst.afw.table.SourceCatalog(schema) 

822 outputCatalog.reserve(len(df)) 

823 

824 for diaObjectId, ra, dec in df[['ra', 'dec']].itertuples(): 

825 outputRecord = outputCatalog.addNew() 

826 outputRecord.setId(diaObjectId) 

827 outputRecord.setCoord(lsst.geom.SpherePoint(ra, dec, lsst.geom.degrees)) 

828 return outputCatalog