Coverage for python/lsst/meas/base/forcedPhotCcd.py: 26%

274 statements  

« prev     ^ index     » next       coverage.py v6.4.2, created at 2022-08-03 02:55 -0700

1# This file is part of meas_base. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22import warnings 

23 

24import pandas as pd 

25import numpy as np 

26 

27import lsst.pex.config 

28import lsst.pex.exceptions 

29import lsst.pipe.base 

30import lsst.geom 

31import lsst.afw.detection 

32import lsst.afw.geom 

33import lsst.afw.image 

34import lsst.afw.table 

35import lsst.sphgeom 

36 

37from lsst.obs.base import ExposureIdInfo 

38from lsst.pipe.base import PipelineTaskConnections 

39import lsst.pipe.base.connectionTypes as cT 

40 

41import lsst.pipe.base as pipeBase 

42from lsst.skymap import BaseSkyMap 

43 

44from .forcedMeasurement import ForcedMeasurementTask 

45from .applyApCorr import ApplyApCorrTask 

46from .catalogCalculation import CatalogCalculationTask 

47 

48__all__ = ("ForcedPhotCcdConfig", "ForcedPhotCcdTask", 

49 "ForcedPhotCcdFromDataFrameTask", "ForcedPhotCcdFromDataFrameConfig") 

50 

51 

52class ForcedPhotCcdConnections(PipelineTaskConnections, 

53 dimensions=("instrument", "visit", "detector", "skymap", "tract"), 

54 defaultTemplates={"inputCoaddName": "deep", 

55 "inputName": "calexp", 

56 "skyWcsName": "jointcal", 

57 "photoCalibName": "fgcm"}): 

58 inputSchema = cT.InitInput( 

59 doc="Schema for the input measurement catalogs.", 

60 name="{inputCoaddName}Coadd_ref_schema", 

61 storageClass="SourceCatalog", 

62 ) 

63 outputSchema = cT.InitOutput( 

64 doc="Schema for the output forced measurement catalogs.", 

65 name="forced_src_schema", 

66 storageClass="SourceCatalog", 

67 ) 

68 exposure = cT.Input( 

69 doc="Input exposure to perform photometry on.", 

70 name="{inputName}", 

71 storageClass="ExposureF", 

72 dimensions=["instrument", "visit", "detector"], 

73 ) 

74 refCat = cT.Input( 

75 doc="Catalog of shapes and positions at which to force photometry.", 

76 name="{inputCoaddName}Coadd_ref", 

77 storageClass="SourceCatalog", 

78 dimensions=["skymap", "tract", "patch"], 

79 multiple=True, 

80 deferLoad=True, 

81 ) 

82 skyMap = cT.Input( 

83 doc="SkyMap dataset that defines the coordinate system of the reference catalog.", 

84 name=BaseSkyMap.SKYMAP_DATASET_TYPE_NAME, 

85 storageClass="SkyMap", 

86 dimensions=["skymap"], 

87 ) 

88 skyCorr = cT.Input( 

89 doc="Input Sky Correction to be subtracted from the calexp if doApplySkyCorr=True", 

90 name="skyCorr", 

91 storageClass="Background", 

92 dimensions=("instrument", "visit", "detector"), 

93 ) 

94 externalSkyWcsTractCatalog = cT.Input( 

95 doc=("Per-tract, per-visit wcs calibrations. These catalogs use the detector " 

96 "id for the catalog id, sorted on id for fast lookup."), 

97 name="{skyWcsName}SkyWcsCatalog", 

98 storageClass="ExposureCatalog", 

99 dimensions=["instrument", "visit", "tract"], 

100 ) 

101 externalSkyWcsGlobalCatalog = cT.Input( 

102 doc=("Per-visit wcs calibrations computed globally (with no tract information). " 

103 "These catalogs use the detector id for the catalog id, sorted on id for " 

104 "fast lookup."), 

105 name="{skyWcsName}SkyWcsCatalog", 

106 storageClass="ExposureCatalog", 

107 dimensions=["instrument", "visit"], 

108 ) 

109 externalPhotoCalibTractCatalog = cT.Input( 

110 doc=("Per-tract, per-visit photometric calibrations. These catalogs use the " 

111 "detector id for the catalog id, sorted on id for fast lookup."), 

112 name="{photoCalibName}PhotoCalibCatalog", 

113 storageClass="ExposureCatalog", 

114 dimensions=["instrument", "visit", "tract"], 

115 ) 

116 externalPhotoCalibGlobalCatalog = cT.Input( 

117 doc=("Per-visit photometric calibrations computed globally (with no tract " 

118 "information). These catalogs use the detector id for the catalog id, " 

119 "sorted on id for fast lookup."), 

120 name="{photoCalibName}PhotoCalibCatalog", 

121 storageClass="ExposureCatalog", 

122 dimensions=["instrument", "visit"], 

123 ) 

124 finalizedPsfApCorrCatalog = cT.Input( 

125 doc=("Per-visit finalized psf models and aperture correction maps. " 

126 "These catalogs use the detector id for the catalog id, " 

127 "sorted on id for fast lookup."), 

128 name="finalized_psf_ap_corr_catalog", 

129 storageClass="ExposureCatalog", 

130 dimensions=["instrument", "visit"], 

131 ) 

132 measCat = cT.Output( 

133 doc="Output forced photometry catalog.", 

134 name="forced_src", 

135 storageClass="SourceCatalog", 

136 dimensions=["instrument", "visit", "detector", "skymap", "tract"], 

137 ) 

138 

139 def __init__(self, *, config=None): 

140 super().__init__(config=config) 

141 if not config.doApplySkyCorr: 

142 self.inputs.remove("skyCorr") 

143 if config.doApplyExternalSkyWcs: 

144 if config.useGlobalExternalSkyWcs: 

145 self.inputs.remove("externalSkyWcsTractCatalog") 

146 else: 

147 self.inputs.remove("externalSkyWcsGlobalCatalog") 

148 else: 

149 self.inputs.remove("externalSkyWcsTractCatalog") 

150 self.inputs.remove("externalSkyWcsGlobalCatalog") 

151 if config.doApplyExternalPhotoCalib: 

152 if config.useGlobalExternalPhotoCalib: 

153 self.inputs.remove("externalPhotoCalibTractCatalog") 

154 else: 

155 self.inputs.remove("externalPhotoCalibGlobalCatalog") 

156 else: 

157 self.inputs.remove("externalPhotoCalibTractCatalog") 

158 self.inputs.remove("externalPhotoCalibGlobalCatalog") 

159 if not config.doApplyFinalizedPsf: 

160 self.inputs.remove("finalizedPsfApCorrCatalog") 

161 

162 

163class ForcedPhotCcdConfig(pipeBase.PipelineTaskConfig, 

164 pipelineConnections=ForcedPhotCcdConnections): 

165 """Config class for forced measurement driver task.""" 

166 measurement = lsst.pex.config.ConfigurableField( 

167 target=ForcedMeasurementTask, 

168 doc="subtask to do forced measurement" 

169 ) 

170 coaddName = lsst.pex.config.Field( 

171 doc="coadd name: typically one of deep or goodSeeing", 

172 dtype=str, 

173 default="deep", 

174 ) 

175 doApCorr = lsst.pex.config.Field( 

176 dtype=bool, 

177 default=True, 

178 doc="Run subtask to apply aperture corrections" 

179 ) 

180 applyApCorr = lsst.pex.config.ConfigurableField( 

181 target=ApplyApCorrTask, 

182 doc="Subtask to apply aperture corrections" 

183 ) 

184 catalogCalculation = lsst.pex.config.ConfigurableField( 

185 target=CatalogCalculationTask, 

186 doc="Subtask to run catalogCalculation plugins on catalog" 

187 ) 

188 doApplyUberCal = lsst.pex.config.Field( 

189 dtype=bool, 

190 doc="Apply meas_mosaic ubercal results to input calexps?", 

191 default=False, 

192 deprecated="Deprecated by DM-23352; use doApplyExternalPhotoCalib and doApplyExternalSkyWcs instead", 

193 ) 

194 doApplyExternalPhotoCalib = lsst.pex.config.Field( 

195 dtype=bool, 

196 default=False, 

197 doc=("Whether to apply external photometric calibration via an " 

198 "`lsst.afw.image.PhotoCalib` object."), 

199 ) 

200 useGlobalExternalPhotoCalib = lsst.pex.config.Field( 

201 dtype=bool, 

202 default=True, 

203 doc=("When using doApplyExternalPhotoCalib, use 'global' calibrations " 

204 "that are not run per-tract. When False, use per-tract photometric " 

205 "calibration files.") 

206 ) 

207 doApplyExternalSkyWcs = lsst.pex.config.Field( 

208 dtype=bool, 

209 default=False, 

210 doc=("Whether to apply external astrometric calibration via an " 

211 "`lsst.afw.geom.SkyWcs` object."), 

212 ) 

213 useGlobalExternalSkyWcs = lsst.pex.config.Field( 

214 dtype=bool, 

215 default=False, 

216 doc=("When using doApplyExternalSkyWcs, use 'global' calibrations " 

217 "that are not run per-tract. When False, use per-tract wcs " 

218 "files.") 

219 ) 

220 doApplyFinalizedPsf = lsst.pex.config.Field( 

221 dtype=bool, 

222 default=False, 

223 doc="Whether to apply finalized psf models and aperture correction map.", 

224 ) 

225 doApplySkyCorr = lsst.pex.config.Field( 

226 dtype=bool, 

227 default=False, 

228 doc="Apply sky correction?", 

229 ) 

230 includePhotoCalibVar = lsst.pex.config.Field( 

231 dtype=bool, 

232 default=False, 

233 doc="Add photometric calibration variance to warp variance plane?", 

234 ) 

235 footprintSource = lsst.pex.config.ChoiceField( 

236 dtype=str, 

237 doc="Where to obtain footprints to install in the measurement catalog, prior to measurement.", 

238 allowed={ 

239 "transformed": "Transform footprints from the reference catalog (downgrades HeavyFootprints).", 

240 "psf": ("Use the scaled shape of the PSF at the position of each source (does not generate " 

241 "HeavyFootprints)."), 

242 }, 

243 optional=True, 

244 default="transformed", 

245 ) 

246 psfFootprintScaling = lsst.pex.config.Field( 

247 dtype=float, 

248 doc="Scaling factor to apply to the PSF shape when footprintSource='psf' (ignored otherwise).", 

249 default=3.0, 

250 ) 

251 

252 def setDefaults(self): 

253 # Docstring inherited. 

254 # Make catalogCalculation a no-op by default as no modelFlux is setup by default in 

255 # ForcedMeasurementTask 

256 super().setDefaults() 

257 self.measurement.plugins.names |= ['base_LocalPhotoCalib', 'base_LocalWcs'] 

258 self.catalogCalculation.plugins.names = [] 

259 

260 

261class ForcedPhotCcdTask(pipeBase.PipelineTask): 

262 """A pipeline task for performing forced measurement on CCD images. 

263 

264 Parameters 

265 ---------- 

266 butler : `None` 

267 Deprecated and unused. Should always be `None`. 

268 refSchema : `lsst.afw.table.Schema`, optional 

269 The schema of the reference catalog, passed to the constructor of the 

270 references subtask. Optional, but must be specified if ``initInputs`` 

271 is not; if both are specified, ``initInputs`` takes precedence. 

272 initInputs : `dict` 

273 Dictionary that can contain a key ``inputSchema`` containing the 

274 schema. If present will override the value of ``refSchema``. 

275 **kwds 

276 Keyword arguments are passed to the supertask constructor. 

277 """ 

278 

279 ConfigClass = ForcedPhotCcdConfig 

280 _DefaultName = "forcedPhotCcd" 

281 dataPrefix = "" 

282 

283 def __init__(self, butler=None, refSchema=None, initInputs=None, **kwds): 

284 super().__init__(**kwds) 

285 

286 if butler is not None: 

287 warnings.warn("The 'butler' parameter is no longer used and can be safely removed.", 

288 category=FutureWarning, stacklevel=2) 

289 butler = None 

290 

291 if initInputs is not None: 

292 refSchema = initInputs['inputSchema'].schema 

293 

294 if refSchema is None: 

295 raise ValueError("No reference schema provided.") 

296 

297 self.makeSubtask("measurement", refSchema=refSchema) 

298 # It is necessary to get the schema internal to the forced measurement task until such a time 

299 # that the schema is not owned by the measurement task, but is passed in by an external caller 

300 if self.config.doApCorr: 

301 self.makeSubtask("applyApCorr", schema=self.measurement.schema) 

302 self.makeSubtask('catalogCalculation', schema=self.measurement.schema) 

303 self.outputSchema = lsst.afw.table.SourceCatalog(self.measurement.schema) 

304 

305 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

306 inputs = butlerQC.get(inputRefs) 

307 

308 tract = butlerQC.quantum.dataId['tract'] 

309 skyMap = inputs.pop('skyMap') 

310 inputs['refWcs'] = skyMap[tract].getWcs() 

311 

312 # Connections only exist if they are configured to be used. 

313 skyCorr = inputs.pop('skyCorr', None) 

314 if self.config.useGlobalExternalSkyWcs: 

315 externalSkyWcsCatalog = inputs.pop('externalSkyWcsGlobalCatalog', None) 

316 else: 

317 externalSkyWcsCatalog = inputs.pop('externalSkyWcsTractCatalog', None) 

318 if self.config.useGlobalExternalPhotoCalib: 

319 externalPhotoCalibCatalog = inputs.pop('externalPhotoCalibGlobalCatalog', None) 

320 else: 

321 externalPhotoCalibCatalog = inputs.pop('externalPhotoCalibTractCatalog', None) 

322 finalizedPsfApCorrCatalog = inputs.pop('finalizedPsfApCorrCatalog', None) 

323 

324 inputs['exposure'] = self.prepareCalibratedExposure( 

325 inputs['exposure'], 

326 skyCorr=skyCorr, 

327 externalSkyWcsCatalog=externalSkyWcsCatalog, 

328 externalPhotoCalibCatalog=externalPhotoCalibCatalog, 

329 finalizedPsfApCorrCatalog=finalizedPsfApCorrCatalog 

330 ) 

331 

332 inputs['refCat'] = self.mergeAndFilterReferences(inputs['exposure'], inputs['refCat'], 

333 inputs['refWcs']) 

334 

335 inputs['measCat'], inputs['exposureId'] = self.generateMeasCat(inputRefs.exposure.dataId, 

336 inputs['exposure'], 

337 inputs['refCat'], inputs['refWcs'], 

338 "visit_detector") 

339 self.attachFootprints(inputs['measCat'], inputs['refCat'], inputs['exposure'], inputs['refWcs']) 

340 outputs = self.run(**inputs) 

341 butlerQC.put(outputs, outputRefs) 

342 

343 def prepareCalibratedExposure(self, exposure, skyCorr=None, externalSkyWcsCatalog=None, 

344 externalPhotoCalibCatalog=None, finalizedPsfApCorrCatalog=None): 

345 """Prepare a calibrated exposure and apply external calibrations 

346 and sky corrections if so configured. 

347 

348 Parameters 

349 ---------- 

350 exposure : `lsst.afw.image.exposure.Exposure` 

351 Input exposure to adjust calibrations. 

352 skyCorr : `lsst.afw.math.backgroundList`, optional 

353 Sky correction frame to apply if doApplySkyCorr=True. 

354 externalSkyWcsCatalog : `lsst.afw.table.ExposureCatalog`, optional 

355 Exposure catalog with external skyWcs to be applied 

356 if config.doApplyExternalSkyWcs=True. Catalog uses the detector id 

357 for the catalog id, sorted on id for fast lookup. 

358 externalPhotoCalibCatalog : `lsst.afw.table.ExposureCatalog`, optional 

359 Exposure catalog with external photoCalib to be applied 

360 if config.doApplyExternalPhotoCalib=True. Catalog uses the detector 

361 id for the catalog id, sorted on id for fast lookup. 

362 finalizedPsfApCorrCatalog : `lsst.afw.table.ExposureCatalog`, optional 

363 Exposure catalog with finalized psf models and aperture correction 

364 maps to be applied if config.doApplyFinalizedPsf=True. Catalog uses 

365 the detector id for the catalog id, sorted on id for fast lookup. 

366 

367 Returns 

368 ------- 

369 exposure : `lsst.afw.image.exposure.Exposure` 

370 Exposure with adjusted calibrations. 

371 """ 

372 detectorId = exposure.getInfo().getDetector().getId() 

373 

374 if externalPhotoCalibCatalog is not None: 

375 row = externalPhotoCalibCatalog.find(detectorId) 

376 if row is None: 

377 self.log.warning("Detector id %s not found in externalPhotoCalibCatalog; " 

378 "Using original photoCalib.", detectorId) 

379 else: 

380 photoCalib = row.getPhotoCalib() 

381 if photoCalib is None: 

382 self.log.warning("Detector id %s has None for photoCalib in externalPhotoCalibCatalog; " 

383 "Using original photoCalib.", detectorId) 

384 else: 

385 exposure.setPhotoCalib(photoCalib) 

386 

387 if externalSkyWcsCatalog is not None: 

388 row = externalSkyWcsCatalog.find(detectorId) 

389 if row is None: 

390 self.log.warning("Detector id %s not found in externalSkyWcsCatalog; " 

391 "Using original skyWcs.", detectorId) 

392 else: 

393 skyWcs = row.getWcs() 

394 if skyWcs is None: 

395 self.log.warning("Detector id %s has None for skyWcs in externalSkyWcsCatalog; " 

396 "Using original skyWcs.", detectorId) 

397 else: 

398 exposure.setWcs(skyWcs) 

399 

400 if finalizedPsfApCorrCatalog is not None: 

401 row = finalizedPsfApCorrCatalog.find(detectorId) 

402 if row is None: 

403 self.log.warning("Detector id %s not found in finalizedPsfApCorrCatalog; " 

404 "Using original psf.", detectorId) 

405 else: 

406 psf = row.getPsf() 

407 apCorrMap = row.getApCorrMap() 

408 if psf is None or apCorrMap is None: 

409 self.log.warning("Detector id %s has None for psf/apCorrMap in " 

410 "finalizedPsfApCorrCatalog; Using original psf.", detectorId) 

411 else: 

412 exposure.setPsf(psf) 

413 exposure.setApCorrMap(apCorrMap) 

414 

415 if skyCorr is not None: 

416 exposure.maskedImage -= skyCorr.getImage() 

417 

418 return exposure 

419 

420 def mergeAndFilterReferences(self, exposure, refCats, refWcs): 

421 """Filter reference catalog so that all sources are within the 

422 boundaries of the exposure. 

423 

424 Parameters 

425 ---------- 

426 exposure : `lsst.afw.image.exposure.Exposure` 

427 Exposure to generate the catalog for. 

428 refCats : sequence of `lsst.daf.butler.DeferredDatasetHandle` 

429 Handles for catalogs of shapes and positions at which to force 

430 photometry. 

431 refWcs : `lsst.afw.image.SkyWcs` 

432 Reference world coordinate system. 

433 

434 Returns 

435 ------- 

436 refSources : `lsst.afw.table.SourceCatalog` 

437 Filtered catalog of forced sources to measure. 

438 

439 Notes 

440 ----- 

441 The majority of this code is based on the methods of 

442 lsst.meas.algorithms.loadReferenceObjects.ReferenceObjectLoader 

443 

444 """ 

445 

446 # Step 1: Determine bounds of the exposure photometry will 

447 # be performed on. 

448 expWcs = exposure.getWcs() 

449 expRegion = exposure.getBBox(lsst.afw.image.PARENT) 

450 expBBox = lsst.geom.Box2D(expRegion) 

451 expBoxCorners = expBBox.getCorners() 

452 expSkyCorners = [expWcs.pixelToSky(corner).getVector() for 

453 corner in expBoxCorners] 

454 expPolygon = lsst.sphgeom.ConvexPolygon(expSkyCorners) 

455 

456 # Step 2: Filter out reference catalog sources that are 

457 # not contained within the exposure boundaries, or whose 

458 # parents are not within the exposure boundaries. Note 

459 # that within a single input refCat, the parents always 

460 # appear before the children. 

461 mergedRefCat = None 

462 for refCat in refCats: 

463 refCat = refCat.get() 

464 if mergedRefCat is None: 

465 mergedRefCat = lsst.afw.table.SourceCatalog(refCat.table) 

466 containedIds = {0} # zero as a parent ID means "this is a parent" 

467 for record in refCat: 

468 if expPolygon.contains(record.getCoord().getVector()) and record.getParent() in containedIds: 

469 record.setFootprint(record.getFootprint()) 

470 mergedRefCat.append(record) 

471 containedIds.add(record.getId()) 

472 if mergedRefCat is None: 

473 raise RuntimeError("No reference objects for forced photometry.") 

474 mergedRefCat.sort(lsst.afw.table.SourceTable.getParentKey()) 

475 return mergedRefCat 

476 

477 def generateMeasCat(self, exposureDataId, exposure, refCat, refWcs, idPackerName): 

478 """Generate a measurement catalog. 

479 

480 Parameters 

481 ---------- 

482 exposureDataId : `DataId` 

483 Butler dataId for this exposure. 

484 exposure : `lsst.afw.image.exposure.Exposure` 

485 Exposure to generate the catalog for. 

486 refCat : `lsst.afw.table.SourceCatalog` 

487 Catalog of shapes and positions at which to force photometry. 

488 refWcs : `lsst.afw.image.SkyWcs` 

489 Reference world coordinate system. 

490 This parameter is not currently used. 

491 idPackerName : `str` 

492 Type of ID packer to construct from the registry. 

493 

494 Returns 

495 ------- 

496 measCat : `lsst.afw.table.SourceCatalog` 

497 Catalog of forced sources to measure. 

498 expId : `int` 

499 Unique binary id associated with the input exposure 

500 """ 

501 exposureIdInfo = ExposureIdInfo.fromDataId(exposureDataId, idPackerName) 

502 idFactory = exposureIdInfo.makeSourceIdFactory() 

503 

504 measCat = self.measurement.generateMeasCat(exposure, refCat, refWcs, 

505 idFactory=idFactory) 

506 return measCat, exposureIdInfo.expId 

507 

508 def run(self, measCat, exposure, refCat, refWcs, exposureId=None): 

509 """Perform forced measurement on a single exposure. 

510 

511 Parameters 

512 ---------- 

513 measCat : `lsst.afw.table.SourceCatalog` 

514 The measurement catalog, based on the sources listed in the 

515 reference catalog. 

516 exposure : `lsst.afw.image.Exposure` 

517 The measurement image upon which to perform forced detection. 

518 refCat : `lsst.afw.table.SourceCatalog` 

519 The reference catalog of sources to measure. 

520 refWcs : `lsst.afw.image.SkyWcs` 

521 The WCS for the references. 

522 exposureId : `int` 

523 Optional unique exposureId used for random seed in measurement 

524 task. 

525 

526 Returns 

527 ------- 

528 result : `lsst.pipe.base.Struct` 

529 Structure with fields: 

530 

531 ``measCat`` 

532 Catalog of forced measurement results 

533 (`lsst.afw.table.SourceCatalog`). 

534 """ 

535 self.measurement.run(measCat, exposure, refCat, refWcs, exposureId=exposureId) 

536 if self.config.doApCorr: 

537 self.applyApCorr.run( 

538 catalog=measCat, 

539 apCorrMap=exposure.getInfo().getApCorrMap() 

540 ) 

541 self.catalogCalculation.run(measCat) 

542 

543 return pipeBase.Struct(measCat=measCat) 

544 

545 def attachFootprints(self, sources, refCat, exposure, refWcs): 

546 r"""Attach footprints to blank sources prior to measurements. 

547 

548 Notes 

549 ----- 

550 `~lsst.afw.detection.Footprint`\ s for forced photometry must be in the 

551 pixel coordinate system of the image being measured, while the actual 

552 detections may start out in a different coordinate system. 

553 

554 Subclasses of this class may implement this method to define how 

555 those `~lsst.afw.detection.Footprint`\ s should be generated. 

556 

557 This default implementation transforms depends on the 

558 ``footprintSource`` configuration parameter. 

559 """ 

560 if self.config.footprintSource == "transformed": 

561 return self.measurement.attachTransformedFootprints(sources, refCat, exposure, refWcs) 

562 elif self.config.footprintSource == "psf": 

563 return self.measurement.attachPsfShapeFootprints(sources, exposure, 

564 scaling=self.config.psfFootprintScaling) 

565 

566 def getSchemaCatalogs(self): 

567 """The schema catalogs that will be used by this task. 

568 

569 Returns 

570 ------- 

571 schemaCatalogs : `dict` 

572 Dictionary mapping dataset type to schema catalog. 

573 

574 Notes 

575 ----- 

576 There is only one schema for each type of forced measurement. The 

577 dataset type for this measurement is defined in the mapper. 

578 """ 

579 catalog = lsst.afw.table.SourceCatalog(self.measurement.schema) 

580 catalog.getTable().setMetadata(self.measurement.algMetadata) 

581 datasetType = self.dataPrefix + "forced_src" 

582 return {datasetType: catalog} 

583 

584 

585class ForcedPhotCcdFromDataFrameConnections(PipelineTaskConnections, 

586 dimensions=("instrument", "visit", "detector", "skymap", "tract"), 

587 defaultTemplates={"inputCoaddName": "goodSeeing", 

588 "inputName": "calexp", 

589 "skyWcsName": "jointcal", 

590 "photoCalibName": "fgcm"}): 

591 refCat = cT.Input( 

592 doc="Catalog of positions at which to force photometry.", 

593 name="{inputCoaddName}Diff_fullDiaObjTable", 

594 storageClass="DataFrame", 

595 dimensions=["skymap", "tract", "patch"], 

596 multiple=True, 

597 deferLoad=True, 

598 ) 

599 exposure = cT.Input( 

600 doc="Input exposure to perform photometry on.", 

601 name="{inputName}", 

602 storageClass="ExposureF", 

603 dimensions=["instrument", "visit", "detector"], 

604 ) 

605 skyCorr = cT.Input( 

606 doc="Input Sky Correction to be subtracted from the calexp if doApplySkyCorr=True", 

607 name="skyCorr", 

608 storageClass="Background", 

609 dimensions=("instrument", "visit", "detector"), 

610 ) 

611 externalSkyWcsTractCatalog = cT.Input( 

612 doc=("Per-tract, per-visit wcs calibrations. These catalogs use the detector " 

613 "id for the catalog id, sorted on id for fast lookup."), 

614 name="{skyWcsName}SkyWcsCatalog", 

615 storageClass="ExposureCatalog", 

616 dimensions=["instrument", "visit", "tract"], 

617 ) 

618 externalSkyWcsGlobalCatalog = cT.Input( 

619 doc=("Per-visit wcs calibrations computed globally (with no tract information). " 

620 "These catalogs use the detector id for the catalog id, sorted on id for " 

621 "fast lookup."), 

622 name="{skyWcsName}SkyWcsCatalog", 

623 storageClass="ExposureCatalog", 

624 dimensions=["instrument", "visit"], 

625 ) 

626 externalPhotoCalibTractCatalog = cT.Input( 

627 doc=("Per-tract, per-visit photometric calibrations. These catalogs use the " 

628 "detector id for the catalog id, sorted on id for fast lookup."), 

629 name="{photoCalibName}PhotoCalibCatalog", 

630 storageClass="ExposureCatalog", 

631 dimensions=["instrument", "visit", "tract"], 

632 ) 

633 externalPhotoCalibGlobalCatalog = cT.Input( 

634 doc=("Per-visit photometric calibrations computed globally (with no tract " 

635 "information). These catalogs use the detector id for the catalog id, " 

636 "sorted on id for fast lookup."), 

637 name="{photoCalibName}PhotoCalibCatalog", 

638 storageClass="ExposureCatalog", 

639 dimensions=["instrument", "visit"], 

640 ) 

641 finalizedPsfApCorrCatalog = cT.Input( 

642 doc=("Per-visit finalized psf models and aperture correction maps. " 

643 "These catalogs use the detector id for the catalog id, " 

644 "sorted on id for fast lookup."), 

645 name="finalized_psf_ap_corr_catalog", 

646 storageClass="ExposureCatalog", 

647 dimensions=["instrument", "visit"], 

648 ) 

649 measCat = cT.Output( 

650 doc="Output forced photometry catalog.", 

651 name="forced_src_diaObject", 

652 storageClass="SourceCatalog", 

653 dimensions=["instrument", "visit", "detector", "skymap", "tract"], 

654 ) 

655 outputSchema = cT.InitOutput( 

656 doc="Schema for the output forced measurement catalogs.", 

657 name="forced_src_diaObject_schema", 

658 storageClass="SourceCatalog", 

659 ) 

660 

661 def __init__(self, *, config=None): 

662 super().__init__(config=config) 

663 if not config.doApplySkyCorr: 

664 self.inputs.remove("skyCorr") 

665 if config.doApplyExternalSkyWcs: 

666 if config.useGlobalExternalSkyWcs: 

667 self.inputs.remove("externalSkyWcsTractCatalog") 

668 else: 

669 self.inputs.remove("externalSkyWcsGlobalCatalog") 

670 else: 

671 self.inputs.remove("externalSkyWcsTractCatalog") 

672 self.inputs.remove("externalSkyWcsGlobalCatalog") 

673 if config.doApplyExternalPhotoCalib: 

674 if config.useGlobalExternalPhotoCalib: 

675 self.inputs.remove("externalPhotoCalibTractCatalog") 

676 else: 

677 self.inputs.remove("externalPhotoCalibGlobalCatalog") 

678 else: 

679 self.inputs.remove("externalPhotoCalibTractCatalog") 

680 self.inputs.remove("externalPhotoCalibGlobalCatalog") 

681 if not config.doApplyFinalizedPsf: 

682 self.inputs.remove("finalizedPsfApCorrCatalog") 

683 

684 

685class ForcedPhotCcdFromDataFrameConfig(ForcedPhotCcdConfig, 

686 pipelineConnections=ForcedPhotCcdFromDataFrameConnections): 

687 def setDefaults(self): 

688 super().setDefaults() 

689 self.footprintSource = "psf" 

690 self.measurement.doReplaceWithNoise = False 

691 self.measurement.plugins.names = ["base_LocalPhotoCalib", "base_LocalWcs", "base_LocalBackground", 

692 "base_TransformedCentroidFromCoord", "base_PsfFlux", 

693 "base_PixelFlags"] 

694 self.measurement.copyColumns = {'id': 'diaObjectId', 'coord_ra': 'coord_ra', 'coord_dec': 'coord_dec'} 

695 self.measurement.slots.centroid = "base_TransformedCentroidFromCoord" 

696 self.measurement.slots.psfFlux = "base_PsfFlux" 

697 self.measurement.slots.shape = None 

698 

699 def validate(self): 

700 super().validate() 

701 if self.footprintSource == "transformed": 

702 raise ValueError("Cannot transform footprints from reference catalog, " 

703 "because DataFrames can't hold footprints.") 

704 

705 

706class ForcedPhotCcdFromDataFrameTask(ForcedPhotCcdTask): 

707 """Force Photometry on a per-detector exposure with coords from a DataFrame 

708 

709 Uses input from a DataFrame instead of SourceCatalog 

710 like the base class ForcedPhotCcd does. 

711 Writes out a SourceCatalog so that the downstream 

712 WriteForcedSourceTableTask can be reused with output from this Task. 

713 """ 

714 _DefaultName = "forcedPhotCcdFromDataFrame" 

715 ConfigClass = ForcedPhotCcdFromDataFrameConfig 

716 

717 def __init__(self, butler=None, refSchema=None, initInputs=None, **kwds): 

718 # Parent's init assumes that we have a reference schema; Cannot reuse 

719 pipeBase.PipelineTask.__init__(self, **kwds) 

720 

721 if butler is not None: 

722 warnings.warn("The 'butler' parameter is no longer used and can be safely removed.", 

723 category=FutureWarning, stacklevel=2) 

724 butler = None 

725 

726 self.makeSubtask("measurement", refSchema=lsst.afw.table.SourceTable.makeMinimalSchema()) 

727 

728 if self.config.doApCorr: 

729 self.makeSubtask("applyApCorr", schema=self.measurement.schema) 

730 self.makeSubtask('catalogCalculation', schema=self.measurement.schema) 

731 self.outputSchema = lsst.afw.table.SourceCatalog(self.measurement.schema) 

732 

733 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

734 inputs = butlerQC.get(inputRefs) 

735 

736 # When run with dataframes, we do not need a reference wcs. 

737 inputs['refWcs'] = None 

738 

739 # Connections only exist if they are configured to be used. 

740 skyCorr = inputs.pop('skyCorr', None) 

741 if self.config.useGlobalExternalSkyWcs: 

742 externalSkyWcsCatalog = inputs.pop('externalSkyWcsGlobalCatalog', None) 

743 else: 

744 externalSkyWcsCatalog = inputs.pop('externalSkyWcsTractCatalog', None) 

745 if self.config.useGlobalExternalPhotoCalib: 

746 externalPhotoCalibCatalog = inputs.pop('externalPhotoCalibGlobalCatalog', None) 

747 else: 

748 externalPhotoCalibCatalog = inputs.pop('externalPhotoCalibTractCatalog', None) 

749 finalizedPsfApCorrCatalog = inputs.pop('finalizedPsfApCorrCatalog', None) 

750 

751 inputs['exposure'] = self.prepareCalibratedExposure( 

752 inputs['exposure'], 

753 skyCorr=skyCorr, 

754 externalSkyWcsCatalog=externalSkyWcsCatalog, 

755 externalPhotoCalibCatalog=externalPhotoCalibCatalog, 

756 finalizedPsfApCorrCatalog=finalizedPsfApCorrCatalog 

757 ) 

758 

759 self.log.info("Filtering ref cats: %s", ','.join([str(i.dataId) for i in inputs['refCat']])) 

760 refCat = self.df2RefCat([i.get(parameters={"columns": ['diaObjectId', 'ra', 'decl']}) 

761 for i in inputs['refCat']], 

762 inputs['exposure'].getBBox(), inputs['exposure'].getWcs()) 

763 inputs['refCat'] = refCat 

764 # generateMeasCat does not use the refWcs. 

765 inputs['measCat'], inputs['exposureId'] = self.generateMeasCat(inputRefs.exposure.dataId, 

766 inputs['exposure'], inputs['refCat'], 

767 inputs['refWcs'], 

768 "visit_detector") 

769 # attachFootprints only uses refWcs in ``transformed`` mode, which is not 

770 # supported in the DataFrame-backed task. 

771 self.attachFootprints(inputs["measCat"], inputs["refCat"], inputs["exposure"], inputs["refWcs"]) 

772 outputs = self.run(**inputs) 

773 

774 butlerQC.put(outputs, outputRefs) 

775 

776 def df2RefCat(self, dfList, exposureBBox, exposureWcs): 

777 """Convert list of DataFrames to reference catalog 

778 

779 Concatenate list of DataFrames presumably from multiple patches and 

780 downselect rows that overlap the exposureBBox using the exposureWcs. 

781 

782 Parameters 

783 ---------- 

784 dfList : `list` of `pandas.DataFrame` 

785 Each element containst diaObjects with ra/decl position in degrees 

786 Columns 'diaObjectId', 'ra', 'decl' are expected 

787 exposureBBox : `lsst.geom.Box2I` 

788 Bounding box on which to select rows that overlap 

789 exposureWcs : `lsst.afw.geom.SkyWcs` 

790 World coordinate system to convert sky coords in ref cat to 

791 pixel coords with which to compare with exposureBBox 

792 

793 Returns 

794 ------- 

795 refCat : `lsst.afw.table.SourceTable` 

796 Source Catalog with minimal schema that overlaps exposureBBox 

797 """ 

798 df = pd.concat(dfList) 

799 # translate ra/decl coords in dataframe to detector pixel coords 

800 # to down select rows that overlap the detector bbox 

801 mapping = exposureWcs.getTransform().getMapping() 

802 x, y = mapping.applyInverse(np.array(df[['ra', 'decl']].values*2*np.pi/360).T) 

803 inBBox = lsst.geom.Box2D(exposureBBox).contains(x, y) 

804 refCat = self.df2SourceCat(df[inBBox]) 

805 return refCat 

806 

807 def df2SourceCat(self, df): 

808 """Create minimal schema SourceCatalog from a pandas DataFrame. 

809 

810 The forced measurement subtask expects this as input. 

811 

812 Parameters 

813 ---------- 

814 df : `pandas.DataFrame` 

815 DiaObjects with locations and ids. 

816 

817 Returns 

818 ------- 

819 outputCatalog : `lsst.afw.table.SourceTable` 

820 Output catalog with minimal schema. 

821 """ 

822 schema = lsst.afw.table.SourceTable.makeMinimalSchema() 

823 outputCatalog = lsst.afw.table.SourceCatalog(schema) 

824 outputCatalog.reserve(len(df)) 

825 

826 for diaObjectId, ra, decl in df[['ra', 'decl']].itertuples(): 

827 outputRecord = outputCatalog.addNew() 

828 outputRecord.setId(diaObjectId) 

829 outputRecord.setCoord(lsst.geom.SpherePoint(ra, decl, lsst.geom.degrees)) 

830 return outputCatalog