Coverage for python/lsst/meas/base/forcedPhotCcd.py: 25%

283 statements  

« prev     ^ index     » next       coverage.py v7.1.0, created at 2023-02-05 18:07 -0800

1# This file is part of meas_base. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22import collections 

23import logging 

24import pandas as pd 

25import numpy as np 

26 

27import lsst.pex.config 

28import lsst.pex.exceptions 

29import lsst.pipe.base 

30import lsst.geom 

31import lsst.afw.detection 

32import lsst.afw.geom 

33import lsst.afw.image 

34import lsst.afw.table 

35import lsst.sphgeom 

36 

37from lsst.obs.base import ExposureIdInfo 

38from lsst.pipe.base import PipelineTaskConnections 

39import lsst.pipe.base.connectionTypes as cT 

40 

41import lsst.pipe.base as pipeBase 

42from lsst.skymap import BaseSkyMap 

43 

44from .references import MultiBandReferencesTask 

45from .forcedMeasurement import ForcedMeasurementTask 

46from .applyApCorr import ApplyApCorrTask 

47from .catalogCalculation import CatalogCalculationTask 

48 

49try: 

50 from lsst.meas.mosaic import applyMosaicResults 

51except ImportError: 

52 applyMosaicResults = None 

53 

54__all__ = ("PerTractCcdDataIdContainer", "ForcedPhotCcdConfig", "ForcedPhotCcdTask", "imageOverlapsTract", 

55 "ForcedPhotCcdFromDataFrameTask", "ForcedPhotCcdFromDataFrameConfig") 

56 

57 

58class PerTractCcdDataIdContainer(pipeBase.DataIdContainer): 

59 """A data ID container which combines raw data IDs with a tract. 

60 

61 Notes 

62 ----- 

63 Required because we need to add "tract" to the raw data ID keys (defined as 

64 whatever we use for ``src``) when no tract is provided (so that the user is 

65 not required to know which tracts are spanned by the raw data ID). 

66 

67 This subclass of `~lsst.pipe.base.DataIdContainer` assumes that a calexp is 

68 being measured using the detection information, a set of reference 

69 catalogs, from the set of coadds which intersect with the calexp. It needs 

70 the calexp id (e.g. visit, raft, sensor), but is also uses the tract to 

71 decide what set of coadds to use. The references from the tract whose 

72 patches intersect with the calexp are used. 

73 """ 

74 

75 def makeDataRefList(self, namespace): 

76 """Make self.refList from self.idList 

77 """ 

78 if self.datasetType is None: 

79 raise RuntimeError("Must call setDatasetType first") 

80 log = logging.getLogger("meas.base.forcedPhotCcd.PerTractCcdDataIdContainer") 

81 skymap = None 

82 visitTract = collections.defaultdict(set) # Set of tracts for each visit 

83 visitRefs = collections.defaultdict(list) # List of data references for each visit 

84 for dataId in self.idList: 

85 if "tract" not in dataId: 

86 # Discover which tracts the data overlaps 

87 log.info("Reading WCS for components of dataId=%s to determine tracts", dict(dataId)) 

88 if skymap is None: 

89 skymap = namespace.butler.get(namespace.config.coaddName + "Coadd_skyMap") 

90 

91 for ref in namespace.butler.subset("calexp", dataId=dataId): 

92 if not ref.datasetExists("calexp"): 

93 continue 

94 

95 visit = ref.dataId["visit"] 

96 visitRefs[visit].append(ref) 

97 

98 md = ref.get("calexp_md", immediate=True) 

99 wcs = lsst.afw.geom.makeSkyWcs(md) 

100 box = lsst.geom.Box2D(lsst.afw.image.bboxFromMetadata(md)) 

101 # Going with just the nearest tract. Since we're throwing all tracts for the visit 

102 # together, this shouldn't be a problem unless the tracts are much smaller than a CCD. 

103 tract = skymap.findTract(wcs.pixelToSky(box.getCenter())) 

104 if imageOverlapsTract(tract, wcs, box): 

105 visitTract[visit].add(tract.getId()) 

106 else: 

107 self.refList.extend(ref for ref in namespace.butler.subset(self.datasetType, dataId=dataId)) 

108 

109 # Ensure all components of a visit are kept together by putting them all in the same set of tracts 

110 for visit, tractSet in visitTract.items(): 

111 for ref in visitRefs[visit]: 

112 for tract in tractSet: 

113 self.refList.append(namespace.butler.dataRef(datasetType=self.datasetType, 

114 dataId=ref.dataId, tract=tract)) 

115 if visitTract: 

116 tractCounter = collections.Counter() 

117 for tractSet in visitTract.values(): 

118 tractCounter.update(tractSet) 

119 log.info("Number of visits for each tract: %s", dict(tractCounter)) 

120 

121 

122def imageOverlapsTract(tract, imageWcs, imageBox): 

123 """Return whether the given bounding box overlaps the tract given a WCS. 

124 

125 Parameters 

126 ---------- 

127 tract : `lsst.skymap.TractInfo` 

128 TractInfo specifying a tract. 

129 imageWcs : `lsst.afw.geom.SkyWcs` 

130 World coordinate system for the image. 

131 imageBox : `lsst.geom.Box2I` 

132 Bounding box for the image. 

133 

134 Returns 

135 ------- 

136 overlap : `bool` 

137 `True` if the bounding box overlaps the tract; `False` otherwise. 

138 """ 

139 tractPoly = tract.getOuterSkyPolygon() 

140 

141 imagePixelCorners = lsst.geom.Box2D(imageBox).getCorners() 

142 try: 

143 imageSkyCorners = imageWcs.pixelToSky(imagePixelCorners) 

144 except lsst.pex.exceptions.LsstCppException as e: 

145 # Protecting ourselves from awful Wcs solutions in input images 

146 if (not isinstance(e.message, lsst.pex.exceptions.DomainErrorException) 

147 and not isinstance(e.message, lsst.pex.exceptions.RuntimeErrorException)): 

148 raise 

149 return False 

150 

151 imagePoly = lsst.sphgeom.ConvexPolygon.convexHull([coord.getVector() for coord in imageSkyCorners]) 

152 return tractPoly.intersects(imagePoly) # "intersects" also covers "contains" or "is contained by" 

153 

154 

155class ForcedPhotCcdConnections(PipelineTaskConnections, 

156 dimensions=("instrument", "visit", "detector", "skymap", "tract"), 

157 defaultTemplates={"inputCoaddName": "deep", 

158 "inputName": "calexp"}): 

159 inputSchema = cT.InitInput( 

160 doc="Schema for the input measurement catalogs.", 

161 name="{inputCoaddName}Coadd_ref_schema", 

162 storageClass="SourceCatalog", 

163 ) 

164 outputSchema = cT.InitOutput( 

165 doc="Schema for the output forced measurement catalogs.", 

166 name="forced_src_schema", 

167 storageClass="SourceCatalog", 

168 ) 

169 exposure = cT.Input( 

170 doc="Input exposure to perform photometry on.", 

171 name="{inputName}", 

172 storageClass="ExposureF", 

173 dimensions=["instrument", "visit", "detector"], 

174 ) 

175 refCat = cT.Input( 

176 doc="Catalog of shapes and positions at which to force photometry.", 

177 name="{inputCoaddName}Coadd_ref", 

178 storageClass="SourceCatalog", 

179 dimensions=["skymap", "tract", "patch"], 

180 multiple=True, 

181 deferLoad=True, 

182 ) 

183 skyMap = cT.Input( 

184 doc="SkyMap dataset that defines the coordinate system of the reference catalog.", 

185 name=BaseSkyMap.SKYMAP_DATASET_TYPE_NAME, 

186 storageClass="SkyMap", 

187 dimensions=["skymap"], 

188 ) 

189 measCat = cT.Output( 

190 doc="Output forced photometry catalog.", 

191 name="forced_src", 

192 storageClass="SourceCatalog", 

193 dimensions=["instrument", "visit", "detector", "skymap", "tract"], 

194 ) 

195 

196 

197class ForcedPhotCcdConfig(pipeBase.PipelineTaskConfig, 

198 pipelineConnections=ForcedPhotCcdConnections): 

199 """Config class for forced measurement driver task.""" 

200 references = lsst.pex.config.ConfigurableField( 

201 target=MultiBandReferencesTask, 

202 doc="subtask to retrieve reference source catalog" 

203 ) 

204 measurement = lsst.pex.config.ConfigurableField( 

205 target=ForcedMeasurementTask, 

206 doc="subtask to do forced measurement" 

207 ) 

208 coaddName = lsst.pex.config.Field( 

209 doc="coadd name: typically one of deep or goodSeeing", 

210 dtype=str, 

211 default="deep", 

212 ) 

213 doApCorr = lsst.pex.config.Field( 

214 dtype=bool, 

215 default=True, 

216 doc="Run subtask to apply aperture corrections" 

217 ) 

218 applyApCorr = lsst.pex.config.ConfigurableField( 

219 target=ApplyApCorrTask, 

220 doc="Subtask to apply aperture corrections" 

221 ) 

222 catalogCalculation = lsst.pex.config.ConfigurableField( 

223 target=CatalogCalculationTask, 

224 doc="Subtask to run catalogCalculation plugins on catalog" 

225 ) 

226 doApplyUberCal = lsst.pex.config.Field( 

227 dtype=bool, 

228 doc="Apply meas_mosaic ubercal results to input calexps?", 

229 default=False, 

230 deprecated="Deprecated by DM-23352; use doApplyExternalPhotoCalib and doApplyExternalSkyWcs instead", 

231 ) 

232 doApplyExternalPhotoCalib = lsst.pex.config.Field( 

233 dtype=bool, 

234 default=False, 

235 doc=("Whether to apply external photometric calibration via an " 

236 "`lsst.afw.image.PhotoCalib` object. Uses the " 

237 "``externalPhotoCalibName`` field to determine which calibration " 

238 "to load."), 

239 ) 

240 doApplyExternalSkyWcs = lsst.pex.config.Field( 

241 dtype=bool, 

242 default=False, 

243 doc=("Whether to apply external astrometric calibration via an " 

244 "`lsst.afw.geom.SkyWcs` object. Uses ``externalSkyWcsName`` " 

245 "field to determine which calibration to load."), 

246 ) 

247 doApplySkyCorr = lsst.pex.config.Field( 

248 dtype=bool, 

249 default=False, 

250 doc="Apply sky correction?", 

251 ) 

252 includePhotoCalibVar = lsst.pex.config.Field( 

253 dtype=bool, 

254 default=False, 

255 doc="Add photometric calibration variance to warp variance plane?", 

256 ) 

257 externalPhotoCalibName = lsst.pex.config.ChoiceField( 

258 dtype=str, 

259 doc=("Type of external PhotoCalib if ``doApplyExternalPhotoCalib`` is True. " 

260 "Unused for Gen3 middleware."), 

261 default="jointcal", 

262 allowed={ 

263 "jointcal": "Use jointcal_photoCalib", 

264 "fgcm": "Use fgcm_photoCalib", 

265 "fgcm_tract": "Use fgcm_tract_photoCalib" 

266 }, 

267 ) 

268 externalSkyWcsName = lsst.pex.config.ChoiceField( 

269 dtype=str, 

270 doc="Type of external SkyWcs if ``doApplyExternalSkyWcs`` is True. Unused for Gen3 middleware.", 

271 default="jointcal", 

272 allowed={ 

273 "jointcal": "Use jointcal_wcs" 

274 }, 

275 ) 

276 footprintSource = lsst.pex.config.ChoiceField( 

277 dtype=str, 

278 doc="Where to obtain footprints to install in the measurement catalog, prior to measurement.", 

279 allowed={ 

280 "transformed": "Transform footprints from the reference catalog (downgrades HeavyFootprints).", 

281 "psf": ("Use the scaled shape of the PSF at the position of each source (does not generate " 

282 "HeavyFootprints)."), 

283 }, 

284 optional=True, 

285 default="transformed", 

286 ) 

287 psfFootprintScaling = lsst.pex.config.Field( 

288 dtype=float, 

289 doc="Scaling factor to apply to the PSF shape when footprintSource='psf' (ignored otherwise).", 

290 default=3.0, 

291 ) 

292 

293 def setDefaults(self): 

294 # Docstring inherited. 

295 # Make catalogCalculation a no-op by default as no modelFlux is setup by default in 

296 # ForcedMeasurementTask 

297 super().setDefaults() 

298 self.measurement.plugins.names |= ['base_LocalPhotoCalib', 'base_LocalWcs'] 

299 self.catalogCalculation.plugins.names = [] 

300 

301 

302class ForcedPhotCcdTask(pipeBase.PipelineTask, pipeBase.CmdLineTask): 

303 """A command-line driver for performing forced measurement on CCD images. 

304 

305 Parameters 

306 ---------- 

307 butler : `lsst.daf.persistence.butler.Butler`, optional 

308 A Butler which will be passed to the references subtask to allow it to 

309 load its schema from disk. Optional, but must be specified if 

310 ``refSchema`` is not; if both are specified, ``refSchema`` takes 

311 precedence. 

312 refSchema : `lsst.afw.table.Schema`, optional 

313 The schema of the reference catalog, passed to the constructor of the 

314 references subtask. Optional, but must be specified if ``butler`` is 

315 not; if both are specified, ``refSchema`` takes precedence. 

316 **kwds 

317 Keyword arguments are passed to the supertask constructor. 

318 

319 Notes 

320 ----- 

321 The `runDataRef` method takes a `~lsst.daf.persistence.ButlerDataRef` argument 

322 that corresponds to a single CCD. This should contain the data ID keys that 

323 correspond to the ``forced_src`` dataset (the output dataset for this 

324 task), which are typically all those used to specify the ``calexp`` dataset 

325 (``visit``, ``raft``, ``sensor`` for LSST data) as well as a coadd tract. 

326 The tract is used to look up the appropriate coadd measurement catalogs to 

327 use as references (e.g. ``deepCoadd_src``; see 

328 :lsst-task:`lsst.meas.base.references.CoaddSrcReferencesTask` for more 

329 information). While the tract must be given as part of the dataRef, the 

330 patches are determined automatically from the bounding box and WCS of the 

331 calexp to be measured, and the filter used to fetch references is set via 

332 the ``filter`` option in the configuration of 

333 :lsst-task:`lsst.meas.base.references.BaseReferencesTask`). 

334 """ 

335 

336 ConfigClass = ForcedPhotCcdConfig 

337 RunnerClass = pipeBase.ButlerInitializedTaskRunner 

338 _DefaultName = "forcedPhotCcd" 

339 dataPrefix = "" 

340 

341 def __init__(self, butler=None, refSchema=None, initInputs=None, **kwds): 

342 super().__init__(**kwds) 

343 

344 if initInputs is not None: 

345 refSchema = initInputs['inputSchema'].schema 

346 

347 self.makeSubtask("references", butler=butler, schema=refSchema) 

348 if refSchema is None: 

349 refSchema = self.references.schema 

350 self.makeSubtask("measurement", refSchema=refSchema) 

351 # It is necessary to get the schema internal to the forced measurement task until such a time 

352 # that the schema is not owned by the measurement task, but is passed in by an external caller 

353 if self.config.doApCorr: 

354 self.makeSubtask("applyApCorr", schema=self.measurement.schema) 

355 self.makeSubtask('catalogCalculation', schema=self.measurement.schema) 

356 self.outputSchema = lsst.afw.table.SourceCatalog(self.measurement.schema) 

357 

358 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

359 inputs = butlerQC.get(inputRefs) 

360 

361 tract = butlerQC.quantum.dataId['tract'] 

362 skyMap = inputs.pop("skyMap") 

363 inputs['refWcs'] = skyMap[tract].getWcs() 

364 

365 inputs['refCat'] = self.mergeAndFilterReferences(inputs['exposure'], inputs['refCat'], 

366 inputs['refWcs']) 

367 

368 inputs['measCat'], inputs['exposureId'] = self.generateMeasCat(inputRefs.exposure.dataId, 

369 inputs['exposure'], 

370 inputs['refCat'], inputs['refWcs'], 

371 "visit_detector") 

372 self.attachFootprints(inputs['measCat'], inputs['refCat'], inputs['exposure'], inputs['refWcs']) 

373 # TODO: apply external calibrations (DM-17062) 

374 outputs = self.run(**inputs) 

375 butlerQC.put(outputs, outputRefs) 

376 

377 def mergeAndFilterReferences(self, exposure, refCats, refWcs): 

378 """Filter reference catalog so that all sources are within the 

379 boundaries of the exposure. 

380 

381 Parameters 

382 ---------- 

383 exposure : `lsst.afw.image.exposure.Exposure` 

384 Exposure to generate the catalog for. 

385 refCats : sequence of `lsst.daf.butler.DeferredDatasetHandle` 

386 Handles for catalogs of shapes and positions at which to force 

387 photometry. 

388 refWcs : `lsst.afw.image.SkyWcs` 

389 Reference world coordinate system. 

390 

391 Returns 

392 ------- 

393 refSources : `lsst.afw.table.SourceCatalog` 

394 Filtered catalog of forced sources to measure. 

395 

396 Notes 

397 ----- 

398 Filtering the reference catalog is currently handled by Gen2 

399 specific methods. To function for Gen3, this method copies 

400 code segments to do the filtering and transformation. The 

401 majority of this code is based on the methods of 

402 lsst.meas.algorithms.loadReferenceObjects.ReferenceObjectLoader 

403 

404 """ 

405 

406 # Step 1: Determine bounds of the exposure photometry will 

407 # be performed on. 

408 expWcs = exposure.getWcs() 

409 expRegion = exposure.getBBox(lsst.afw.image.PARENT) 

410 expBBox = lsst.geom.Box2D(expRegion) 

411 expBoxCorners = expBBox.getCorners() 

412 expSkyCorners = [expWcs.pixelToSky(corner).getVector() for 

413 corner in expBoxCorners] 

414 expPolygon = lsst.sphgeom.ConvexPolygon(expSkyCorners) 

415 

416 # Step 2: Filter out reference catalog sources that are 

417 # not contained within the exposure boundaries, or whose 

418 # parents are not within the exposure boundaries. Note 

419 # that within a single input refCat, the parents always 

420 # appear before the children. 

421 mergedRefCat = None 

422 for refCat in refCats: 

423 refCat = refCat.get() 

424 if mergedRefCat is None: 

425 mergedRefCat = lsst.afw.table.SourceCatalog(refCat.table) 

426 containedIds = {0} # zero as a parent ID means "this is a parent" 

427 for record in refCat: 

428 if expPolygon.contains(record.getCoord().getVector()) and record.getParent() in containedIds: 

429 record.setFootprint(record.getFootprint()) 

430 mergedRefCat.append(record) 

431 containedIds.add(record.getId()) 

432 if mergedRefCat is None: 

433 raise RuntimeError("No reference objects for forced photometry.") 

434 mergedRefCat.sort(lsst.afw.table.SourceTable.getParentKey()) 

435 return mergedRefCat 

436 

437 def generateMeasCat(self, exposureDataId, exposure, refCat, refWcs, idPackerName): 

438 """Generate a measurement catalog for Gen3. 

439 

440 Parameters 

441 ---------- 

442 exposureDataId : `DataId` 

443 Butler dataId for this exposure. 

444 exposure : `lsst.afw.image.exposure.Exposure` 

445 Exposure to generate the catalog for. 

446 refCat : `lsst.afw.table.SourceCatalog` 

447 Catalog of shapes and positions at which to force photometry. 

448 refWcs : `lsst.afw.image.SkyWcs` 

449 Reference world coordinate system. 

450 idPackerName : `str` 

451 Type of ID packer to construct from the registry. 

452 

453 Returns 

454 ------- 

455 measCat : `lsst.afw.table.SourceCatalog` 

456 Catalog of forced sources to measure. 

457 expId : `int` 

458 Unique binary id associated with the input exposure 

459 """ 

460 exposureIdInfo = ExposureIdInfo.fromDataId(exposureDataId, idPackerName) 

461 idFactory = exposureIdInfo.makeSourceIdFactory() 

462 

463 measCat = self.measurement.generateMeasCat(exposure, refCat, refWcs, 

464 idFactory=idFactory) 

465 return measCat, exposureIdInfo.expId 

466 

467 def runDataRef(self, dataRef, psfCache=None): 

468 """Perform forced measurement on a single exposure. 

469 

470 Parameters 

471 ---------- 

472 dataRef : `lsst.daf.persistence.ButlerDataRef` 

473 Passed to the ``references`` subtask to obtain the reference WCS, 

474 the ``getExposure`` method (implemented by derived classes) to 

475 read the measurment image, and the ``fetchReferences`` method to 

476 get the exposure and load the reference catalog (see 

477 :lsst-task`lsst.meas.base.references.CoaddSrcReferencesTask`). 

478 Refer to derived class documentation for details of the datasets 

479 and data ID keys which are used. 

480 psfCache : `int`, optional 

481 Size of PSF cache, or `None`. The size of the PSF cache can have 

482 a significant effect upon the runtime for complicated PSF models. 

483 

484 Notes 

485 ----- 

486 Sources are generated with ``generateMeasCat`` in the ``measurement`` 

487 subtask. These are passed to ``measurement``'s ``run`` method, which 

488 fills the source catalog with the forced measurement results. The 

489 sources are then passed to the ``writeOutputs`` method (implemented by 

490 derived classes) which writes the outputs. 

491 """ 

492 refWcs = self.references.getWcs(dataRef) 

493 exposure = self.getExposure(dataRef) 

494 if psfCache is not None: 

495 exposure.getPsf().setCacheSize(psfCache) 

496 refCat = self.fetchReferences(dataRef, exposure) 

497 measCat = self.measurement.generateMeasCat(exposure, refCat, refWcs, 

498 idFactory=self.makeIdFactory(dataRef)) 

499 self.log.info("Performing forced measurement on %s", dataRef.dataId) 

500 self.attachFootprints(measCat, refCat, exposure, refWcs) 

501 

502 exposureId = self.getExposureId(dataRef) 

503 

504 forcedPhotResult = self.run(measCat, exposure, refCat, refWcs, exposureId=exposureId) 

505 

506 self.writeOutput(dataRef, forcedPhotResult.measCat) 

507 

508 def run(self, measCat, exposure, refCat, refWcs, exposureId=None): 

509 """Perform forced measurement on a single exposure. 

510 

511 Parameters 

512 ---------- 

513 measCat : `lsst.afw.table.SourceCatalog` 

514 The measurement catalog, based on the sources listed in the 

515 reference catalog. 

516 exposure : `lsst.afw.image.Exposure` 

517 The measurement image upon which to perform forced detection. 

518 refCat : `lsst.afw.table.SourceCatalog` 

519 The reference catalog of sources to measure. 

520 refWcs : `lsst.afw.image.SkyWcs` 

521 The WCS for the references. 

522 exposureId : `int` 

523 Optional unique exposureId used for random seed in measurement 

524 task. 

525 

526 Returns 

527 ------- 

528 result : `lsst.pipe.base.Struct` 

529 Structure with fields: 

530 

531 ``measCat`` 

532 Catalog of forced measurement results 

533 (`lsst.afw.table.SourceCatalog`). 

534 """ 

535 self.measurement.run(measCat, exposure, refCat, refWcs, exposureId=exposureId) 

536 if self.config.doApCorr: 

537 self.applyApCorr.run( 

538 catalog=measCat, 

539 apCorrMap=exposure.getInfo().getApCorrMap() 

540 ) 

541 self.catalogCalculation.run(measCat) 

542 

543 return pipeBase.Struct(measCat=measCat) 

544 

545 def makeIdFactory(self, dataRef): 

546 """Create an object that generates globally unique source IDs. 

547 

548 Source IDs are created based on a per-CCD ID and the ID of the CCD 

549 itself. 

550 

551 Parameters 

552 ---------- 

553 dataRef : `lsst.daf.persistence.ButlerDataRef` 

554 Butler data reference. The ``ccdExposureId_bits`` and 

555 ``ccdExposureId`` datasets are accessed. The data ID must have the 

556 keys that correspond to ``ccdExposureId``, which are generally the 

557 same as those that correspond to ``calexp`` (``visit``, ``raft``, 

558 ``sensor`` for LSST data). 

559 """ 

560 exposureIdInfo = ExposureIdInfo(int(dataRef.get("ccdExposureId")), dataRef.get("ccdExposureId_bits")) 

561 return exposureIdInfo.makeSourceIdFactory() 

562 

563 def getExposureId(self, dataRef): 

564 return int(dataRef.get("ccdExposureId", immediate=True)) 

565 

566 def fetchReferences(self, dataRef, exposure): 

567 """Get sources that overlap the exposure. 

568 

569 Parameters 

570 ---------- 

571 dataRef : `lsst.daf.persistence.ButlerDataRef` 

572 Butler data reference corresponding to the image to be measured; 

573 should have ``tract``, ``patch``, and ``filter`` keys. 

574 exposure : `lsst.afw.image.Exposure` 

575 The image to be measured (used only to obtain a WCS and bounding 

576 box). 

577 

578 Returns 

579 ------- 

580 referencs : `lsst.afw.table.SourceCatalog` 

581 Catalog of sources that overlap the exposure 

582 

583 Notes 

584 ----- 

585 The returned catalog is sorted by ID and guarantees that all included 

586 children have their parent included and that all Footprints are valid. 

587 

588 All work is delegated to the references subtask; see 

589 :lsst-task:`lsst.meas.base.references.CoaddSrcReferencesTask` 

590 for information about the default behavior. 

591 """ 

592 references = lsst.afw.table.SourceCatalog(self.references.schema) 

593 badParents = set() 

594 unfiltered = self.references.fetchInBox(dataRef, exposure.getBBox(), exposure.getWcs()) 

595 for record in unfiltered: 

596 if record.getFootprint() is None or record.getFootprint().getArea() == 0: 

597 if record.getParent() != 0: 

598 self.log.warning("Skipping reference %s (child of %s) with bad Footprint", 

599 record.getId(), record.getParent()) 

600 else: 

601 self.log.warning("Skipping reference parent %s with bad Footprint", record.getId()) 

602 badParents.add(record.getId()) 

603 elif record.getParent() not in badParents: 

604 references.append(record) 

605 # catalog must be sorted by parent ID for lsst.afw.table.getChildren to work 

606 references.sort(lsst.afw.table.SourceTable.getParentKey()) 

607 return references 

608 

609 def attachFootprints(self, sources, refCat, exposure, refWcs): 

610 r"""Attach footprints to blank sources prior to measurements. 

611 

612 Notes 

613 ----- 

614 `~lsst.afw.detection.Footprint`\ s for forced photometry must be in the 

615 pixel coordinate system of the image being measured, while the actual 

616 detections may start out in a different coordinate system. 

617 

618 Subclasses of this class may implement this method to define how 

619 those `~lsst.afw.detection.Footprint`\ s should be generated. 

620 

621 This default implementation transforms depends on the 

622 ``footprintSource`` configuration parameter. 

623 """ 

624 if self.config.footprintSource == "transformed": 

625 return self.measurement.attachTransformedFootprints(sources, refCat, exposure, refWcs) 

626 elif self.config.footprintSource == "psf": 

627 return self.measurement.attachPsfShapeFootprints(sources, exposure, 

628 scaling=self.config.psfFootprintScaling) 

629 

630 def getExposure(self, dataRef): 

631 """Read input exposure for measurement. 

632 

633 Parameters 

634 ---------- 

635 dataRef : `lsst.daf.persistence.ButlerDataRef` 

636 Butler data reference. 

637 """ 

638 exposure = dataRef.get(self.dataPrefix + "calexp", immediate=True) 

639 

640 if self.config.doApplyExternalPhotoCalib: 

641 source = f"{self.config.externalPhotoCalibName}_photoCalib" 

642 self.log.info("Applying external photoCalib from %s", source) 

643 photoCalib = dataRef.get(source) 

644 exposure.setPhotoCalib(photoCalib) # No need for calibrateImage; having the photoCalib suffices 

645 

646 if self.config.doApplyExternalSkyWcs: 

647 source = f"{self.config.externalSkyWcsName}_wcs" 

648 self.log.info("Applying external skyWcs from %s", source) 

649 skyWcs = dataRef.get(source) 

650 exposure.setWcs(skyWcs) 

651 

652 if self.config.doApplySkyCorr: 

653 self.log.info("Apply sky correction") 

654 skyCorr = dataRef.get("skyCorr") 

655 exposure.maskedImage -= skyCorr.getImage() 

656 

657 return exposure 

658 

659 def writeOutput(self, dataRef, sources): 

660 """Write forced source table 

661 

662 Parameters 

663 ---------- 

664 dataRef : `lsst.daf.persistence.ButlerDataRef` 

665 Butler data reference. The forced_src dataset (with 

666 self.dataPrefix prepended) is all that will be modified. 

667 sources : `lsst.afw.table.SourceCatalog` 

668 Catalog of sources to save. 

669 """ 

670 dataRef.put(sources, self.dataPrefix + "forced_src", flags=lsst.afw.table.SOURCE_IO_NO_FOOTPRINTS) 

671 

672 def getSchemaCatalogs(self): 

673 """The schema catalogs that will be used by this task. 

674 

675 Returns 

676 ------- 

677 schemaCatalogs : `dict` 

678 Dictionary mapping dataset type to schema catalog. 

679 

680 Notes 

681 ----- 

682 There is only one schema for each type of forced measurement. The 

683 dataset type for this measurement is defined in the mapper. 

684 """ 

685 catalog = lsst.afw.table.SourceCatalog(self.measurement.schema) 

686 catalog.getTable().setMetadata(self.measurement.algMetadata) 

687 datasetType = self.dataPrefix + "forced_src" 

688 return {datasetType: catalog} 

689 

690 def _getConfigName(self): 

691 # Documented in superclass. 

692 return self.dataPrefix + "forcedPhotCcd_config" 

693 

694 def _getMetadataName(self): 

695 # Documented in superclass 

696 return self.dataPrefix + "forcedPhotCcd_metadata" 

697 

698 @classmethod 

699 def _makeArgumentParser(cls): 

700 parser = pipeBase.ArgumentParser(name=cls._DefaultName) 

701 parser.add_id_argument("--id", "forced_src", help="data ID with raw CCD keys [+ tract optionally], " 

702 "e.g. --id visit=12345 ccd=1,2 [tract=0]", 

703 ContainerClass=PerTractCcdDataIdContainer) 

704 return parser 

705 

706 

707class ForcedPhotCcdFromDataFrameConnections(PipelineTaskConnections, 

708 dimensions=("instrument", "visit", "detector", "skymap", "tract"), 

709 defaultTemplates={"inputCoaddName": "goodSeeing", 

710 "inputName": "calexp"}): 

711 refCat = cT.Input( 

712 doc="Catalog of positions at which to force photometry.", 

713 name="{inputCoaddName}Diff_fullDiaObjTable", 

714 storageClass="DataFrame", 

715 dimensions=["skymap", "tract", "patch"], 

716 multiple=True, 

717 deferLoad=True, 

718 ) 

719 exposure = cT.Input( 

720 doc="Input exposure to perform photometry on.", 

721 name="{inputName}", 

722 storageClass="ExposureF", 

723 dimensions=["instrument", "visit", "detector"], 

724 ) 

725 measCat = cT.Output( 

726 doc="Output forced photometry catalog.", 

727 name="forced_src_diaObject", 

728 storageClass="SourceCatalog", 

729 dimensions=["instrument", "visit", "detector", "skymap", "tract"], 

730 ) 

731 outputSchema = cT.InitOutput( 

732 doc="Schema for the output forced measurement catalogs.", 

733 name="forced_src_diaObject_schema", 

734 storageClass="SourceCatalog", 

735 ) 

736 

737 

738class ForcedPhotCcdFromDataFrameConfig(ForcedPhotCcdConfig, 

739 pipelineConnections=ForcedPhotCcdFromDataFrameConnections): 

740 def setDefaults(self): 

741 super().setDefaults() 

742 self.footprintSource = "psf" 

743 self.measurement.doReplaceWithNoise = False 

744 self.measurement.plugins.names = ["base_LocalPhotoCalib", "base_LocalWcs", "base_LocalBackground", 

745 "base_TransformedCentroidFromCoord", "base_PsfFlux", 

746 "base_PixelFlags"] 

747 self.measurement.copyColumns = {'id': 'diaObjectId', 'coord_ra': 'coord_ra', 'coord_dec': 'coord_dec'} 

748 self.measurement.slots.centroid = "base_TransformedCentroidFromCoord" 

749 self.measurement.slots.psfFlux = "base_PsfFlux" 

750 self.measurement.slots.shape = None 

751 

752 def validate(self): 

753 super().validate() 

754 if self.footprintSource == "transformed": 

755 raise ValueError("Cannot transform footprints from reference catalog, " 

756 "because DataFrames can't hold footprints.") 

757 

758 

759class ForcedPhotCcdFromDataFrameTask(ForcedPhotCcdTask): 

760 """Force Photometry on a per-detector exposure with coords from a DataFrame 

761 

762 Uses input from a DataFrame instead of SourceCatalog 

763 like the base class ForcedPhotCcd does. 

764 Writes out a SourceCatalog so that the downstream 

765 WriteForcedSourceTableTask can be reused with output from this Task. 

766 """ 

767 _DefaultName = "forcedPhotCcdFromDataFrame" 

768 ConfigClass = ForcedPhotCcdFromDataFrameConfig 

769 

770 def __init__(self, butler=None, refSchema=None, initInputs=None, **kwds): 

771 # Parent's init assumes that we have a reference schema; Cannot reuse 

772 pipeBase.PipelineTask.__init__(self, **kwds) 

773 

774 self.makeSubtask("measurement", refSchema=lsst.afw.table.SourceTable.makeMinimalSchema()) 

775 

776 if self.config.doApCorr: 

777 self.makeSubtask("applyApCorr", schema=self.measurement.schema) 

778 self.makeSubtask('catalogCalculation', schema=self.measurement.schema) 

779 self.outputSchema = lsst.afw.table.SourceCatalog(self.measurement.schema) 

780 

781 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

782 inputs = butlerQC.get(inputRefs) 

783 self.log.info("Filtering ref cats: %s", ','.join([str(i.dataId) for i in inputs['refCat']])) 

784 refCat = self.df2RefCat([i.get(parameters={"columns": ['diaObjectId', 'ra', 'decl']}) 

785 for i in inputs['refCat']], 

786 inputs['exposure'].getBBox(), inputs['exposure'].getWcs()) 

787 inputs['refCat'] = refCat 

788 inputs['refWcs'] = inputs['exposure'].getWcs() 

789 inputs['measCat'], inputs['exposureId'] = self.generateMeasCat(inputRefs.exposure.dataId, 

790 inputs['exposure'], inputs['refCat'], 

791 inputs['refWcs'], 

792 "visit_detector") 

793 self.attachFootprints(inputs["measCat"], inputs["refCat"], inputs["exposure"], inputs["refWcs"]) 

794 outputs = self.run(**inputs) 

795 butlerQC.put(outputs, outputRefs) 

796 

797 def df2RefCat(self, dfList, exposureBBox, exposureWcs): 

798 """Convert list of DataFrames to reference catalog 

799 

800 Concatenate list of DataFrames presumably from multiple patches and 

801 downselect rows that overlap the exposureBBox using the exposureWcs. 

802 

803 Parameters 

804 ---------- 

805 dfList : `list` of `pandas.DataFrame` 

806 Each element containst diaObjects with ra/decl position in degrees 

807 Columns 'diaObjectId', 'ra', 'decl' are expected 

808 exposureBBox : `lsst.geom.Box2I` 

809 Bounding box on which to select rows that overlap 

810 exposureWcs : `lsst.afw.geom.SkyWcs` 

811 World coordinate system to convert sky coords in ref cat to 

812 pixel coords with which to compare with exposureBBox 

813 

814 Returns 

815 ------- 

816 refCat : `lsst.afw.table.SourceTable` 

817 Source Catalog with minimal schema that overlaps exposureBBox 

818 """ 

819 df = pd.concat(dfList) 

820 # translate ra/decl coords in dataframe to detector pixel coords 

821 # to down select rows that overlap the detector bbox 

822 mapping = exposureWcs.getTransform().getMapping() 

823 x, y = mapping.applyInverse(np.array(df[['ra', 'decl']].values*2*np.pi/360).T) 

824 inBBox = lsst.geom.Box2D(exposureBBox).contains(x, y) 

825 refCat = self.df2SourceCat(df[inBBox]) 

826 return refCat 

827 

828 def df2SourceCat(self, df): 

829 """Create minimal schema SourceCatalog from a pandas DataFrame. 

830 

831 The forced measurement subtask expects this as input. 

832 

833 Parameters 

834 ---------- 

835 df : `pandas.DataFrame` 

836 DiaObjects with locations and ids. 

837 

838 Returns 

839 ------- 

840 outputCatalog : `lsst.afw.table.SourceTable` 

841 Output catalog with minimal schema. 

842 """ 

843 schema = lsst.afw.table.SourceTable.makeMinimalSchema() 

844 outputCatalog = lsst.afw.table.SourceCatalog(schema) 

845 outputCatalog.reserve(len(df)) 

846 

847 for diaObjectId, ra, decl in df[['ra', 'decl']].itertuples(): 

848 outputRecord = outputCatalog.addNew() 

849 outputRecord.setId(diaObjectId) 

850 outputRecord.setCoord(lsst.geom.SpherePoint(ra, decl, lsst.geom.degrees)) 

851 return outputCatalog