Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of meas_base. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22import collections 

23import logging 

24import pandas as pd 

25import numpy as np 

26 

27import lsst.pex.config 

28import lsst.pex.exceptions 

29import lsst.pipe.base 

30import lsst.geom 

31import lsst.afw.geom 

32import lsst.afw.image 

33import lsst.afw.table 

34import lsst.sphgeom 

35 

36from lsst.pipe.base import PipelineTaskConnections 

37import lsst.pipe.base.connectionTypes as cT 

38 

39import lsst.pipe.base as pipeBase 

40from lsst.skymap import BaseSkyMap 

41 

42from .references import MultiBandReferencesTask 

43from .forcedMeasurement import ForcedMeasurementTask 

44from .applyApCorr import ApplyApCorrTask 

45from .catalogCalculation import CatalogCalculationTask 

46 

47try: 

48 from lsst.meas.mosaic import applyMosaicResults 

49except ImportError: 

50 applyMosaicResults = None 

51 

52__all__ = ("PerTractCcdDataIdContainer", "ForcedPhotCcdConfig", "ForcedPhotCcdTask", "imageOverlapsTract", 

53 "ForcedPhotCcdFromDataFrameTask", "ForcedPhotCcdFromDataFrameConfig") 

54 

55 

56class PerTractCcdDataIdContainer(pipeBase.DataIdContainer): 

57 """A data ID container which combines raw data IDs with a tract. 

58 

59 Notes 

60 ----- 

61 Required because we need to add "tract" to the raw data ID keys (defined as 

62 whatever we use for ``src``) when no tract is provided (so that the user is 

63 not required to know which tracts are spanned by the raw data ID). 

64 

65 This subclass of `~lsst.pipe.base.DataIdContainer` assumes that a calexp is 

66 being measured using the detection information, a set of reference 

67 catalogs, from the set of coadds which intersect with the calexp. It needs 

68 the calexp id (e.g. visit, raft, sensor), but is also uses the tract to 

69 decide what set of coadds to use. The references from the tract whose 

70 patches intersect with the calexp are used. 

71 """ 

72 

73 def makeDataRefList(self, namespace): 

74 """Make self.refList from self.idList 

75 """ 

76 if self.datasetType is None: 

77 raise RuntimeError("Must call setDatasetType first") 

78 log = logging.getLogger("meas.base.forcedPhotCcd.PerTractCcdDataIdContainer") 

79 skymap = None 

80 visitTract = collections.defaultdict(set) # Set of tracts for each visit 

81 visitRefs = collections.defaultdict(list) # List of data references for each visit 

82 for dataId in self.idList: 

83 if "tract" not in dataId: 

84 # Discover which tracts the data overlaps 

85 log.info("Reading WCS for components of dataId=%s to determine tracts", dict(dataId)) 

86 if skymap is None: 

87 skymap = namespace.butler.get(namespace.config.coaddName + "Coadd_skyMap") 

88 

89 for ref in namespace.butler.subset("calexp", dataId=dataId): 

90 if not ref.datasetExists("calexp"): 

91 continue 

92 

93 visit = ref.dataId["visit"] 

94 visitRefs[visit].append(ref) 

95 

96 md = ref.get("calexp_md", immediate=True) 

97 wcs = lsst.afw.geom.makeSkyWcs(md) 

98 box = lsst.geom.Box2D(lsst.afw.image.bboxFromMetadata(md)) 

99 # Going with just the nearest tract. Since we're throwing all tracts for the visit 

100 # together, this shouldn't be a problem unless the tracts are much smaller than a CCD. 

101 tract = skymap.findTract(wcs.pixelToSky(box.getCenter())) 

102 if imageOverlapsTract(tract, wcs, box): 

103 visitTract[visit].add(tract.getId()) 

104 else: 

105 self.refList.extend(ref for ref in namespace.butler.subset(self.datasetType, dataId=dataId)) 

106 

107 # Ensure all components of a visit are kept together by putting them all in the same set of tracts 

108 for visit, tractSet in visitTract.items(): 

109 for ref in visitRefs[visit]: 

110 for tract in tractSet: 

111 self.refList.append(namespace.butler.dataRef(datasetType=self.datasetType, 

112 dataId=ref.dataId, tract=tract)) 

113 if visitTract: 

114 tractCounter = collections.Counter() 

115 for tractSet in visitTract.values(): 

116 tractCounter.update(tractSet) 

117 log.info("Number of visits for each tract: %s", dict(tractCounter)) 

118 

119 

120def imageOverlapsTract(tract, imageWcs, imageBox): 

121 """Return whether the given bounding box overlaps the tract given a WCS. 

122 

123 Parameters 

124 ---------- 

125 tract : `lsst.skymap.TractInfo` 

126 TractInfo specifying a tract. 

127 imageWcs : `lsst.afw.geom.SkyWcs` 

128 World coordinate system for the image. 

129 imageBox : `lsst.geom.Box2I` 

130 Bounding box for the image. 

131 

132 Returns 

133 ------- 

134 overlap : `bool` 

135 `True` if the bounding box overlaps the tract; `False` otherwise. 

136 """ 

137 tractPoly = tract.getOuterSkyPolygon() 

138 

139 imagePixelCorners = lsst.geom.Box2D(imageBox).getCorners() 

140 try: 

141 imageSkyCorners = imageWcs.pixelToSky(imagePixelCorners) 

142 except lsst.pex.exceptions.LsstCppException as e: 

143 # Protecting ourselves from awful Wcs solutions in input images 

144 if (not isinstance(e.message, lsst.pex.exceptions.DomainErrorException) 

145 and not isinstance(e.message, lsst.pex.exceptions.RuntimeErrorException)): 

146 raise 

147 return False 

148 

149 imagePoly = lsst.sphgeom.ConvexPolygon.convexHull([coord.getVector() for coord in imageSkyCorners]) 

150 return tractPoly.intersects(imagePoly) # "intersects" also covers "contains" or "is contained by" 

151 

152 

153class ForcedPhotCcdConnections(PipelineTaskConnections, 

154 dimensions=("instrument", "visit", "detector", "skymap", "tract"), 

155 defaultTemplates={"inputCoaddName": "deep", 

156 "inputName": "calexp"}): 

157 inputSchema = cT.InitInput( 

158 doc="Schema for the input measurement catalogs.", 

159 name="{inputCoaddName}Coadd_ref_schema", 

160 storageClass="SourceCatalog", 

161 ) 

162 outputSchema = cT.InitOutput( 

163 doc="Schema for the output forced measurement catalogs.", 

164 name="forced_src_schema", 

165 storageClass="SourceCatalog", 

166 ) 

167 exposure = cT.Input( 

168 doc="Input exposure to perform photometry on.", 

169 name="{inputName}", 

170 storageClass="ExposureF", 

171 dimensions=["instrument", "visit", "detector"], 

172 ) 

173 refCat = cT.Input( 

174 doc="Catalog of shapes and positions at which to force photometry.", 

175 name="{inputCoaddName}Coadd_ref", 

176 storageClass="SourceCatalog", 

177 dimensions=["skymap", "tract", "patch"], 

178 multiple=True, 

179 deferLoad=True, 

180 ) 

181 skyMap = cT.Input( 

182 doc="SkyMap dataset that defines the coordinate system of the reference catalog.", 

183 name=BaseSkyMap.SKYMAP_DATASET_TYPE_NAME, 

184 storageClass="SkyMap", 

185 dimensions=["skymap"], 

186 ) 

187 measCat = cT.Output( 

188 doc="Output forced photometry catalog.", 

189 name="forced_src", 

190 storageClass="SourceCatalog", 

191 dimensions=["instrument", "visit", "detector", "skymap", "tract"], 

192 ) 

193 

194 

195class ForcedPhotCcdConfig(pipeBase.PipelineTaskConfig, 

196 pipelineConnections=ForcedPhotCcdConnections): 

197 """Config class for forced measurement driver task.""" 

198 references = lsst.pex.config.ConfigurableField( 

199 target=MultiBandReferencesTask, 

200 doc="subtask to retrieve reference source catalog" 

201 ) 

202 measurement = lsst.pex.config.ConfigurableField( 

203 target=ForcedMeasurementTask, 

204 doc="subtask to do forced measurement" 

205 ) 

206 coaddName = lsst.pex.config.Field( 

207 doc="coadd name: typically one of deep or goodSeeing", 

208 dtype=str, 

209 default="deep", 

210 ) 

211 doApCorr = lsst.pex.config.Field( 

212 dtype=bool, 

213 default=True, 

214 doc="Run subtask to apply aperture corrections" 

215 ) 

216 applyApCorr = lsst.pex.config.ConfigurableField( 

217 target=ApplyApCorrTask, 

218 doc="Subtask to apply aperture corrections" 

219 ) 

220 catalogCalculation = lsst.pex.config.ConfigurableField( 

221 target=CatalogCalculationTask, 

222 doc="Subtask to run catalogCalculation plugins on catalog" 

223 ) 

224 doApplyUberCal = lsst.pex.config.Field( 

225 dtype=bool, 

226 doc="Apply meas_mosaic ubercal results to input calexps?", 

227 default=False, 

228 deprecated="Deprecated by DM-23352; use doApplyExternalPhotoCalib and doApplyExternalSkyWcs instead", 

229 ) 

230 doApplyExternalPhotoCalib = lsst.pex.config.Field( 

231 dtype=bool, 

232 default=False, 

233 doc=("Whether to apply external photometric calibration via an " 

234 "`lsst.afw.image.PhotoCalib` object. Uses the " 

235 "``externalPhotoCalibName`` field to determine which calibration " 

236 "to load."), 

237 ) 

238 doApplyExternalSkyWcs = lsst.pex.config.Field( 

239 dtype=bool, 

240 default=False, 

241 doc=("Whether to apply external astrometric calibration via an " 

242 "`lsst.afw.geom.SkyWcs` object. Uses ``externalSkyWcsName`` " 

243 "field to determine which calibration to load."), 

244 ) 

245 doApplySkyCorr = lsst.pex.config.Field( 

246 dtype=bool, 

247 default=False, 

248 doc="Apply sky correction?", 

249 ) 

250 includePhotoCalibVar = lsst.pex.config.Field( 

251 dtype=bool, 

252 default=False, 

253 doc="Add photometric calibration variance to warp variance plane?", 

254 ) 

255 externalPhotoCalibName = lsst.pex.config.ChoiceField( 

256 dtype=str, 

257 doc=("Type of external PhotoCalib if ``doApplyExternalPhotoCalib`` is True. " 

258 "Unused for Gen3 middleware."), 

259 default="jointcal", 

260 allowed={ 

261 "jointcal": "Use jointcal_photoCalib", 

262 "fgcm": "Use fgcm_photoCalib", 

263 "fgcm_tract": "Use fgcm_tract_photoCalib" 

264 }, 

265 ) 

266 externalSkyWcsName = lsst.pex.config.ChoiceField( 

267 dtype=str, 

268 doc="Type of external SkyWcs if ``doApplyExternalSkyWcs`` is True. Unused for Gen3 middleware.", 

269 default="jointcal", 

270 allowed={ 

271 "jointcal": "Use jointcal_wcs" 

272 }, 

273 ) 

274 

275 def setDefaults(self): 

276 # Docstring inherited. 

277 # Make catalogCalculation a no-op by default as no modelFlux is setup by default in 

278 # ForcedMeasurementTask 

279 super().setDefaults() 

280 self.measurement.plugins.names |= ['base_LocalPhotoCalib', 'base_LocalWcs'] 

281 self.catalogCalculation.plugins.names = [] 

282 

283 

284class ForcedPhotCcdTask(pipeBase.PipelineTask, pipeBase.CmdLineTask): 

285 """A command-line driver for performing forced measurement on CCD images. 

286 

287 Parameters 

288 ---------- 

289 butler : `lsst.daf.persistence.butler.Butler`, optional 

290 A Butler which will be passed to the references subtask to allow it to 

291 load its schema from disk. Optional, but must be specified if 

292 ``refSchema`` is not; if both are specified, ``refSchema`` takes 

293 precedence. 

294 refSchema : `lsst.afw.table.Schema`, optional 

295 The schema of the reference catalog, passed to the constructor of the 

296 references subtask. Optional, but must be specified if ``butler`` is 

297 not; if both are specified, ``refSchema`` takes precedence. 

298 **kwds 

299 Keyword arguments are passed to the supertask constructor. 

300 

301 Notes 

302 ----- 

303 The `runDataRef` method takes a `~lsst.daf.persistence.ButlerDataRef` argument 

304 that corresponds to a single CCD. This should contain the data ID keys that 

305 correspond to the ``forced_src`` dataset (the output dataset for this 

306 task), which are typically all those used to specify the ``calexp`` dataset 

307 (``visit``, ``raft``, ``sensor`` for LSST data) as well as a coadd tract. 

308 The tract is used to look up the appropriate coadd measurement catalogs to 

309 use as references (e.g. ``deepCoadd_src``; see 

310 :lsst-task:`lsst.meas.base.references.CoaddSrcReferencesTask` for more 

311 information). While the tract must be given as part of the dataRef, the 

312 patches are determined automatically from the bounding box and WCS of the 

313 calexp to be measured, and the filter used to fetch references is set via 

314 the ``filter`` option in the configuration of 

315 :lsst-task:`lsst.meas.base.references.BaseReferencesTask`). 

316 """ 

317 

318 ConfigClass = ForcedPhotCcdConfig 

319 RunnerClass = pipeBase.ButlerInitializedTaskRunner 

320 _DefaultName = "forcedPhotCcd" 

321 dataPrefix = "" 

322 

323 def __init__(self, butler=None, refSchema=None, initInputs=None, **kwds): 

324 super().__init__(**kwds) 

325 

326 if initInputs is not None: 

327 refSchema = initInputs['inputSchema'].schema 

328 

329 self.makeSubtask("references", butler=butler, schema=refSchema) 

330 if refSchema is None: 

331 refSchema = self.references.schema 

332 self.makeSubtask("measurement", refSchema=refSchema) 

333 # It is necessary to get the schema internal to the forced measurement task until such a time 

334 # that the schema is not owned by the measurement task, but is passed in by an external caller 

335 if self.config.doApCorr: 

336 self.makeSubtask("applyApCorr", schema=self.measurement.schema) 

337 self.makeSubtask('catalogCalculation', schema=self.measurement.schema) 

338 self.outputSchema = lsst.afw.table.SourceCatalog(self.measurement.schema) 

339 

340 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

341 inputs = butlerQC.get(inputRefs) 

342 

343 tract = butlerQC.quantum.dataId['tract'] 

344 skyMap = inputs.pop("skyMap") 

345 inputs['refWcs'] = skyMap[tract].getWcs() 

346 

347 inputs['refCat'] = self.mergeAndFilterReferences(inputs['exposure'], inputs['refCat'], 

348 inputs['refWcs']) 

349 

350 inputs['measCat'], inputs['exposureId'] = self.generateMeasCat(inputRefs.exposure.dataId, 

351 inputs['exposure'], 

352 inputs['refCat'], inputs['refWcs'], 

353 "visit_detector") 

354 self.attachFootprints(inputs['measCat'], inputs['refCat'], inputs['exposure'], inputs['refWcs']) 

355 # TODO: apply external calibrations (DM-17062) 

356 outputs = self.run(**inputs) 

357 butlerQC.put(outputs, outputRefs) 

358 

359 def mergeAndFilterReferences(self, exposure, refCats, refWcs): 

360 """Filter reference catalog so that all sources are within the 

361 boundaries of the exposure. 

362 

363 Parameters 

364 ---------- 

365 exposure : `lsst.afw.image.exposure.Exposure` 

366 Exposure to generate the catalog for. 

367 refCats : sequence of `lsst.daf.butler.DeferredDatasetHandle` 

368 Handles for catalogs of shapes and positions at which to force 

369 photometry. 

370 refWcs : `lsst.afw.image.SkyWcs` 

371 Reference world coordinate system. 

372 

373 Returns 

374 ------- 

375 refSources : `lsst.afw.table.SourceCatalog` 

376 Filtered catalog of forced sources to measure. 

377 

378 Notes 

379 ----- 

380 Filtering the reference catalog is currently handled by Gen2 

381 specific methods. To function for Gen3, this method copies 

382 code segments to do the filtering and transformation. The 

383 majority of this code is based on the methods of 

384 lsst.meas.algorithms.loadReferenceObjects.ReferenceObjectLoader 

385 

386 """ 

387 

388 # Step 1: Determine bounds of the exposure photometry will 

389 # be performed on. 

390 expWcs = exposure.getWcs() 

391 expRegion = exposure.getBBox(lsst.afw.image.PARENT) 

392 expBBox = lsst.geom.Box2D(expRegion) 

393 expBoxCorners = expBBox.getCorners() 

394 expSkyCorners = [expWcs.pixelToSky(corner).getVector() for 

395 corner in expBoxCorners] 

396 expPolygon = lsst.sphgeom.ConvexPolygon(expSkyCorners) 

397 

398 # Step 2: Filter out reference catalog sources that are 

399 # not contained within the exposure boundaries, or whose 

400 # parents are not within the exposure boundaries. Note 

401 # that within a single input refCat, the parents always 

402 # appear before the children. 

403 mergedRefCat = None 

404 for refCat in refCats: 

405 refCat = refCat.get() 

406 if mergedRefCat is None: 

407 mergedRefCat = lsst.afw.table.SourceCatalog(refCat.table) 

408 containedIds = {0} # zero as a parent ID means "this is a parent" 

409 for record in refCat: 

410 if expPolygon.contains(record.getCoord().getVector()) and record.getParent() in containedIds: 

411 record.setFootprint(record.getFootprint().transform(refWcs, expWcs, expRegion)) 

412 mergedRefCat.append(record) 

413 containedIds.add(record.getId()) 

414 if mergedRefCat is None: 

415 raise RuntimeError("No reference objects for forced photometry.") 

416 mergedRefCat.sort(lsst.afw.table.SourceTable.getParentKey()) 

417 return mergedRefCat 

418 

419 def generateMeasCat(self, exposureDataId, exposure, refCat, refWcs, idPackerName): 

420 """Generate a measurement catalog for Gen3. 

421 

422 Parameters 

423 ---------- 

424 exposureDataId : `DataId` 

425 Butler dataId for this exposure. 

426 exposure : `lsst.afw.image.exposure.Exposure` 

427 Exposure to generate the catalog for. 

428 refCat : `lsst.afw.table.SourceCatalog` 

429 Catalog of shapes and positions at which to force photometry. 

430 refWcs : `lsst.afw.image.SkyWcs` 

431 Reference world coordinate system. 

432 idPackerName : `str` 

433 Type of ID packer to construct from the registry. 

434 

435 Returns 

436 ------- 

437 measCat : `lsst.afw.table.SourceCatalog` 

438 Catalog of forced sources to measure. 

439 expId : `int` 

440 Unique binary id associated with the input exposure 

441 """ 

442 expId, expBits = exposureDataId.pack(idPackerName, returnMaxBits=True) 

443 idFactory = lsst.afw.table.IdFactory.makeSource(expId, 64 - expBits) 

444 

445 measCat = self.measurement.generateMeasCat(exposure, refCat, refWcs, 

446 idFactory=idFactory) 

447 return measCat, expId 

448 

449 def runDataRef(self, dataRef, psfCache=None): 

450 """Perform forced measurement on a single exposure. 

451 

452 Parameters 

453 ---------- 

454 dataRef : `lsst.daf.persistence.ButlerDataRef` 

455 Passed to the ``references`` subtask to obtain the reference WCS, 

456 the ``getExposure`` method (implemented by derived classes) to 

457 read the measurment image, and the ``fetchReferences`` method to 

458 get the exposure and load the reference catalog (see 

459 :lsst-task`lsst.meas.base.references.CoaddSrcReferencesTask`). 

460 Refer to derived class documentation for details of the datasets 

461 and data ID keys which are used. 

462 psfCache : `int`, optional 

463 Size of PSF cache, or `None`. The size of the PSF cache can have 

464 a significant effect upon the runtime for complicated PSF models. 

465 

466 Notes 

467 ----- 

468 Sources are generated with ``generateMeasCat`` in the ``measurement`` 

469 subtask. These are passed to ``measurement``'s ``run`` method, which 

470 fills the source catalog with the forced measurement results. The 

471 sources are then passed to the ``writeOutputs`` method (implemented by 

472 derived classes) which writes the outputs. 

473 """ 

474 refWcs = self.references.getWcs(dataRef) 

475 exposure = self.getExposure(dataRef) 

476 if psfCache is not None: 

477 exposure.getPsf().setCacheSize(psfCache) 

478 refCat = self.fetchReferences(dataRef, exposure) 

479 

480 measCat = self.measurement.generateMeasCat(exposure, refCat, refWcs, 

481 idFactory=self.makeIdFactory(dataRef)) 

482 self.log.info("Performing forced measurement on %s", dataRef.dataId) 

483 self.attachFootprints(measCat, refCat, exposure, refWcs) 

484 

485 exposureId = self.getExposureId(dataRef) 

486 

487 forcedPhotResult = self.run(measCat, exposure, refCat, refWcs, exposureId=exposureId) 

488 

489 self.writeOutput(dataRef, forcedPhotResult.measCat) 

490 

491 def run(self, measCat, exposure, refCat, refWcs, exposureId=None): 

492 """Perform forced measurement on a single exposure. 

493 

494 Parameters 

495 ---------- 

496 measCat : `lsst.afw.table.SourceCatalog` 

497 The measurement catalog, based on the sources listed in the 

498 reference catalog. 

499 exposure : `lsst.afw.image.Exposure` 

500 The measurement image upon which to perform forced detection. 

501 refCat : `lsst.afw.table.SourceCatalog` 

502 The reference catalog of sources to measure. 

503 refWcs : `lsst.afw.image.SkyWcs` 

504 The WCS for the references. 

505 exposureId : `int` 

506 Optional unique exposureId used for random seed in measurement 

507 task. 

508 

509 Returns 

510 ------- 

511 result : `lsst.pipe.base.Struct` 

512 Structure with fields: 

513 

514 ``measCat`` 

515 Catalog of forced measurement results 

516 (`lsst.afw.table.SourceCatalog`). 

517 """ 

518 self.measurement.run(measCat, exposure, refCat, refWcs, exposureId=exposureId) 

519 if self.config.doApCorr: 

520 self.applyApCorr.run( 

521 catalog=measCat, 

522 apCorrMap=exposure.getInfo().getApCorrMap() 

523 ) 

524 self.catalogCalculation.run(measCat) 

525 

526 return pipeBase.Struct(measCat=measCat) 

527 

528 def makeIdFactory(self, dataRef): 

529 """Create an object that generates globally unique source IDs. 

530 

531 Source IDs are created based on a per-CCD ID and the ID of the CCD 

532 itself. 

533 

534 Parameters 

535 ---------- 

536 dataRef : `lsst.daf.persistence.ButlerDataRef` 

537 Butler data reference. The ``ccdExposureId_bits`` and 

538 ``ccdExposureId`` datasets are accessed. The data ID must have the 

539 keys that correspond to ``ccdExposureId``, which are generally the 

540 same as those that correspond to ``calexp`` (``visit``, ``raft``, 

541 ``sensor`` for LSST data). 

542 """ 

543 expBits = dataRef.get("ccdExposureId_bits") 

544 expId = int(dataRef.get("ccdExposureId")) 

545 return lsst.afw.table.IdFactory.makeSource(expId, 64 - expBits) 

546 

547 def getExposureId(self, dataRef): 

548 return int(dataRef.get("ccdExposureId", immediate=True)) 

549 

550 def fetchReferences(self, dataRef, exposure): 

551 """Get sources that overlap the exposure. 

552 

553 Parameters 

554 ---------- 

555 dataRef : `lsst.daf.persistence.ButlerDataRef` 

556 Butler data reference corresponding to the image to be measured; 

557 should have ``tract``, ``patch``, and ``filter`` keys. 

558 exposure : `lsst.afw.image.Exposure` 

559 The image to be measured (used only to obtain a WCS and bounding 

560 box). 

561 

562 Returns 

563 ------- 

564 referencs : `lsst.afw.table.SourceCatalog` 

565 Catalog of sources that overlap the exposure 

566 

567 Notes 

568 ----- 

569 The returned catalog is sorted by ID and guarantees that all included 

570 children have their parent included and that all Footprints are valid. 

571 

572 All work is delegated to the references subtask; see 

573 :lsst-task:`lsst.meas.base.references.CoaddSrcReferencesTask` 

574 for information about the default behavior. 

575 """ 

576 references = lsst.afw.table.SourceCatalog(self.references.schema) 

577 badParents = set() 

578 unfiltered = self.references.fetchInBox(dataRef, exposure.getBBox(), exposure.getWcs()) 

579 for record in unfiltered: 

580 if record.getFootprint() is None or record.getFootprint().getArea() == 0: 

581 if record.getParent() != 0: 

582 self.log.warning("Skipping reference %s (child of %s) with bad Footprint", 

583 record.getId(), record.getParent()) 

584 else: 

585 self.log.warning("Skipping reference parent %s with bad Footprint", record.getId()) 

586 badParents.add(record.getId()) 

587 elif record.getParent() not in badParents: 

588 references.append(record) 

589 # catalog must be sorted by parent ID for lsst.afw.table.getChildren to work 

590 references.sort(lsst.afw.table.SourceTable.getParentKey()) 

591 return references 

592 

593 def attachFootprints(self, sources, refCat, exposure, refWcs): 

594 r"""Attach footprints to blank sources prior to measurements. 

595 

596 Notes 

597 ----- 

598 `~lsst.afw.detection.Footprint`\ s for forced photometry must be in the 

599 pixel coordinate system of the image being measured, while the actual 

600 detections may start out in a different coordinate system. 

601 

602 Subclasses of this class must implement this method to define how 

603 those `~lsst.afw.detection.Footprint`\ s should be generated. 

604 

605 This default implementation transforms the 

606 `~lsst.afw.detection.Footprint`\ s from the reference catalog from the 

607 reference WCS to the exposure's WcS, which downgrades 

608 `lsst.afw.detection.heavyFootprint.HeavyFootprint`\ s into regular 

609 `~lsst.afw.detection.Footprint`\ s, destroying deblend information. 

610 """ 

611 return self.measurement.attachTransformedFootprints(sources, refCat, exposure, refWcs) 

612 

613 def getExposure(self, dataRef): 

614 """Read input exposure for measurement. 

615 

616 Parameters 

617 ---------- 

618 dataRef : `lsst.daf.persistence.ButlerDataRef` 

619 Butler data reference. 

620 """ 

621 exposure = dataRef.get(self.dataPrefix + "calexp", immediate=True) 

622 

623 if self.config.doApplyExternalPhotoCalib: 

624 source = f"{self.config.externalPhotoCalibName}_photoCalib" 

625 self.log.info("Applying external photoCalib from %s", source) 

626 photoCalib = dataRef.get(source) 

627 exposure.setPhotoCalib(photoCalib) # No need for calibrateImage; having the photoCalib suffices 

628 

629 if self.config.doApplyExternalSkyWcs: 

630 source = f"{self.config.externalSkyWcsName}_wcs" 

631 self.log.info("Applying external skyWcs from %s", source) 

632 skyWcs = dataRef.get(source) 

633 exposure.setWcs(skyWcs) 

634 

635 if self.config.doApplySkyCorr: 

636 self.log.info("Apply sky correction") 

637 skyCorr = dataRef.get("skyCorr") 

638 exposure.maskedImage -= skyCorr.getImage() 

639 

640 return exposure 

641 

642 def writeOutput(self, dataRef, sources): 

643 """Write forced source table 

644 

645 Parameters 

646 ---------- 

647 dataRef : `lsst.daf.persistence.ButlerDataRef` 

648 Butler data reference. The forced_src dataset (with 

649 self.dataPrefix prepended) is all that will be modified. 

650 sources : `lsst.afw.table.SourceCatalog` 

651 Catalog of sources to save. 

652 """ 

653 dataRef.put(sources, self.dataPrefix + "forced_src", flags=lsst.afw.table.SOURCE_IO_NO_FOOTPRINTS) 

654 

655 def getSchemaCatalogs(self): 

656 """The schema catalogs that will be used by this task. 

657 

658 Returns 

659 ------- 

660 schemaCatalogs : `dict` 

661 Dictionary mapping dataset type to schema catalog. 

662 

663 Notes 

664 ----- 

665 There is only one schema for each type of forced measurement. The 

666 dataset type for this measurement is defined in the mapper. 

667 """ 

668 catalog = lsst.afw.table.SourceCatalog(self.measurement.schema) 

669 catalog.getTable().setMetadata(self.measurement.algMetadata) 

670 datasetType = self.dataPrefix + "forced_src" 

671 return {datasetType: catalog} 

672 

673 def _getConfigName(self): 

674 # Documented in superclass. 

675 return self.dataPrefix + "forcedPhotCcd_config" 

676 

677 def _getMetadataName(self): 

678 # Documented in superclass 

679 return self.dataPrefix + "forcedPhotCcd_metadata" 

680 

681 @classmethod 

682 def _makeArgumentParser(cls): 

683 parser = pipeBase.ArgumentParser(name=cls._DefaultName) 

684 parser.add_id_argument("--id", "forced_src", help="data ID with raw CCD keys [+ tract optionally], " 

685 "e.g. --id visit=12345 ccd=1,2 [tract=0]", 

686 ContainerClass=PerTractCcdDataIdContainer) 

687 return parser 

688 

689 

690class ForcedPhotCcdFromDataFrameConnections(PipelineTaskConnections, 

691 dimensions=("instrument", "visit", "detector", "skymap", "tract"), 

692 defaultTemplates={"inputCoaddName": "goodSeeing", 

693 "inputName": "calexp"}): 

694 refCat = cT.Input( 

695 doc="Catalog of positions at which to force photometry.", 

696 name="{inputCoaddName}Diff_fullDiaObjTable", 

697 storageClass="DataFrame", 

698 dimensions=["skymap", "tract", "patch"], 

699 multiple=True, 

700 deferLoad=True, 

701 ) 

702 exposure = cT.Input( 

703 doc="Input exposure to perform photometry on.", 

704 name="{inputName}", 

705 storageClass="ExposureF", 

706 dimensions=["instrument", "visit", "detector"], 

707 ) 

708 measCat = cT.Output( 

709 doc="Output forced photometry catalog.", 

710 name="forced_src_diaObject", 

711 storageClass="SourceCatalog", 

712 dimensions=["instrument", "visit", "detector", "skymap", "tract"], 

713 ) 

714 outputSchema = cT.InitOutput( 

715 doc="Schema for the output forced measurement catalogs.", 

716 name="forced_src_diaObject_schema", 

717 storageClass="SourceCatalog", 

718 ) 

719 

720 

721class ForcedPhotCcdFromDataFrameConfig(ForcedPhotCcdConfig, 

722 pipelineConnections=ForcedPhotCcdFromDataFrameConnections): 

723 def setDefaults(self): 

724 self.measurement.doReplaceWithNoise = False 

725 self.measurement.plugins = ["base_TransformedCentroidFromCoord", "base_PsfFlux"] 

726 self.measurement.copyColumns = {'id': 'diaObjectId', 'coord_ra': 'coord_ra', 'coord_dec': 'coord_dec'} 

727 self.measurement.slots.centroid = "base_TransformedCentroidFromCoord" 

728 self.measurement.slots.psfFlux = "base_PsfFlux" 

729 self.measurement.slots.shape = None 

730 self.catalogCalculation.plugins.names = [] 

731 

732 

733class ForcedPhotCcdFromDataFrameTask(ForcedPhotCcdTask): 

734 """Force Photometry on a per-detector exposure with coords from a DataFrame 

735 

736 Uses input from a DataFrame instead of SourceCatalog 

737 like the base class ForcedPhotCcd does. 

738 Writes out a SourceCatalog so that the downstream WriteForcedSourceTableTask 

739 can be reused with output from this Task. 

740 """ 

741 _DefaultName = "forcedPhotCcdFromDataFrame" 

742 ConfigClass = ForcedPhotCcdFromDataFrameConfig 

743 

744 def __init__(self, butler=None, refSchema=None, initInputs=None, **kwds): 

745 # Parent's init assumes that we have a reference schema; Cannot reuse 

746 pipeBase.PipelineTask.__init__(self, **kwds) 

747 

748 self.makeSubtask("measurement", refSchema=lsst.afw.table.SourceTable.makeMinimalSchema()) 

749 

750 if self.config.doApCorr: 

751 self.makeSubtask("applyApCorr", schema=self.measurement.schema) 

752 self.makeSubtask('catalogCalculation', schema=self.measurement.schema) 

753 self.outputSchema = lsst.afw.table.SourceCatalog(self.measurement.schema) 

754 

755 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

756 inputs = butlerQC.get(inputRefs) 

757 self.log.info("Filtering ref cats: %s", ','.join([str(i.dataId) for i in inputs['refCat']])) 

758 refCat = self.df2RefCat([i.get(parameters={"columns": ['diaObjectId', 'ra', 'decl']}) 

759 for i in inputs['refCat']], 

760 inputs['exposure'].getBBox(), inputs['exposure'].getWcs()) 

761 inputs['refCat'] = refCat 

762 inputs['refWcs'] = inputs['exposure'].getWcs() 

763 inputs['measCat'], inputs['exposureId'] = self.generateMeasCat(inputRefs.exposure.dataId, 

764 inputs['exposure'], inputs['refCat'], 

765 inputs['refWcs'], 

766 "visit_detector") 

767 outputs = self.run(**inputs) 

768 butlerQC.put(outputs, outputRefs) 

769 

770 def df2RefCat(self, dfList, exposureBBox, exposureWcs): 

771 """Convert list of DataFrames to reference catalog 

772 

773 Concatenate list of DataFrames presumably from multiple patches and 

774 downselect rows that overlap the exposureBBox using the exposureWcs. 

775 

776 Parameters 

777 ---------- 

778 dfList : `list` of `pandas.DataFrame` 

779 Each element containst diaObjects with ra/decl position in degrees 

780 Columns 'diaObjectId', 'ra', 'decl' are expected 

781 exposureBBox : `lsst.geom.Box2I` 

782 Bounding box on which to select rows that overlap 

783 exposureWcs : `lsst.afw.geom.SkyWcs` 

784 World coordinate system to convert sky coords in ref cat to 

785 pixel coords with which to compare with exposureBBox 

786 

787 Returns 

788 ------- 

789 refCat : `lsst.afw.table.SourceTable` 

790 Source Catalog with minimal schema that overlaps exposureBBox 

791 """ 

792 df = pd.concat(dfList) 

793 # translate ra/decl coords in dataframe to detector pixel coords 

794 # to down select rows that overlap the detector bbox 

795 mapping = exposureWcs.getTransform().getMapping() 

796 x, y = mapping.applyInverse(np.array(df[['ra', 'decl']].values*2*np.pi/360).T) 

797 inBBox = exposureBBox.contains(x, y) 

798 refCat = self.df2SourceCat(df[inBBox]) 

799 return refCat 

800 

801 def df2SourceCat(self, df): 

802 """Create minimal schema SourceCatalog from a pandas DataFrame. 

803 

804 The forced measurement subtask expects this as input. 

805 

806 Parameters 

807 ---------- 

808 df : `pandas.DataFrame` 

809 DiaObjects with locations and ids. 

810 

811 Returns 

812 ------- 

813 outputCatalog : `lsst.afw.table.SourceTable` 

814 Output catalog with minimal schema. 

815 """ 

816 schema = lsst.afw.table.SourceTable.makeMinimalSchema() 

817 outputCatalog = lsst.afw.table.SourceCatalog(schema) 

818 outputCatalog.reserve(len(df)) 

819 

820 for id, diaObjectId, ra, decl in df[['diaObjectId', 'ra', 'decl']].itertuples(): 

821 outputRecord = outputCatalog.addNew() 

822 outputRecord.setId(diaObjectId) 

823 outputRecord.setCoord(lsst.geom.SpherePoint(ra, decl, lsst.geom.degrees)) 

824 return outputCatalog