Coverage for python/lsst/pipe/tasks/processCcdWithFakes.py: 24%

212 statements  

« prev     ^ index     » next       coverage.py v6.4.4, created at 2022-09-30 10:43 +0000

1# This file is part of pipe_tasks. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22""" 

23Insert fake sources into calexps 

24""" 

25 

26__all__ = ["ProcessCcdWithFakesConfig", "ProcessCcdWithFakesTask", 

27 "ProcessCcdWithVariableFakesConfig", "ProcessCcdWithVariableFakesTask"] 

28 

29import numpy as np 

30import pandas as pd 

31 

32import lsst.pex.config as pexConfig 

33import lsst.pipe.base as pipeBase 

34 

35from .insertFakes import InsertFakesTask 

36from lsst.afw.table import SourceTable 

37from lsst.obs.base import ExposureIdInfo 

38from lsst.pipe.base import PipelineTask, PipelineTaskConfig, PipelineTaskConnections 

39import lsst.pipe.base.connectionTypes as cT 

40import lsst.afw.table as afwTable 

41from lsst.skymap import BaseSkyMap 

42from lsst.pipe.tasks.calibrate import CalibrateTask 

43 

44 

45class ProcessCcdWithFakesConnections(PipelineTaskConnections, 

46 dimensions=("instrument", "visit", "detector"), 

47 defaultTemplates={"coaddName": "deep", 

48 "wcsName": "jointcal", 

49 "photoCalibName": "jointcal", 

50 "fakesType": "fakes_"}): 

51 skyMap = cT.Input( 

52 doc="Input definition of geometry/bbox and projection/wcs for " 

53 "template exposures. Needed to test which tract to generate ", 

54 name=BaseSkyMap.SKYMAP_DATASET_TYPE_NAME, 

55 dimensions=("skymap",), 

56 storageClass="SkyMap", 

57 ) 

58 

59 exposure = cT.Input( 

60 doc="Exposure into which fakes are to be added.", 

61 name="calexp", 

62 storageClass="ExposureF", 

63 dimensions=("instrument", "visit", "detector") 

64 ) 

65 

66 fakeCats = cT.Input( 

67 doc="Set of catalogs of fake sources to draw inputs from. We " 

68 "concatenate the tract catalogs for detectorVisits that cover " 

69 "multiple tracts.", 

70 name="{fakesType}fakeSourceCat", 

71 storageClass="DataFrame", 

72 dimensions=("tract", "skymap"), 

73 deferLoad=True, 

74 multiple=True, 

75 ) 

76 

77 externalSkyWcsTractCatalog = cT.Input( 

78 doc=("Per-tract, per-visit wcs calibrations. These catalogs use the detector " 

79 "id for the catalog id, sorted on id for fast lookup."), 

80 name="{wcsName}SkyWcsCatalog", 

81 storageClass="ExposureCatalog", 

82 dimensions=("instrument", "visit", "tract", "skymap"), 

83 deferLoad=True, 

84 multiple=True, 

85 ) 

86 

87 externalSkyWcsGlobalCatalog = cT.Input( 

88 doc=("Per-visit wcs calibrations computed globally (with no tract information). " 

89 "These catalogs use the detector id for the catalog id, sorted on id for " 

90 "fast lookup."), 

91 name="{wcsName}SkyWcsCatalog", 

92 storageClass="ExposureCatalog", 

93 dimensions=("instrument", "visit"), 

94 ) 

95 

96 externalPhotoCalibTractCatalog = cT.Input( 

97 doc=("Per-tract, per-visit photometric calibrations. These catalogs use the " 

98 "detector id for the catalog id, sorted on id for fast lookup."), 

99 name="{photoCalibName}PhotoCalibCatalog", 

100 storageClass="ExposureCatalog", 

101 dimensions=("instrument", "visit", "tract"), 

102 deferLoad=True, 

103 multiple=True, 

104 ) 

105 

106 externalPhotoCalibGlobalCatalog = cT.Input( 

107 doc=("Per-visit photometric calibrations. These catalogs use the " 

108 "detector id for the catalog id, sorted on id for fast lookup."), 

109 name="{photoCalibName}PhotoCalibCatalog", 

110 storageClass="ExposureCatalog", 

111 dimensions=("instrument", "visit"), 

112 ) 

113 

114 icSourceCat = cT.Input( 

115 doc="Catalog of calibration sources", 

116 name="icSrc", 

117 storageClass="SourceCatalog", 

118 dimensions=("instrument", "visit", "detector") 

119 ) 

120 

121 sfdSourceCat = cT.Input( 

122 doc="Catalog of calibration sources", 

123 name="src", 

124 storageClass="SourceCatalog", 

125 dimensions=("instrument", "visit", "detector") 

126 ) 

127 

128 outputExposure = cT.Output( 

129 doc="Exposure with fake sources added.", 

130 name="{fakesType}calexp", 

131 storageClass="ExposureF", 

132 dimensions=("instrument", "visit", "detector") 

133 ) 

134 

135 outputCat = cT.Output( 

136 doc="Source catalog produced in calibrate task with fakes also measured.", 

137 name="{fakesType}src", 

138 storageClass="SourceCatalog", 

139 dimensions=("instrument", "visit", "detector"), 

140 ) 

141 

142 def __init__(self, *, config=None): 

143 super().__init__(config=config) 

144 

145 if not config.doApplyExternalGlobalPhotoCalib: 

146 self.inputs.remove("externalPhotoCalibGlobalCatalog") 

147 if not config.doApplyExternalTractPhotoCalib: 

148 self.inputs.remove("externalPhotoCalibTractCatalog") 

149 

150 if not config.doApplyExternalGlobalSkyWcs: 

151 self.inputs.remove("externalSkyWcsGlobalCatalog") 

152 if not config.doApplyExternalTractSkyWcs: 

153 self.inputs.remove("externalSkyWcsTractCatalog") 

154 

155 

156class ProcessCcdWithFakesConfig(PipelineTaskConfig, 

157 pipelineConnections=ProcessCcdWithFakesConnections): 

158 """Config for inserting fake sources 

159 

160 Notes 

161 ----- 

162 The default column names are those from the UW sims database. 

163 """ 

164 

165 doApplyExternalGlobalPhotoCalib = pexConfig.Field( 

166 dtype=bool, 

167 default=False, 

168 doc="Whether to apply an external photometric calibration via an " 

169 "`lsst.afw.image.PhotoCalib` object. Uses the " 

170 "`externalPhotoCalibName` config option to determine which " 

171 "calibration to use. Uses a global calibration." 

172 ) 

173 

174 doApplyExternalTractPhotoCalib = pexConfig.Field( 

175 dtype=bool, 

176 default=False, 

177 doc="Whether to apply an external photometric calibration via an " 

178 "`lsst.afw.image.PhotoCalib` object. Uses the " 

179 "`externalPhotoCalibName` config option to determine which " 

180 "calibration to use. Uses a per tract calibration." 

181 ) 

182 

183 externalPhotoCalibName = pexConfig.ChoiceField( 

184 doc="What type of external photo calib to use.", 

185 dtype=str, 

186 default="jointcal", 

187 allowed={"jointcal": "Use jointcal_photoCalib", 

188 "fgcm": "Use fgcm_photoCalib", 

189 "fgcm_tract": "Use fgcm_tract_photoCalib"} 

190 ) 

191 

192 doApplyExternalGlobalSkyWcs = pexConfig.Field( 

193 dtype=bool, 

194 default=False, 

195 doc="Whether to apply an external astrometric calibration via an " 

196 "`lsst.afw.geom.SkyWcs` object. Uses the " 

197 "`externalSkyWcsName` config option to determine which " 

198 "calibration to use. Uses a global calibration." 

199 ) 

200 

201 doApplyExternalTractSkyWcs = pexConfig.Field( 

202 dtype=bool, 

203 default=False, 

204 doc="Whether to apply an external astrometric calibration via an " 

205 "`lsst.afw.geom.SkyWcs` object. Uses the " 

206 "`externalSkyWcsName` config option to determine which " 

207 "calibration to use. Uses a per tract calibration." 

208 ) 

209 

210 externalSkyWcsName = pexConfig.ChoiceField( 

211 doc="What type of updated WCS calib to use.", 

212 dtype=str, 

213 default="jointcal", 

214 allowed={"jointcal": "Use jointcal_wcs"} 

215 ) 

216 

217 coaddName = pexConfig.Field( 

218 doc="The name of the type of coadd used", 

219 dtype=str, 

220 default="deep", 

221 ) 

222 

223 srcFieldsToCopy = pexConfig.ListField( 

224 dtype=str, 

225 default=("calib_photometry_reserved", "calib_photometry_used", "calib_astrometry_used", 

226 "calib_psf_candidate", "calib_psf_used", "calib_psf_reserved"), 

227 doc=("Fields to copy from the `src` catalog to the output catalog " 

228 "for matching sources Any missing fields will trigger a " 

229 "RuntimeError exception.") 

230 ) 

231 

232 matchRadiusPix = pexConfig.Field( 

233 dtype=float, 

234 default=3, 

235 doc=("Match radius for matching icSourceCat objects to sourceCat objects (pixels)"), 

236 ) 

237 

238 doMatchVisit = pexConfig.Field( 

239 dtype=bool, 

240 default=False, 

241 doc="Match visit to trim the fakeCat" 

242 ) 

243 

244 calibrate = pexConfig.ConfigurableField(target=CalibrateTask, 

245 doc="The calibration task to use.") 

246 

247 insertFakes = pexConfig.ConfigurableField(target=InsertFakesTask, 

248 doc="Configuration for the fake sources") 

249 

250 def setDefaults(self): 

251 super().setDefaults() 

252 self.calibrate.measurement.plugins["base_PixelFlags"].masksFpAnywhere.append("FAKE") 

253 self.calibrate.measurement.plugins["base_PixelFlags"].masksFpCenter.append("FAKE") 

254 self.calibrate.doAstrometry = False 

255 self.calibrate.doWriteMatches = False 

256 self.calibrate.doPhotoCal = False 

257 self.calibrate.detection.reEstimateBackground = False 

258 

259 

260class ProcessCcdWithFakesTask(PipelineTask): 

261 """Insert fake objects into calexps. 

262 

263 Add fake stars and galaxies to the given calexp, specified in the dataRef. Galaxy parameters are read in 

264 from the specified file and then modelled using galsim. Re-runs characterize image and calibrate image to 

265 give a new background estimation and measurement of the calexp. 

266 

267 `ProcessFakeSourcesTask` inherits six functions from insertFakesTask that make images of the fake 

268 sources and then add them to the calexp. 

269 

270 `addPixCoords` 

271 Use the WCS information to add the pixel coordinates of each source 

272 Adds an ``x`` and ``y`` column to the catalog of fake sources. 

273 `trimFakeCat` 

274 Trim the fake cat to about the size of the input image. 

275 `mkFakeGalsimGalaxies` 

276 Use Galsim to make fake double sersic galaxies for each set of galaxy parameters in the input file. 

277 `mkFakeStars` 

278 Use the PSF information from the calexp to make a fake star using the magnitude information from the 

279 input file. 

280 `cleanCat` 

281 Remove rows of the input fake catalog which have half light radius, of either the bulge or the disk, 

282 that are 0. 

283 `addFakeSources` 

284 Add the fake sources to the calexp. 

285 

286 Notes 

287 ----- 

288 The ``calexp`` with fake souces added to it is written out as the datatype ``calexp_fakes``. 

289 """ 

290 

291 _DefaultName = "processCcdWithFakes" 

292 ConfigClass = ProcessCcdWithFakesConfig 

293 

294 def __init__(self, schema=None, butler=None, **kwargs): 

295 """Initalize things! This should go above in the class docstring 

296 """ 

297 

298 super().__init__(**kwargs) 

299 

300 if schema is None: 

301 schema = SourceTable.makeMinimalSchema() 

302 self.schema = schema 

303 self.makeSubtask("insertFakes") 

304 self.makeSubtask("calibrate") 

305 

306 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

307 inputs = butlerQC.get(inputRefs) 

308 detectorId = inputs["exposure"].getInfo().getDetector().getId() 

309 

310 if 'exposureIdInfo' not in inputs.keys(): 

311 expId, expBits = butlerQC.quantum.dataId.pack("visit_detector", returnMaxBits=True) 

312 inputs['exposureIdInfo'] = ExposureIdInfo(expId, expBits) 

313 

314 expWcs = inputs["exposure"].getWcs() 

315 tractId = inputs["skyMap"].findTract( 

316 expWcs.pixelToSky(inputs["exposure"].getBBox().getCenter())).tract_id 

317 if not self.config.doApplyExternalGlobalSkyWcs and not self.config.doApplyExternalTractSkyWcs: 

318 inputs["wcs"] = expWcs 

319 elif self.config.doApplyExternalGlobalSkyWcs: 

320 externalSkyWcsCatalog = inputs["externalSkyWcsGlobalCatalog"] 

321 row = externalSkyWcsCatalog.find(detectorId) 

322 inputs["wcs"] = row.getWcs() 

323 elif self.config.doApplyExternalTractSkyWcs: 

324 externalSkyWcsCatalogList = inputs["externalSkyWcsTractCatalog"] 

325 externalSkyWcsCatalog = None 

326 for externalSkyWcsCatalogRef in externalSkyWcsCatalogList: 

327 if externalSkyWcsCatalogRef.dataId["tract"] == tractId: 

328 externalSkyWcsCatalog = externalSkyWcsCatalogRef.get( 

329 datasetType=self.config.connections.externalSkyWcsTractCatalog) 

330 break 

331 if externalSkyWcsCatalog is None: 

332 usedTract = externalSkyWcsCatalogList[-1].dataId["tract"] 

333 self.log.warn( 

334 f"Warning, external SkyWcs for tract {tractId} not found. Using tract {usedTract} " 

335 "instead.") 

336 externalSkyWcsCatalog = externalSkyWcsCatalogList[-1].get( 

337 datasetType=self.config.connections.externalSkyWcsTractCatalog) 

338 row = externalSkyWcsCatalog.find(detectorId) 

339 inputs["wcs"] = row.getWcs() 

340 

341 if not self.config.doApplyExternalGlobalPhotoCalib and not self.config.doApplyExternalTractPhotoCalib: 

342 inputs["photoCalib"] = inputs["exposure"].getPhotoCalib() 

343 elif self.config.doApplyExternalGlobalPhotoCalib: 

344 externalPhotoCalibCatalog = inputs["externalPhotoCalibGlobalCatalog"] 

345 row = externalPhotoCalibCatalog.find(detectorId) 

346 inputs["photoCalib"] = row.getPhotoCalib() 

347 elif self.config.doApplyExternalTractPhotoCalib: 

348 externalPhotoCalibCatalogList = inputs["externalPhotoCalibTractCatalog"] 

349 externalPhotoCalibCatalog = None 

350 for externalPhotoCalibCatalogRef in externalPhotoCalibCatalogList: 

351 if externalPhotoCalibCatalogRef.dataId["tract"] == tractId: 

352 externalPhotoCalibCatalog = externalPhotoCalibCatalogRef.get( 

353 datasetType=self.config.connections.externalPhotoCalibTractCatalog) 

354 break 

355 if externalPhotoCalibCatalog is None: 

356 usedTract = externalPhotoCalibCatalogList[-1].dataId["tract"] 

357 self.log.warn( 

358 f"Warning, external PhotoCalib for tract {tractId} not found. Using tract {usedTract} " 

359 "instead.") 

360 externalPhotoCalibCatalog = externalPhotoCalibCatalogList[-1].get( 

361 datasetType=self.config.connections.externalPhotoCalibTractCatalog) 

362 row = externalPhotoCalibCatalog.find(detectorId) 

363 inputs["photoCalib"] = row.getPhotoCalib() 

364 

365 outputs = self.run(**inputs) 

366 butlerQC.put(outputs, outputRefs) 

367 

368 def run(self, fakeCats, exposure, skyMap, wcs=None, photoCalib=None, exposureIdInfo=None, 

369 icSourceCat=None, sfdSourceCat=None, externalSkyWcsGlobalCatalog=None, 

370 externalSkyWcsTractCatalog=None, externalPhotoCalibGlobalCatalog=None, 

371 externalPhotoCalibTractCatalog=None): 

372 """Add fake sources to a calexp and then run detection, deblending and measurement. 

373 

374 Parameters 

375 ---------- 

376 fakeCats : `list` of `lsst.daf.butler.DeferredDatasetHandle` 

377 Set of tract level fake catalogs that potentially cover 

378 this detectorVisit. 

379 exposure : `lsst.afw.image.exposure.exposure.ExposureF` 

380 The exposure to add the fake sources to 

381 skyMap : `lsst.skymap.SkyMap` 

382 SkyMap defining the tracts and patches the fakes are stored over. 

383 wcs : `lsst.afw.geom.SkyWcs` 

384 WCS to use to add fake sources 

385 photoCalib : `lsst.afw.image.photoCalib.PhotoCalib` 

386 Photometric calibration to be used to calibrate the fake sources 

387 exposureIdInfo : `lsst.obs.base.ExposureIdInfo` 

388 icSourceCat : `lsst.afw.table.SourceCatalog` 

389 Default : None 

390 Catalog to take the information about which sources were used for calibration from. 

391 sfdSourceCat : `lsst.afw.table.SourceCatalog` 

392 Default : None 

393 Catalog produced by singleFrameDriver, needed to copy some calibration flags from. 

394 

395 Returns 

396 ------- 

397 resultStruct : `lsst.pipe.base.struct.Struct` 

398 contains : outputExposure : `lsst.afw.image.exposure.exposure.ExposureF` 

399 outputCat : `lsst.afw.table.source.source.SourceCatalog` 

400 

401 Notes 

402 ----- 

403 Adds pixel coordinates for each source to the fakeCat and removes objects with bulge or disk half 

404 light radius = 0 (if ``config.cleanCat = True``). These columns are called ``x`` and ``y`` and are in 

405 pixels. 

406 

407 Adds the ``Fake`` mask plane to the exposure which is then set by `addFakeSources` to mark where fake 

408 sources have been added. Uses the information in the ``fakeCat`` to make fake galaxies (using galsim) 

409 and fake stars, using the PSF models from the PSF information for the calexp. These are then added to 

410 the calexp and the calexp with fakes included returned. 

411 

412 The galsim galaxies are made using a double sersic profile, one for the bulge and one for the disk, 

413 this is then convolved with the PSF at that point. 

414 

415 If exposureIdInfo is not provided then the SourceCatalog IDs will not be globally unique. 

416 """ 

417 fakeCat = self.composeFakeCat(fakeCats, skyMap) 

418 

419 if wcs is None: 

420 wcs = exposure.getWcs() 

421 

422 if photoCalib is None: 

423 photoCalib = exposure.getPhotoCalib() 

424 

425 if self.config.doMatchVisit: 

426 fakeCat = self.getVisitMatchedFakeCat(fakeCat, exposure) 

427 

428 self.insertFakes.run(fakeCat, exposure, wcs, photoCalib) 

429 

430 # detect, deblend and measure sources 

431 if exposureIdInfo is None: 

432 exposureIdInfo = ExposureIdInfo() 

433 returnedStruct = self.calibrate.run(exposure, exposureIdInfo=exposureIdInfo) 

434 sourceCat = returnedStruct.sourceCat 

435 

436 sourceCat = self.copyCalibrationFields(sfdSourceCat, sourceCat, self.config.srcFieldsToCopy) 

437 

438 resultStruct = pipeBase.Struct(outputExposure=exposure, outputCat=sourceCat) 

439 return resultStruct 

440 

441 def composeFakeCat(self, fakeCats, skyMap): 

442 """Concatenate the fakeCats from tracts that may cover the exposure. 

443 

444 Parameters 

445 ---------- 

446 fakeCats : `list` of `lst.daf.butler.DeferredDatasetHandle` 

447 Set of fake cats to concatenate. 

448 skyMap : `lsst.skymap.SkyMap` 

449 SkyMap defining the geometry of the tracts and patches. 

450 

451 Returns 

452 ------- 

453 combinedFakeCat : `pandas.DataFrame` 

454 All fakes that cover the inner polygon of the tracts in this 

455 quantum. 

456 """ 

457 if len(fakeCats) == 1: 

458 return fakeCats[0].get( 

459 datasetType=self.config.connections.fakeCats) 

460 outputCat = [] 

461 for fakeCatRef in fakeCats: 

462 cat = fakeCatRef.get( 

463 datasetType=self.config.connections.fakeCats) 

464 tractId = fakeCatRef.dataId["tract"] 

465 # Make sure all data is within the inner part of the tract. 

466 outputCat.append(cat[ 

467 skyMap.findTractIdArray(cat[self.config.insertFakes.ra_col], 

468 cat[self.config.insertFakes.dec_col], 

469 degrees=False) 

470 == tractId]) 

471 

472 return pd.concat(outputCat) 

473 

474 def getVisitMatchedFakeCat(self, fakeCat, exposure): 

475 """Trim the fakeCat to select particular visit 

476 

477 Parameters 

478 ---------- 

479 fakeCat : `pandas.core.frame.DataFrame` 

480 The catalog of fake sources to add to the exposure 

481 exposure : `lsst.afw.image.exposure.exposure.ExposureF` 

482 The exposure to add the fake sources to 

483 

484 Returns 

485 ------- 

486 movingFakeCat : `pandas.DataFrame` 

487 All fakes that belong to the visit 

488 """ 

489 selected = exposure.getInfo().getVisitInfo().getId() == fakeCat["visit"] 

490 

491 return fakeCat[selected] 

492 

493 def copyCalibrationFields(self, calibCat, sourceCat, fieldsToCopy): 

494 """Match sources in calibCat and sourceCat and copy the specified fields 

495 

496 Parameters 

497 ---------- 

498 calibCat : `lsst.afw.table.SourceCatalog` 

499 Catalog from which to copy fields. 

500 sourceCat : `lsst.afw.table.SourceCatalog` 

501 Catalog to which to copy fields. 

502 fieldsToCopy : `lsst.pex.config.listField.List` 

503 Fields to copy from calibCat to SoourceCat. 

504 

505 Returns 

506 ------- 

507 newCat : `lsst.afw.table.SourceCatalog` 

508 Catalog which includes the copied fields. 

509 

510 The fields copied are those specified by `fieldsToCopy` that actually exist 

511 in the schema of `calibCat`. 

512 

513 This version was based on and adapted from the one in calibrateTask. 

514 """ 

515 

516 # Make a new SourceCatalog with the data from sourceCat so that we can add the new columns to it 

517 sourceSchemaMapper = afwTable.SchemaMapper(sourceCat.schema) 

518 sourceSchemaMapper.addMinimalSchema(sourceCat.schema, True) 

519 

520 calibSchemaMapper = afwTable.SchemaMapper(calibCat.schema, sourceCat.schema) 

521 

522 # Add the desired columns from the option fieldsToCopy 

523 missingFieldNames = [] 

524 for fieldName in fieldsToCopy: 

525 if fieldName in calibCat.schema: 

526 schemaItem = calibCat.schema.find(fieldName) 

527 calibSchemaMapper.editOutputSchema().addField(schemaItem.getField()) 

528 schema = calibSchemaMapper.editOutputSchema() 

529 calibSchemaMapper.addMapping(schemaItem.getKey(), schema.find(fieldName).getField()) 

530 else: 

531 missingFieldNames.append(fieldName) 

532 if missingFieldNames: 

533 raise RuntimeError(f"calibCat is missing fields {missingFieldNames} specified in " 

534 "fieldsToCopy") 

535 

536 if "calib_detected" not in calibSchemaMapper.getOutputSchema(): 

537 self.calibSourceKey = calibSchemaMapper.addOutputField(afwTable.Field["Flag"]("calib_detected", 

538 "Source was detected as an icSource")) 

539 else: 

540 self.calibSourceKey = None 

541 

542 schema = calibSchemaMapper.getOutputSchema() 

543 newCat = afwTable.SourceCatalog(schema) 

544 newCat.reserve(len(sourceCat)) 

545 newCat.extend(sourceCat, sourceSchemaMapper) 

546 

547 # Set the aliases so it doesn't complain. 

548 for k, v in sourceCat.schema.getAliasMap().items(): 

549 newCat.schema.getAliasMap().set(k, v) 

550 

551 select = newCat["deblend_nChild"] == 0 

552 matches = afwTable.matchXy(newCat[select], calibCat, self.config.matchRadiusPix) 

553 # Check that no sourceCat sources are listed twice (we already know 

554 # that each match has a unique calibCat source ID, due to using 

555 # that ID as the key in bestMatches) 

556 numMatches = len(matches) 

557 numUniqueSources = len(set(m[1].getId() for m in matches)) 

558 if numUniqueSources != numMatches: 

559 self.log.warning("%d calibCat sources matched only %d sourceCat sources", numMatches, 

560 numUniqueSources) 

561 

562 self.log.info("Copying flags from calibCat to sourceCat for %s sources", numMatches) 

563 

564 # For each match: set the calibSourceKey flag and copy the desired 

565 # fields 

566 for src, calibSrc, d in matches: 

567 if self.calibSourceKey: 

568 src.setFlag(self.calibSourceKey, True) 

569 # src.assign copies the footprint from calibSrc, which we don't want 

570 # (DM-407) 

571 # so set calibSrc's footprint to src's footprint before src.assign, 

572 # then restore it 

573 calibSrcFootprint = calibSrc.getFootprint() 

574 try: 

575 calibSrc.setFootprint(src.getFootprint()) 

576 src.assign(calibSrc, calibSchemaMapper) 

577 finally: 

578 calibSrc.setFootprint(calibSrcFootprint) 

579 

580 return newCat 

581 

582 

583class ProcessCcdWithVariableFakesConnections(ProcessCcdWithFakesConnections): 

584 ccdVisitFakeMagnitudes = cT.Output( 

585 doc="Catalog of fakes with magnitudes scattered for this ccdVisit.", 

586 name="{fakesType}ccdVisitFakeMagnitudes", 

587 storageClass="DataFrame", 

588 dimensions=("instrument", "visit", "detector"), 

589 ) 

590 

591 

592class ProcessCcdWithVariableFakesConfig(ProcessCcdWithFakesConfig, 

593 pipelineConnections=ProcessCcdWithVariableFakesConnections): 

594 scatterSize = pexConfig.RangeField( 

595 dtype=float, 

596 default=0.4, 

597 min=0, 

598 max=100, 

599 doc="Amount of scatter to add to the visit magnitude for variable " 

600 "sources." 

601 ) 

602 

603 

604class ProcessCcdWithVariableFakesTask(ProcessCcdWithFakesTask): 

605 """As ProcessCcdWithFakes except add variablity to the fakes catalog 

606 magnitude in the observed band for this ccdVisit. 

607 

608 Additionally, write out the modified magnitudes to the Butler. 

609 """ 

610 

611 _DefaultName = "processCcdWithVariableFakes" 

612 ConfigClass = ProcessCcdWithVariableFakesConfig 

613 

614 def run(self, fakeCats, exposure, skyMap, wcs=None, photoCalib=None, exposureIdInfo=None, 

615 icSourceCat=None, sfdSourceCat=None): 

616 """Add fake sources to a calexp and then run detection, deblending and measurement. 

617 

618 Parameters 

619 ---------- 

620 fakeCat : `pandas.core.frame.DataFrame` 

621 The catalog of fake sources to add to the exposure 

622 exposure : `lsst.afw.image.exposure.exposure.ExposureF` 

623 The exposure to add the fake sources to 

624 skyMap : `lsst.skymap.SkyMap` 

625 SkyMap defining the tracts and patches the fakes are stored over. 

626 wcs : `lsst.afw.geom.SkyWcs` 

627 WCS to use to add fake sources 

628 photoCalib : `lsst.afw.image.photoCalib.PhotoCalib` 

629 Photometric calibration to be used to calibrate the fake sources 

630 exposureIdInfo : `lsst.obs.base.ExposureIdInfo` 

631 icSourceCat : `lsst.afw.table.SourceCatalog` 

632 Default : None 

633 Catalog to take the information about which sources were used for calibration from. 

634 sfdSourceCat : `lsst.afw.table.SourceCatalog` 

635 Default : None 

636 Catalog produced by singleFrameDriver, needed to copy some calibration flags from. 

637 

638 Returns 

639 ------- 

640 resultStruct : `lsst.pipe.base.struct.Struct` 

641 Results Strcut containing: 

642 

643 - outputExposure : Exposure with added fakes 

644 (`lsst.afw.image.exposure.exposure.ExposureF`) 

645 - outputCat : Catalog with detected fakes 

646 (`lsst.afw.table.source.source.SourceCatalog`) 

647 - ccdVisitFakeMagnitudes : Magnitudes that these fakes were 

648 inserted with after being scattered (`pandas.DataFrame`) 

649 

650 Notes 

651 ----- 

652 Adds pixel coordinates for each source to the fakeCat and removes objects with bulge or disk half 

653 light radius = 0 (if ``config.cleanCat = True``). These columns are called ``x`` and ``y`` and are in 

654 pixels. 

655 

656 Adds the ``Fake`` mask plane to the exposure which is then set by `addFakeSources` to mark where fake 

657 sources have been added. Uses the information in the ``fakeCat`` to make fake galaxies (using galsim) 

658 and fake stars, using the PSF models from the PSF information for the calexp. These are then added to 

659 the calexp and the calexp with fakes included returned. 

660 

661 The galsim galaxies are made using a double sersic profile, one for the bulge and one for the disk, 

662 this is then convolved with the PSF at that point. 

663 

664 If exposureIdInfo is not provided then the SourceCatalog IDs will not be globally unique. 

665 """ 

666 fakeCat = self.composeFakeCat(fakeCats, skyMap) 

667 

668 if wcs is None: 

669 wcs = exposure.getWcs() 

670 

671 if photoCalib is None: 

672 photoCalib = exposure.getPhotoCalib() 

673 

674 if exposureIdInfo is None: 

675 exposureIdInfo = ExposureIdInfo() 

676 

677 band = exposure.getFilter().bandLabel 

678 ccdVisitMagnitudes = self.addVariablity(fakeCat, band, exposure, photoCalib, exposureIdInfo) 

679 

680 self.insertFakes.run(fakeCat, exposure, wcs, photoCalib) 

681 

682 # detect, deblend and measure sources 

683 returnedStruct = self.calibrate.run(exposure, exposureIdInfo=exposureIdInfo) 

684 sourceCat = returnedStruct.sourceCat 

685 

686 sourceCat = self.copyCalibrationFields(sfdSourceCat, sourceCat, self.config.srcFieldsToCopy) 

687 

688 resultStruct = pipeBase.Struct(outputExposure=exposure, 

689 outputCat=sourceCat, 

690 ccdVisitFakeMagnitudes=ccdVisitMagnitudes) 

691 return resultStruct 

692 

693 def addVariablity(self, fakeCat, band, exposure, photoCalib, exposureIdInfo): 

694 """Add scatter to the fake catalog visit magnitudes. 

695 

696 Currently just adds a simple Gaussian scatter around the static fake 

697 magnitude. This function could be modified to return any number of 

698 fake variability. 

699 

700 Parameters 

701 ---------- 

702 fakeCat : `pandas.DataFrame` 

703 Catalog of fakes to modify magnitudes of. 

704 band : `str` 

705 Current observing band to modify. 

706 exposure : `lsst.afw.image.ExposureF` 

707 Exposure fakes will be added to. 

708 photoCalib : `lsst.afw.image.PhotoCalib` 

709 Photometric calibration object of ``exposure``. 

710 exposureIdInfo : `lsst.obs.base.ExposureIdInfo` 

711 Exposure id information and metadata. 

712 

713 Returns 

714 ------- 

715 dataFrame : `pandas.DataFrame` 

716 DataFrame containing the values of the magnitudes to that will 

717 be inserted into this ccdVisit. 

718 """ 

719 expId = exposureIdInfo.expId 

720 rng = np.random.default_rng(expId) 

721 magScatter = rng.normal(loc=0, 

722 scale=self.config.scatterSize, 

723 size=len(fakeCat)) 

724 visitMagnitudes = fakeCat[self.insertFakes.config.mag_col % band] + magScatter 

725 fakeCat.loc[:, self.insertFakes.config.mag_col % band] = visitMagnitudes 

726 return pd.DataFrame(data={"variableMag": visitMagnitudes})