Coverage for python/lsst/pipe/tasks/processCcdWithFakes.py: 19%

232 statements  

« prev     ^ index     » next       coverage.py v7.2.7, created at 2023-06-13 17:19 -0700

1# This file is part of pipe_tasks. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22""" 

23Insert fake sources into calexps 

24""" 

25 

26__all__ = ["ProcessCcdWithFakesConfig", "ProcessCcdWithFakesTask", 

27 "ProcessCcdWithVariableFakesConfig", "ProcessCcdWithVariableFakesTask"] 

28 

29import numpy as np 

30import pandas as pd 

31 

32import lsst.pex.config as pexConfig 

33import lsst.pipe.base as pipeBase 

34 

35from .insertFakes import InsertFakesTask 

36from lsst.afw.table import SourceTable 

37from lsst.obs.base import ExposureIdInfo 

38from lsst.pipe.base import PipelineTask, PipelineTaskConfig, PipelineTaskConnections 

39import lsst.pipe.base.connectionTypes as cT 

40import lsst.afw.table as afwTable 

41from lsst.skymap import BaseSkyMap 

42from lsst.pipe.tasks.calibrate import CalibrateTask 

43 

44 

45class ProcessCcdWithFakesConnections(PipelineTaskConnections, 

46 dimensions=("instrument", "visit", "detector"), 

47 defaultTemplates={"coaddName": "deep", 

48 "wcsName": "jointcal", 

49 "photoCalibName": "jointcal", 

50 "fakesType": "fakes_"}): 

51 skyMap = cT.Input( 

52 doc="Input definition of geometry/bbox and projection/wcs for " 

53 "template exposures. Needed to test which tract to generate ", 

54 name=BaseSkyMap.SKYMAP_DATASET_TYPE_NAME, 

55 dimensions=("skymap",), 

56 storageClass="SkyMap", 

57 ) 

58 

59 exposure = cT.Input( 

60 doc="Exposure into which fakes are to be added.", 

61 name="calexp", 

62 storageClass="ExposureF", 

63 dimensions=("instrument", "visit", "detector") 

64 ) 

65 

66 fakeCats = cT.Input( 

67 doc="Set of catalogs of fake sources to draw inputs from. We " 

68 "concatenate the tract catalogs for detectorVisits that cover " 

69 "multiple tracts.", 

70 name="{fakesType}fakeSourceCat", 

71 storageClass="DataFrame", 

72 dimensions=("tract", "skymap"), 

73 deferLoad=True, 

74 multiple=True, 

75 ) 

76 

77 externalSkyWcsTractCatalog = cT.Input( 

78 doc=("Per-tract, per-visit wcs calibrations. These catalogs use the detector " 

79 "id for the catalog id, sorted on id for fast lookup."), 

80 name="{wcsName}SkyWcsCatalog", 

81 storageClass="ExposureCatalog", 

82 dimensions=("instrument", "visit", "tract", "skymap"), 

83 deferLoad=True, 

84 multiple=True, 

85 ) 

86 

87 externalSkyWcsGlobalCatalog = cT.Input( 

88 doc=("Per-visit wcs calibrations computed globally (with no tract information). " 

89 "These catalogs use the detector id for the catalog id, sorted on id for " 

90 "fast lookup."), 

91 name="finalVisitSummary", 

92 storageClass="ExposureCatalog", 

93 dimensions=("instrument", "visit"), 

94 ) 

95 

96 externalPhotoCalibTractCatalog = cT.Input( 

97 doc=("Per-tract, per-visit photometric calibrations. These catalogs use the " 

98 "detector id for the catalog id, sorted on id for fast lookup."), 

99 name="{photoCalibName}PhotoCalibCatalog", 

100 storageClass="ExposureCatalog", 

101 dimensions=("instrument", "visit", "tract"), 

102 deferLoad=True, 

103 multiple=True, 

104 ) 

105 

106 externalPhotoCalibGlobalCatalog = cT.Input( 

107 doc=("Per-visit photometric calibrations. These catalogs use the " 

108 "detector id for the catalog id, sorted on id for fast lookup."), 

109 name="finalVisitSummary", 

110 storageClass="ExposureCatalog", 

111 dimensions=("instrument", "visit"), 

112 ) 

113 

114 icSourceCat = cT.Input( 

115 doc="Catalog of calibration sources", 

116 name="icSrc", 

117 storageClass="SourceCatalog", 

118 dimensions=("instrument", "visit", "detector") 

119 ) 

120 

121 sfdSourceCat = cT.Input( 

122 doc="Catalog of calibration sources", 

123 name="src", 

124 storageClass="SourceCatalog", 

125 dimensions=("instrument", "visit", "detector") 

126 ) 

127 

128 outputExposure = cT.Output( 

129 doc="Exposure with fake sources added.", 

130 name="{fakesType}calexp", 

131 storageClass="ExposureF", 

132 dimensions=("instrument", "visit", "detector") 

133 ) 

134 

135 outputCat = cT.Output( 

136 doc="Source catalog produced in calibrate task with fakes also measured.", 

137 name="{fakesType}src", 

138 storageClass="SourceCatalog", 

139 dimensions=("instrument", "visit", "detector"), 

140 ) 

141 

142 def __init__(self, *, config=None): 

143 super().__init__(config=config) 

144 

145 if not config.doApplyExternalGlobalPhotoCalib: 

146 self.inputs.remove("externalPhotoCalibGlobalCatalog") 

147 if not config.doApplyExternalTractPhotoCalib: 

148 self.inputs.remove("externalPhotoCalibTractCatalog") 

149 

150 if not config.doApplyExternalGlobalSkyWcs: 

151 self.inputs.remove("externalSkyWcsGlobalCatalog") 

152 if not config.doApplyExternalTractSkyWcs: 

153 self.inputs.remove("externalSkyWcsTractCatalog") 

154 

155 

156class ProcessCcdWithFakesConfig(PipelineTaskConfig, 

157 pipelineConnections=ProcessCcdWithFakesConnections): 

158 """Config for inserting fake sources 

159 

160 Notes 

161 ----- 

162 The default column names are those from the UW sims database. 

163 """ 

164 

165 doApplyExternalGlobalPhotoCalib = pexConfig.Field( 

166 dtype=bool, 

167 default=False, 

168 doc="Whether to apply an external photometric calibration via an " 

169 "`lsst.afw.image.PhotoCalib` object. Uses the " 

170 "`externalPhotoCalibName` config option to determine which " 

171 "calibration to use. Uses a global calibration." 

172 ) 

173 

174 doApplyExternalTractPhotoCalib = pexConfig.Field( 

175 dtype=bool, 

176 default=False, 

177 doc="Whether to apply an external photometric calibration via an " 

178 "`lsst.afw.image.PhotoCalib` object. Uses the " 

179 "`externalPhotoCalibName` config option to determine which " 

180 "calibration to use. Uses a per tract calibration." 

181 ) 

182 

183 externalPhotoCalibName = pexConfig.ChoiceField( 

184 doc="What type of external photo calib to use.", 

185 dtype=str, 

186 default="jointcal", 

187 allowed={"jointcal": "Use jointcal_photoCalib", 

188 "fgcm": "Use fgcm_photoCalib", 

189 "fgcm_tract": "Use fgcm_tract_photoCalib"} 

190 ) 

191 

192 doApplyExternalGlobalSkyWcs = pexConfig.Field( 

193 dtype=bool, 

194 default=False, 

195 doc="Whether to apply an external astrometric calibration via an " 

196 "`lsst.afw.geom.SkyWcs` object. Uses the " 

197 "`externalSkyWcsName` config option to determine which " 

198 "calibration to use. Uses a global calibration." 

199 ) 

200 

201 doApplyExternalTractSkyWcs = pexConfig.Field( 

202 dtype=bool, 

203 default=False, 

204 doc="Whether to apply an external astrometric calibration via an " 

205 "`lsst.afw.geom.SkyWcs` object. Uses the " 

206 "`externalSkyWcsName` config option to determine which " 

207 "calibration to use. Uses a per tract calibration." 

208 ) 

209 

210 externalSkyWcsName = pexConfig.ChoiceField( 

211 doc="What type of updated WCS calib to use.", 

212 dtype=str, 

213 default="jointcal", 

214 allowed={"jointcal": "Use jointcal_wcs"} 

215 ) 

216 

217 coaddName = pexConfig.Field( 

218 doc="The name of the type of coadd used", 

219 dtype=str, 

220 default="deep", 

221 ) 

222 

223 srcFieldsToCopy = pexConfig.ListField( 

224 dtype=str, 

225 default=("calib_photometry_reserved", "calib_photometry_used", "calib_astrometry_used", 

226 "calib_psf_candidate", "calib_psf_used", "calib_psf_reserved"), 

227 doc=("Fields to copy from the `src` catalog to the output catalog " 

228 "for matching sources Any missing fields will trigger a " 

229 "RuntimeError exception.") 

230 ) 

231 

232 matchRadiusPix = pexConfig.Field( 

233 dtype=float, 

234 default=3, 

235 doc=("Match radius for matching icSourceCat objects to sourceCat objects (pixels)"), 

236 ) 

237 

238 doMatchVisit = pexConfig.Field( 

239 dtype=bool, 

240 default=False, 

241 doc="Match visit to trim the fakeCat" 

242 ) 

243 

244 calibrate = pexConfig.ConfigurableField(target=CalibrateTask, 

245 doc="The calibration task to use.") 

246 

247 insertFakes = pexConfig.ConfigurableField(target=InsertFakesTask, 

248 doc="Configuration for the fake sources") 

249 

250 def setDefaults(self): 

251 super().setDefaults() 

252 self.calibrate.measurement.plugins["base_PixelFlags"].masksFpAnywhere.append("FAKE") 

253 self.calibrate.measurement.plugins["base_PixelFlags"].masksFpCenter.append("FAKE") 

254 self.calibrate.doAstrometry = False 

255 self.calibrate.doWriteMatches = False 

256 self.calibrate.doPhotoCal = False 

257 self.calibrate.doComputeSummaryStats = False 

258 self.calibrate.detection.reEstimateBackground = False 

259 

260 

261class ProcessCcdWithFakesTask(PipelineTask): 

262 """Insert fake objects into calexps. 

263 

264 Add fake stars and galaxies to the given calexp, specified in the dataRef. Galaxy parameters are read in 

265 from the specified file and then modelled using galsim. Re-runs characterize image and calibrate image to 

266 give a new background estimation and measurement of the calexp. 

267 

268 `ProcessFakeSourcesTask` inherits six functions from insertFakesTask that make images of the fake 

269 sources and then add them to the calexp. 

270 

271 `addPixCoords` 

272 Use the WCS information to add the pixel coordinates of each source 

273 Adds an ``x`` and ``y`` column to the catalog of fake sources. 

274 `trimFakeCat` 

275 Trim the fake cat to about the size of the input image. 

276 `mkFakeGalsimGalaxies` 

277 Use Galsim to make fake double sersic galaxies for each set of galaxy parameters in the input file. 

278 `mkFakeStars` 

279 Use the PSF information from the calexp to make a fake star using the magnitude information from the 

280 input file. 

281 `cleanCat` 

282 Remove rows of the input fake catalog which have half light radius, of either the bulge or the disk, 

283 that are 0. 

284 `addFakeSources` 

285 Add the fake sources to the calexp. 

286 

287 Notes 

288 ----- 

289 The ``calexp`` with fake souces added to it is written out as the datatype ``calexp_fakes``. 

290 """ 

291 

292 _DefaultName = "processCcdWithFakes" 

293 ConfigClass = ProcessCcdWithFakesConfig 

294 

295 def __init__(self, schema=None, butler=None, **kwargs): 

296 """Initalize things! This should go above in the class docstring 

297 """ 

298 

299 super().__init__(**kwargs) 

300 

301 if schema is None: 

302 schema = SourceTable.makeMinimalSchema() 

303 self.schema = schema 

304 self.makeSubtask("insertFakes") 

305 self.makeSubtask("calibrate") 

306 

307 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

308 inputs = butlerQC.get(inputRefs) 

309 detectorId = inputs["exposure"].getInfo().getDetector().getId() 

310 

311 if 'exposureIdInfo' not in inputs.keys(): 

312 expId, expBits = butlerQC.quantum.dataId.pack("visit_detector", returnMaxBits=True) 

313 inputs['exposureIdInfo'] = ExposureIdInfo(expId, expBits) 

314 

315 expWcs = inputs["exposure"].getWcs() 

316 tractId = None 

317 if not self.config.doApplyExternalGlobalSkyWcs and not self.config.doApplyExternalTractSkyWcs: 

318 if expWcs is None: 

319 self.log.info("No WCS for exposure %s so cannot insert fake sources. Skipping detector.", 

320 butlerQC.quantum.dataId) 

321 return None 

322 else: 

323 inputs["wcs"] = expWcs 

324 elif self.config.doApplyExternalGlobalSkyWcs: 

325 externalSkyWcsCatalog = inputs["externalSkyWcsGlobalCatalog"] 

326 row = externalSkyWcsCatalog.find(detectorId) 

327 if row is None: 

328 self.log.info("No %s external global sky WCS for exposure %s so cannot insert fake " 

329 "sources. Skipping detector.", self.config.externalSkyWcsName, 

330 butlerQC.quantum.dataId) 

331 return None 

332 inputs["wcs"] = row.getWcs() 

333 elif self.config.doApplyExternalTractSkyWcs: 

334 externalSkyWcsCatalogList = inputs["externalSkyWcsTractCatalog"] 

335 if tractId is None: 

336 tractId = externalSkyWcsCatalogList[0].dataId["tract"] 

337 externalSkyWcsCatalog = None 

338 for externalSkyWcsCatalogRef in externalSkyWcsCatalogList: 

339 if externalSkyWcsCatalogRef.dataId["tract"] == tractId: 

340 externalSkyWcsCatalog = externalSkyWcsCatalogRef.get( 

341 datasetType=self.config.connections.externalSkyWcsTractCatalog) 

342 break 

343 if externalSkyWcsCatalog is None: 

344 usedTract = externalSkyWcsCatalogList[-1].dataId["tract"] 

345 self.log.warn( 

346 f"Warning, external SkyWcs for tract {tractId} not found. Using tract {usedTract} " 

347 "instead.") 

348 externalSkyWcsCatalog = externalSkyWcsCatalogList[-1].get( 

349 datasetType=self.config.connections.externalSkyWcsTractCatalog) 

350 row = externalSkyWcsCatalog.find(detectorId) 

351 if row is None: 

352 self.log.info("No %s external tract sky WCS for exposure %s so cannot insert fake " 

353 "sources. Skipping detector.", self.config.externalSkyWcsName, 

354 butlerQC.quantum.dataId) 

355 return None 

356 inputs["wcs"] = row.getWcs() 

357 

358 if not self.config.doApplyExternalGlobalPhotoCalib and not self.config.doApplyExternalTractPhotoCalib: 

359 inputs["photoCalib"] = inputs["exposure"].getPhotoCalib() 

360 elif self.config.doApplyExternalGlobalPhotoCalib: 

361 externalPhotoCalibCatalog = inputs["externalPhotoCalibGlobalCatalog"] 

362 row = externalPhotoCalibCatalog.find(detectorId) 

363 if row is None: 

364 self.log.info("No %s external global photoCalib for exposure %s so cannot insert fake " 

365 "sources. Skipping detector.", self.config.externalPhotoCalibName, 

366 butlerQC.quantum.dataId) 

367 return None 

368 inputs["photoCalib"] = row.getPhotoCalib() 

369 elif self.config.doApplyExternalTractPhotoCalib: 

370 externalPhotoCalibCatalogList = inputs["externalPhotoCalibTractCatalog"] 

371 if tractId is None: 

372 tractId = externalPhotoCalibCatalogList[0].dataId["tract"] 

373 externalPhotoCalibCatalog = None 

374 for externalPhotoCalibCatalogRef in externalPhotoCalibCatalogList: 

375 if externalPhotoCalibCatalogRef.dataId["tract"] == tractId: 

376 externalPhotoCalibCatalog = externalPhotoCalibCatalogRef.get( 

377 datasetType=self.config.connections.externalPhotoCalibTractCatalog) 

378 break 

379 if externalPhotoCalibCatalog is None: 

380 usedTract = externalPhotoCalibCatalogList[-1].dataId["tract"] 

381 self.log.warn( 

382 f"Warning, external PhotoCalib for tract {tractId} not found. Using tract {usedTract} " 

383 "instead.") 

384 externalPhotoCalibCatalog = externalPhotoCalibCatalogList[-1].get( 

385 datasetType=self.config.connections.externalPhotoCalibTractCatalog) 

386 row = externalPhotoCalibCatalog.find(detectorId) 

387 if row is None: 

388 self.log.info("No %s external tract photoCalib for exposure %s so cannot insert fake " 

389 "sources. Skipping detector.", self.config.externalPhotoCalibName, 

390 butlerQC.quantum.dataId) 

391 return None 

392 inputs["photoCalib"] = row.getPhotoCalib() 

393 

394 outputs = self.run(**inputs) 

395 butlerQC.put(outputs, outputRefs) 

396 

397 def run(self, fakeCats, exposure, skyMap, wcs=None, photoCalib=None, exposureIdInfo=None, 

398 icSourceCat=None, sfdSourceCat=None, externalSkyWcsGlobalCatalog=None, 

399 externalSkyWcsTractCatalog=None, externalPhotoCalibGlobalCatalog=None, 

400 externalPhotoCalibTractCatalog=None): 

401 """Add fake sources to a calexp and then run detection, deblending and measurement. 

402 

403 Parameters 

404 ---------- 

405 fakeCats : `list` of `lsst.daf.butler.DeferredDatasetHandle` 

406 Set of tract level fake catalogs that potentially cover 

407 this detectorVisit. 

408 exposure : `lsst.afw.image.exposure.exposure.ExposureF` 

409 The exposure to add the fake sources to 

410 skyMap : `lsst.skymap.SkyMap` 

411 SkyMap defining the tracts and patches the fakes are stored over. 

412 wcs : `lsst.afw.geom.SkyWcs` 

413 WCS to use to add fake sources 

414 photoCalib : `lsst.afw.image.photoCalib.PhotoCalib` 

415 Photometric calibration to be used to calibrate the fake sources 

416 exposureIdInfo : `lsst.obs.base.ExposureIdInfo` 

417 icSourceCat : `lsst.afw.table.SourceCatalog` 

418 Default : None 

419 Catalog to take the information about which sources were used for calibration from. 

420 sfdSourceCat : `lsst.afw.table.SourceCatalog` 

421 Default : None 

422 Catalog produced by singleFrameDriver, needed to copy some calibration flags from. 

423 

424 Returns 

425 ------- 

426 resultStruct : `lsst.pipe.base.struct.Struct` 

427 contains : outputExposure : `lsst.afw.image.exposure.exposure.ExposureF` 

428 outputCat : `lsst.afw.table.source.source.SourceCatalog` 

429 

430 Notes 

431 ----- 

432 Adds pixel coordinates for each source to the fakeCat and removes objects with bulge or disk half 

433 light radius = 0 (if ``config.cleanCat = True``). These columns are called ``x`` and ``y`` and are in 

434 pixels. 

435 

436 Adds the ``Fake`` mask plane to the exposure which is then set by `addFakeSources` to mark where fake 

437 sources have been added. Uses the information in the ``fakeCat`` to make fake galaxies (using galsim) 

438 and fake stars, using the PSF models from the PSF information for the calexp. These are then added to 

439 the calexp and the calexp with fakes included returned. 

440 

441 The galsim galaxies are made using a double sersic profile, one for the bulge and one for the disk, 

442 this is then convolved with the PSF at that point. 

443 

444 If exposureIdInfo is not provided then the SourceCatalog IDs will not be globally unique. 

445 """ 

446 fakeCat = self.composeFakeCat(fakeCats, skyMap) 

447 

448 if wcs is None: 

449 wcs = exposure.getWcs() 

450 

451 if photoCalib is None: 

452 photoCalib = exposure.getPhotoCalib() 

453 

454 if self.config.doMatchVisit: 

455 fakeCat = self.getVisitMatchedFakeCat(fakeCat, exposure) 

456 

457 self.insertFakes.run(fakeCat, exposure, wcs, photoCalib) 

458 

459 # detect, deblend and measure sources 

460 if exposureIdInfo is None: 

461 exposureIdInfo = ExposureIdInfo() 

462 returnedStruct = self.calibrate.run(exposure, exposureIdInfo=exposureIdInfo) 

463 sourceCat = returnedStruct.sourceCat 

464 

465 sourceCat = self.copyCalibrationFields(sfdSourceCat, sourceCat, self.config.srcFieldsToCopy) 

466 

467 resultStruct = pipeBase.Struct(outputExposure=exposure, outputCat=sourceCat) 

468 return resultStruct 

469 

470 def composeFakeCat(self, fakeCats, skyMap): 

471 """Concatenate the fakeCats from tracts that may cover the exposure. 

472 

473 Parameters 

474 ---------- 

475 fakeCats : `list` of `lst.daf.butler.DeferredDatasetHandle` 

476 Set of fake cats to concatenate. 

477 skyMap : `lsst.skymap.SkyMap` 

478 SkyMap defining the geometry of the tracts and patches. 

479 

480 Returns 

481 ------- 

482 combinedFakeCat : `pandas.DataFrame` 

483 All fakes that cover the inner polygon of the tracts in this 

484 quantum. 

485 """ 

486 if len(fakeCats) == 1: 

487 return fakeCats[0].get( 

488 datasetType=self.config.connections.fakeCats) 

489 outputCat = [] 

490 for fakeCatRef in fakeCats: 

491 cat = fakeCatRef.get( 

492 datasetType=self.config.connections.fakeCats) 

493 tractId = fakeCatRef.dataId["tract"] 

494 # Make sure all data is within the inner part of the tract. 

495 outputCat.append(cat[ 

496 skyMap.findTractIdArray(cat[self.config.insertFakes.ra_col], 

497 cat[self.config.insertFakes.dec_col], 

498 degrees=False) 

499 == tractId]) 

500 

501 return pd.concat(outputCat) 

502 

503 def getVisitMatchedFakeCat(self, fakeCat, exposure): 

504 """Trim the fakeCat to select particular visit 

505 

506 Parameters 

507 ---------- 

508 fakeCat : `pandas.core.frame.DataFrame` 

509 The catalog of fake sources to add to the exposure 

510 exposure : `lsst.afw.image.exposure.exposure.ExposureF` 

511 The exposure to add the fake sources to 

512 

513 Returns 

514 ------- 

515 movingFakeCat : `pandas.DataFrame` 

516 All fakes that belong to the visit 

517 """ 

518 selected = exposure.getInfo().getVisitInfo().getId() == fakeCat["visit"] 

519 

520 return fakeCat[selected] 

521 

522 def copyCalibrationFields(self, calibCat, sourceCat, fieldsToCopy): 

523 """Match sources in calibCat and sourceCat and copy the specified fields 

524 

525 Parameters 

526 ---------- 

527 calibCat : `lsst.afw.table.SourceCatalog` 

528 Catalog from which to copy fields. 

529 sourceCat : `lsst.afw.table.SourceCatalog` 

530 Catalog to which to copy fields. 

531 fieldsToCopy : `lsst.pex.config.listField.List` 

532 Fields to copy from calibCat to SoourceCat. 

533 

534 Returns 

535 ------- 

536 newCat : `lsst.afw.table.SourceCatalog` 

537 Catalog which includes the copied fields. 

538 

539 The fields copied are those specified by `fieldsToCopy` that actually exist 

540 in the schema of `calibCat`. 

541 

542 This version was based on and adapted from the one in calibrateTask. 

543 """ 

544 

545 # Make a new SourceCatalog with the data from sourceCat so that we can add the new columns to it 

546 sourceSchemaMapper = afwTable.SchemaMapper(sourceCat.schema) 

547 sourceSchemaMapper.addMinimalSchema(sourceCat.schema, True) 

548 

549 calibSchemaMapper = afwTable.SchemaMapper(calibCat.schema, sourceCat.schema) 

550 

551 # Add the desired columns from the option fieldsToCopy 

552 missingFieldNames = [] 

553 for fieldName in fieldsToCopy: 

554 if fieldName in calibCat.schema: 

555 schemaItem = calibCat.schema.find(fieldName) 

556 calibSchemaMapper.editOutputSchema().addField(schemaItem.getField()) 

557 schema = calibSchemaMapper.editOutputSchema() 

558 calibSchemaMapper.addMapping(schemaItem.getKey(), schema.find(fieldName).getField()) 

559 else: 

560 missingFieldNames.append(fieldName) 

561 if missingFieldNames: 

562 raise RuntimeError(f"calibCat is missing fields {missingFieldNames} specified in " 

563 "fieldsToCopy") 

564 

565 if "calib_detected" not in calibSchemaMapper.getOutputSchema(): 

566 self.calibSourceKey = calibSchemaMapper.addOutputField(afwTable.Field["Flag"]("calib_detected", 

567 "Source was detected as an icSource")) 

568 else: 

569 self.calibSourceKey = None 

570 

571 schema = calibSchemaMapper.getOutputSchema() 

572 newCat = afwTable.SourceCatalog(schema) 

573 newCat.reserve(len(sourceCat)) 

574 newCat.extend(sourceCat, sourceSchemaMapper) 

575 

576 # Set the aliases so it doesn't complain. 

577 for k, v in sourceCat.schema.getAliasMap().items(): 

578 newCat.schema.getAliasMap().set(k, v) 

579 

580 select = newCat["deblend_nChild"] == 0 

581 matches = afwTable.matchXy(newCat[select], calibCat, self.config.matchRadiusPix) 

582 # Check that no sourceCat sources are listed twice (we already know 

583 # that each match has a unique calibCat source ID, due to using 

584 # that ID as the key in bestMatches) 

585 numMatches = len(matches) 

586 numUniqueSources = len(set(m[1].getId() for m in matches)) 

587 if numUniqueSources != numMatches: 

588 self.log.warning("%d calibCat sources matched only %d sourceCat sources", numMatches, 

589 numUniqueSources) 

590 

591 self.log.info("Copying flags from calibCat to sourceCat for %s sources", numMatches) 

592 

593 # For each match: set the calibSourceKey flag and copy the desired 

594 # fields 

595 for src, calibSrc, d in matches: 

596 if self.calibSourceKey: 

597 src.setFlag(self.calibSourceKey, True) 

598 # src.assign copies the footprint from calibSrc, which we don't want 

599 # (DM-407) 

600 # so set calibSrc's footprint to src's footprint before src.assign, 

601 # then restore it 

602 calibSrcFootprint = calibSrc.getFootprint() 

603 try: 

604 calibSrc.setFootprint(src.getFootprint()) 

605 src.assign(calibSrc, calibSchemaMapper) 

606 finally: 

607 calibSrc.setFootprint(calibSrcFootprint) 

608 

609 return newCat 

610 

611 

612class ProcessCcdWithVariableFakesConnections(ProcessCcdWithFakesConnections): 

613 ccdVisitFakeMagnitudes = cT.Output( 

614 doc="Catalog of fakes with magnitudes scattered for this ccdVisit.", 

615 name="{fakesType}ccdVisitFakeMagnitudes", 

616 storageClass="DataFrame", 

617 dimensions=("instrument", "visit", "detector"), 

618 ) 

619 

620 

621class ProcessCcdWithVariableFakesConfig(ProcessCcdWithFakesConfig, 

622 pipelineConnections=ProcessCcdWithVariableFakesConnections): 

623 scatterSize = pexConfig.RangeField( 

624 dtype=float, 

625 default=0.4, 

626 min=0, 

627 max=100, 

628 doc="Amount of scatter to add to the visit magnitude for variable " 

629 "sources." 

630 ) 

631 

632 

633class ProcessCcdWithVariableFakesTask(ProcessCcdWithFakesTask): 

634 """As ProcessCcdWithFakes except add variablity to the fakes catalog 

635 magnitude in the observed band for this ccdVisit. 

636 

637 Additionally, write out the modified magnitudes to the Butler. 

638 """ 

639 

640 _DefaultName = "processCcdWithVariableFakes" 

641 ConfigClass = ProcessCcdWithVariableFakesConfig 

642 

643 def run(self, fakeCats, exposure, skyMap, wcs=None, photoCalib=None, exposureIdInfo=None, 

644 icSourceCat=None, sfdSourceCat=None): 

645 """Add fake sources to a calexp and then run detection, deblending and measurement. 

646 

647 Parameters 

648 ---------- 

649 fakeCat : `pandas.core.frame.DataFrame` 

650 The catalog of fake sources to add to the exposure 

651 exposure : `lsst.afw.image.exposure.exposure.ExposureF` 

652 The exposure to add the fake sources to 

653 skyMap : `lsst.skymap.SkyMap` 

654 SkyMap defining the tracts and patches the fakes are stored over. 

655 wcs : `lsst.afw.geom.SkyWcs` 

656 WCS to use to add fake sources 

657 photoCalib : `lsst.afw.image.photoCalib.PhotoCalib` 

658 Photometric calibration to be used to calibrate the fake sources 

659 exposureIdInfo : `lsst.obs.base.ExposureIdInfo` 

660 icSourceCat : `lsst.afw.table.SourceCatalog` 

661 Default : None 

662 Catalog to take the information about which sources were used for calibration from. 

663 sfdSourceCat : `lsst.afw.table.SourceCatalog` 

664 Default : None 

665 Catalog produced by singleFrameDriver, needed to copy some calibration flags from. 

666 

667 Returns 

668 ------- 

669 resultStruct : `lsst.pipe.base.struct.Struct` 

670 Results Strcut containing: 

671 

672 - outputExposure : Exposure with added fakes 

673 (`lsst.afw.image.exposure.exposure.ExposureF`) 

674 - outputCat : Catalog with detected fakes 

675 (`lsst.afw.table.source.source.SourceCatalog`) 

676 - ccdVisitFakeMagnitudes : Magnitudes that these fakes were 

677 inserted with after being scattered (`pandas.DataFrame`) 

678 

679 Notes 

680 ----- 

681 Adds pixel coordinates for each source to the fakeCat and removes objects with bulge or disk half 

682 light radius = 0 (if ``config.cleanCat = True``). These columns are called ``x`` and ``y`` and are in 

683 pixels. 

684 

685 Adds the ``Fake`` mask plane to the exposure which is then set by `addFakeSources` to mark where fake 

686 sources have been added. Uses the information in the ``fakeCat`` to make fake galaxies (using galsim) 

687 and fake stars, using the PSF models from the PSF information for the calexp. These are then added to 

688 the calexp and the calexp with fakes included returned. 

689 

690 The galsim galaxies are made using a double sersic profile, one for the bulge and one for the disk, 

691 this is then convolved with the PSF at that point. 

692 

693 If exposureIdInfo is not provided then the SourceCatalog IDs will not be globally unique. 

694 """ 

695 fakeCat = self.composeFakeCat(fakeCats, skyMap) 

696 

697 if wcs is None: 

698 wcs = exposure.getWcs() 

699 

700 if photoCalib is None: 

701 photoCalib = exposure.getPhotoCalib() 

702 

703 if exposureIdInfo is None: 

704 exposureIdInfo = ExposureIdInfo() 

705 

706 band = exposure.getFilter().bandLabel 

707 ccdVisitMagnitudes = self.addVariablity(fakeCat, band, exposure, photoCalib, exposureIdInfo) 

708 

709 self.insertFakes.run(fakeCat, exposure, wcs, photoCalib) 

710 

711 # detect, deblend and measure sources 

712 returnedStruct = self.calibrate.run(exposure, exposureIdInfo=exposureIdInfo) 

713 sourceCat = returnedStruct.sourceCat 

714 

715 sourceCat = self.copyCalibrationFields(sfdSourceCat, sourceCat, self.config.srcFieldsToCopy) 

716 

717 resultStruct = pipeBase.Struct(outputExposure=exposure, 

718 outputCat=sourceCat, 

719 ccdVisitFakeMagnitudes=ccdVisitMagnitudes) 

720 return resultStruct 

721 

722 def addVariablity(self, fakeCat, band, exposure, photoCalib, exposureIdInfo): 

723 """Add scatter to the fake catalog visit magnitudes. 

724 

725 Currently just adds a simple Gaussian scatter around the static fake 

726 magnitude. This function could be modified to return any number of 

727 fake variability. 

728 

729 Parameters 

730 ---------- 

731 fakeCat : `pandas.DataFrame` 

732 Catalog of fakes to modify magnitudes of. 

733 band : `str` 

734 Current observing band to modify. 

735 exposure : `lsst.afw.image.ExposureF` 

736 Exposure fakes will be added to. 

737 photoCalib : `lsst.afw.image.PhotoCalib` 

738 Photometric calibration object of ``exposure``. 

739 exposureIdInfo : `lsst.obs.base.ExposureIdInfo` 

740 Exposure id information and metadata. 

741 

742 Returns 

743 ------- 

744 dataFrame : `pandas.DataFrame` 

745 DataFrame containing the values of the magnitudes to that will 

746 be inserted into this ccdVisit. 

747 """ 

748 expId = exposureIdInfo.expId 

749 rng = np.random.default_rng(expId) 

750 magScatter = rng.normal(loc=0, 

751 scale=self.config.scatterSize, 

752 size=len(fakeCat)) 

753 visitMagnitudes = fakeCat[self.insertFakes.config.mag_col % band] + magScatter 

754 fakeCat.loc[:, self.insertFakes.config.mag_col % band] = visitMagnitudes 

755 return pd.DataFrame(data={"variableMag": visitMagnitudes})