Coverage for python/lsst/pipe/tasks/processCcdWithFakes.py: 19%

231 statements  

« prev     ^ index     » next       coverage.py v6.5.0, created at 2022-10-07 11:10 +0000

1# This file is part of pipe_tasks. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22""" 

23Insert fake sources into calexps 

24""" 

25 

26__all__ = ["ProcessCcdWithFakesConfig", "ProcessCcdWithFakesTask", 

27 "ProcessCcdWithVariableFakesConfig", "ProcessCcdWithVariableFakesTask"] 

28 

29import numpy as np 

30import pandas as pd 

31 

32import lsst.pex.config as pexConfig 

33import lsst.pipe.base as pipeBase 

34 

35from .insertFakes import InsertFakesTask 

36from lsst.afw.table import SourceTable 

37from lsst.obs.base import ExposureIdInfo 

38from lsst.pipe.base import PipelineTask, PipelineTaskConfig, PipelineTaskConnections 

39import lsst.pipe.base.connectionTypes as cT 

40import lsst.afw.table as afwTable 

41from lsst.skymap import BaseSkyMap 

42from lsst.pipe.tasks.calibrate import CalibrateTask 

43 

44 

45class ProcessCcdWithFakesConnections(PipelineTaskConnections, 

46 dimensions=("instrument", "visit", "detector"), 

47 defaultTemplates={"coaddName": "deep", 

48 "wcsName": "jointcal", 

49 "photoCalibName": "jointcal", 

50 "fakesType": "fakes_"}): 

51 skyMap = cT.Input( 

52 doc="Input definition of geometry/bbox and projection/wcs for " 

53 "template exposures. Needed to test which tract to generate ", 

54 name=BaseSkyMap.SKYMAP_DATASET_TYPE_NAME, 

55 dimensions=("skymap",), 

56 storageClass="SkyMap", 

57 ) 

58 

59 exposure = cT.Input( 

60 doc="Exposure into which fakes are to be added.", 

61 name="calexp", 

62 storageClass="ExposureF", 

63 dimensions=("instrument", "visit", "detector") 

64 ) 

65 

66 fakeCats = cT.Input( 

67 doc="Set of catalogs of fake sources to draw inputs from. We " 

68 "concatenate the tract catalogs for detectorVisits that cover " 

69 "multiple tracts.", 

70 name="{fakesType}fakeSourceCat", 

71 storageClass="DataFrame", 

72 dimensions=("tract", "skymap"), 

73 deferLoad=True, 

74 multiple=True, 

75 ) 

76 

77 externalSkyWcsTractCatalog = cT.Input( 

78 doc=("Per-tract, per-visit wcs calibrations. These catalogs use the detector " 

79 "id for the catalog id, sorted on id for fast lookup."), 

80 name="{wcsName}SkyWcsCatalog", 

81 storageClass="ExposureCatalog", 

82 dimensions=("instrument", "visit", "tract", "skymap"), 

83 deferLoad=True, 

84 multiple=True, 

85 ) 

86 

87 externalSkyWcsGlobalCatalog = cT.Input( 

88 doc=("Per-visit wcs calibrations computed globally (with no tract information). " 

89 "These catalogs use the detector id for the catalog id, sorted on id for " 

90 "fast lookup."), 

91 name="{wcsName}SkyWcsCatalog", 

92 storageClass="ExposureCatalog", 

93 dimensions=("instrument", "visit"), 

94 ) 

95 

96 externalPhotoCalibTractCatalog = cT.Input( 

97 doc=("Per-tract, per-visit photometric calibrations. These catalogs use the " 

98 "detector id for the catalog id, sorted on id for fast lookup."), 

99 name="{photoCalibName}PhotoCalibCatalog", 

100 storageClass="ExposureCatalog", 

101 dimensions=("instrument", "visit", "tract"), 

102 deferLoad=True, 

103 multiple=True, 

104 ) 

105 

106 externalPhotoCalibGlobalCatalog = cT.Input( 

107 doc=("Per-visit photometric calibrations. These catalogs use the " 

108 "detector id for the catalog id, sorted on id for fast lookup."), 

109 name="{photoCalibName}PhotoCalibCatalog", 

110 storageClass="ExposureCatalog", 

111 dimensions=("instrument", "visit"), 

112 ) 

113 

114 icSourceCat = cT.Input( 

115 doc="Catalog of calibration sources", 

116 name="icSrc", 

117 storageClass="SourceCatalog", 

118 dimensions=("instrument", "visit", "detector") 

119 ) 

120 

121 sfdSourceCat = cT.Input( 

122 doc="Catalog of calibration sources", 

123 name="src", 

124 storageClass="SourceCatalog", 

125 dimensions=("instrument", "visit", "detector") 

126 ) 

127 

128 outputExposure = cT.Output( 

129 doc="Exposure with fake sources added.", 

130 name="{fakesType}calexp", 

131 storageClass="ExposureF", 

132 dimensions=("instrument", "visit", "detector") 

133 ) 

134 

135 outputCat = cT.Output( 

136 doc="Source catalog produced in calibrate task with fakes also measured.", 

137 name="{fakesType}src", 

138 storageClass="SourceCatalog", 

139 dimensions=("instrument", "visit", "detector"), 

140 ) 

141 

142 def __init__(self, *, config=None): 

143 super().__init__(config=config) 

144 

145 if not config.doApplyExternalGlobalPhotoCalib: 

146 self.inputs.remove("externalPhotoCalibGlobalCatalog") 

147 if not config.doApplyExternalTractPhotoCalib: 

148 self.inputs.remove("externalPhotoCalibTractCatalog") 

149 

150 if not config.doApplyExternalGlobalSkyWcs: 

151 self.inputs.remove("externalSkyWcsGlobalCatalog") 

152 if not config.doApplyExternalTractSkyWcs: 

153 self.inputs.remove("externalSkyWcsTractCatalog") 

154 

155 

156class ProcessCcdWithFakesConfig(PipelineTaskConfig, 

157 pipelineConnections=ProcessCcdWithFakesConnections): 

158 """Config for inserting fake sources 

159 

160 Notes 

161 ----- 

162 The default column names are those from the UW sims database. 

163 """ 

164 

165 doApplyExternalGlobalPhotoCalib = pexConfig.Field( 

166 dtype=bool, 

167 default=False, 

168 doc="Whether to apply an external photometric calibration via an " 

169 "`lsst.afw.image.PhotoCalib` object. Uses the " 

170 "`externalPhotoCalibName` config option to determine which " 

171 "calibration to use. Uses a global calibration." 

172 ) 

173 

174 doApplyExternalTractPhotoCalib = pexConfig.Field( 

175 dtype=bool, 

176 default=False, 

177 doc="Whether to apply an external photometric calibration via an " 

178 "`lsst.afw.image.PhotoCalib` object. Uses the " 

179 "`externalPhotoCalibName` config option to determine which " 

180 "calibration to use. Uses a per tract calibration." 

181 ) 

182 

183 externalPhotoCalibName = pexConfig.ChoiceField( 

184 doc="What type of external photo calib to use.", 

185 dtype=str, 

186 default="jointcal", 

187 allowed={"jointcal": "Use jointcal_photoCalib", 

188 "fgcm": "Use fgcm_photoCalib", 

189 "fgcm_tract": "Use fgcm_tract_photoCalib"} 

190 ) 

191 

192 doApplyExternalGlobalSkyWcs = pexConfig.Field( 

193 dtype=bool, 

194 default=False, 

195 doc="Whether to apply an external astrometric calibration via an " 

196 "`lsst.afw.geom.SkyWcs` object. Uses the " 

197 "`externalSkyWcsName` config option to determine which " 

198 "calibration to use. Uses a global calibration." 

199 ) 

200 

201 doApplyExternalTractSkyWcs = pexConfig.Field( 

202 dtype=bool, 

203 default=False, 

204 doc="Whether to apply an external astrometric calibration via an " 

205 "`lsst.afw.geom.SkyWcs` object. Uses the " 

206 "`externalSkyWcsName` config option to determine which " 

207 "calibration to use. Uses a per tract calibration." 

208 ) 

209 

210 externalSkyWcsName = pexConfig.ChoiceField( 

211 doc="What type of updated WCS calib to use.", 

212 dtype=str, 

213 default="jointcal", 

214 allowed={"jointcal": "Use jointcal_wcs"} 

215 ) 

216 

217 coaddName = pexConfig.Field( 

218 doc="The name of the type of coadd used", 

219 dtype=str, 

220 default="deep", 

221 ) 

222 

223 srcFieldsToCopy = pexConfig.ListField( 

224 dtype=str, 

225 default=("calib_photometry_reserved", "calib_photometry_used", "calib_astrometry_used", 

226 "calib_psf_candidate", "calib_psf_used", "calib_psf_reserved"), 

227 doc=("Fields to copy from the `src` catalog to the output catalog " 

228 "for matching sources Any missing fields will trigger a " 

229 "RuntimeError exception.") 

230 ) 

231 

232 matchRadiusPix = pexConfig.Field( 

233 dtype=float, 

234 default=3, 

235 doc=("Match radius for matching icSourceCat objects to sourceCat objects (pixels)"), 

236 ) 

237 

238 doMatchVisit = pexConfig.Field( 

239 dtype=bool, 

240 default=False, 

241 doc="Match visit to trim the fakeCat" 

242 ) 

243 

244 calibrate = pexConfig.ConfigurableField(target=CalibrateTask, 

245 doc="The calibration task to use.") 

246 

247 insertFakes = pexConfig.ConfigurableField(target=InsertFakesTask, 

248 doc="Configuration for the fake sources") 

249 

250 def setDefaults(self): 

251 super().setDefaults() 

252 self.calibrate.measurement.plugins["base_PixelFlags"].masksFpAnywhere.append("FAKE") 

253 self.calibrate.measurement.plugins["base_PixelFlags"].masksFpCenter.append("FAKE") 

254 self.calibrate.doAstrometry = False 

255 self.calibrate.doWriteMatches = False 

256 self.calibrate.doPhotoCal = False 

257 self.calibrate.detection.reEstimateBackground = False 

258 

259 

260class ProcessCcdWithFakesTask(PipelineTask): 

261 """Insert fake objects into calexps. 

262 

263 Add fake stars and galaxies to the given calexp, specified in the dataRef. Galaxy parameters are read in 

264 from the specified file and then modelled using galsim. Re-runs characterize image and calibrate image to 

265 give a new background estimation and measurement of the calexp. 

266 

267 `ProcessFakeSourcesTask` inherits six functions from insertFakesTask that make images of the fake 

268 sources and then add them to the calexp. 

269 

270 `addPixCoords` 

271 Use the WCS information to add the pixel coordinates of each source 

272 Adds an ``x`` and ``y`` column to the catalog of fake sources. 

273 `trimFakeCat` 

274 Trim the fake cat to about the size of the input image. 

275 `mkFakeGalsimGalaxies` 

276 Use Galsim to make fake double sersic galaxies for each set of galaxy parameters in the input file. 

277 `mkFakeStars` 

278 Use the PSF information from the calexp to make a fake star using the magnitude information from the 

279 input file. 

280 `cleanCat` 

281 Remove rows of the input fake catalog which have half light radius, of either the bulge or the disk, 

282 that are 0. 

283 `addFakeSources` 

284 Add the fake sources to the calexp. 

285 

286 Notes 

287 ----- 

288 The ``calexp`` with fake souces added to it is written out as the datatype ``calexp_fakes``. 

289 """ 

290 

291 _DefaultName = "processCcdWithFakes" 

292 ConfigClass = ProcessCcdWithFakesConfig 

293 

294 def __init__(self, schema=None, butler=None, **kwargs): 

295 """Initalize things! This should go above in the class docstring 

296 """ 

297 

298 super().__init__(**kwargs) 

299 

300 if schema is None: 

301 schema = SourceTable.makeMinimalSchema() 

302 self.schema = schema 

303 self.makeSubtask("insertFakes") 

304 self.makeSubtask("calibrate") 

305 

306 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

307 inputs = butlerQC.get(inputRefs) 

308 detectorId = inputs["exposure"].getInfo().getDetector().getId() 

309 

310 if 'exposureIdInfo' not in inputs.keys(): 

311 expId, expBits = butlerQC.quantum.dataId.pack("visit_detector", returnMaxBits=True) 

312 inputs['exposureIdInfo'] = ExposureIdInfo(expId, expBits) 

313 

314 expWcs = inputs["exposure"].getWcs() 

315 tractId = None 

316 if not self.config.doApplyExternalGlobalSkyWcs and not self.config.doApplyExternalTractSkyWcs: 

317 if expWcs is None: 

318 self.log.info("No WCS for exposure %s so cannot insert fake sources. Skipping detector.", 

319 butlerQC.quantum.dataId) 

320 return None 

321 else: 

322 inputs["wcs"] = expWcs 

323 elif self.config.doApplyExternalGlobalSkyWcs: 

324 externalSkyWcsCatalog = inputs["externalSkyWcsGlobalCatalog"] 

325 row = externalSkyWcsCatalog.find(detectorId) 

326 if row is None: 

327 self.log.info("No %s external global sky WCS for exposure %s so cannot insert fake " 

328 "sources. Skipping detector.", self.config.externalSkyWcsName, 

329 butlerQC.quantum.dataId) 

330 return None 

331 inputs["wcs"] = row.getWcs() 

332 elif self.config.doApplyExternalTractSkyWcs: 

333 externalSkyWcsCatalogList = inputs["externalSkyWcsTractCatalog"] 

334 if tractId is None: 

335 tractId = externalSkyWcsCatalogList[0].dataId["tract"] 

336 externalSkyWcsCatalog = None 

337 for externalSkyWcsCatalogRef in externalSkyWcsCatalogList: 

338 if externalSkyWcsCatalogRef.dataId["tract"] == tractId: 

339 externalSkyWcsCatalog = externalSkyWcsCatalogRef.get( 

340 datasetType=self.config.connections.externalSkyWcsTractCatalog) 

341 break 

342 if externalSkyWcsCatalog is None: 

343 usedTract = externalSkyWcsCatalogList[-1].dataId["tract"] 

344 self.log.warn( 

345 f"Warning, external SkyWcs for tract {tractId} not found. Using tract {usedTract} " 

346 "instead.") 

347 externalSkyWcsCatalog = externalSkyWcsCatalogList[-1].get( 

348 datasetType=self.config.connections.externalSkyWcsTractCatalog) 

349 row = externalSkyWcsCatalog.find(detectorId) 

350 if row is None: 

351 self.log.info("No %s external tract sky WCS for exposure %s so cannot insert fake " 

352 "sources. Skipping detector.", self.config.externalSkyWcsName, 

353 butlerQC.quantum.dataId) 

354 return None 

355 inputs["wcs"] = row.getWcs() 

356 

357 if not self.config.doApplyExternalGlobalPhotoCalib and not self.config.doApplyExternalTractPhotoCalib: 

358 inputs["photoCalib"] = inputs["exposure"].getPhotoCalib() 

359 elif self.config.doApplyExternalGlobalPhotoCalib: 

360 externalPhotoCalibCatalog = inputs["externalPhotoCalibGlobalCatalog"] 

361 row = externalPhotoCalibCatalog.find(detectorId) 

362 if row is None: 

363 self.log.info("No %s external global photoCalib for exposure %s so cannot insert fake " 

364 "sources. Skipping detector.", self.config.externalPhotoCalibName, 

365 butlerQC.quantum.dataId) 

366 return None 

367 inputs["photoCalib"] = row.getPhotoCalib() 

368 elif self.config.doApplyExternalTractPhotoCalib: 

369 externalPhotoCalibCatalogList = inputs["externalPhotoCalibTractCatalog"] 

370 if tractId is None: 

371 tractId = externalPhotoCalibCatalogList[0].dataId["tract"] 

372 externalPhotoCalibCatalog = None 

373 for externalPhotoCalibCatalogRef in externalPhotoCalibCatalogList: 

374 if externalPhotoCalibCatalogRef.dataId["tract"] == tractId: 

375 externalPhotoCalibCatalog = externalPhotoCalibCatalogRef.get( 

376 datasetType=self.config.connections.externalPhotoCalibTractCatalog) 

377 break 

378 if externalPhotoCalibCatalog is None: 

379 usedTract = externalPhotoCalibCatalogList[-1].dataId["tract"] 

380 self.log.warn( 

381 f"Warning, external PhotoCalib for tract {tractId} not found. Using tract {usedTract} " 

382 "instead.") 

383 externalPhotoCalibCatalog = externalPhotoCalibCatalogList[-1].get( 

384 datasetType=self.config.connections.externalPhotoCalibTractCatalog) 

385 row = externalPhotoCalibCatalog.find(detectorId) 

386 if row is None: 

387 self.log.info("No %s external tract photoCalib for exposure %s so cannot insert fake " 

388 "sources. Skipping detector.", self.config.externalPhotoCalibName, 

389 butlerQC.quantum.dataId) 

390 return None 

391 inputs["photoCalib"] = row.getPhotoCalib() 

392 

393 outputs = self.run(**inputs) 

394 butlerQC.put(outputs, outputRefs) 

395 

396 def run(self, fakeCats, exposure, skyMap, wcs=None, photoCalib=None, exposureIdInfo=None, 

397 icSourceCat=None, sfdSourceCat=None, externalSkyWcsGlobalCatalog=None, 

398 externalSkyWcsTractCatalog=None, externalPhotoCalibGlobalCatalog=None, 

399 externalPhotoCalibTractCatalog=None): 

400 """Add fake sources to a calexp and then run detection, deblending and measurement. 

401 

402 Parameters 

403 ---------- 

404 fakeCats : `list` of `lsst.daf.butler.DeferredDatasetHandle` 

405 Set of tract level fake catalogs that potentially cover 

406 this detectorVisit. 

407 exposure : `lsst.afw.image.exposure.exposure.ExposureF` 

408 The exposure to add the fake sources to 

409 skyMap : `lsst.skymap.SkyMap` 

410 SkyMap defining the tracts and patches the fakes are stored over. 

411 wcs : `lsst.afw.geom.SkyWcs` 

412 WCS to use to add fake sources 

413 photoCalib : `lsst.afw.image.photoCalib.PhotoCalib` 

414 Photometric calibration to be used to calibrate the fake sources 

415 exposureIdInfo : `lsst.obs.base.ExposureIdInfo` 

416 icSourceCat : `lsst.afw.table.SourceCatalog` 

417 Default : None 

418 Catalog to take the information about which sources were used for calibration from. 

419 sfdSourceCat : `lsst.afw.table.SourceCatalog` 

420 Default : None 

421 Catalog produced by singleFrameDriver, needed to copy some calibration flags from. 

422 

423 Returns 

424 ------- 

425 resultStruct : `lsst.pipe.base.struct.Struct` 

426 contains : outputExposure : `lsst.afw.image.exposure.exposure.ExposureF` 

427 outputCat : `lsst.afw.table.source.source.SourceCatalog` 

428 

429 Notes 

430 ----- 

431 Adds pixel coordinates for each source to the fakeCat and removes objects with bulge or disk half 

432 light radius = 0 (if ``config.cleanCat = True``). These columns are called ``x`` and ``y`` and are in 

433 pixels. 

434 

435 Adds the ``Fake`` mask plane to the exposure which is then set by `addFakeSources` to mark where fake 

436 sources have been added. Uses the information in the ``fakeCat`` to make fake galaxies (using galsim) 

437 and fake stars, using the PSF models from the PSF information for the calexp. These are then added to 

438 the calexp and the calexp with fakes included returned. 

439 

440 The galsim galaxies are made using a double sersic profile, one for the bulge and one for the disk, 

441 this is then convolved with the PSF at that point. 

442 

443 If exposureIdInfo is not provided then the SourceCatalog IDs will not be globally unique. 

444 """ 

445 fakeCat = self.composeFakeCat(fakeCats, skyMap) 

446 

447 if wcs is None: 

448 wcs = exposure.getWcs() 

449 

450 if photoCalib is None: 

451 photoCalib = exposure.getPhotoCalib() 

452 

453 if self.config.doMatchVisit: 

454 fakeCat = self.getVisitMatchedFakeCat(fakeCat, exposure) 

455 

456 self.insertFakes.run(fakeCat, exposure, wcs, photoCalib) 

457 

458 # detect, deblend and measure sources 

459 if exposureIdInfo is None: 

460 exposureIdInfo = ExposureIdInfo() 

461 returnedStruct = self.calibrate.run(exposure, exposureIdInfo=exposureIdInfo) 

462 sourceCat = returnedStruct.sourceCat 

463 

464 sourceCat = self.copyCalibrationFields(sfdSourceCat, sourceCat, self.config.srcFieldsToCopy) 

465 

466 resultStruct = pipeBase.Struct(outputExposure=exposure, outputCat=sourceCat) 

467 return resultStruct 

468 

469 def composeFakeCat(self, fakeCats, skyMap): 

470 """Concatenate the fakeCats from tracts that may cover the exposure. 

471 

472 Parameters 

473 ---------- 

474 fakeCats : `list` of `lst.daf.butler.DeferredDatasetHandle` 

475 Set of fake cats to concatenate. 

476 skyMap : `lsst.skymap.SkyMap` 

477 SkyMap defining the geometry of the tracts and patches. 

478 

479 Returns 

480 ------- 

481 combinedFakeCat : `pandas.DataFrame` 

482 All fakes that cover the inner polygon of the tracts in this 

483 quantum. 

484 """ 

485 if len(fakeCats) == 1: 

486 return fakeCats[0].get( 

487 datasetType=self.config.connections.fakeCats) 

488 outputCat = [] 

489 for fakeCatRef in fakeCats: 

490 cat = fakeCatRef.get( 

491 datasetType=self.config.connections.fakeCats) 

492 tractId = fakeCatRef.dataId["tract"] 

493 # Make sure all data is within the inner part of the tract. 

494 outputCat.append(cat[ 

495 skyMap.findTractIdArray(cat[self.config.insertFakes.ra_col], 

496 cat[self.config.insertFakes.dec_col], 

497 degrees=False) 

498 == tractId]) 

499 

500 return pd.concat(outputCat) 

501 

502 def getVisitMatchedFakeCat(self, fakeCat, exposure): 

503 """Trim the fakeCat to select particular visit 

504 

505 Parameters 

506 ---------- 

507 fakeCat : `pandas.core.frame.DataFrame` 

508 The catalog of fake sources to add to the exposure 

509 exposure : `lsst.afw.image.exposure.exposure.ExposureF` 

510 The exposure to add the fake sources to 

511 

512 Returns 

513 ------- 

514 movingFakeCat : `pandas.DataFrame` 

515 All fakes that belong to the visit 

516 """ 

517 selected = exposure.getInfo().getVisitInfo().getId() == fakeCat["visit"] 

518 

519 return fakeCat[selected] 

520 

521 def copyCalibrationFields(self, calibCat, sourceCat, fieldsToCopy): 

522 """Match sources in calibCat and sourceCat and copy the specified fields 

523 

524 Parameters 

525 ---------- 

526 calibCat : `lsst.afw.table.SourceCatalog` 

527 Catalog from which to copy fields. 

528 sourceCat : `lsst.afw.table.SourceCatalog` 

529 Catalog to which to copy fields. 

530 fieldsToCopy : `lsst.pex.config.listField.List` 

531 Fields to copy from calibCat to SoourceCat. 

532 

533 Returns 

534 ------- 

535 newCat : `lsst.afw.table.SourceCatalog` 

536 Catalog which includes the copied fields. 

537 

538 The fields copied are those specified by `fieldsToCopy` that actually exist 

539 in the schema of `calibCat`. 

540 

541 This version was based on and adapted from the one in calibrateTask. 

542 """ 

543 

544 # Make a new SourceCatalog with the data from sourceCat so that we can add the new columns to it 

545 sourceSchemaMapper = afwTable.SchemaMapper(sourceCat.schema) 

546 sourceSchemaMapper.addMinimalSchema(sourceCat.schema, True) 

547 

548 calibSchemaMapper = afwTable.SchemaMapper(calibCat.schema, sourceCat.schema) 

549 

550 # Add the desired columns from the option fieldsToCopy 

551 missingFieldNames = [] 

552 for fieldName in fieldsToCopy: 

553 if fieldName in calibCat.schema: 

554 schemaItem = calibCat.schema.find(fieldName) 

555 calibSchemaMapper.editOutputSchema().addField(schemaItem.getField()) 

556 schema = calibSchemaMapper.editOutputSchema() 

557 calibSchemaMapper.addMapping(schemaItem.getKey(), schema.find(fieldName).getField()) 

558 else: 

559 missingFieldNames.append(fieldName) 

560 if missingFieldNames: 

561 raise RuntimeError(f"calibCat is missing fields {missingFieldNames} specified in " 

562 "fieldsToCopy") 

563 

564 if "calib_detected" not in calibSchemaMapper.getOutputSchema(): 

565 self.calibSourceKey = calibSchemaMapper.addOutputField(afwTable.Field["Flag"]("calib_detected", 

566 "Source was detected as an icSource")) 

567 else: 

568 self.calibSourceKey = None 

569 

570 schema = calibSchemaMapper.getOutputSchema() 

571 newCat = afwTable.SourceCatalog(schema) 

572 newCat.reserve(len(sourceCat)) 

573 newCat.extend(sourceCat, sourceSchemaMapper) 

574 

575 # Set the aliases so it doesn't complain. 

576 for k, v in sourceCat.schema.getAliasMap().items(): 

577 newCat.schema.getAliasMap().set(k, v) 

578 

579 select = newCat["deblend_nChild"] == 0 

580 matches = afwTable.matchXy(newCat[select], calibCat, self.config.matchRadiusPix) 

581 # Check that no sourceCat sources are listed twice (we already know 

582 # that each match has a unique calibCat source ID, due to using 

583 # that ID as the key in bestMatches) 

584 numMatches = len(matches) 

585 numUniqueSources = len(set(m[1].getId() for m in matches)) 

586 if numUniqueSources != numMatches: 

587 self.log.warning("%d calibCat sources matched only %d sourceCat sources", numMatches, 

588 numUniqueSources) 

589 

590 self.log.info("Copying flags from calibCat to sourceCat for %s sources", numMatches) 

591 

592 # For each match: set the calibSourceKey flag and copy the desired 

593 # fields 

594 for src, calibSrc, d in matches: 

595 if self.calibSourceKey: 

596 src.setFlag(self.calibSourceKey, True) 

597 # src.assign copies the footprint from calibSrc, which we don't want 

598 # (DM-407) 

599 # so set calibSrc's footprint to src's footprint before src.assign, 

600 # then restore it 

601 calibSrcFootprint = calibSrc.getFootprint() 

602 try: 

603 calibSrc.setFootprint(src.getFootprint()) 

604 src.assign(calibSrc, calibSchemaMapper) 

605 finally: 

606 calibSrc.setFootprint(calibSrcFootprint) 

607 

608 return newCat 

609 

610 

611class ProcessCcdWithVariableFakesConnections(ProcessCcdWithFakesConnections): 

612 ccdVisitFakeMagnitudes = cT.Output( 

613 doc="Catalog of fakes with magnitudes scattered for this ccdVisit.", 

614 name="{fakesType}ccdVisitFakeMagnitudes", 

615 storageClass="DataFrame", 

616 dimensions=("instrument", "visit", "detector"), 

617 ) 

618 

619 

620class ProcessCcdWithVariableFakesConfig(ProcessCcdWithFakesConfig, 

621 pipelineConnections=ProcessCcdWithVariableFakesConnections): 

622 scatterSize = pexConfig.RangeField( 

623 dtype=float, 

624 default=0.4, 

625 min=0, 

626 max=100, 

627 doc="Amount of scatter to add to the visit magnitude for variable " 

628 "sources." 

629 ) 

630 

631 

632class ProcessCcdWithVariableFakesTask(ProcessCcdWithFakesTask): 

633 """As ProcessCcdWithFakes except add variablity to the fakes catalog 

634 magnitude in the observed band for this ccdVisit. 

635 

636 Additionally, write out the modified magnitudes to the Butler. 

637 """ 

638 

639 _DefaultName = "processCcdWithVariableFakes" 

640 ConfigClass = ProcessCcdWithVariableFakesConfig 

641 

642 def run(self, fakeCats, exposure, skyMap, wcs=None, photoCalib=None, exposureIdInfo=None, 

643 icSourceCat=None, sfdSourceCat=None): 

644 """Add fake sources to a calexp and then run detection, deblending and measurement. 

645 

646 Parameters 

647 ---------- 

648 fakeCat : `pandas.core.frame.DataFrame` 

649 The catalog of fake sources to add to the exposure 

650 exposure : `lsst.afw.image.exposure.exposure.ExposureF` 

651 The exposure to add the fake sources to 

652 skyMap : `lsst.skymap.SkyMap` 

653 SkyMap defining the tracts and patches the fakes are stored over. 

654 wcs : `lsst.afw.geom.SkyWcs` 

655 WCS to use to add fake sources 

656 photoCalib : `lsst.afw.image.photoCalib.PhotoCalib` 

657 Photometric calibration to be used to calibrate the fake sources 

658 exposureIdInfo : `lsst.obs.base.ExposureIdInfo` 

659 icSourceCat : `lsst.afw.table.SourceCatalog` 

660 Default : None 

661 Catalog to take the information about which sources were used for calibration from. 

662 sfdSourceCat : `lsst.afw.table.SourceCatalog` 

663 Default : None 

664 Catalog produced by singleFrameDriver, needed to copy some calibration flags from. 

665 

666 Returns 

667 ------- 

668 resultStruct : `lsst.pipe.base.struct.Struct` 

669 Results Strcut containing: 

670 

671 - outputExposure : Exposure with added fakes 

672 (`lsst.afw.image.exposure.exposure.ExposureF`) 

673 - outputCat : Catalog with detected fakes 

674 (`lsst.afw.table.source.source.SourceCatalog`) 

675 - ccdVisitFakeMagnitudes : Magnitudes that these fakes were 

676 inserted with after being scattered (`pandas.DataFrame`) 

677 

678 Notes 

679 ----- 

680 Adds pixel coordinates for each source to the fakeCat and removes objects with bulge or disk half 

681 light radius = 0 (if ``config.cleanCat = True``). These columns are called ``x`` and ``y`` and are in 

682 pixels. 

683 

684 Adds the ``Fake`` mask plane to the exposure which is then set by `addFakeSources` to mark where fake 

685 sources have been added. Uses the information in the ``fakeCat`` to make fake galaxies (using galsim) 

686 and fake stars, using the PSF models from the PSF information for the calexp. These are then added to 

687 the calexp and the calexp with fakes included returned. 

688 

689 The galsim galaxies are made using a double sersic profile, one for the bulge and one for the disk, 

690 this is then convolved with the PSF at that point. 

691 

692 If exposureIdInfo is not provided then the SourceCatalog IDs will not be globally unique. 

693 """ 

694 fakeCat = self.composeFakeCat(fakeCats, skyMap) 

695 

696 if wcs is None: 

697 wcs = exposure.getWcs() 

698 

699 if photoCalib is None: 

700 photoCalib = exposure.getPhotoCalib() 

701 

702 if exposureIdInfo is None: 

703 exposureIdInfo = ExposureIdInfo() 

704 

705 band = exposure.getFilter().bandLabel 

706 ccdVisitMagnitudes = self.addVariablity(fakeCat, band, exposure, photoCalib, exposureIdInfo) 

707 

708 self.insertFakes.run(fakeCat, exposure, wcs, photoCalib) 

709 

710 # detect, deblend and measure sources 

711 returnedStruct = self.calibrate.run(exposure, exposureIdInfo=exposureIdInfo) 

712 sourceCat = returnedStruct.sourceCat 

713 

714 sourceCat = self.copyCalibrationFields(sfdSourceCat, sourceCat, self.config.srcFieldsToCopy) 

715 

716 resultStruct = pipeBase.Struct(outputExposure=exposure, 

717 outputCat=sourceCat, 

718 ccdVisitFakeMagnitudes=ccdVisitMagnitudes) 

719 return resultStruct 

720 

721 def addVariablity(self, fakeCat, band, exposure, photoCalib, exposureIdInfo): 

722 """Add scatter to the fake catalog visit magnitudes. 

723 

724 Currently just adds a simple Gaussian scatter around the static fake 

725 magnitude. This function could be modified to return any number of 

726 fake variability. 

727 

728 Parameters 

729 ---------- 

730 fakeCat : `pandas.DataFrame` 

731 Catalog of fakes to modify magnitudes of. 

732 band : `str` 

733 Current observing band to modify. 

734 exposure : `lsst.afw.image.ExposureF` 

735 Exposure fakes will be added to. 

736 photoCalib : `lsst.afw.image.PhotoCalib` 

737 Photometric calibration object of ``exposure``. 

738 exposureIdInfo : `lsst.obs.base.ExposureIdInfo` 

739 Exposure id information and metadata. 

740 

741 Returns 

742 ------- 

743 dataFrame : `pandas.DataFrame` 

744 DataFrame containing the values of the magnitudes to that will 

745 be inserted into this ccdVisit. 

746 """ 

747 expId = exposureIdInfo.expId 

748 rng = np.random.default_rng(expId) 

749 magScatter = rng.normal(loc=0, 

750 scale=self.config.scatterSize, 

751 size=len(fakeCat)) 

752 visitMagnitudes = fakeCat[self.insertFakes.config.mag_col % band] + magScatter 

753 fakeCat.loc[:, self.insertFakes.config.mag_col % band] = visitMagnitudes 

754 return pd.DataFrame(data={"variableMag": visitMagnitudes})