Coverage for python/lsst/pipe/tasks/processCcdWithFakes.py: 19%

232 statements  

« prev     ^ index     » next       coverage.py v6.5.0, created at 2023-03-09 04:15 -0800

1# This file is part of pipe_tasks. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22""" 

23Insert fake sources into calexps 

24""" 

25 

26__all__ = ["ProcessCcdWithFakesConfig", "ProcessCcdWithFakesTask", 

27 "ProcessCcdWithVariableFakesConfig", "ProcessCcdWithVariableFakesTask"] 

28 

29import numpy as np 

30import pandas as pd 

31 

32import lsst.pex.config as pexConfig 

33import lsst.pipe.base as pipeBase 

34 

35from .insertFakes import InsertFakesTask 

36from lsst.afw.table import SourceTable 

37from lsst.obs.base import ExposureIdInfo 

38from lsst.pipe.base import PipelineTask, PipelineTaskConfig, PipelineTaskConnections 

39import lsst.pipe.base.connectionTypes as cT 

40import lsst.afw.table as afwTable 

41from lsst.skymap import BaseSkyMap 

42from lsst.pipe.tasks.calibrate import CalibrateTask 

43 

44 

45class ProcessCcdWithFakesConnections(PipelineTaskConnections, 

46 dimensions=("instrument", "visit", "detector"), 

47 defaultTemplates={"coaddName": "deep", 

48 "wcsName": "jointcal", 

49 "photoCalibName": "jointcal", 

50 "fakesType": "fakes_"}): 

51 skyMap = cT.Input( 

52 doc="Input definition of geometry/bbox and projection/wcs for " 

53 "template exposures. Needed to test which tract to generate ", 

54 name=BaseSkyMap.SKYMAP_DATASET_TYPE_NAME, 

55 dimensions=("skymap",), 

56 storageClass="SkyMap", 

57 ) 

58 

59 exposure = cT.Input( 

60 doc="Exposure into which fakes are to be added.", 

61 name="calexp", 

62 storageClass="ExposureF", 

63 dimensions=("instrument", "visit", "detector") 

64 ) 

65 

66 fakeCats = cT.Input( 

67 doc="Set of catalogs of fake sources to draw inputs from. We " 

68 "concatenate the tract catalogs for detectorVisits that cover " 

69 "multiple tracts.", 

70 name="{fakesType}fakeSourceCat", 

71 storageClass="DataFrame", 

72 dimensions=("tract", "skymap"), 

73 deferLoad=True, 

74 multiple=True, 

75 ) 

76 

77 externalSkyWcsTractCatalog = cT.Input( 

78 doc=("Per-tract, per-visit wcs calibrations. These catalogs use the detector " 

79 "id for the catalog id, sorted on id for fast lookup."), 

80 name="{wcsName}SkyWcsCatalog", 

81 storageClass="ExposureCatalog", 

82 dimensions=("instrument", "visit", "tract", "skymap"), 

83 deferLoad=True, 

84 multiple=True, 

85 ) 

86 

87 externalSkyWcsGlobalCatalog = cT.Input( 

88 doc=("Per-visit wcs calibrations computed globally (with no tract information). " 

89 "These catalogs use the detector id for the catalog id, sorted on id for " 

90 "fast lookup."), 

91 name="finalVisitSummary", 

92 storageClass="ExposureCatalog", 

93 dimensions=("instrument", "visit"), 

94 ) 

95 

96 externalPhotoCalibTractCatalog = cT.Input( 

97 doc=("Per-tract, per-visit photometric calibrations. These catalogs use the " 

98 "detector id for the catalog id, sorted on id for fast lookup."), 

99 name="{photoCalibName}PhotoCalibCatalog", 

100 storageClass="ExposureCatalog", 

101 dimensions=("instrument", "visit", "tract"), 

102 deferLoad=True, 

103 multiple=True, 

104 ) 

105 

106 externalPhotoCalibGlobalCatalog = cT.Input( 

107 doc=("Per-visit photometric calibrations. These catalogs use the " 

108 "detector id for the catalog id, sorted on id for fast lookup."), 

109 name="finalVisitSummary", 

110 storageClass="ExposureCatalog", 

111 dimensions=("instrument", "visit"), 

112 ) 

113 

114 icSourceCat = cT.Input( 

115 doc="Catalog of calibration sources", 

116 name="icSrc", 

117 storageClass="SourceCatalog", 

118 dimensions=("instrument", "visit", "detector") 

119 ) 

120 

121 sfdSourceCat = cT.Input( 

122 doc="Catalog of calibration sources", 

123 name="src", 

124 storageClass="SourceCatalog", 

125 dimensions=("instrument", "visit", "detector") 

126 ) 

127 

128 outputExposure = cT.Output( 

129 doc="Exposure with fake sources added.", 

130 name="{fakesType}calexp", 

131 storageClass="ExposureF", 

132 dimensions=("instrument", "visit", "detector") 

133 ) 

134 

135 outputCat = cT.Output( 

136 doc="Source catalog produced in calibrate task with fakes also measured.", 

137 name="{fakesType}src", 

138 storageClass="SourceCatalog", 

139 dimensions=("instrument", "visit", "detector"), 

140 ) 

141 

142 def __init__(self, *, config=None): 

143 super().__init__(config=config) 

144 

145 if not config.doApplyExternalGlobalPhotoCalib: 

146 self.inputs.remove("externalPhotoCalibGlobalCatalog") 

147 if not config.doApplyExternalTractPhotoCalib: 

148 self.inputs.remove("externalPhotoCalibTractCatalog") 

149 

150 if not config.doApplyExternalGlobalSkyWcs: 

151 self.inputs.remove("externalSkyWcsGlobalCatalog") 

152 if not config.doApplyExternalTractSkyWcs: 

153 self.inputs.remove("externalSkyWcsTractCatalog") 

154 

155 

156class ProcessCcdWithFakesConfig(PipelineTaskConfig, 

157 pipelineConnections=ProcessCcdWithFakesConnections): 

158 """Config for inserting fake sources 

159 

160 Notes 

161 ----- 

162 The default column names are those from the UW sims database. 

163 """ 

164 

165 doApplyExternalGlobalPhotoCalib = pexConfig.Field( 

166 dtype=bool, 

167 default=False, 

168 doc="Whether to apply an external photometric calibration via an " 

169 "`lsst.afw.image.PhotoCalib` object. Uses the " 

170 "`externalPhotoCalibName` config option to determine which " 

171 "calibration to use. Uses a global calibration." 

172 ) 

173 

174 doApplyExternalTractPhotoCalib = pexConfig.Field( 

175 dtype=bool, 

176 default=False, 

177 doc="Whether to apply an external photometric calibration via an " 

178 "`lsst.afw.image.PhotoCalib` object. Uses the " 

179 "`externalPhotoCalibName` config option to determine which " 

180 "calibration to use. Uses a per tract calibration." 

181 ) 

182 

183 externalPhotoCalibName = pexConfig.ChoiceField( 

184 doc="What type of external photo calib to use.", 

185 dtype=str, 

186 default="jointcal", 

187 allowed={"jointcal": "Use jointcal_photoCalib", 

188 "fgcm": "Use fgcm_photoCalib", 

189 "fgcm_tract": "Use fgcm_tract_photoCalib"} 

190 ) 

191 

192 doApplyExternalGlobalSkyWcs = pexConfig.Field( 

193 dtype=bool, 

194 default=False, 

195 doc="Whether to apply an external astrometric calibration via an " 

196 "`lsst.afw.geom.SkyWcs` object. Uses the " 

197 "`externalSkyWcsName` config option to determine which " 

198 "calibration to use. Uses a global calibration." 

199 ) 

200 

201 doApplyExternalTractSkyWcs = pexConfig.Field( 

202 dtype=bool, 

203 default=False, 

204 doc="Whether to apply an external astrometric calibration via an " 

205 "`lsst.afw.geom.SkyWcs` object. Uses the " 

206 "`externalSkyWcsName` config option to determine which " 

207 "calibration to use. Uses a per tract calibration." 

208 ) 

209 

210 externalSkyWcsName = pexConfig.ChoiceField( 

211 doc="What type of updated WCS calib to use.", 

212 dtype=str, 

213 default="jointcal", 

214 allowed={"jointcal": "Use jointcal_wcs"} 

215 ) 

216 

217 coaddName = pexConfig.Field( 

218 doc="The name of the type of coadd used", 

219 dtype=str, 

220 default="deep", 

221 ) 

222 

223 srcFieldsToCopy = pexConfig.ListField( 

224 dtype=str, 

225 default=("calib_photometry_reserved", "calib_photometry_used", "calib_astrometry_used", 

226 "calib_psf_candidate", "calib_psf_used", "calib_psf_reserved"), 

227 doc=("Fields to copy from the `src` catalog to the output catalog " 

228 "for matching sources Any missing fields will trigger a " 

229 "RuntimeError exception.") 

230 ) 

231 

232 matchRadiusPix = pexConfig.Field( 

233 dtype=float, 

234 default=3, 

235 doc=("Match radius for matching icSourceCat objects to sourceCat objects (pixels)"), 

236 ) 

237 

238 doMatchVisit = pexConfig.Field( 

239 dtype=bool, 

240 default=False, 

241 doc="Match visit to trim the fakeCat" 

242 ) 

243 

244 calibrate = pexConfig.ConfigurableField(target=CalibrateTask, 

245 doc="The calibration task to use.") 

246 

247 insertFakes = pexConfig.ConfigurableField(target=InsertFakesTask, 

248 doc="Configuration for the fake sources") 

249 

250 def setDefaults(self): 

251 super().setDefaults() 

252 self.calibrate.measurement.plugins["base_PixelFlags"].masksFpAnywhere.append("FAKE") 

253 self.calibrate.measurement.plugins["base_PixelFlags"].masksFpCenter.append("FAKE") 

254 self.calibrate.doAstrometry = False 

255 self.calibrate.doWriteMatches = False 

256 self.calibrate.doPhotoCal = False 

257 self.calibrate.doComputeSummaryStats = False 

258 self.calibrate.detection.reEstimateBackground = False 

259 

260 

261class ProcessCcdWithFakesTask(PipelineTask): 

262 """Insert fake objects into calexps. 

263 

264 Add fake stars and galaxies to the given calexp, specified in the dataRef. Galaxy parameters are read in 

265 from the specified file and then modelled using galsim. Re-runs characterize image and calibrate image to 

266 give a new background estimation and measurement of the calexp. 

267 

268 `ProcessFakeSourcesTask` inherits six functions from insertFakesTask that make images of the fake 

269 sources and then add them to the calexp. 

270 

271 `addPixCoords` 

272 Use the WCS information to add the pixel coordinates of each source 

273 Adds an ``x`` and ``y`` column to the catalog of fake sources. 

274 `trimFakeCat` 

275 Trim the fake cat to about the size of the input image. 

276 `mkFakeGalsimGalaxies` 

277 Use Galsim to make fake double sersic galaxies for each set of galaxy parameters in the input file. 

278 `mkFakeStars` 

279 Use the PSF information from the calexp to make a fake star using the magnitude information from the 

280 input file. 

281 `cleanCat` 

282 Remove rows of the input fake catalog which have half light radius, of either the bulge or the disk, 

283 that are 0. 

284 `addFakeSources` 

285 Add the fake sources to the calexp. 

286 

287 Notes 

288 ----- 

289 The ``calexp`` with fake souces added to it is written out as the datatype ``calexp_fakes``. 

290 """ 

291 

292 _DefaultName = "processCcdWithFakes" 

293 ConfigClass = ProcessCcdWithFakesConfig 

294 

295 def __init__(self, schema=None, butler=None, **kwargs): 

296 """Initalize things! This should go above in the class docstring 

297 """ 

298 

299 super().__init__(**kwargs) 

300 

301 if schema is None: 

302 schema = SourceTable.makeMinimalSchema() 

303 self.schema = schema 

304 self.makeSubtask("insertFakes") 

305 self.makeSubtask("calibrate") 

306 

307 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

308 inputs = butlerQC.get(inputRefs) 

309 detectorId = inputs["exposure"].getInfo().getDetector().getId() 

310 

311 if 'exposureIdInfo' not in inputs.keys(): 

312 expId, expBits = butlerQC.quantum.dataId.pack("visit_detector", returnMaxBits=True) 

313 inputs['exposureIdInfo'] = ExposureIdInfo(expId, expBits) 

314 

315 expWcs = inputs["exposure"].getWcs() 

316 tractId = None 

317 if not self.config.doApplyExternalGlobalSkyWcs and not self.config.doApplyExternalTractSkyWcs: 

318 if expWcs is None: 

319 self.log.info("No WCS for exposure %s so cannot insert fake sources. Skipping detector.", 

320 butlerQC.quantum.dataId) 

321 return None 

322 else: 

323 inputs["wcs"] = expWcs 

324 elif self.config.doApplyExternalGlobalSkyWcs: 

325 externalSkyWcsCatalog = inputs["externalSkyWcsGlobalCatalog"] 

326 row = externalSkyWcsCatalog.find(detectorId) 

327 if row is None: 

328 self.log.info("No %s external global sky WCS for exposure %s so cannot insert fake " 

329 "sources. Skipping detector.", self.config.externalSkyWcsName, 

330 butlerQC.quantum.dataId) 

331 return None 

332 inputs["wcs"] = row.getWcs() 

333 elif self.config.doApplyExternalTractSkyWcs: 

334 externalSkyWcsCatalogList = inputs["externalSkyWcsTractCatalog"] 

335 if tractId is None: 

336 tractId = externalSkyWcsCatalogList[0].dataId["tract"] 

337 externalSkyWcsCatalog = None 

338 for externalSkyWcsCatalogRef in externalSkyWcsCatalogList: 

339 if externalSkyWcsCatalogRef.dataId["tract"] == tractId: 

340 externalSkyWcsCatalog = externalSkyWcsCatalogRef.get() 

341 break 

342 if externalSkyWcsCatalog is None: 

343 usedTract = externalSkyWcsCatalogList[-1].dataId["tract"] 

344 self.log.warn( 

345 f"Warning, external SkyWcs for tract {tractId} not found. Using tract {usedTract} " 

346 "instead.") 

347 externalSkyWcsCatalog = externalSkyWcsCatalogList[-1].get() 

348 row = externalSkyWcsCatalog.find(detectorId) 

349 if row is None: 

350 self.log.info("No %s external tract sky WCS for exposure %s so cannot insert fake " 

351 "sources. Skipping detector.", self.config.externalSkyWcsName, 

352 butlerQC.quantum.dataId) 

353 return None 

354 inputs["wcs"] = row.getWcs() 

355 

356 if not self.config.doApplyExternalGlobalPhotoCalib and not self.config.doApplyExternalTractPhotoCalib: 

357 inputs["photoCalib"] = inputs["exposure"].getPhotoCalib() 

358 elif self.config.doApplyExternalGlobalPhotoCalib: 

359 externalPhotoCalibCatalog = inputs["externalPhotoCalibGlobalCatalog"] 

360 row = externalPhotoCalibCatalog.find(detectorId) 

361 if row is None: 

362 self.log.info("No %s external global photoCalib for exposure %s so cannot insert fake " 

363 "sources. Skipping detector.", self.config.externalPhotoCalibName, 

364 butlerQC.quantum.dataId) 

365 return None 

366 inputs["photoCalib"] = row.getPhotoCalib() 

367 elif self.config.doApplyExternalTractPhotoCalib: 

368 externalPhotoCalibCatalogList = inputs["externalPhotoCalibTractCatalog"] 

369 if tractId is None: 

370 tractId = externalPhotoCalibCatalogList[0].dataId["tract"] 

371 externalPhotoCalibCatalog = None 

372 for externalPhotoCalibCatalogRef in externalPhotoCalibCatalogList: 

373 if externalPhotoCalibCatalogRef.dataId["tract"] == tractId: 

374 externalPhotoCalibCatalog = externalPhotoCalibCatalogRef.get() 

375 break 

376 if externalPhotoCalibCatalog is None: 

377 usedTract = externalPhotoCalibCatalogList[-1].dataId["tract"] 

378 self.log.warn( 

379 f"Warning, external PhotoCalib for tract {tractId} not found. Using tract {usedTract} " 

380 "instead.") 

381 externalPhotoCalibCatalog = externalPhotoCalibCatalogList[-1].get() 

382 row = externalPhotoCalibCatalog.find(detectorId) 

383 if row is None: 

384 self.log.info("No %s external tract photoCalib for exposure %s so cannot insert fake " 

385 "sources. Skipping detector.", self.config.externalPhotoCalibName, 

386 butlerQC.quantum.dataId) 

387 return None 

388 inputs["photoCalib"] = row.getPhotoCalib() 

389 

390 outputs = self.run(**inputs) 

391 butlerQC.put(outputs, outputRefs) 

392 

393 def run(self, fakeCats, exposure, skyMap, wcs=None, photoCalib=None, exposureIdInfo=None, 

394 icSourceCat=None, sfdSourceCat=None, externalSkyWcsGlobalCatalog=None, 

395 externalSkyWcsTractCatalog=None, externalPhotoCalibGlobalCatalog=None, 

396 externalPhotoCalibTractCatalog=None): 

397 """Add fake sources to a calexp and then run detection, deblending and measurement. 

398 

399 Parameters 

400 ---------- 

401 fakeCats : `list` of `lsst.daf.butler.DeferredDatasetHandle` 

402 Set of tract level fake catalogs that potentially cover 

403 this detectorVisit. 

404 exposure : `lsst.afw.image.exposure.exposure.ExposureF` 

405 The exposure to add the fake sources to 

406 skyMap : `lsst.skymap.SkyMap` 

407 SkyMap defining the tracts and patches the fakes are stored over. 

408 wcs : `lsst.afw.geom.SkyWcs` 

409 WCS to use to add fake sources 

410 photoCalib : `lsst.afw.image.photoCalib.PhotoCalib` 

411 Photometric calibration to be used to calibrate the fake sources 

412 exposureIdInfo : `lsst.obs.base.ExposureIdInfo` 

413 icSourceCat : `lsst.afw.table.SourceCatalog` 

414 Default : None 

415 Catalog to take the information about which sources were used for calibration from. 

416 sfdSourceCat : `lsst.afw.table.SourceCatalog` 

417 Default : None 

418 Catalog produced by singleFrameDriver, needed to copy some calibration flags from. 

419 

420 Returns 

421 ------- 

422 resultStruct : `lsst.pipe.base.struct.Struct` 

423 contains : outputExposure : `lsst.afw.image.exposure.exposure.ExposureF` 

424 outputCat : `lsst.afw.table.source.source.SourceCatalog` 

425 

426 Notes 

427 ----- 

428 Adds pixel coordinates for each source to the fakeCat and removes objects with bulge or disk half 

429 light radius = 0 (if ``config.cleanCat = True``). These columns are called ``x`` and ``y`` and are in 

430 pixels. 

431 

432 Adds the ``Fake`` mask plane to the exposure which is then set by `addFakeSources` to mark where fake 

433 sources have been added. Uses the information in the ``fakeCat`` to make fake galaxies (using galsim) 

434 and fake stars, using the PSF models from the PSF information for the calexp. These are then added to 

435 the calexp and the calexp with fakes included returned. 

436 

437 The galsim galaxies are made using a double sersic profile, one for the bulge and one for the disk, 

438 this is then convolved with the PSF at that point. 

439 

440 If exposureIdInfo is not provided then the SourceCatalog IDs will not be globally unique. 

441 """ 

442 fakeCat = self.composeFakeCat(fakeCats, skyMap) 

443 

444 if wcs is None: 

445 wcs = exposure.getWcs() 

446 

447 if photoCalib is None: 

448 photoCalib = exposure.getPhotoCalib() 

449 

450 if self.config.doMatchVisit: 

451 fakeCat = self.getVisitMatchedFakeCat(fakeCat, exposure) 

452 

453 self.insertFakes.run(fakeCat, exposure, wcs, photoCalib) 

454 

455 # detect, deblend and measure sources 

456 if exposureIdInfo is None: 

457 exposureIdInfo = ExposureIdInfo() 

458 returnedStruct = self.calibrate.run(exposure, exposureIdInfo=exposureIdInfo) 

459 sourceCat = returnedStruct.sourceCat 

460 

461 sourceCat = self.copyCalibrationFields(sfdSourceCat, sourceCat, self.config.srcFieldsToCopy) 

462 

463 resultStruct = pipeBase.Struct(outputExposure=exposure, outputCat=sourceCat) 

464 return resultStruct 

465 

466 def composeFakeCat(self, fakeCats, skyMap): 

467 """Concatenate the fakeCats from tracts that may cover the exposure. 

468 

469 Parameters 

470 ---------- 

471 fakeCats : `list` of `lsst.daf.butler.DeferredDatasetHandle` 

472 Set of fake cats to concatenate. 

473 skyMap : `lsst.skymap.SkyMap` 

474 SkyMap defining the geometry of the tracts and patches. 

475 

476 Returns 

477 ------- 

478 combinedFakeCat : `pandas.DataFrame` 

479 All fakes that cover the inner polygon of the tracts in this 

480 quantum. 

481 """ 

482 if len(fakeCats) == 1: 

483 return fakeCats[0].get() 

484 outputCat = [] 

485 for fakeCatRef in fakeCats: 

486 cat = fakeCatRef.get() 

487 tractId = fakeCatRef.dataId["tract"] 

488 # Make sure all data is within the inner part of the tract. 

489 outputCat.append(cat[ 

490 skyMap.findTractIdArray(cat[self.config.insertFakes.ra_col], 

491 cat[self.config.insertFakes.dec_col], 

492 degrees=False) 

493 == tractId]) 

494 

495 return pd.concat(outputCat) 

496 

497 def getVisitMatchedFakeCat(self, fakeCat, exposure): 

498 """Trim the fakeCat to select particular visit 

499 

500 Parameters 

501 ---------- 

502 fakeCat : `pandas.core.frame.DataFrame` 

503 The catalog of fake sources to add to the exposure 

504 exposure : `lsst.afw.image.exposure.exposure.ExposureF` 

505 The exposure to add the fake sources to 

506 

507 Returns 

508 ------- 

509 movingFakeCat : `pandas.DataFrame` 

510 All fakes that belong to the visit 

511 """ 

512 selected = exposure.getInfo().getVisitInfo().getId() == fakeCat["visit"] 

513 

514 return fakeCat[selected] 

515 

516 def copyCalibrationFields(self, calibCat, sourceCat, fieldsToCopy): 

517 """Match sources in calibCat and sourceCat and copy the specified fields 

518 

519 Parameters 

520 ---------- 

521 calibCat : `lsst.afw.table.SourceCatalog` 

522 Catalog from which to copy fields. 

523 sourceCat : `lsst.afw.table.SourceCatalog` 

524 Catalog to which to copy fields. 

525 fieldsToCopy : `lsst.pex.config.listField.List` 

526 Fields to copy from calibCat to SoourceCat. 

527 

528 Returns 

529 ------- 

530 newCat : `lsst.afw.table.SourceCatalog` 

531 Catalog which includes the copied fields. 

532 

533 The fields copied are those specified by `fieldsToCopy` that actually exist 

534 in the schema of `calibCat`. 

535 

536 This version was based on and adapted from the one in calibrateTask. 

537 """ 

538 

539 # Make a new SourceCatalog with the data from sourceCat so that we can add the new columns to it 

540 sourceSchemaMapper = afwTable.SchemaMapper(sourceCat.schema) 

541 sourceSchemaMapper.addMinimalSchema(sourceCat.schema, True) 

542 

543 calibSchemaMapper = afwTable.SchemaMapper(calibCat.schema, sourceCat.schema) 

544 

545 # Add the desired columns from the option fieldsToCopy 

546 missingFieldNames = [] 

547 for fieldName in fieldsToCopy: 

548 if fieldName in calibCat.schema: 

549 schemaItem = calibCat.schema.find(fieldName) 

550 calibSchemaMapper.editOutputSchema().addField(schemaItem.getField()) 

551 schema = calibSchemaMapper.editOutputSchema() 

552 calibSchemaMapper.addMapping(schemaItem.getKey(), schema.find(fieldName).getField()) 

553 else: 

554 missingFieldNames.append(fieldName) 

555 if missingFieldNames: 

556 raise RuntimeError(f"calibCat is missing fields {missingFieldNames} specified in " 

557 "fieldsToCopy") 

558 

559 if "calib_detected" not in calibSchemaMapper.getOutputSchema(): 

560 self.calibSourceKey = calibSchemaMapper.addOutputField(afwTable.Field["Flag"]("calib_detected", 

561 "Source was detected as an icSource")) 

562 else: 

563 self.calibSourceKey = None 

564 

565 schema = calibSchemaMapper.getOutputSchema() 

566 newCat = afwTable.SourceCatalog(schema) 

567 newCat.reserve(len(sourceCat)) 

568 newCat.extend(sourceCat, sourceSchemaMapper) 

569 

570 # Set the aliases so it doesn't complain. 

571 for k, v in sourceCat.schema.getAliasMap().items(): 

572 newCat.schema.getAliasMap().set(k, v) 

573 

574 select = newCat["deblend_nChild"] == 0 

575 matches = afwTable.matchXy(newCat[select], calibCat, self.config.matchRadiusPix) 

576 # Check that no sourceCat sources are listed twice (we already know 

577 # that each match has a unique calibCat source ID, due to using 

578 # that ID as the key in bestMatches) 

579 numMatches = len(matches) 

580 numUniqueSources = len(set(m[1].getId() for m in matches)) 

581 if numUniqueSources != numMatches: 

582 self.log.warning("%d calibCat sources matched only %d sourceCat sources", numMatches, 

583 numUniqueSources) 

584 

585 self.log.info("Copying flags from calibCat to sourceCat for %s sources", numMatches) 

586 

587 # For each match: set the calibSourceKey flag and copy the desired 

588 # fields 

589 for src, calibSrc, d in matches: 

590 if self.calibSourceKey: 

591 src.setFlag(self.calibSourceKey, True) 

592 # src.assign copies the footprint from calibSrc, which we don't want 

593 # (DM-407) 

594 # so set calibSrc's footprint to src's footprint before src.assign, 

595 # then restore it 

596 calibSrcFootprint = calibSrc.getFootprint() 

597 try: 

598 calibSrc.setFootprint(src.getFootprint()) 

599 src.assign(calibSrc, calibSchemaMapper) 

600 finally: 

601 calibSrc.setFootprint(calibSrcFootprint) 

602 

603 return newCat 

604 

605 

606class ProcessCcdWithVariableFakesConnections(ProcessCcdWithFakesConnections): 

607 ccdVisitFakeMagnitudes = cT.Output( 

608 doc="Catalog of fakes with magnitudes scattered for this ccdVisit.", 

609 name="{fakesType}ccdVisitFakeMagnitudes", 

610 storageClass="DataFrame", 

611 dimensions=("instrument", "visit", "detector"), 

612 ) 

613 

614 

615class ProcessCcdWithVariableFakesConfig(ProcessCcdWithFakesConfig, 

616 pipelineConnections=ProcessCcdWithVariableFakesConnections): 

617 scatterSize = pexConfig.RangeField( 

618 dtype=float, 

619 default=0.4, 

620 min=0, 

621 max=100, 

622 doc="Amount of scatter to add to the visit magnitude for variable " 

623 "sources." 

624 ) 

625 

626 

627class ProcessCcdWithVariableFakesTask(ProcessCcdWithFakesTask): 

628 """As ProcessCcdWithFakes except add variablity to the fakes catalog 

629 magnitude in the observed band for this ccdVisit. 

630 

631 Additionally, write out the modified magnitudes to the Butler. 

632 """ 

633 

634 _DefaultName = "processCcdWithVariableFakes" 

635 ConfigClass = ProcessCcdWithVariableFakesConfig 

636 

637 def run(self, fakeCats, exposure, skyMap, wcs=None, photoCalib=None, exposureIdInfo=None, 

638 icSourceCat=None, sfdSourceCat=None): 

639 """Add fake sources to a calexp and then run detection, deblending and measurement. 

640 

641 Parameters 

642 ---------- 

643 fakeCat : `pandas.core.frame.DataFrame` 

644 The catalog of fake sources to add to the exposure 

645 exposure : `lsst.afw.image.exposure.exposure.ExposureF` 

646 The exposure to add the fake sources to 

647 skyMap : `lsst.skymap.SkyMap` 

648 SkyMap defining the tracts and patches the fakes are stored over. 

649 wcs : `lsst.afw.geom.SkyWcs` 

650 WCS to use to add fake sources 

651 photoCalib : `lsst.afw.image.photoCalib.PhotoCalib` 

652 Photometric calibration to be used to calibrate the fake sources 

653 exposureIdInfo : `lsst.obs.base.ExposureIdInfo` 

654 icSourceCat : `lsst.afw.table.SourceCatalog` 

655 Default : None 

656 Catalog to take the information about which sources were used for calibration from. 

657 sfdSourceCat : `lsst.afw.table.SourceCatalog` 

658 Default : None 

659 Catalog produced by singleFrameDriver, needed to copy some calibration flags from. 

660 

661 Returns 

662 ------- 

663 resultStruct : `lsst.pipe.base.struct.Struct` 

664 Results Strcut containing: 

665 

666 - outputExposure : Exposure with added fakes 

667 (`lsst.afw.image.exposure.exposure.ExposureF`) 

668 - outputCat : Catalog with detected fakes 

669 (`lsst.afw.table.source.source.SourceCatalog`) 

670 - ccdVisitFakeMagnitudes : Magnitudes that these fakes were 

671 inserted with after being scattered (`pandas.DataFrame`) 

672 

673 Notes 

674 ----- 

675 Adds pixel coordinates for each source to the fakeCat and removes objects with bulge or disk half 

676 light radius = 0 (if ``config.cleanCat = True``). These columns are called ``x`` and ``y`` and are in 

677 pixels. 

678 

679 Adds the ``Fake`` mask plane to the exposure which is then set by `addFakeSources` to mark where fake 

680 sources have been added. Uses the information in the ``fakeCat`` to make fake galaxies (using galsim) 

681 and fake stars, using the PSF models from the PSF information for the calexp. These are then added to 

682 the calexp and the calexp with fakes included returned. 

683 

684 The galsim galaxies are made using a double sersic profile, one for the bulge and one for the disk, 

685 this is then convolved with the PSF at that point. 

686 

687 If exposureIdInfo is not provided then the SourceCatalog IDs will not be globally unique. 

688 """ 

689 fakeCat = self.composeFakeCat(fakeCats, skyMap) 

690 

691 if wcs is None: 

692 wcs = exposure.getWcs() 

693 

694 if photoCalib is None: 

695 photoCalib = exposure.getPhotoCalib() 

696 

697 if exposureIdInfo is None: 

698 exposureIdInfo = ExposureIdInfo() 

699 

700 band = exposure.getFilter().bandLabel 

701 ccdVisitMagnitudes = self.addVariablity(fakeCat, band, exposure, photoCalib, exposureIdInfo) 

702 

703 self.insertFakes.run(fakeCat, exposure, wcs, photoCalib) 

704 

705 # detect, deblend and measure sources 

706 returnedStruct = self.calibrate.run(exposure, exposureIdInfo=exposureIdInfo) 

707 sourceCat = returnedStruct.sourceCat 

708 

709 sourceCat = self.copyCalibrationFields(sfdSourceCat, sourceCat, self.config.srcFieldsToCopy) 

710 

711 resultStruct = pipeBase.Struct(outputExposure=exposure, 

712 outputCat=sourceCat, 

713 ccdVisitFakeMagnitudes=ccdVisitMagnitudes) 

714 return resultStruct 

715 

716 def addVariablity(self, fakeCat, band, exposure, photoCalib, exposureIdInfo): 

717 """Add scatter to the fake catalog visit magnitudes. 

718 

719 Currently just adds a simple Gaussian scatter around the static fake 

720 magnitude. This function could be modified to return any number of 

721 fake variability. 

722 

723 Parameters 

724 ---------- 

725 fakeCat : `pandas.DataFrame` 

726 Catalog of fakes to modify magnitudes of. 

727 band : `str` 

728 Current observing band to modify. 

729 exposure : `lsst.afw.image.ExposureF` 

730 Exposure fakes will be added to. 

731 photoCalib : `lsst.afw.image.PhotoCalib` 

732 Photometric calibration object of ``exposure``. 

733 exposureIdInfo : `lsst.obs.base.ExposureIdInfo` 

734 Exposure id information and metadata. 

735 

736 Returns 

737 ------- 

738 dataFrame : `pandas.DataFrame` 

739 DataFrame containing the values of the magnitudes to that will 

740 be inserted into this ccdVisit. 

741 """ 

742 expId = exposureIdInfo.expId 

743 rng = np.random.default_rng(expId) 

744 magScatter = rng.normal(loc=0, 

745 scale=self.config.scatterSize, 

746 size=len(fakeCat)) 

747 visitMagnitudes = fakeCat[self.insertFakes.config.mag_col % band] + magScatter 

748 fakeCat.loc[:, self.insertFakes.config.mag_col % band] = visitMagnitudes 

749 return pd.DataFrame(data={"variableMag": visitMagnitudes})