Coverage for python/lsst/pipe/tasks/processCcdWithFakes.py: 24%

212 statements  

« prev     ^ index     » next       coverage.py v6.4.4, created at 2022-09-11 02:03 -0700

1# This file is part of pipe tasks 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22""" 

23Insert fake sources into calexps 

24""" 

25import numpy as np 

26import pandas as pd 

27 

28import lsst.pex.config as pexConfig 

29import lsst.pipe.base as pipeBase 

30 

31from .insertFakes import InsertFakesTask 

32from lsst.afw.table import SourceTable 

33from lsst.obs.base import ExposureIdInfo 

34from lsst.pipe.base import PipelineTask, PipelineTaskConfig, PipelineTaskConnections 

35import lsst.pipe.base.connectionTypes as cT 

36import lsst.afw.table as afwTable 

37from lsst.skymap import BaseSkyMap 

38from lsst.pipe.tasks.calibrate import CalibrateTask 

39 

40__all__ = ["ProcessCcdWithFakesConfig", "ProcessCcdWithFakesTask", 

41 "ProcessCcdWithVariableFakesConfig", "ProcessCcdWithVariableFakesTask"] 

42 

43 

44class ProcessCcdWithFakesConnections(PipelineTaskConnections, 

45 dimensions=("instrument", "visit", "detector"), 

46 defaultTemplates={"coaddName": "deep", 

47 "wcsName": "jointcal", 

48 "photoCalibName": "jointcal", 

49 "fakesType": "fakes_"}): 

50 skyMap = cT.Input( 

51 doc="Input definition of geometry/bbox and projection/wcs for " 

52 "template exposures. Needed to test which tract to generate ", 

53 name=BaseSkyMap.SKYMAP_DATASET_TYPE_NAME, 

54 dimensions=("skymap",), 

55 storageClass="SkyMap", 

56 ) 

57 

58 exposure = cT.Input( 

59 doc="Exposure into which fakes are to be added.", 

60 name="calexp", 

61 storageClass="ExposureF", 

62 dimensions=("instrument", "visit", "detector") 

63 ) 

64 

65 fakeCats = cT.Input( 

66 doc="Set of catalogs of fake sources to draw inputs from. We " 

67 "concatenate the tract catalogs for detectorVisits that cover " 

68 "multiple tracts.", 

69 name="{fakesType}fakeSourceCat", 

70 storageClass="DataFrame", 

71 dimensions=("tract", "skymap"), 

72 deferLoad=True, 

73 multiple=True, 

74 ) 

75 

76 externalSkyWcsTractCatalog = cT.Input( 

77 doc=("Per-tract, per-visit wcs calibrations. These catalogs use the detector " 

78 "id for the catalog id, sorted on id for fast lookup."), 

79 name="{wcsName}SkyWcsCatalog", 

80 storageClass="ExposureCatalog", 

81 dimensions=("instrument", "visit", "tract", "skymap"), 

82 deferLoad=True, 

83 multiple=True, 

84 ) 

85 

86 externalSkyWcsGlobalCatalog = cT.Input( 

87 doc=("Per-visit wcs calibrations computed globally (with no tract information). " 

88 "These catalogs use the detector id for the catalog id, sorted on id for " 

89 "fast lookup."), 

90 name="{wcsName}SkyWcsCatalog", 

91 storageClass="ExposureCatalog", 

92 dimensions=("instrument", "visit"), 

93 ) 

94 

95 externalPhotoCalibTractCatalog = cT.Input( 

96 doc=("Per-tract, per-visit photometric calibrations. These catalogs use the " 

97 "detector id for the catalog id, sorted on id for fast lookup."), 

98 name="{photoCalibName}PhotoCalibCatalog", 

99 storageClass="ExposureCatalog", 

100 dimensions=("instrument", "visit", "tract"), 

101 deferLoad=True, 

102 multiple=True, 

103 ) 

104 

105 externalPhotoCalibGlobalCatalog = cT.Input( 

106 doc=("Per-visit photometric calibrations. These catalogs use the " 

107 "detector id for the catalog id, sorted on id for fast lookup."), 

108 name="{photoCalibName}PhotoCalibCatalog", 

109 storageClass="ExposureCatalog", 

110 dimensions=("instrument", "visit"), 

111 ) 

112 

113 icSourceCat = cT.Input( 

114 doc="Catalog of calibration sources", 

115 name="icSrc", 

116 storageClass="SourceCatalog", 

117 dimensions=("instrument", "visit", "detector") 

118 ) 

119 

120 sfdSourceCat = cT.Input( 

121 doc="Catalog of calibration sources", 

122 name="src", 

123 storageClass="SourceCatalog", 

124 dimensions=("instrument", "visit", "detector") 

125 ) 

126 

127 outputExposure = cT.Output( 

128 doc="Exposure with fake sources added.", 

129 name="{fakesType}calexp", 

130 storageClass="ExposureF", 

131 dimensions=("instrument", "visit", "detector") 

132 ) 

133 

134 outputCat = cT.Output( 

135 doc="Source catalog produced in calibrate task with fakes also measured.", 

136 name="{fakesType}src", 

137 storageClass="SourceCatalog", 

138 dimensions=("instrument", "visit", "detector"), 

139 ) 

140 

141 def __init__(self, *, config=None): 

142 super().__init__(config=config) 

143 

144 if not config.doApplyExternalGlobalPhotoCalib: 

145 self.inputs.remove("externalPhotoCalibGlobalCatalog") 

146 if not config.doApplyExternalTractPhotoCalib: 

147 self.inputs.remove("externalPhotoCalibTractCatalog") 

148 

149 if not config.doApplyExternalGlobalSkyWcs: 

150 self.inputs.remove("externalSkyWcsGlobalCatalog") 

151 if not config.doApplyExternalTractSkyWcs: 

152 self.inputs.remove("externalSkyWcsTractCatalog") 

153 

154 

155class ProcessCcdWithFakesConfig(PipelineTaskConfig, 

156 pipelineConnections=ProcessCcdWithFakesConnections): 

157 """Config for inserting fake sources 

158 

159 Notes 

160 ----- 

161 The default column names are those from the UW sims database. 

162 """ 

163 

164 doApplyExternalGlobalPhotoCalib = pexConfig.Field( 

165 dtype=bool, 

166 default=False, 

167 doc="Whether to apply an external photometric calibration via an " 

168 "`lsst.afw.image.PhotoCalib` object. Uses the " 

169 "`externalPhotoCalibName` config option to determine which " 

170 "calibration to use. Uses a global calibration." 

171 ) 

172 

173 doApplyExternalTractPhotoCalib = pexConfig.Field( 

174 dtype=bool, 

175 default=False, 

176 doc="Whether to apply an external photometric calibration via an " 

177 "`lsst.afw.image.PhotoCalib` object. Uses the " 

178 "`externalPhotoCalibName` config option to determine which " 

179 "calibration to use. Uses a per tract calibration." 

180 ) 

181 

182 externalPhotoCalibName = pexConfig.ChoiceField( 

183 doc="What type of external photo calib to use.", 

184 dtype=str, 

185 default="jointcal", 

186 allowed={"jointcal": "Use jointcal_photoCalib", 

187 "fgcm": "Use fgcm_photoCalib", 

188 "fgcm_tract": "Use fgcm_tract_photoCalib"} 

189 ) 

190 

191 doApplyExternalGlobalSkyWcs = pexConfig.Field( 

192 dtype=bool, 

193 default=False, 

194 doc="Whether to apply an external astrometric calibration via an " 

195 "`lsst.afw.geom.SkyWcs` object. Uses the " 

196 "`externalSkyWcsName` config option to determine which " 

197 "calibration to use. Uses a global calibration." 

198 ) 

199 

200 doApplyExternalTractSkyWcs = pexConfig.Field( 

201 dtype=bool, 

202 default=False, 

203 doc="Whether to apply an external astrometric calibration via an " 

204 "`lsst.afw.geom.SkyWcs` object. Uses the " 

205 "`externalSkyWcsName` config option to determine which " 

206 "calibration to use. Uses a per tract calibration." 

207 ) 

208 

209 externalSkyWcsName = pexConfig.ChoiceField( 

210 doc="What type of updated WCS calib to use.", 

211 dtype=str, 

212 default="jointcal", 

213 allowed={"jointcal": "Use jointcal_wcs"} 

214 ) 

215 

216 coaddName = pexConfig.Field( 

217 doc="The name of the type of coadd used", 

218 dtype=str, 

219 default="deep", 

220 ) 

221 

222 srcFieldsToCopy = pexConfig.ListField( 

223 dtype=str, 

224 default=("calib_photometry_reserved", "calib_photometry_used", "calib_astrometry_used", 

225 "calib_psf_candidate", "calib_psf_used", "calib_psf_reserved"), 

226 doc=("Fields to copy from the `src` catalog to the output catalog " 

227 "for matching sources Any missing fields will trigger a " 

228 "RuntimeError exception.") 

229 ) 

230 

231 matchRadiusPix = pexConfig.Field( 

232 dtype=float, 

233 default=3, 

234 doc=("Match radius for matching icSourceCat objects to sourceCat objects (pixels)"), 

235 ) 

236 

237 doMatchVisit = pexConfig.Field( 

238 dtype=bool, 

239 default=False, 

240 doc="Match visit to trim the fakeCat" 

241 ) 

242 

243 calibrate = pexConfig.ConfigurableField(target=CalibrateTask, 

244 doc="The calibration task to use.") 

245 

246 insertFakes = pexConfig.ConfigurableField(target=InsertFakesTask, 

247 doc="Configuration for the fake sources") 

248 

249 def setDefaults(self): 

250 super().setDefaults() 

251 self.calibrate.measurement.plugins["base_PixelFlags"].masksFpAnywhere.append("FAKE") 

252 self.calibrate.measurement.plugins["base_PixelFlags"].masksFpCenter.append("FAKE") 

253 self.calibrate.doAstrometry = False 

254 self.calibrate.doWriteMatches = False 

255 self.calibrate.doPhotoCal = False 

256 self.calibrate.detection.reEstimateBackground = False 

257 

258 

259class ProcessCcdWithFakesTask(PipelineTask): 

260 """Insert fake objects into calexps. 

261 

262 Add fake stars and galaxies to the given calexp, specified in the dataRef. Galaxy parameters are read in 

263 from the specified file and then modelled using galsim. Re-runs characterize image and calibrate image to 

264 give a new background estimation and measurement of the calexp. 

265 

266 `ProcessFakeSourcesTask` inherits six functions from insertFakesTask that make images of the fake 

267 sources and then add them to the calexp. 

268 

269 `addPixCoords` 

270 Use the WCS information to add the pixel coordinates of each source 

271 Adds an ``x`` and ``y`` column to the catalog of fake sources. 

272 `trimFakeCat` 

273 Trim the fake cat to about the size of the input image. 

274 `mkFakeGalsimGalaxies` 

275 Use Galsim to make fake double sersic galaxies for each set of galaxy parameters in the input file. 

276 `mkFakeStars` 

277 Use the PSF information from the calexp to make a fake star using the magnitude information from the 

278 input file. 

279 `cleanCat` 

280 Remove rows of the input fake catalog which have half light radius, of either the bulge or the disk, 

281 that are 0. 

282 `addFakeSources` 

283 Add the fake sources to the calexp. 

284 

285 Notes 

286 ----- 

287 The ``calexp`` with fake souces added to it is written out as the datatype ``calexp_fakes``. 

288 """ 

289 

290 _DefaultName = "processCcdWithFakes" 

291 ConfigClass = ProcessCcdWithFakesConfig 

292 

293 def __init__(self, schema=None, butler=None, **kwargs): 

294 """Initalize things! This should go above in the class docstring 

295 """ 

296 

297 super().__init__(**kwargs) 

298 

299 if schema is None: 

300 schema = SourceTable.makeMinimalSchema() 

301 self.schema = schema 

302 self.makeSubtask("insertFakes") 

303 self.makeSubtask("calibrate") 

304 

305 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

306 inputs = butlerQC.get(inputRefs) 

307 detectorId = inputs["exposure"].getInfo().getDetector().getId() 

308 

309 if 'exposureIdInfo' not in inputs.keys(): 

310 expId, expBits = butlerQC.quantum.dataId.pack("visit_detector", returnMaxBits=True) 

311 inputs['exposureIdInfo'] = ExposureIdInfo(expId, expBits) 

312 

313 expWcs = inputs["exposure"].getWcs() 

314 tractId = inputs["skyMap"].findTract( 

315 expWcs.pixelToSky(inputs["exposure"].getBBox().getCenter())).tract_id 

316 if not self.config.doApplyExternalGlobalSkyWcs and not self.config.doApplyExternalTractSkyWcs: 

317 inputs["wcs"] = expWcs 

318 elif self.config.doApplyExternalGlobalSkyWcs: 

319 externalSkyWcsCatalog = inputs["externalSkyWcsGlobalCatalog"] 

320 row = externalSkyWcsCatalog.find(detectorId) 

321 inputs["wcs"] = row.getWcs() 

322 elif self.config.doApplyExternalTractSkyWcs: 

323 externalSkyWcsCatalogList = inputs["externalSkyWcsTractCatalog"] 

324 externalSkyWcsCatalog = None 

325 for externalSkyWcsCatalogRef in externalSkyWcsCatalogList: 

326 if externalSkyWcsCatalogRef.dataId["tract"] == tractId: 

327 externalSkyWcsCatalog = externalSkyWcsCatalogRef.get( 

328 datasetType=self.config.connections.externalSkyWcsTractCatalog) 

329 break 

330 if externalSkyWcsCatalog is None: 

331 usedTract = externalSkyWcsCatalogList[-1].dataId["tract"] 

332 self.log.warn( 

333 f"Warning, external SkyWcs for tract {tractId} not found. Using tract {usedTract} " 

334 "instead.") 

335 externalSkyWcsCatalog = externalSkyWcsCatalogList[-1].get( 

336 datasetType=self.config.connections.externalSkyWcsTractCatalog) 

337 row = externalSkyWcsCatalog.find(detectorId) 

338 inputs["wcs"] = row.getWcs() 

339 

340 if not self.config.doApplyExternalGlobalPhotoCalib and not self.config.doApplyExternalTractPhotoCalib: 

341 inputs["photoCalib"] = inputs["exposure"].getPhotoCalib() 

342 elif self.config.doApplyExternalGlobalPhotoCalib: 

343 externalPhotoCalibCatalog = inputs["externalPhotoCalibGlobalCatalog"] 

344 row = externalPhotoCalibCatalog.find(detectorId) 

345 inputs["photoCalib"] = row.getPhotoCalib() 

346 elif self.config.doApplyExternalTractPhotoCalib: 

347 externalPhotoCalibCatalogList = inputs["externalPhotoCalibTractCatalog"] 

348 externalPhotoCalibCatalog = None 

349 for externalPhotoCalibCatalogRef in externalPhotoCalibCatalogList: 

350 if externalPhotoCalibCatalogRef.dataId["tract"] == tractId: 

351 externalPhotoCalibCatalog = externalPhotoCalibCatalogRef.get( 

352 datasetType=self.config.connections.externalPhotoCalibTractCatalog) 

353 break 

354 if externalPhotoCalibCatalog is None: 

355 usedTract = externalPhotoCalibCatalogList[-1].dataId["tract"] 

356 self.log.warn( 

357 f"Warning, external PhotoCalib for tract {tractId} not found. Using tract {usedTract} " 

358 "instead.") 

359 externalPhotoCalibCatalog = externalPhotoCalibCatalogList[-1].get( 

360 datasetType=self.config.connections.externalPhotoCalibTractCatalog) 

361 row = externalPhotoCalibCatalog.find(detectorId) 

362 inputs["photoCalib"] = row.getPhotoCalib() 

363 

364 outputs = self.run(**inputs) 

365 butlerQC.put(outputs, outputRefs) 

366 

367 def run(self, fakeCats, exposure, skyMap, wcs=None, photoCalib=None, exposureIdInfo=None, 

368 icSourceCat=None, sfdSourceCat=None, externalSkyWcsGlobalCatalog=None, 

369 externalSkyWcsTractCatalog=None, externalPhotoCalibGlobalCatalog=None, 

370 externalPhotoCalibTractCatalog=None): 

371 """Add fake sources to a calexp and then run detection, deblending and measurement. 

372 

373 Parameters 

374 ---------- 

375 fakeCats : `list` of `lsst.daf.butler.DeferredDatasetHandle` 

376 Set of tract level fake catalogs that potentially cover 

377 this detectorVisit. 

378 exposure : `lsst.afw.image.exposure.exposure.ExposureF` 

379 The exposure to add the fake sources to 

380 skyMap : `lsst.skymap.SkyMap` 

381 SkyMap defining the tracts and patches the fakes are stored over. 

382 wcs : `lsst.afw.geom.SkyWcs` 

383 WCS to use to add fake sources 

384 photoCalib : `lsst.afw.image.photoCalib.PhotoCalib` 

385 Photometric calibration to be used to calibrate the fake sources 

386 exposureIdInfo : `lsst.obs.base.ExposureIdInfo` 

387 icSourceCat : `lsst.afw.table.SourceCatalog` 

388 Default : None 

389 Catalog to take the information about which sources were used for calibration from. 

390 sfdSourceCat : `lsst.afw.table.SourceCatalog` 

391 Default : None 

392 Catalog produced by singleFrameDriver, needed to copy some calibration flags from. 

393 

394 Returns 

395 ------- 

396 resultStruct : `lsst.pipe.base.struct.Struct` 

397 contains : outputExposure : `lsst.afw.image.exposure.exposure.ExposureF` 

398 outputCat : `lsst.afw.table.source.source.SourceCatalog` 

399 

400 Notes 

401 ----- 

402 Adds pixel coordinates for each source to the fakeCat and removes objects with bulge or disk half 

403 light radius = 0 (if ``config.cleanCat = True``). These columns are called ``x`` and ``y`` and are in 

404 pixels. 

405 

406 Adds the ``Fake`` mask plane to the exposure which is then set by `addFakeSources` to mark where fake 

407 sources have been added. Uses the information in the ``fakeCat`` to make fake galaxies (using galsim) 

408 and fake stars, using the PSF models from the PSF information for the calexp. These are then added to 

409 the calexp and the calexp with fakes included returned. 

410 

411 The galsim galaxies are made using a double sersic profile, one for the bulge and one for the disk, 

412 this is then convolved with the PSF at that point. 

413 

414 If exposureIdInfo is not provided then the SourceCatalog IDs will not be globally unique. 

415 """ 

416 fakeCat = self.composeFakeCat(fakeCats, skyMap) 

417 

418 if wcs is None: 

419 wcs = exposure.getWcs() 

420 

421 if photoCalib is None: 

422 photoCalib = exposure.getPhotoCalib() 

423 

424 if self.config.doMatchVisit: 

425 fakeCat = self.getVisitMatchedFakeCat(fakeCat, exposure) 

426 

427 self.insertFakes.run(fakeCat, exposure, wcs, photoCalib) 

428 

429 # detect, deblend and measure sources 

430 if exposureIdInfo is None: 

431 exposureIdInfo = ExposureIdInfo() 

432 returnedStruct = self.calibrate.run(exposure, exposureIdInfo=exposureIdInfo) 

433 sourceCat = returnedStruct.sourceCat 

434 

435 sourceCat = self.copyCalibrationFields(sfdSourceCat, sourceCat, self.config.srcFieldsToCopy) 

436 

437 resultStruct = pipeBase.Struct(outputExposure=exposure, outputCat=sourceCat) 

438 return resultStruct 

439 

440 def composeFakeCat(self, fakeCats, skyMap): 

441 """Concatenate the fakeCats from tracts that may cover the exposure. 

442 

443 Parameters 

444 ---------- 

445 fakeCats : `list` of `lst.daf.butler.DeferredDatasetHandle` 

446 Set of fake cats to concatenate. 

447 skyMap : `lsst.skymap.SkyMap` 

448 SkyMap defining the geometry of the tracts and patches. 

449 

450 Returns 

451 ------- 

452 combinedFakeCat : `pandas.DataFrame` 

453 All fakes that cover the inner polygon of the tracts in this 

454 quantum. 

455 """ 

456 if len(fakeCats) == 1: 

457 return fakeCats[0].get( 

458 datasetType=self.config.connections.fakeCats) 

459 outputCat = [] 

460 for fakeCatRef in fakeCats: 

461 cat = fakeCatRef.get( 

462 datasetType=self.config.connections.fakeCats) 

463 tractId = fakeCatRef.dataId["tract"] 

464 # Make sure all data is within the inner part of the tract. 

465 outputCat.append(cat[ 

466 skyMap.findTractIdArray(cat[self.config.insertFakes.ra_col], 

467 cat[self.config.insertFakes.dec_col], 

468 degrees=False) 

469 == tractId]) 

470 

471 return pd.concat(outputCat) 

472 

473 def getVisitMatchedFakeCat(self, fakeCat, exposure): 

474 """Trim the fakeCat to select particular visit 

475 

476 Parameters 

477 ---------- 

478 fakeCat : `pandas.core.frame.DataFrame` 

479 The catalog of fake sources to add to the exposure 

480 exposure : `lsst.afw.image.exposure.exposure.ExposureF` 

481 The exposure to add the fake sources to 

482 

483 Returns 

484 ------- 

485 movingFakeCat : `pandas.DataFrame` 

486 All fakes that belong to the visit 

487 """ 

488 selected = exposure.getInfo().getVisitInfo().getId() == fakeCat["visit"] 

489 

490 return fakeCat[selected] 

491 

492 def copyCalibrationFields(self, calibCat, sourceCat, fieldsToCopy): 

493 """Match sources in calibCat and sourceCat and copy the specified fields 

494 

495 Parameters 

496 ---------- 

497 calibCat : `lsst.afw.table.SourceCatalog` 

498 Catalog from which to copy fields. 

499 sourceCat : `lsst.afw.table.SourceCatalog` 

500 Catalog to which to copy fields. 

501 fieldsToCopy : `lsst.pex.config.listField.List` 

502 Fields to copy from calibCat to SoourceCat. 

503 

504 Returns 

505 ------- 

506 newCat : `lsst.afw.table.SourceCatalog` 

507 Catalog which includes the copied fields. 

508 

509 The fields copied are those specified by `fieldsToCopy` that actually exist 

510 in the schema of `calibCat`. 

511 

512 This version was based on and adapted from the one in calibrateTask. 

513 """ 

514 

515 # Make a new SourceCatalog with the data from sourceCat so that we can add the new columns to it 

516 sourceSchemaMapper = afwTable.SchemaMapper(sourceCat.schema) 

517 sourceSchemaMapper.addMinimalSchema(sourceCat.schema, True) 

518 

519 calibSchemaMapper = afwTable.SchemaMapper(calibCat.schema, sourceCat.schema) 

520 

521 # Add the desired columns from the option fieldsToCopy 

522 missingFieldNames = [] 

523 for fieldName in fieldsToCopy: 

524 if fieldName in calibCat.schema: 

525 schemaItem = calibCat.schema.find(fieldName) 

526 calibSchemaMapper.editOutputSchema().addField(schemaItem.getField()) 

527 schema = calibSchemaMapper.editOutputSchema() 

528 calibSchemaMapper.addMapping(schemaItem.getKey(), schema.find(fieldName).getField()) 

529 else: 

530 missingFieldNames.append(fieldName) 

531 if missingFieldNames: 

532 raise RuntimeError(f"calibCat is missing fields {missingFieldNames} specified in " 

533 "fieldsToCopy") 

534 

535 if "calib_detected" not in calibSchemaMapper.getOutputSchema(): 

536 self.calibSourceKey = calibSchemaMapper.addOutputField(afwTable.Field["Flag"]("calib_detected", 

537 "Source was detected as an icSource")) 

538 else: 

539 self.calibSourceKey = None 

540 

541 schema = calibSchemaMapper.getOutputSchema() 

542 newCat = afwTable.SourceCatalog(schema) 

543 newCat.reserve(len(sourceCat)) 

544 newCat.extend(sourceCat, sourceSchemaMapper) 

545 

546 # Set the aliases so it doesn't complain. 

547 for k, v in sourceCat.schema.getAliasMap().items(): 

548 newCat.schema.getAliasMap().set(k, v) 

549 

550 select = newCat["deblend_nChild"] == 0 

551 matches = afwTable.matchXy(newCat[select], calibCat, self.config.matchRadiusPix) 

552 # Check that no sourceCat sources are listed twice (we already know 

553 # that each match has a unique calibCat source ID, due to using 

554 # that ID as the key in bestMatches) 

555 numMatches = len(matches) 

556 numUniqueSources = len(set(m[1].getId() for m in matches)) 

557 if numUniqueSources != numMatches: 

558 self.log.warning("%d calibCat sources matched only %d sourceCat sources", numMatches, 

559 numUniqueSources) 

560 

561 self.log.info("Copying flags from calibCat to sourceCat for %s sources", numMatches) 

562 

563 # For each match: set the calibSourceKey flag and copy the desired 

564 # fields 

565 for src, calibSrc, d in matches: 

566 if self.calibSourceKey: 

567 src.setFlag(self.calibSourceKey, True) 

568 # src.assign copies the footprint from calibSrc, which we don't want 

569 # (DM-407) 

570 # so set calibSrc's footprint to src's footprint before src.assign, 

571 # then restore it 

572 calibSrcFootprint = calibSrc.getFootprint() 

573 try: 

574 calibSrc.setFootprint(src.getFootprint()) 

575 src.assign(calibSrc, calibSchemaMapper) 

576 finally: 

577 calibSrc.setFootprint(calibSrcFootprint) 

578 

579 return newCat 

580 

581 

582class ProcessCcdWithVariableFakesConnections(ProcessCcdWithFakesConnections): 

583 ccdVisitFakeMagnitudes = cT.Output( 

584 doc="Catalog of fakes with magnitudes scattered for this ccdVisit.", 

585 name="{fakesType}ccdVisitFakeMagnitudes", 

586 storageClass="DataFrame", 

587 dimensions=("instrument", "visit", "detector"), 

588 ) 

589 

590 

591class ProcessCcdWithVariableFakesConfig(ProcessCcdWithFakesConfig, 

592 pipelineConnections=ProcessCcdWithVariableFakesConnections): 

593 scatterSize = pexConfig.RangeField( 

594 dtype=float, 

595 default=0.4, 

596 min=0, 

597 max=100, 

598 doc="Amount of scatter to add to the visit magnitude for variable " 

599 "sources." 

600 ) 

601 

602 

603class ProcessCcdWithVariableFakesTask(ProcessCcdWithFakesTask): 

604 """As ProcessCcdWithFakes except add variablity to the fakes catalog 

605 magnitude in the observed band for this ccdVisit. 

606 

607 Additionally, write out the modified magnitudes to the Butler. 

608 """ 

609 

610 _DefaultName = "processCcdWithVariableFakes" 

611 ConfigClass = ProcessCcdWithVariableFakesConfig 

612 

613 def run(self, fakeCats, exposure, skyMap, wcs=None, photoCalib=None, exposureIdInfo=None, 

614 icSourceCat=None, sfdSourceCat=None): 

615 """Add fake sources to a calexp and then run detection, deblending and measurement. 

616 

617 Parameters 

618 ---------- 

619 fakeCat : `pandas.core.frame.DataFrame` 

620 The catalog of fake sources to add to the exposure 

621 exposure : `lsst.afw.image.exposure.exposure.ExposureF` 

622 The exposure to add the fake sources to 

623 skyMap : `lsst.skymap.SkyMap` 

624 SkyMap defining the tracts and patches the fakes are stored over. 

625 wcs : `lsst.afw.geom.SkyWcs` 

626 WCS to use to add fake sources 

627 photoCalib : `lsst.afw.image.photoCalib.PhotoCalib` 

628 Photometric calibration to be used to calibrate the fake sources 

629 exposureIdInfo : `lsst.obs.base.ExposureIdInfo` 

630 icSourceCat : `lsst.afw.table.SourceCatalog` 

631 Default : None 

632 Catalog to take the information about which sources were used for calibration from. 

633 sfdSourceCat : `lsst.afw.table.SourceCatalog` 

634 Default : None 

635 Catalog produced by singleFrameDriver, needed to copy some calibration flags from. 

636 

637 Returns 

638 ------- 

639 resultStruct : `lsst.pipe.base.struct.Struct` 

640 Results Strcut containing: 

641 

642 - outputExposure : Exposure with added fakes 

643 (`lsst.afw.image.exposure.exposure.ExposureF`) 

644 - outputCat : Catalog with detected fakes 

645 (`lsst.afw.table.source.source.SourceCatalog`) 

646 - ccdVisitFakeMagnitudes : Magnitudes that these fakes were 

647 inserted with after being scattered (`pandas.DataFrame`) 

648 

649 Notes 

650 ----- 

651 Adds pixel coordinates for each source to the fakeCat and removes objects with bulge or disk half 

652 light radius = 0 (if ``config.cleanCat = True``). These columns are called ``x`` and ``y`` and are in 

653 pixels. 

654 

655 Adds the ``Fake`` mask plane to the exposure which is then set by `addFakeSources` to mark where fake 

656 sources have been added. Uses the information in the ``fakeCat`` to make fake galaxies (using galsim) 

657 and fake stars, using the PSF models from the PSF information for the calexp. These are then added to 

658 the calexp and the calexp with fakes included returned. 

659 

660 The galsim galaxies are made using a double sersic profile, one for the bulge and one for the disk, 

661 this is then convolved with the PSF at that point. 

662 

663 If exposureIdInfo is not provided then the SourceCatalog IDs will not be globally unique. 

664 """ 

665 fakeCat = self.composeFakeCat(fakeCats, skyMap) 

666 

667 if wcs is None: 

668 wcs = exposure.getWcs() 

669 

670 if photoCalib is None: 

671 photoCalib = exposure.getPhotoCalib() 

672 

673 if exposureIdInfo is None: 

674 exposureIdInfo = ExposureIdInfo() 

675 

676 band = exposure.getFilter().bandLabel 

677 ccdVisitMagnitudes = self.addVariablity(fakeCat, band, exposure, photoCalib, exposureIdInfo) 

678 

679 self.insertFakes.run(fakeCat, exposure, wcs, photoCalib) 

680 

681 # detect, deblend and measure sources 

682 returnedStruct = self.calibrate.run(exposure, exposureIdInfo=exposureIdInfo) 

683 sourceCat = returnedStruct.sourceCat 

684 

685 sourceCat = self.copyCalibrationFields(sfdSourceCat, sourceCat, self.config.srcFieldsToCopy) 

686 

687 resultStruct = pipeBase.Struct(outputExposure=exposure, 

688 outputCat=sourceCat, 

689 ccdVisitFakeMagnitudes=ccdVisitMagnitudes) 

690 return resultStruct 

691 

692 def addVariablity(self, fakeCat, band, exposure, photoCalib, exposureIdInfo): 

693 """Add scatter to the fake catalog visit magnitudes. 

694 

695 Currently just adds a simple Gaussian scatter around the static fake 

696 magnitude. This function could be modified to return any number of 

697 fake variability. 

698 

699 Parameters 

700 ---------- 

701 fakeCat : `pandas.DataFrame` 

702 Catalog of fakes to modify magnitudes of. 

703 band : `str` 

704 Current observing band to modify. 

705 exposure : `lsst.afw.image.ExposureF` 

706 Exposure fakes will be added to. 

707 photoCalib : `lsst.afw.image.PhotoCalib` 

708 Photometric calibration object of ``exposure``. 

709 exposureIdInfo : `lsst.obs.base.ExposureIdInfo` 

710 Exposure id information and metadata. 

711 

712 Returns 

713 ------- 

714 dataFrame : `pandas.DataFrame` 

715 DataFrame containing the values of the magnitudes to that will 

716 be inserted into this ccdVisit. 

717 """ 

718 expId = exposureIdInfo.expId 

719 rng = np.random.default_rng(expId) 

720 magScatter = rng.normal(loc=0, 

721 scale=self.config.scatterSize, 

722 size=len(fakeCat)) 

723 visitMagnitudes = fakeCat[self.insertFakes.config.mag_col % band] + magScatter 

724 fakeCat.loc[:, self.insertFakes.config.mag_col % band] = visitMagnitudes 

725 return pd.DataFrame(data={"variableMag": visitMagnitudes})