Coverage for python/lsst/pipe/tasks/processCcdWithFakes.py: 18%

235 statements  

« prev     ^ index     » next       coverage.py v7.2.7, created at 2023-07-18 10:36 +0000

1# This file is part of pipe_tasks. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22""" 

23Insert fake sources into calexps 

24""" 

25 

26__all__ = ["ProcessCcdWithFakesConfig", "ProcessCcdWithFakesTask", 

27 "ProcessCcdWithVariableFakesConfig", "ProcessCcdWithVariableFakesTask"] 

28 

29import numpy as np 

30import pandas as pd 

31 

32import lsst.pex.config as pexConfig 

33import lsst.pipe.base as pipeBase 

34 

35from .insertFakes import InsertFakesTask 

36from lsst.afw.table import SourceTable 

37from lsst.meas.base import IdGenerator, DetectorVisitIdGeneratorConfig 

38from lsst.pipe.base import PipelineTask, PipelineTaskConfig, PipelineTaskConnections 

39import lsst.pipe.base.connectionTypes as cT 

40import lsst.afw.table as afwTable 

41from lsst.skymap import BaseSkyMap 

42from lsst.pipe.tasks.calibrate import CalibrateTask 

43 

44 

45class ProcessCcdWithFakesConnections(PipelineTaskConnections, 

46 dimensions=("instrument", "visit", "detector"), 

47 defaultTemplates={"coaddName": "deep", 

48 "wcsName": "gbdesAstrometricFit", 

49 "photoCalibName": "jointcal", 

50 "fakesType": "fakes_"}): 

51 skyMap = cT.Input( 

52 doc="Input definition of geometry/bbox and projection/wcs for " 

53 "template exposures. Needed to test which tract to generate ", 

54 name=BaseSkyMap.SKYMAP_DATASET_TYPE_NAME, 

55 dimensions=("skymap",), 

56 storageClass="SkyMap", 

57 ) 

58 

59 exposure = cT.Input( 

60 doc="Exposure into which fakes are to be added.", 

61 name="calexp", 

62 storageClass="ExposureF", 

63 dimensions=("instrument", "visit", "detector") 

64 ) 

65 

66 fakeCats = cT.Input( 

67 doc="Set of catalogs of fake sources to draw inputs from. We " 

68 "concatenate the tract catalogs for detectorVisits that cover " 

69 "multiple tracts.", 

70 name="{fakesType}fakeSourceCat", 

71 storageClass="DataFrame", 

72 dimensions=("tract", "skymap"), 

73 deferLoad=True, 

74 multiple=True, 

75 ) 

76 

77 externalSkyWcsTractCatalog = cT.Input( 

78 doc=("Per-tract, per-visit wcs calibrations. These catalogs use the detector " 

79 "id for the catalog id, sorted on id for fast lookup."), 

80 name="{wcsName}SkyWcsCatalog", 

81 storageClass="ExposureCatalog", 

82 dimensions=("instrument", "visit", "tract", "skymap"), 

83 deferLoad=True, 

84 multiple=True, 

85 ) 

86 

87 externalSkyWcsGlobalCatalog = cT.Input( 

88 doc=("Per-visit wcs calibrations computed globally (with no tract information). " 

89 "These catalogs use the detector id for the catalog id, sorted on id for " 

90 "fast lookup."), 

91 name="finalVisitSummary", 

92 storageClass="ExposureCatalog", 

93 dimensions=("instrument", "visit"), 

94 ) 

95 

96 externalPhotoCalibTractCatalog = cT.Input( 

97 doc=("Per-tract, per-visit photometric calibrations. These catalogs use the " 

98 "detector id for the catalog id, sorted on id for fast lookup."), 

99 name="{photoCalibName}PhotoCalibCatalog", 

100 storageClass="ExposureCatalog", 

101 dimensions=("instrument", "visit", "tract"), 

102 deferLoad=True, 

103 multiple=True, 

104 ) 

105 

106 externalPhotoCalibGlobalCatalog = cT.Input( 

107 doc=("Per-visit photometric calibrations. These catalogs use the " 

108 "detector id for the catalog id, sorted on id for fast lookup."), 

109 name="finalVisitSummary", 

110 storageClass="ExposureCatalog", 

111 dimensions=("instrument", "visit"), 

112 ) 

113 

114 icSourceCat = cT.Input( 

115 doc="Catalog of calibration sources", 

116 name="icSrc", 

117 storageClass="SourceCatalog", 

118 dimensions=("instrument", "visit", "detector") 

119 ) 

120 

121 sfdSourceCat = cT.Input( 

122 doc="Catalog of calibration sources", 

123 name="src", 

124 storageClass="SourceCatalog", 

125 dimensions=("instrument", "visit", "detector") 

126 ) 

127 

128 outputExposure = cT.Output( 

129 doc="Exposure with fake sources added.", 

130 name="{fakesType}calexp", 

131 storageClass="ExposureF", 

132 dimensions=("instrument", "visit", "detector") 

133 ) 

134 

135 outputCat = cT.Output( 

136 doc="Source catalog produced in calibrate task with fakes also measured.", 

137 name="{fakesType}src", 

138 storageClass="SourceCatalog", 

139 dimensions=("instrument", "visit", "detector"), 

140 ) 

141 

142 def __init__(self, *, config=None): 

143 super().__init__(config=config) 

144 

145 if not config.doApplyExternalGlobalPhotoCalib: 

146 self.inputs.remove("externalPhotoCalibGlobalCatalog") 

147 if not config.doApplyExternalTractPhotoCalib: 

148 self.inputs.remove("externalPhotoCalibTractCatalog") 

149 

150 if not config.doApplyExternalGlobalSkyWcs: 

151 self.inputs.remove("externalSkyWcsGlobalCatalog") 

152 if not config.doApplyExternalTractSkyWcs: 

153 self.inputs.remove("externalSkyWcsTractCatalog") 

154 

155 

156class ProcessCcdWithFakesConfig(PipelineTaskConfig, 

157 pipelineConnections=ProcessCcdWithFakesConnections): 

158 """Config for inserting fake sources 

159 

160 Notes 

161 ----- 

162 The default column names are those from the UW sims database. 

163 """ 

164 

165 doApplyExternalGlobalPhotoCalib = pexConfig.Field( 

166 dtype=bool, 

167 default=False, 

168 doc="Whether to apply an external photometric calibration via an " 

169 "`lsst.afw.image.PhotoCalib` object. Uses the " 

170 "`externalPhotoCalibName` config option to determine which " 

171 "calibration to use. Uses a global calibration." 

172 ) 

173 

174 doApplyExternalTractPhotoCalib = pexConfig.Field( 

175 dtype=bool, 

176 default=False, 

177 doc="Whether to apply an external photometric calibration via an " 

178 "`lsst.afw.image.PhotoCalib` object. Uses the " 

179 "`externalPhotoCalibName` config option to determine which " 

180 "calibration to use. Uses a per tract calibration." 

181 ) 

182 

183 externalPhotoCalibName = pexConfig.ChoiceField( 

184 doc="What type of external photo calib to use.", 

185 dtype=str, 

186 default="jointcal", 

187 allowed={"jointcal": "Use jointcal_photoCalib", 

188 "fgcm": "Use fgcm_photoCalib", 

189 "fgcm_tract": "Use fgcm_tract_photoCalib"} 

190 ) 

191 

192 doApplyExternalGlobalSkyWcs = pexConfig.Field( 

193 dtype=bool, 

194 default=False, 

195 doc="Whether to apply an external astrometric calibration via an " 

196 "`lsst.afw.geom.SkyWcs` object. Uses the " 

197 "`externalSkyWcsName` config option to determine which " 

198 "calibration to use. Uses a global calibration." 

199 ) 

200 

201 doApplyExternalTractSkyWcs = pexConfig.Field( 

202 dtype=bool, 

203 default=False, 

204 doc="Whether to apply an external astrometric calibration via an " 

205 "`lsst.afw.geom.SkyWcs` object. Uses the " 

206 "`externalSkyWcsName` config option to determine which " 

207 "calibration to use. Uses a per tract calibration." 

208 ) 

209 

210 externalSkyWcsName = pexConfig.ChoiceField( 

211 doc="What type of updated WCS calib to use.", 

212 dtype=str, 

213 default="gbdesAstrometricFit", 

214 allowed={"gbdesAstrometricFit": "Use gbdesAstrometricFit_wcs"} 

215 ) 

216 

217 coaddName = pexConfig.Field( 

218 doc="The name of the type of coadd used", 

219 dtype=str, 

220 default="deep", 

221 ) 

222 

223 srcFieldsToCopy = pexConfig.ListField( 

224 dtype=str, 

225 default=("calib_photometry_reserved", "calib_photometry_used", "calib_astrometry_used", 

226 "calib_psf_candidate", "calib_psf_used", "calib_psf_reserved"), 

227 doc=("Fields to copy from the `src` catalog to the output catalog " 

228 "for matching sources Any missing fields will trigger a " 

229 "RuntimeError exception.") 

230 ) 

231 

232 matchRadiusPix = pexConfig.Field( 

233 dtype=float, 

234 default=3, 

235 doc=("Match radius for matching icSourceCat objects to sourceCat objects (pixels)"), 

236 ) 

237 

238 doMatchVisit = pexConfig.Field( 

239 dtype=bool, 

240 default=False, 

241 doc="Match visit to trim the fakeCat" 

242 ) 

243 

244 calibrate = pexConfig.ConfigurableField(target=CalibrateTask, 

245 doc="The calibration task to use.") 

246 

247 insertFakes = pexConfig.ConfigurableField(target=InsertFakesTask, 

248 doc="Configuration for the fake sources") 

249 

250 idGenerator = DetectorVisitIdGeneratorConfig.make_field() 

251 

252 def setDefaults(self): 

253 super().setDefaults() 

254 self.calibrate.measurement.plugins["base_PixelFlags"].masksFpAnywhere.append("FAKE") 

255 self.calibrate.measurement.plugins["base_PixelFlags"].masksFpCenter.append("FAKE") 

256 self.calibrate.doAstrometry = False 

257 self.calibrate.doWriteMatches = False 

258 self.calibrate.doPhotoCal = False 

259 self.calibrate.doComputeSummaryStats = False 

260 self.calibrate.detection.reEstimateBackground = False 

261 

262 

263class ProcessCcdWithFakesTask(PipelineTask): 

264 """Insert fake objects into calexps. 

265 

266 Add fake stars and galaxies to the given calexp, specified in the dataRef. Galaxy parameters are read in 

267 from the specified file and then modelled using galsim. Re-runs characterize image and calibrate image to 

268 give a new background estimation and measurement of the calexp. 

269 

270 `ProcessFakeSourcesTask` inherits six functions from insertFakesTask that make images of the fake 

271 sources and then add them to the calexp. 

272 

273 `addPixCoords` 

274 Use the WCS information to add the pixel coordinates of each source 

275 Adds an ``x`` and ``y`` column to the catalog of fake sources. 

276 `trimFakeCat` 

277 Trim the fake cat to about the size of the input image. 

278 `mkFakeGalsimGalaxies` 

279 Use Galsim to make fake double sersic galaxies for each set of galaxy parameters in the input file. 

280 `mkFakeStars` 

281 Use the PSF information from the calexp to make a fake star using the magnitude information from the 

282 input file. 

283 `cleanCat` 

284 Remove rows of the input fake catalog which have half light radius, of either the bulge or the disk, 

285 that are 0. 

286 `addFakeSources` 

287 Add the fake sources to the calexp. 

288 

289 Notes 

290 ----- 

291 The ``calexp`` with fake souces added to it is written out as the datatype ``calexp_fakes``. 

292 """ 

293 

294 _DefaultName = "processCcdWithFakes" 

295 ConfigClass = ProcessCcdWithFakesConfig 

296 

297 def __init__(self, schema=None, butler=None, **kwargs): 

298 """Initalize things! This should go above in the class docstring 

299 """ 

300 

301 super().__init__(**kwargs) 

302 

303 if schema is None: 

304 schema = SourceTable.makeMinimalSchema() 

305 self.schema = schema 

306 self.makeSubtask("insertFakes") 

307 self.makeSubtask("calibrate") 

308 

309 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

310 inputs = butlerQC.get(inputRefs) 

311 detectorId = inputs["exposure"].getInfo().getDetector().getId() 

312 

313 if 'idGenerator' not in inputs.keys(): 

314 inputs['idGenerator'] = self.config.idGenerator.apply(butlerQC.quantum.dataId) 

315 

316 expWcs = inputs["exposure"].getWcs() 

317 tractId = None 

318 if not self.config.doApplyExternalGlobalSkyWcs and not self.config.doApplyExternalTractSkyWcs: 

319 if expWcs is None: 

320 self.log.info("No WCS for exposure %s so cannot insert fake sources. Skipping detector.", 

321 butlerQC.quantum.dataId) 

322 return None 

323 else: 

324 inputs["wcs"] = expWcs 

325 elif self.config.doApplyExternalGlobalSkyWcs: 

326 externalSkyWcsCatalog = inputs["externalSkyWcsGlobalCatalog"] 

327 row = externalSkyWcsCatalog.find(detectorId) 

328 if row is None: 

329 self.log.info("No %s external global sky WCS for exposure %s so cannot insert fake " 

330 "sources. Skipping detector.", self.config.externalSkyWcsName, 

331 butlerQC.quantum.dataId) 

332 return None 

333 inputs["wcs"] = row.getWcs() 

334 elif self.config.doApplyExternalTractSkyWcs: 

335 externalSkyWcsCatalogList = inputs["externalSkyWcsTractCatalog"] 

336 if tractId is None: 

337 tractId = externalSkyWcsCatalogList[0].dataId["tract"] 

338 externalSkyWcsCatalog = None 

339 for externalSkyWcsCatalogRef in externalSkyWcsCatalogList: 

340 if externalSkyWcsCatalogRef.dataId["tract"] == tractId: 

341 externalSkyWcsCatalog = externalSkyWcsCatalogRef.get() 

342 break 

343 if externalSkyWcsCatalog is None: 

344 usedTract = externalSkyWcsCatalogList[-1].dataId["tract"] 

345 self.log.warn( 

346 f"Warning, external SkyWcs for tract {tractId} not found. Using tract {usedTract} " 

347 "instead.") 

348 externalSkyWcsCatalog = externalSkyWcsCatalogList[-1].get() 

349 row = externalSkyWcsCatalog.find(detectorId) 

350 if row is None: 

351 self.log.info("No %s external tract sky WCS for exposure %s so cannot insert fake " 

352 "sources. Skipping detector.", self.config.externalSkyWcsName, 

353 butlerQC.quantum.dataId) 

354 return None 

355 inputs["wcs"] = row.getWcs() 

356 

357 if not self.config.doApplyExternalGlobalPhotoCalib and not self.config.doApplyExternalTractPhotoCalib: 

358 inputs["photoCalib"] = inputs["exposure"].getPhotoCalib() 

359 elif self.config.doApplyExternalGlobalPhotoCalib: 

360 externalPhotoCalibCatalog = inputs["externalPhotoCalibGlobalCatalog"] 

361 row = externalPhotoCalibCatalog.find(detectorId) 

362 if row is None: 

363 self.log.info("No %s external global photoCalib for exposure %s so cannot insert fake " 

364 "sources. Skipping detector.", self.config.externalPhotoCalibName, 

365 butlerQC.quantum.dataId) 

366 return None 

367 inputs["photoCalib"] = row.getPhotoCalib() 

368 elif self.config.doApplyExternalTractPhotoCalib: 

369 externalPhotoCalibCatalogList = inputs["externalPhotoCalibTractCatalog"] 

370 if tractId is None: 

371 tractId = externalPhotoCalibCatalogList[0].dataId["tract"] 

372 externalPhotoCalibCatalog = None 

373 for externalPhotoCalibCatalogRef in externalPhotoCalibCatalogList: 

374 if externalPhotoCalibCatalogRef.dataId["tract"] == tractId: 

375 externalPhotoCalibCatalog = externalPhotoCalibCatalogRef.get() 

376 break 

377 if externalPhotoCalibCatalog is None: 

378 usedTract = externalPhotoCalibCatalogList[-1].dataId["tract"] 

379 self.log.warn( 

380 f"Warning, external PhotoCalib for tract {tractId} not found. Using tract {usedTract} " 

381 "instead.") 

382 externalPhotoCalibCatalog = externalPhotoCalibCatalogList[-1].get() 

383 row = externalPhotoCalibCatalog.find(detectorId) 

384 if row is None: 

385 self.log.info("No %s external tract photoCalib for exposure %s so cannot insert fake " 

386 "sources. Skipping detector.", self.config.externalPhotoCalibName, 

387 butlerQC.quantum.dataId) 

388 return None 

389 inputs["photoCalib"] = row.getPhotoCalib() 

390 

391 outputs = self.run(**inputs) 

392 butlerQC.put(outputs, outputRefs) 

393 

394 def run(self, fakeCats, exposure, skyMap, wcs=None, photoCalib=None, exposureIdInfo=None, 

395 icSourceCat=None, sfdSourceCat=None, externalSkyWcsGlobalCatalog=None, 

396 externalSkyWcsTractCatalog=None, externalPhotoCalibGlobalCatalog=None, 

397 externalPhotoCalibTractCatalog=None, idGenerator=None): 

398 """Add fake sources to a calexp and then run detection, deblending and 

399 measurement. 

400 

401 Parameters 

402 ---------- 

403 fakeCats : `list` of `lsst.daf.butler.DeferredDatasetHandle` 

404 Set of tract level fake catalogs that potentially cover this 

405 detectorVisit. 

406 exposure : `lsst.afw.image.exposure.exposure.ExposureF` 

407 The exposure to add the fake sources to. 

408 skyMap : `lsst.skymap.SkyMap` 

409 SkyMap defining the tracts and patches the fakes are stored over. 

410 wcs : `lsst.afw.geom.SkyWcs`, optional 

411 WCS to use to add fake sources. 

412 photoCalib : `lsst.afw.image.photoCalib.PhotoCalib`, optional 

413 Photometric calibration to be used to calibrate the fake sources. 

414 exposureIdInfo : `lsst.obs.base.ExposureIdInfo`, optional 

415 Object that carries ID information for this image/catalog. 

416 Deprecated in favor of ``idGenerator``. 

417 icSourceCat : `lsst.afw.table.SourceCatalog`, optional 

418 Catalog to take the information about which sources were used for 

419 calibration from. 

420 sfdSourceCat : `lsst.afw.table.SourceCatalog`, optional 

421 Catalog produced by singleFrameDriver, needed to copy some 

422 calibration flags from. 

423 externalSkyWcsGlobalCatalog : `lsst.afw.table.ExposureCatalog`, \ 

424 optional 

425 Exposure catalog with external skyWcs to be applied per config. 

426 externalSkyWcsTractCatalog : `lsst.afw.table.ExposureCatalog`, optional 

427 Exposure catalog with external skyWcs to be applied per config. 

428 externalPhotoCalibGlobalCatalog : `lsst.afw.table.ExposureCatalog`, \ 

429 optional 

430 Exposure catalog with external photoCalib to be applied per config 

431 externalPhotoCalibTractCatalog : `lsst.afw.table.ExposureCatalog`, \ 

432 optional 

433 Exposure catalog with external photoCalib to be applied per config. 

434 idGenerator : `lsst.meas.base.IdGenerator`, optional 

435 Object that generates Source IDs and random seeds. 

436 

437 Returns 

438 ------- 

439 resultStruct : `lsst.pipe.base.struct.Struct` 

440 Result struct containing: 

441 

442 - outputExposure: `lsst.afw.image.exposure.exposure.ExposureF` 

443 - outputCat: `lsst.afw.table.source.source.SourceCatalog` 

444 

445 Notes 

446 ----- 

447 Adds pixel coordinates for each source to the fakeCat and removes 

448 objects with bulge or disk half light radius = 0 (if ``config.cleanCat 

449 = True``). These columns are called ``x`` and ``y`` and are in pixels. 

450 

451 Adds the ``Fake`` mask plane to the exposure which is then set by 

452 `addFakeSources` to mark where fake sources have been added. Uses the 

453 information in the ``fakeCat`` to make fake galaxies (using galsim) and 

454 fake stars, using the PSF models from the PSF information for the 

455 calexp. These are then added to the calexp and the calexp with fakes 

456 included returned. 

457 

458 The galsim galaxies are made using a double sersic profile, one for the 

459 bulge and one for the disk, this is then convolved with the PSF at that 

460 point. 

461 """ 

462 fakeCat = self.composeFakeCat(fakeCats, skyMap) 

463 

464 if wcs is None: 

465 wcs = exposure.getWcs() 

466 

467 if photoCalib is None: 

468 photoCalib = exposure.getPhotoCalib() 

469 

470 if self.config.doMatchVisit: 

471 fakeCat = self.getVisitMatchedFakeCat(fakeCat, exposure) 

472 

473 self.insertFakes.run(fakeCat, exposure, wcs, photoCalib) 

474 

475 # detect, deblend and measure sources 

476 if idGenerator is None: 

477 if exposureIdInfo is not None: 

478 idGenerator = IdGenerator._from_exposure_id_info(exposureIdInfo) 

479 else: 

480 idGenerator = IdGenerator() 

481 returnedStruct = self.calibrate.run(exposure, idGenerator=idGenerator) 

482 sourceCat = returnedStruct.sourceCat 

483 

484 sourceCat = self.copyCalibrationFields(sfdSourceCat, sourceCat, self.config.srcFieldsToCopy) 

485 

486 resultStruct = pipeBase.Struct(outputExposure=exposure, outputCat=sourceCat) 

487 return resultStruct 

488 

489 def composeFakeCat(self, fakeCats, skyMap): 

490 """Concatenate the fakeCats from tracts that may cover the exposure. 

491 

492 Parameters 

493 ---------- 

494 fakeCats : `list` of `lsst.daf.butler.DeferredDatasetHandle` 

495 Set of fake cats to concatenate. 

496 skyMap : `lsst.skymap.SkyMap` 

497 SkyMap defining the geometry of the tracts and patches. 

498 

499 Returns 

500 ------- 

501 combinedFakeCat : `pandas.DataFrame` 

502 All fakes that cover the inner polygon of the tracts in this 

503 quantum. 

504 """ 

505 if len(fakeCats) == 1: 

506 return fakeCats[0].get() 

507 outputCat = [] 

508 for fakeCatRef in fakeCats: 

509 cat = fakeCatRef.get() 

510 tractId = fakeCatRef.dataId["tract"] 

511 # Make sure all data is within the inner part of the tract. 

512 outputCat.append(cat[ 

513 skyMap.findTractIdArray(cat[self.config.insertFakes.ra_col], 

514 cat[self.config.insertFakes.dec_col], 

515 degrees=False) 

516 == tractId]) 

517 

518 return pd.concat(outputCat) 

519 

520 def getVisitMatchedFakeCat(self, fakeCat, exposure): 

521 """Trim the fakeCat to select particular visit 

522 

523 Parameters 

524 ---------- 

525 fakeCat : `pandas.core.frame.DataFrame` 

526 The catalog of fake sources to add to the exposure 

527 exposure : `lsst.afw.image.exposure.exposure.ExposureF` 

528 The exposure to add the fake sources to 

529 

530 Returns 

531 ------- 

532 movingFakeCat : `pandas.DataFrame` 

533 All fakes that belong to the visit 

534 """ 

535 selected = exposure.getInfo().getVisitInfo().getId() == fakeCat["visit"] 

536 

537 return fakeCat[selected] 

538 

539 def copyCalibrationFields(self, calibCat, sourceCat, fieldsToCopy): 

540 """Match sources in calibCat and sourceCat and copy the specified fields 

541 

542 Parameters 

543 ---------- 

544 calibCat : `lsst.afw.table.SourceCatalog` 

545 Catalog from which to copy fields. 

546 sourceCat : `lsst.afw.table.SourceCatalog` 

547 Catalog to which to copy fields. 

548 fieldsToCopy : `lsst.pex.config.listField.List` 

549 Fields to copy from calibCat to SoourceCat. 

550 

551 Returns 

552 ------- 

553 newCat : `lsst.afw.table.SourceCatalog` 

554 Catalog which includes the copied fields. 

555 

556 The fields copied are those specified by `fieldsToCopy` that actually exist 

557 in the schema of `calibCat`. 

558 

559 This version was based on and adapted from the one in calibrateTask. 

560 """ 

561 

562 # Make a new SourceCatalog with the data from sourceCat so that we can add the new columns to it 

563 sourceSchemaMapper = afwTable.SchemaMapper(sourceCat.schema) 

564 sourceSchemaMapper.addMinimalSchema(sourceCat.schema, True) 

565 

566 calibSchemaMapper = afwTable.SchemaMapper(calibCat.schema, sourceCat.schema) 

567 

568 # Add the desired columns from the option fieldsToCopy 

569 missingFieldNames = [] 

570 for fieldName in fieldsToCopy: 

571 if fieldName in calibCat.schema: 

572 schemaItem = calibCat.schema.find(fieldName) 

573 calibSchemaMapper.editOutputSchema().addField(schemaItem.getField()) 

574 schema = calibSchemaMapper.editOutputSchema() 

575 calibSchemaMapper.addMapping(schemaItem.getKey(), schema.find(fieldName).getField()) 

576 else: 

577 missingFieldNames.append(fieldName) 

578 if missingFieldNames: 

579 raise RuntimeError(f"calibCat is missing fields {missingFieldNames} specified in " 

580 "fieldsToCopy") 

581 

582 if "calib_detected" not in calibSchemaMapper.getOutputSchema(): 

583 self.calibSourceKey = calibSchemaMapper.addOutputField(afwTable.Field["Flag"]("calib_detected", 

584 "Source was detected as an icSource")) 

585 else: 

586 self.calibSourceKey = None 

587 

588 schema = calibSchemaMapper.getOutputSchema() 

589 newCat = afwTable.SourceCatalog(schema) 

590 newCat.reserve(len(sourceCat)) 

591 newCat.extend(sourceCat, sourceSchemaMapper) 

592 

593 # Set the aliases so it doesn't complain. 

594 for k, v in sourceCat.schema.getAliasMap().items(): 

595 newCat.schema.getAliasMap().set(k, v) 

596 

597 select = newCat["deblend_nChild"] == 0 

598 matches = afwTable.matchXy(newCat[select], calibCat, self.config.matchRadiusPix) 

599 # Check that no sourceCat sources are listed twice (we already know 

600 # that each match has a unique calibCat source ID, due to using 

601 # that ID as the key in bestMatches) 

602 numMatches = len(matches) 

603 numUniqueSources = len(set(m[1].getId() for m in matches)) 

604 if numUniqueSources != numMatches: 

605 self.log.warning("%d calibCat sources matched only %d sourceCat sources", numMatches, 

606 numUniqueSources) 

607 

608 self.log.info("Copying flags from calibCat to sourceCat for %s sources", numMatches) 

609 

610 # For each match: set the calibSourceKey flag and copy the desired 

611 # fields 

612 for src, calibSrc, d in matches: 

613 if self.calibSourceKey: 

614 src.setFlag(self.calibSourceKey, True) 

615 # src.assign copies the footprint from calibSrc, which we don't want 

616 # (DM-407) 

617 # so set calibSrc's footprint to src's footprint before src.assign, 

618 # then restore it 

619 calibSrcFootprint = calibSrc.getFootprint() 

620 try: 

621 calibSrc.setFootprint(src.getFootprint()) 

622 src.assign(calibSrc, calibSchemaMapper) 

623 finally: 

624 calibSrc.setFootprint(calibSrcFootprint) 

625 

626 return newCat 

627 

628 

629class ProcessCcdWithVariableFakesConnections(ProcessCcdWithFakesConnections): 

630 ccdVisitFakeMagnitudes = cT.Output( 

631 doc="Catalog of fakes with magnitudes scattered for this ccdVisit.", 

632 name="{fakesType}ccdVisitFakeMagnitudes", 

633 storageClass="DataFrame", 

634 dimensions=("instrument", "visit", "detector"), 

635 ) 

636 

637 

638class ProcessCcdWithVariableFakesConfig(ProcessCcdWithFakesConfig, 

639 pipelineConnections=ProcessCcdWithVariableFakesConnections): 

640 scatterSize = pexConfig.RangeField( 

641 dtype=float, 

642 default=0.4, 

643 min=0, 

644 max=100, 

645 doc="Amount of scatter to add to the visit magnitude for variable " 

646 "sources." 

647 ) 

648 

649 

650class ProcessCcdWithVariableFakesTask(ProcessCcdWithFakesTask): 

651 """As ProcessCcdWithFakes except add variablity to the fakes catalog 

652 magnitude in the observed band for this ccdVisit. 

653 

654 Additionally, write out the modified magnitudes to the Butler. 

655 """ 

656 

657 _DefaultName = "processCcdWithVariableFakes" 

658 ConfigClass = ProcessCcdWithVariableFakesConfig 

659 

660 def run(self, fakeCats, exposure, skyMap, wcs=None, photoCalib=None, exposureIdInfo=None, 

661 icSourceCat=None, sfdSourceCat=None, idGenerator=None): 

662 """Add fake sources to a calexp and then run detection, deblending and 

663 measurement. 

664 

665 Parameters 

666 ---------- 

667 fakeCat : `pandas.core.frame.DataFrame` 

668 The catalog of fake sources to add to the exposure. 

669 exposure : `lsst.afw.image.exposure.exposure.ExposureF` 

670 The exposure to add the fake sources to. 

671 skyMap : `lsst.skymap.SkyMap` 

672 SkyMap defining the tracts and patches the fakes are stored over. 

673 wcs : `lsst.afw.geom.SkyWcs`, optional 

674 WCS to use to add fake sources. 

675 photoCalib : `lsst.afw.image.photoCalib.PhotoCalib`, optional 

676 Photometric calibration to be used to calibrate the fake sources. 

677 exposureIdInfo : `lsst.obs.base.ExposureIdInfo`, optional 

678 Object that carries ID information for this image/catalog. 

679 Deprecated in favor of ``idGenerator``. 

680 icSourceCat : `lsst.afw.table.SourceCatalog`, optional 

681 Catalog to take the information about which sources were used for 

682 calibration from. 

683 sfdSourceCat : `lsst.afw.table.SourceCatalog`, optional 

684 Catalog produced by singleFrameDriver, needed to copy some 

685 calibration flags from. 

686 idGenerator : `lsst.meas.base.IdGenerator`, optional 

687 Object that generates Source IDs and random seeds. 

688 

689 Returns 

690 ------- 

691 resultStruct : `lsst.pipe.base.struct.Struct` 

692 Results struct containing: 

693 

694 - outputExposure : Exposure with added fakes 

695 (`lsst.afw.image.exposure.exposure.ExposureF`) 

696 - outputCat : Catalog with detected fakes 

697 (`lsst.afw.table.source.source.SourceCatalog`) 

698 - ccdVisitFakeMagnitudes : Magnitudes that these fakes were 

699 inserted with after being scattered (`pandas.DataFrame`) 

700 

701 Notes 

702 ----- 

703 Adds pixel coordinates for each source to the fakeCat and removes 

704 objects with bulge or disk half light radius = 0 (if ``config.cleanCat 

705 = True``). These columns are called ``x`` and ``y`` and are in pixels. 

706 

707 Adds the ``Fake`` mask plane to the exposure which is then set by 

708 `addFakeSources` to mark where fake sources have been added. Uses the 

709 information in the ``fakeCat`` to make fake galaxies (using galsim) and 

710 fake stars, using the PSF models from the PSF information for the 

711 calexp. These are then added to the calexp and the calexp with fakes 

712 included returned. 

713 

714 The galsim galaxies are made using a double sersic profile, one for the 

715 bulge and one for the disk, this is then convolved with the PSF at that 

716 point. 

717 

718 

719 """ 

720 fakeCat = self.composeFakeCat(fakeCats, skyMap) 

721 

722 if wcs is None: 

723 wcs = exposure.getWcs() 

724 

725 if photoCalib is None: 

726 photoCalib = exposure.getPhotoCalib() 

727 

728 if idGenerator is None: 

729 if exposureIdInfo is not None: 

730 idGenerator = IdGenerator._from_exposure_id_info(exposureIdInfo) 

731 else: 

732 idGenerator = IdGenerator() 

733 

734 band = exposure.getFilter().bandLabel 

735 ccdVisitMagnitudes = self.addVariability( 

736 fakeCat, 

737 band, 

738 exposure, 

739 photoCalib, 

740 idGenerator.catalog_id, 

741 ) 

742 

743 self.insertFakes.run(fakeCat, exposure, wcs, photoCalib) 

744 

745 # detect, deblend and measure sources 

746 returnedStruct = self.calibrate.run(exposure, idGenerator=idGenerator) 

747 sourceCat = returnedStruct.sourceCat 

748 

749 sourceCat = self.copyCalibrationFields(sfdSourceCat, sourceCat, self.config.srcFieldsToCopy) 

750 

751 resultStruct = pipeBase.Struct(outputExposure=exposure, 

752 outputCat=sourceCat, 

753 ccdVisitFakeMagnitudes=ccdVisitMagnitudes) 

754 return resultStruct 

755 

756 def addVariability(self, fakeCat, band, exposure, photoCalib, rngSeed): 

757 """Add scatter to the fake catalog visit magnitudes. 

758 

759 Currently just adds a simple Gaussian scatter around the static fake 

760 magnitude. This function could be modified to return any number of 

761 fake variability. 

762 

763 Parameters 

764 ---------- 

765 fakeCat : `pandas.DataFrame` 

766 Catalog of fakes to modify magnitudes of. 

767 band : `str` 

768 Current observing band to modify. 

769 exposure : `lsst.afw.image.ExposureF` 

770 Exposure fakes will be added to. 

771 photoCalib : `lsst.afw.image.PhotoCalib` 

772 Photometric calibration object of ``exposure``. 

773 rngSeed : `int` 

774 Random number generator seed. 

775 

776 Returns 

777 ------- 

778 dataFrame : `pandas.DataFrame` 

779 DataFrame containing the values of the magnitudes to that will 

780 be inserted into this ccdVisit. 

781 """ 

782 rng = np.random.default_rng(rngSeed) 

783 magScatter = rng.normal(loc=0, 

784 scale=self.config.scatterSize, 

785 size=len(fakeCat)) 

786 visitMagnitudes = fakeCat[self.insertFakes.config.mag_col % band] + magScatter 

787 fakeCat.loc[:, self.insertFakes.config.mag_col % band] = visitMagnitudes 

788 return pd.DataFrame(data={"variableMag": visitMagnitudes})