lsst.pipe.tasks ge58501c5e9+6b7d1cf650
processCcdWithFakes.py
Go to the documentation of this file.
1# This file is part of pipe tasks
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
21
22"""
23Insert fake sources into calexps
24"""
25from astropy.table import Table
26import numpy as np
27import pandas as pd
28
29import lsst.pex.config as pexConfig
30import lsst.pipe.base as pipeBase
31
32from .insertFakes import InsertFakesTask
33from lsst.afw.table import SourceTable
34from lsst.obs.base import ExposureIdInfo
35from lsst.pipe.base import PipelineTask, PipelineTaskConfig, CmdLineTask, PipelineTaskConnections
36import lsst.pipe.base.connectionTypes as cT
37import lsst.afw.table as afwTable
38from lsst.skymap import BaseSkyMap
39from lsst.pipe.tasks.calibrate import CalibrateTask
40
41__all__ = ["ProcessCcdWithFakesConfig", "ProcessCcdWithFakesTask",
42 "ProcessCcdWithVariableFakesConfig", "ProcessCcdWithVariableFakesTask"]
43
44
45class ProcessCcdWithFakesConnections(PipelineTaskConnections,
46 dimensions=("instrument", "visit", "detector"),
47 defaultTemplates={"coaddName": "deep",
48 "wcsName": "jointcal",
49 "photoCalibName": "jointcal",
50 "fakesType": "fakes_"}):
51 skyMap = cT.Input(
52 doc="Input definition of geometry/bbox and projection/wcs for "
53 "template exposures. Needed to test which tract to generate ",
54 name=BaseSkyMap.SKYMAP_DATASET_TYPE_NAME,
55 dimensions=("skymap",),
56 storageClass="SkyMap",
57 )
58
59 exposure = cT.Input(
60 doc="Exposure into which fakes are to be added.",
61 name="calexp",
62 storageClass="ExposureF",
63 dimensions=("instrument", "visit", "detector")
64 )
65
66 fakeCats = cT.Input(
67 doc="Set of catalogs of fake sources to draw inputs from. We "
68 "concatenate the tract catalogs for detectorVisits that cover "
69 "multiple tracts.",
70 name="{fakesType}fakeSourceCat",
71 storageClass="DataFrame",
72 dimensions=("tract", "skymap"),
73 deferLoad=True,
74 multiple=True,
75 )
76
77 externalSkyWcsTractCatalog = cT.Input(
78 doc=("Per-tract, per-visit wcs calibrations. These catalogs use the detector "
79 "id for the catalog id, sorted on id for fast lookup."),
80 name="{wcsName}SkyWcsCatalog",
81 storageClass="ExposureCatalog",
82 dimensions=("instrument", "visit", "tract", "skymap"),
83 deferLoad=True,
84 multiple=True,
85 )
86
87 externalSkyWcsGlobalCatalog = cT.Input(
88 doc=("Per-visit wcs calibrations computed globally (with no tract information). "
89 "These catalogs use the detector id for the catalog id, sorted on id for "
90 "fast lookup."),
91 name="{wcsName}SkyWcsCatalog",
92 storageClass="ExposureCatalog",
93 dimensions=("instrument", "visit"),
94 )
95
96 externalPhotoCalibTractCatalog = cT.Input(
97 doc=("Per-tract, per-visit photometric calibrations. These catalogs use the "
98 "detector id for the catalog id, sorted on id for fast lookup."),
99 name="{photoCalibName}PhotoCalibCatalog",
100 storageClass="ExposureCatalog",
101 dimensions=("instrument", "visit", "tract"),
102 deferLoad=True,
103 multiple=True,
104 )
105
106 externalPhotoCalibGlobalCatalog = cT.Input(
107 doc=("Per-visit photometric calibrations. These catalogs use the "
108 "detector id for the catalog id, sorted on id for fast lookup."),
109 name="{photoCalibName}PhotoCalibCatalog",
110 storageClass="ExposureCatalog",
111 dimensions=("instrument", "visit"),
112 )
113
114 icSourceCat = cT.Input(
115 doc="Catalog of calibration sources",
116 name="icSrc",
117 storageClass="SourceCatalog",
118 dimensions=("instrument", "visit", "detector")
119 )
120
121 sfdSourceCat = cT.Input(
122 doc="Catalog of calibration sources",
123 name="src",
124 storageClass="SourceCatalog",
125 dimensions=("instrument", "visit", "detector")
126 )
127
128 outputExposure = cT.Output(
129 doc="Exposure with fake sources added.",
130 name="{fakesType}calexp",
131 storageClass="ExposureF",
132 dimensions=("instrument", "visit", "detector")
133 )
134
135 outputCat = cT.Output(
136 doc="Source catalog produced in calibrate task with fakes also measured.",
137 name="{fakesType}src",
138 storageClass="SourceCatalog",
139 dimensions=("instrument", "visit", "detector"),
140 )
141
142 def __init__(self, *, config=None):
143 super().__init__(config=config)
144
145 if not config.doApplyExternalGlobalPhotoCalib:
146 self.inputs.remove("externalPhotoCalibGlobalCatalog")
147 if not config.doApplyExternalTractPhotoCalib:
148 self.inputs.remove("externalPhotoCalibTractCatalog")
149
150 if not config.doApplyExternalGlobalSkyWcs:
151 self.inputs.remove("externalSkyWcsGlobalCatalog")
152 if not config.doApplyExternalTractSkyWcs:
153 self.inputs.remove("externalSkyWcsTractCatalog")
154
155
156class ProcessCcdWithFakesConfig(PipelineTaskConfig,
157 pipelineConnections=ProcessCcdWithFakesConnections):
158 """Config for inserting fake sources
159
160 Notes
161 -----
162 The default column names are those from the UW sims database.
163 """
164
165 doApplyExternalGlobalPhotoCalib = pexConfig.Field(
166 dtype=bool,
167 default=False,
168 doc="Whether to apply an external photometric calibration via an "
169 "`lsst.afw.image.PhotoCalib` object. Uses the "
170 "`externalPhotoCalibName` config option to determine which "
171 "calibration to use. Uses a global calibration."
172 )
173
174 doApplyExternalTractPhotoCalib = pexConfig.Field(
175 dtype=bool,
176 default=False,
177 doc="Whether to apply an external photometric calibration via an "
178 "`lsst.afw.image.PhotoCalib` object. Uses the "
179 "`externalPhotoCalibName` config option to determine which "
180 "calibration to use. Uses a per tract calibration."
181 )
182
183 externalPhotoCalibName = pexConfig.ChoiceField(
184 doc="What type of external photo calib to use.",
185 dtype=str,
186 default="jointcal",
187 allowed={"jointcal": "Use jointcal_photoCalib",
188 "fgcm": "Use fgcm_photoCalib",
189 "fgcm_tract": "Use fgcm_tract_photoCalib"}
190 )
191
192 doApplyExternalGlobalSkyWcs = pexConfig.Field(
193 dtype=bool,
194 default=False,
195 doc="Whether to apply an external astrometric calibration via an "
196 "`lsst.afw.geom.SkyWcs` object. Uses the "
197 "`externalSkyWcsName` config option to determine which "
198 "calibration to use. Uses a global calibration."
199 )
200
201 doApplyExternalTractSkyWcs = pexConfig.Field(
202 dtype=bool,
203 default=False,
204 doc="Whether to apply an external astrometric calibration via an "
205 "`lsst.afw.geom.SkyWcs` object. Uses the "
206 "`externalSkyWcsName` config option to determine which "
207 "calibration to use. Uses a per tract calibration."
208 )
209
210 externalSkyWcsName = pexConfig.ChoiceField(
211 doc="What type of updated WCS calib to use.",
212 dtype=str,
213 default="jointcal",
214 allowed={"jointcal": "Use jointcal_wcs"}
215 )
216
217 coaddName = pexConfig.Field(
218 doc="The name of the type of coadd used",
219 dtype=str,
220 default="deep",
221 )
222
223 srcFieldsToCopy = pexConfig.ListField(
224 dtype=str,
225 default=("calib_photometry_reserved", "calib_photometry_used", "calib_astrometry_used",
226 "calib_psf_candidate", "calib_psf_used", "calib_psf_reserved"),
227 doc=("Fields to copy from the `src` catalog to the output catalog "
228 "for matching sources Any missing fields will trigger a "
229 "RuntimeError exception.")
230 )
231
232 matchRadiusPix = pexConfig.Field(
233 dtype=float,
234 default=3,
235 doc=("Match radius for matching icSourceCat objects to sourceCat objects (pixels)"),
236 )
237
238 doMatchVisit = pexConfig.Field(
239 dtype=bool,
240 default=False,
241 doc="Match visit to trim the fakeCat"
242 )
243
244 calibrate = pexConfig.ConfigurableField(target=CalibrateTask,
245 doc="The calibration task to use.")
246
247 insertFakes = pexConfig.ConfigurableField(target=InsertFakesTask,
248 doc="Configuration for the fake sources")
249
250 def setDefaults(self):
251 super().setDefaults()
252 self.calibrate.measurement.plugins["base_PixelFlags"].masksFpAnywhere.append("FAKE")
253 self.calibrate.measurement.plugins["base_PixelFlags"].masksFpCenter.append("FAKE")
254 self.calibrate.doAstrometry = False
255 self.calibrate.doWriteMatches = False
256 self.calibrate.doPhotoCal = False
257 self.calibrate.detection.reEstimateBackground = False
258
259
260class ProcessCcdWithFakesTask(PipelineTask, CmdLineTask):
261 """Insert fake objects into calexps.
262
263 Add fake stars and galaxies to the given calexp, specified in the dataRef. Galaxy parameters are read in
264 from the specified file and then modelled using galsim. Re-runs characterize image and calibrate image to
265 give a new background estimation and measurement of the calexp.
266
267 `ProcessFakeSourcesTask` inherits six functions from insertFakesTask that make images of the fake
268 sources and then add them to the calexp.
269
270 `addPixCoords`
271 Use the WCS information to add the pixel coordinates of each source
272 Adds an ``x`` and ``y`` column to the catalog of fake sources.
273 `trimFakeCat`
274 Trim the fake cat to about the size of the input image.
275 `mkFakeGalsimGalaxies`
276 Use Galsim to make fake double sersic galaxies for each set of galaxy parameters in the input file.
277 `mkFakeStars`
278 Use the PSF information from the calexp to make a fake star using the magnitude information from the
279 input file.
280 `cleanCat`
281 Remove rows of the input fake catalog which have half light radius, of either the bulge or the disk,
282 that are 0.
283 `addFakeSources`
284 Add the fake sources to the calexp.
285
286 Notes
287 -----
288 The ``calexp`` with fake souces added to it is written out as the datatype ``calexp_fakes``.
289 """
290
291 _DefaultName = "processCcdWithFakes"
292 ConfigClass = ProcessCcdWithFakesConfig
293
294 def __init__(self, schema=None, butler=None, **kwargs):
295 """Initalize things! This should go above in the class docstring
296 """
297
298 super().__init__(**kwargs)
299
300 if schema is None:
301 schema = SourceTable.makeMinimalSchema()
302 self.schema = schema
303 self.makeSubtask("insertFakes")
304 self.makeSubtask("calibrate")
305
306 def runDataRef(self, dataRef):
307 """Read in/write out the required data products and add fake sources to the calexp.
308
309 Parameters
310 ----------
312 Data reference defining the ccd to have fakes added to it.
313 Used to access the following data products:
314 calexp
315 jointcal_wcs
316 jointcal_photoCalib
317
318 Notes
319 -----
320 Uses the calibration and WCS information attached to the calexp for the posistioning and calibration
321 of the sources unless the config option config.externalPhotoCalibName or config.externalSkyWcsName
322 are set then it uses the specified outputs. The config defualts for the column names in the catalog
323 of fakes are taken from the University of Washington simulations database.
324 Operates on one ccd at a time.
325 """
326 exposureIdInfo = dataRef.get("expIdInfo")
327
328 if self.config.insertFakes.fakeType == "snapshot":
329 fakeCat = dataRef.get("fakeSourceCat").toDataFrame()
330 elif self.config.insertFakes.fakeType == "static":
331 fakeCat = dataRef.get("deepCoadd_fakeSourceCat").toDataFrame()
332 else:
333 fakeCat = Table.read(self.config.insertFakes.fakeType).to_pandas()
334
335 calexp = dataRef.get("calexp")
336 if self.config.doApplyExternalGlobalSkyWcs or self.config.doApplyExternalTractSkyWcs:
337 self.log.info("Using external wcs from %s", self.config.externalSkyWcsName)
338 wcs = dataRef.get(self.config.externalSkyWcsName + "_wcs")
339 else:
340 wcs = calexp.getWcs()
341
342 if self.config.doApplyExternalGlobalPhotoCalib or self.config.doApplyExternalTractPhotoCalib:
343 self.log.info("Using external photocalib from %s", self.config.externalPhotoCalibName)
344 photoCalib = dataRef.get(self.config.externalPhotoCalibName + "_photoCalib")
345 else:
346 photoCalib = calexp.getPhotoCalib()
347
348 icSourceCat = dataRef.get("icSrc", immediate=True)
349 sfdSourceCat = dataRef.get("src", immediate=True)
350
351 resultStruct = self.run(fakeCat, calexp, wcs=wcs, photoCalib=photoCalib,
352 exposureIdInfo=exposureIdInfo, icSourceCat=icSourceCat,
353 sfdSourceCat=sfdSourceCat)
354
355 dataRef.put(resultStruct.outputExposure, "fakes_calexp")
356 dataRef.put(resultStruct.outputCat, "fakes_src")
357 return resultStruct
358
359 def runQuantum(self, butlerQC, inputRefs, outputRefs):
360 inputs = butlerQC.get(inputRefs)
361 detectorId = inputs["exposure"].getInfo().getDetector().getId()
362
363 if 'exposureIdInfo' not in inputs.keys():
364 expId, expBits = butlerQC.quantum.dataId.pack("visit_detector", returnMaxBits=True)
365 inputs['exposureIdInfo'] = ExposureIdInfo(expId, expBits)
366
367 expWcs = inputs["exposure"].getWcs()
368 tractId = inputs["skyMap"].findTract(
369 expWcs.pixelToSky(inputs["exposure"].getBBox().getCenter())).tract_id
370 if not self.config.doApplyExternalGlobalSkyWcs and not self.config.doApplyExternalTractSkyWcs:
371 inputs["wcs"] = expWcs
372 elif self.config.doApplyExternalGlobalSkyWcs:
373 externalSkyWcsCatalog = inputs["externalSkyWcsGlobalCatalog"]
374 row = externalSkyWcsCatalog.find(detectorId)
375 inputs["wcs"] = row.getWcs()
376 elif self.config.doApplyExternalTractSkyWcs:
377 externalSkyWcsCatalogList = inputs["externalSkyWcsTractCatalog"]
378 externalSkyWcsCatalog = None
379 for externalSkyWcsCatalogRef in externalSkyWcsCatalogList:
380 if externalSkyWcsCatalogRef.dataId["tract"] == tractId:
381 externalSkyWcsCatalog = externalSkyWcsCatalogRef.get(
382 datasetType=self.config.connections.externalSkyWcsTractCatalog)
383 break
384 if externalSkyWcsCatalog is None:
385 usedTract = externalSkyWcsCatalogList[-1].dataId["tract"]
386 self.log.warn(
387 f"Warning, external SkyWcs for tract {tractId} not found. Using tract {usedTract} "
388 "instead.")
389 externalSkyWcsCatalog = externalSkyWcsCatalogList[-1].get(
390 datasetType=self.config.connections.externalSkyWcsTractCatalog)
391 row = externalSkyWcsCatalog.find(detectorId)
392 inputs["wcs"] = row.getWcs()
393
394 if not self.config.doApplyExternalGlobalPhotoCalib and not self.config.doApplyExternalTractPhotoCalib:
395 inputs["photoCalib"] = inputs["exposure"].getPhotoCalib()
396 elif self.config.doApplyExternalGlobalPhotoCalib:
397 externalPhotoCalibCatalog = inputs["externalPhotoCalibGlobalCatalog"]
398 row = externalPhotoCalibCatalog.find(detectorId)
399 inputs["photoCalib"] = row.getPhotoCalib()
400 elif self.config.doApplyExternalTractPhotoCalib:
401 externalPhotoCalibCatalogList = inputs["externalPhotoCalibTractCatalog"]
402 externalPhotoCalibCatalog = None
403 for externalPhotoCalibCatalogRef in externalPhotoCalibCatalogList:
404 if externalPhotoCalibCatalogRef.dataId["tract"] == tractId:
405 externalPhotoCalibCatalog = externalPhotoCalibCatalogRef.get(
406 datasetType=self.config.connections.externalPhotoCalibTractCatalog)
407 break
408 if externalPhotoCalibCatalog is None:
409 usedTract = externalPhotoCalibCatalogList[-1].dataId["tract"]
410 self.log.warn(
411 f"Warning, external PhotoCalib for tract {tractId} not found. Using tract {usedTract} "
412 "instead.")
413 externalPhotoCalibCatalog = externalPhotoCalibCatalogList[-1].get(
414 datasetType=self.config.connections.externalPhotoCalibTractCatalog)
415 row = externalPhotoCalibCatalog.find(detectorId)
416 inputs["photoCalib"] = row.getPhotoCalib()
417
418 outputs = self.run(**inputs)
419 butlerQC.put(outputs, outputRefs)
420
421 def run(self, fakeCats, exposure, skyMap, wcs=None, photoCalib=None, exposureIdInfo=None,
422 icSourceCat=None, sfdSourceCat=None, externalSkyWcsGlobalCatalog=None,
423 externalSkyWcsTractCatalog=None, externalPhotoCalibGlobalCatalog=None,
424 externalPhotoCalibTractCatalog=None):
425 """Add fake sources to a calexp and then run detection, deblending and measurement.
426
427 Parameters
428 ----------
429 fakeCats : `list` of `lsst.daf.butler.DeferredDatasetHandle`
430 Set of tract level fake catalogs that potentially cover
431 this detectorVisit.
432 exposure : `lsst.afw.image.exposure.exposure.ExposureF`
433 The exposure to add the fake sources to
434 skyMap : `lsst.skymap.SkyMap`
435 SkyMap defining the tracts and patches the fakes are stored over.
437 WCS to use to add fake sources
438 photoCalib : `lsst.afw.image.photoCalib.PhotoCalib`
439 Photometric calibration to be used to calibrate the fake sources
440 exposureIdInfo : `lsst.obs.base.ExposureIdInfo`
441 icSourceCat : `lsst.afw.table.SourceCatalog`
442 Default : None
443 Catalog to take the information about which sources were used for calibration from.
444 sfdSourceCat : `lsst.afw.table.SourceCatalog`
445 Default : None
446 Catalog produced by singleFrameDriver, needed to copy some calibration flags from.
447
448 Returns
449 -------
450 resultStruct : `lsst.pipe.base.struct.Struct`
451 contains : outputExposure : `lsst.afw.image.exposure.exposure.ExposureF`
452 outputCat : `lsst.afw.table.source.source.SourceCatalog`
453
454 Notes
455 -----
456 Adds pixel coordinates for each source to the fakeCat and removes objects with bulge or disk half
457 light radius = 0 (if ``config.cleanCat = True``). These columns are called ``x`` and ``y`` and are in
458 pixels.
459
460 Adds the ``Fake`` mask plane to the exposure which is then set by `addFakeSources` to mark where fake
461 sources have been added. Uses the information in the ``fakeCat`` to make fake galaxies (using galsim)
462 and fake stars, using the PSF models from the PSF information for the calexp. These are then added to
463 the calexp and the calexp with fakes included returned.
464
465 The galsim galaxies are made using a double sersic profile, one for the bulge and one for the disk,
466 this is then convolved with the PSF at that point.
467
468 If exposureIdInfo is not provided then the SourceCatalog IDs will not be globally unique.
469 """
470 fakeCat = self.composeFakeCat(fakeCats, skyMap)
471
472 if wcs is None:
473 wcs = exposure.getWcs()
474
475 if photoCalib is None:
476 photoCalib = exposure.getPhotoCalib()
477
478 if self.config.doMatchVisit:
479 fakeCat = self.getVisitMatchedFakeCat(fakeCat, exposure)
480
481 self.insertFakes.run(fakeCat, exposure, wcs, photoCalib)
482
483 # detect, deblend and measure sources
484 if exposureIdInfo is None:
485 exposureIdInfo = ExposureIdInfo()
486 returnedStruct = self.calibrate.run(exposure, exposureIdInfo=exposureIdInfo)
487 sourceCat = returnedStruct.sourceCat
488
489 sourceCat = self.copyCalibrationFields(sfdSourceCat, sourceCat, self.config.srcFieldsToCopy)
490
491 resultStruct = pipeBase.Struct(outputExposure=exposure, outputCat=sourceCat)
492 return resultStruct
493
494 def composeFakeCat(self, fakeCats, skyMap):
495 """Concatenate the fakeCats from tracts that may cover the exposure.
496
497 Parameters
498 ----------
499 fakeCats : `list` of `lst.daf.butler.DeferredDatasetHandle`
500 Set of fake cats to concatenate.
501 skyMap : `lsst.skymap.SkyMap`
502 SkyMap defining the geometry of the tracts and patches.
503
504 Returns
505 -------
506 combinedFakeCat : `pandas.DataFrame`
507 All fakes that cover the inner polygon of the tracts in this
508 quantum.
509 """
510 if len(fakeCats) == 1:
511 return fakeCats[0].get(
512 datasetType=self.config.connections.fakeCats)
513 outputCat = []
514 for fakeCatRef in fakeCats:
515 cat = fakeCatRef.get(
516 datasetType=self.config.connections.fakeCats)
517 tractId = fakeCatRef.dataId["tract"]
518 # Make sure all data is within the inner part of the tract.
519 outputCat.append(cat[
520 skyMap.findTractIdArray(cat[self.config.insertFakes.ra_col],
521 cat[self.config.insertFakes.dec_col],
522 degrees=False)
523 == tractId])
524
525 return pd.concat(outputCat)
526
527 def getVisitMatchedFakeCat(self, fakeCat, exposure):
528 """Trim the fakeCat to select particular visit
529
530 Parameters
531 ----------
532 fakeCat : `pandas.core.frame.DataFrame`
533 The catalog of fake sources to add to the exposure
534 exposure : `lsst.afw.image.exposure.exposure.ExposureF`
535 The exposure to add the fake sources to
536
537 Returns
538 -------
539 movingFakeCat : `pandas.DataFrame`
540 All fakes that belong to the visit
541 """
542 selected = exposure.getInfo().getVisitInfo().getId() == fakeCat["visit"]
543
544 return fakeCat[selected]
545
546 def copyCalibrationFields(self, calibCat, sourceCat, fieldsToCopy):
547 """Match sources in calibCat and sourceCat and copy the specified fields
548
549 Parameters
550 ----------
552 Catalog from which to copy fields.
553 sourceCat : `lsst.afw.table.SourceCatalog`
554 Catalog to which to copy fields.
555 fieldsToCopy : `lsst.pex.config.listField.List`
556 Fields to copy from calibCat to SoourceCat.
557
558 Returns
559 -------
561 Catalog which includes the copied fields.
562
563 The fields copied are those specified by `fieldsToCopy` that actually exist
564 in the schema of `calibCat`.
565
566 This version was based on and adapted from the one in calibrateTask.
567 """
568
569 # Make a new SourceCatalog with the data from sourceCat so that we can add the new columns to it
570 sourceSchemaMapper = afwTable.SchemaMapper(sourceCat.schema)
571 sourceSchemaMapper.addMinimalSchema(sourceCat.schema, True)
572
573 calibSchemaMapper = afwTable.SchemaMapper(calibCat.schema, sourceCat.schema)
574
575 # Add the desired columns from the option fieldsToCopy
576 missingFieldNames = []
577 for fieldName in fieldsToCopy:
578 if fieldName in calibCat.schema:
579 schemaItem = calibCat.schema.find(fieldName)
580 calibSchemaMapper.editOutputSchema().addField(schemaItem.getField())
581 schema = calibSchemaMapper.editOutputSchema()
582 calibSchemaMapper.addMapping(schemaItem.getKey(), schema.find(fieldName).getField())
583 else:
584 missingFieldNames.append(fieldName)
585 if missingFieldNames:
586 raise RuntimeError(f"calibCat is missing fields {missingFieldNames} specified in "
587 "fieldsToCopy")
588
589 if "calib_detected" not in calibSchemaMapper.getOutputSchema():
590 self.calibSourceKey = calibSchemaMapper.addOutputField(afwTable.Field["Flag"]("calib_detected",
591 "Source was detected as an icSource"))
592 else:
593 self.calibSourceKey = None
594
595 schema = calibSchemaMapper.getOutputSchema()
596 newCat = afwTable.SourceCatalog(schema)
597 newCat.reserve(len(sourceCat))
598 newCat.extend(sourceCat, sourceSchemaMapper)
599
600 # Set the aliases so it doesn't complain.
601 for k, v in sourceCat.schema.getAliasMap().items():
602 newCat.schema.getAliasMap().set(k, v)
603
604 select = newCat["deblend_nChild"] == 0
605 matches = afwTable.matchXy(newCat[select], calibCat, self.config.matchRadiusPix)
606 # Check that no sourceCat sources are listed twice (we already know
607 # that each match has a unique calibCat source ID, due to using
608 # that ID as the key in bestMatches)
609 numMatches = len(matches)
610 numUniqueSources = len(set(m[1].getId() for m in matches))
611 if numUniqueSources != numMatches:
612 self.log.warning("%d calibCat sources matched only %d sourceCat sources", numMatches,
613 numUniqueSources)
614
615 self.log.info("Copying flags from calibCat to sourceCat for %s sources", numMatches)
616
617 # For each match: set the calibSourceKey flag and copy the desired
618 # fields
619 for src, calibSrc, d in matches:
620 if self.calibSourceKey:
621 src.setFlag(self.calibSourceKey, True)
622 # src.assign copies the footprint from calibSrc, which we don't want
623 # (DM-407)
624 # so set calibSrc's footprint to src's footprint before src.assign,
625 # then restore it
626 calibSrcFootprint = calibSrc.getFootprint()
627 try:
628 calibSrc.setFootprint(src.getFootprint())
629 src.assign(calibSrc, calibSchemaMapper)
630 finally:
631 calibSrc.setFootprint(calibSrcFootprint)
632
633 return newCat
634
635
636class ProcessCcdWithVariableFakesConnections(ProcessCcdWithFakesConnections):
637 ccdVisitFakeMagnitudes = cT.Output(
638 doc="Catalog of fakes with magnitudes scattered for this ccdVisit.",
639 name="{fakesType}ccdVisitFakeMagnitudes",
640 storageClass="DataFrame",
641 dimensions=("instrument", "visit", "detector"),
642 )
643
644
645class ProcessCcdWithVariableFakesConfig(ProcessCcdWithFakesConfig,
646 pipelineConnections=ProcessCcdWithVariableFakesConnections):
647 scatterSize = pexConfig.RangeField(
648 dtype=float,
649 default=0.4,
650 min=0,
651 max=100,
652 doc="Amount of scatter to add to the visit magnitude for variable "
653 "sources."
654 )
655
656
657class ProcessCcdWithVariableFakesTask(ProcessCcdWithFakesTask):
658 """As ProcessCcdWithFakes except add variablity to the fakes catalog
659 magnitude in the observed band for this ccdVisit.
660
661 Additionally, write out the modified magnitudes to the Butler.
662 """
663
664 _DefaultName = "processCcdWithVariableFakes"
665 ConfigClass = ProcessCcdWithVariableFakesConfig
666
667 def run(self, fakeCats, exposure, skyMap, wcs=None, photoCalib=None, exposureIdInfo=None,
668 icSourceCat=None, sfdSourceCat=None):
669 """Add fake sources to a calexp and then run detection, deblending and measurement.
670
671 Parameters
672 ----------
673 fakeCat : `pandas.core.frame.DataFrame`
674 The catalog of fake sources to add to the exposure
675 exposure : `lsst.afw.image.exposure.exposure.ExposureF`
676 The exposure to add the fake sources to
677 skyMap : `lsst.skymap.SkyMap`
678 SkyMap defining the tracts and patches the fakes are stored over.
680 WCS to use to add fake sources
681 photoCalib : `lsst.afw.image.photoCalib.PhotoCalib`
682 Photometric calibration to be used to calibrate the fake sources
683 exposureIdInfo : `lsst.obs.base.ExposureIdInfo`
684 icSourceCat : `lsst.afw.table.SourceCatalog`
685 Default : None
686 Catalog to take the information about which sources were used for calibration from.
687 sfdSourceCat : `lsst.afw.table.SourceCatalog`
688 Default : None
689 Catalog produced by singleFrameDriver, needed to copy some calibration flags from.
690
691 Returns
692 -------
693 resultStruct : `lsst.pipe.base.struct.Struct`
694 Results Strcut containing:
695
696 - outputExposure : Exposure with added fakes
697 (`lsst.afw.image.exposure.exposure.ExposureF`)
698 - outputCat : Catalog with detected fakes
699 (`lsst.afw.table.source.source.SourceCatalog`)
700 - ccdVisitFakeMagnitudes : Magnitudes that these fakes were
701 inserted with after being scattered (`pandas.DataFrame`)
702
703 Notes
704 -----
705 Adds pixel coordinates for each source to the fakeCat and removes objects with bulge or disk half
706 light radius = 0 (if ``config.cleanCat = True``). These columns are called ``x`` and ``y`` and are in
707 pixels.
708
709 Adds the ``Fake`` mask plane to the exposure which is then set by `addFakeSources` to mark where fake
710 sources have been added. Uses the information in the ``fakeCat`` to make fake galaxies (using galsim)
711 and fake stars, using the PSF models from the PSF information for the calexp. These are then added to
712 the calexp and the calexp with fakes included returned.
713
714 The galsim galaxies are made using a double sersic profile, one for the bulge and one for the disk,
715 this is then convolved with the PSF at that point.
716
717 If exposureIdInfo is not provided then the SourceCatalog IDs will not be globally unique.
718 """
719 fakeCat = self.composeFakeCat(fakeCats, skyMap)
720
721 if wcs is None:
722 wcs = exposure.getWcs()
723
724 if photoCalib is None:
725 photoCalib = exposure.getPhotoCalib()
726
727 if exposureIdInfo is None:
728 exposureIdInfo = ExposureIdInfo()
729
730 band = exposure.getFilter().bandLabel
731 ccdVisitMagnitudes = self.addVariablity(fakeCat, band, exposure, photoCalib, exposureIdInfo)
732
733 self.insertFakes.run(fakeCat, exposure, wcs, photoCalib)
734
735 # detect, deblend and measure sources
736 returnedStruct = self.calibrate.run(exposure, exposureIdInfo=exposureIdInfo)
737 sourceCat = returnedStruct.sourceCat
738
739 sourceCat = self.copyCalibrationFields(sfdSourceCat, sourceCat, self.config.srcFieldsToCopy)
740
741 resultStruct = pipeBase.Struct(outputExposure=exposure,
742 outputCat=sourceCat,
743 ccdVisitFakeMagnitudes=ccdVisitMagnitudes)
744 return resultStruct
745
746 def addVariablity(self, fakeCat, band, exposure, photoCalib, exposureIdInfo):
747 """Add scatter to the fake catalog visit magnitudes.
748
749 Currently just adds a simple Gaussian scatter around the static fake
750 magnitude. This function could be modified to return any number of
751 fake variability.
752
753 Parameters
754 ----------
755 fakeCat : `pandas.DataFrame`
756 Catalog of fakes to modify magnitudes of.
757 band : `str`
758 Current observing band to modify.
759 exposure : `lsst.afw.image.ExposureF`
760 Exposure fakes will be added to.
761 photoCalib : `lsst.afw.image.PhotoCalib`
762 Photometric calibration object of ``exposure``.
763 exposureIdInfo : `lsst.obs.base.ExposureIdInfo`
764 Exposure id information and metadata.
765
766 Returns
767 -------
768 dataFrame : `pandas.DataFrame`
769 DataFrame containing the values of the magnitudes to that will
770 be inserted into this ccdVisit.
771 """
772 expId = exposureIdInfo.expId
773 rng = np.random.default_rng(expId)
774 magScatter = rng.normal(loc=0,
775 scale=self.config.scatterSize,
776 size=len(fakeCat))
777 visitMagnitudes = fakeCat[self.insertFakes.config.mag_col % band] + magScatter
778 fakeCat.loc[:, self.insertFakes.config.mag_col % band] = visitMagnitudes
779 return pd.DataFrame(data={"variableMag": visitMagnitudes})