lsst.pipe.tasks 21.0.0-179-g9276d3b8+598dd35d22
processCcdWithFakes.py
Go to the documentation of this file.
1# This file is part of pipe tasks
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
21
22"""
23Insert fake sources into calexps
24"""
25from astropy.table import Table
26import numpy as np
27import pandas as pd
28
29import lsst.pex.config as pexConfig
30import lsst.pipe.base as pipeBase
31
32from .insertFakes import InsertFakesTask
33from lsst.meas.base import PerTractCcdDataIdContainer
34from lsst.afw.table import SourceTable
35from lsst.obs.base import ExposureIdInfo
36from lsst.pipe.base import PipelineTask, PipelineTaskConfig, CmdLineTask, PipelineTaskConnections
37import lsst.pipe.base.connectionTypes as cT
38import lsst.afw.table as afwTable
39from lsst.skymap import BaseSkyMap
40from lsst.pipe.tasks.calibrate import CalibrateTask
41
42__all__ = ["ProcessCcdWithFakesConfig", "ProcessCcdWithFakesTask",
43 "ProcessCcdWithVariableFakesConfig", "ProcessCcdWithVariableFakesTask"]
44
45
46class ProcessCcdWithFakesConnections(PipelineTaskConnections,
47 dimensions=("instrument", "visit", "detector"),
48 defaultTemplates={"coaddName": "deep",
49 "wcsName": "jointcal",
50 "photoCalibName": "jointcal",
51 "fakesType": "fakes_"}):
52 skyMap = cT.Input(
53 doc="Input definition of geometry/bbox and projection/wcs for "
54 "template exposures. Needed to test which tract to generate ",
55 name=BaseSkyMap.SKYMAP_DATASET_TYPE_NAME,
56 dimensions=("skymap",),
57 storageClass="SkyMap",
58 )
59
60 exposure = cT.Input(
61 doc="Exposure into which fakes are to be added.",
62 name="calexp",
63 storageClass="ExposureF",
64 dimensions=("instrument", "visit", "detector")
65 )
66
67 fakeCats = cT.Input(
68 doc="Set of catalogs of fake sources to draw inputs from. We "
69 "concatenate the tract catalogs for detectorVisits that cover "
70 "multiple tracts.",
71 name="{fakesType}fakeSourceCat",
72 storageClass="DataFrame",
73 dimensions=("tract", "skymap"),
74 deferLoad=True,
75 multiple=True,
76 )
77
78 externalSkyWcsTractCatalog = cT.Input(
79 doc=("Per-tract, per-visit wcs calibrations. These catalogs use the detector "
80 "id for the catalog id, sorted on id for fast lookup."),
81 name="{wcsName}SkyWcsCatalog",
82 storageClass="ExposureCatalog",
83 dimensions=("instrument", "visit", "tract", "skymap"),
84 deferLoad=True,
85 multiple=True,
86 )
87
88 externalSkyWcsGlobalCatalog = cT.Input(
89 doc=("Per-visit wcs calibrations computed globally (with no tract information). "
90 "These catalogs use the detector id for the catalog id, sorted on id for "
91 "fast lookup."),
92 name="{wcsName}SkyWcsCatalog",
93 storageClass="ExposureCatalog",
94 dimensions=("instrument", "visit"),
95 )
96
97 externalPhotoCalibTractCatalog = cT.Input(
98 doc=("Per-tract, per-visit photometric calibrations. These catalogs use the "
99 "detector id for the catalog id, sorted on id for fast lookup."),
100 name="{photoCalibName}PhotoCalibCatalog",
101 storageClass="ExposureCatalog",
102 dimensions=("instrument", "visit", "tract"),
103 deferLoad=True,
104 multiple=True,
105 )
106
107 externalPhotoCalibGlobalCatalog = cT.Input(
108 doc=("Per-visit photometric calibrations. These catalogs use the "
109 "detector id for the catalog id, sorted on id for fast lookup."),
110 name="{photoCalibName}PhotoCalibCatalog",
111 storageClass="ExposureCatalog",
112 dimensions=("instrument", "visit"),
113 )
114
115 icSourceCat = cT.Input(
116 doc="Catalog of calibration sources",
117 name="icSrc",
118 storageClass="SourceCatalog",
119 dimensions=("instrument", "visit", "detector")
120 )
121
122 sfdSourceCat = cT.Input(
123 doc="Catalog of calibration sources",
124 name="src",
125 storageClass="SourceCatalog",
126 dimensions=("instrument", "visit", "detector")
127 )
128
129 outputExposure = cT.Output(
130 doc="Exposure with fake sources added.",
131 name="{fakesType}calexp",
132 storageClass="ExposureF",
133 dimensions=("instrument", "visit", "detector")
134 )
135
136 outputCat = cT.Output(
137 doc="Source catalog produced in calibrate task with fakes also measured.",
138 name="{fakesType}src",
139 storageClass="SourceCatalog",
140 dimensions=("instrument", "visit", "detector"),
141 )
142
143 def __init__(self, *, config=None):
144 super().__init__(config=config)
145
146 if not config.doApplyExternalGlobalPhotoCalib:
147 self.inputs.remove("externalPhotoCalibGlobalCatalog")
148 if not config.doApplyExternalTractPhotoCalib:
149 self.inputs.remove("externalPhotoCalibTractCatalog")
150
151 if not config.doApplyExternalGlobalSkyWcs:
152 self.inputs.remove("externalSkyWcsGlobalCatalog")
153 if not config.doApplyExternalTractSkyWcs:
154 self.inputs.remove("externalSkyWcsTractCatalog")
155
156
157class ProcessCcdWithFakesConfig(PipelineTaskConfig,
158 pipelineConnections=ProcessCcdWithFakesConnections):
159 """Config for inserting fake sources
160
161 Notes
162 -----
163 The default column names are those from the UW sims database.
164 """
165
166 doApplyExternalGlobalPhotoCalib = pexConfig.Field(
167 dtype=bool,
168 default=False,
169 doc="Whether to apply an external photometric calibration via an "
170 "`lsst.afw.image.PhotoCalib` object. Uses the "
171 "`externalPhotoCalibName` config option to determine which "
172 "calibration to use. Uses a global calibration."
173 )
174
175 doApplyExternalTractPhotoCalib = pexConfig.Field(
176 dtype=bool,
177 default=False,
178 doc="Whether to apply an external photometric calibration via an "
179 "`lsst.afw.image.PhotoCalib` object. Uses the "
180 "`externalPhotoCalibName` config option to determine which "
181 "calibration to use. Uses a per tract calibration."
182 )
183
184 externalPhotoCalibName = pexConfig.ChoiceField(
185 doc="What type of external photo calib to use.",
186 dtype=str,
187 default="jointcal",
188 allowed={"jointcal": "Use jointcal_photoCalib",
189 "fgcm": "Use fgcm_photoCalib",
190 "fgcm_tract": "Use fgcm_tract_photoCalib"}
191 )
192
193 doApplyExternalGlobalSkyWcs = pexConfig.Field(
194 dtype=bool,
195 default=False,
196 doc="Whether to apply an external astrometric calibration via an "
197 "`lsst.afw.geom.SkyWcs` object. Uses the "
198 "`externalSkyWcsName` config option to determine which "
199 "calibration to use. Uses a global calibration."
200 )
201
202 doApplyExternalTractSkyWcs = pexConfig.Field(
203 dtype=bool,
204 default=False,
205 doc="Whether to apply an external astrometric calibration via an "
206 "`lsst.afw.geom.SkyWcs` object. Uses the "
207 "`externalSkyWcsName` config option to determine which "
208 "calibration to use. Uses a per tract calibration."
209 )
210
211 externalSkyWcsName = pexConfig.ChoiceField(
212 doc="What type of updated WCS calib to use.",
213 dtype=str,
214 default="jointcal",
215 allowed={"jointcal": "Use jointcal_wcs"}
216 )
217
218 coaddName = pexConfig.Field(
219 doc="The name of the type of coadd used",
220 dtype=str,
221 default="deep",
222 )
223
224 srcFieldsToCopy = pexConfig.ListField(
225 dtype=str,
226 default=("calib_photometry_reserved", "calib_photometry_used", "calib_astrometry_used",
227 "calib_psf_candidate", "calib_psf_used", "calib_psf_reserved"),
228 doc=("Fields to copy from the `src` catalog to the output catalog "
229 "for matching sources Any missing fields will trigger a "
230 "RuntimeError exception.")
231 )
232
233 matchRadiusPix = pexConfig.Field(
234 dtype=float,
235 default=3,
236 doc=("Match radius for matching icSourceCat objects to sourceCat objects (pixels)"),
237 )
238
239 calibrate = pexConfig.ConfigurableField(target=CalibrateTask,
240 doc="The calibration task to use.")
241
242 insertFakes = pexConfig.ConfigurableField(target=InsertFakesTask,
243 doc="Configuration for the fake sources")
244
245 def setDefaults(self):
246 super().setDefaults()
247 self.calibrate.measurement.plugins["base_PixelFlags"].masksFpAnywhere.append("FAKE")
248 self.calibrate.measurement.plugins["base_PixelFlags"].masksFpCenter.append("FAKE")
249 self.calibrate.doAstrometry = False
250 self.calibrate.doWriteMatches = False
251 self.calibrate.doPhotoCal = False
252 self.calibrate.detection.reEstimateBackground = False
253
254
255class ProcessCcdWithFakesTask(PipelineTask, CmdLineTask):
256 """Insert fake objects into calexps.
257
258 Add fake stars and galaxies to the given calexp, specified in the dataRef. Galaxy parameters are read in
259 from the specified file and then modelled using galsim. Re-runs characterize image and calibrate image to
260 give a new background estimation and measurement of the calexp.
261
262 `ProcessFakeSourcesTask` inherits six functions from insertFakesTask that make images of the fake
263 sources and then add them to the calexp.
264
265 `addPixCoords`
266 Use the WCS information to add the pixel coordinates of each source
267 Adds an ``x`` and ``y`` column to the catalog of fake sources.
268 `trimFakeCat`
269 Trim the fake cat to about the size of the input image.
270 `mkFakeGalsimGalaxies`
271 Use Galsim to make fake double sersic galaxies for each set of galaxy parameters in the input file.
272 `mkFakeStars`
273 Use the PSF information from the calexp to make a fake star using the magnitude information from the
274 input file.
275 `cleanCat`
276 Remove rows of the input fake catalog which have half light radius, of either the bulge or the disk,
277 that are 0.
278 `addFakeSources`
279 Add the fake sources to the calexp.
280
281 Notes
282 -----
283 The ``calexp`` with fake souces added to it is written out as the datatype ``calexp_fakes``.
284 """
285
286 _DefaultName = "processCcdWithFakes"
287 ConfigClass = ProcessCcdWithFakesConfig
288
289 def __init__(self, schema=None, butler=None, **kwargs):
290 """Initalize things! This should go above in the class docstring
291 """
292
293 super().__init__(**kwargs)
294
295 if schema is None:
296 schema = SourceTable.makeMinimalSchema()
297 self.schema = schema
298 self.makeSubtask("insertFakes")
299 self.makeSubtask("calibrate")
300
301 def runDataRef(self, dataRef):
302 """Read in/write out the required data products and add fake sources to the calexp.
303
304 Parameters
305 ----------
307 Data reference defining the ccd to have fakes added to it.
308 Used to access the following data products:
309 calexp
310 jointcal_wcs
311 jointcal_photoCalib
312
313 Notes
314 -----
315 Uses the calibration and WCS information attached to the calexp for the posistioning and calibration
316 of the sources unless the config option config.externalPhotoCalibName or config.externalSkyWcsName
317 are set then it uses the specified outputs. The config defualts for the column names in the catalog
318 of fakes are taken from the University of Washington simulations database.
319 Operates on one ccd at a time.
320 """
321 exposureIdInfo = dataRef.get("expIdInfo")
322
323 if self.config.insertFakes.fakeType == "snapshot":
324 fakeCat = dataRef.get("fakeSourceCat").toDataFrame()
325 elif self.config.insertFakes.fakeType == "static":
326 fakeCat = dataRef.get("deepCoadd_fakeSourceCat").toDataFrame()
327 else:
328 fakeCat = Table.read(self.config.insertFakes.fakeType).to_pandas()
329
330 calexp = dataRef.get("calexp")
331 if self.config.doApplyExternalGlobalSkyWcs or self.config.doApplyExternalTractSkyWcs:
332 self.log.info("Using external wcs from %s", self.config.externalSkyWcsName)
333 wcs = dataRef.get(self.config.externalSkyWcsName + "_wcs")
334 else:
335 wcs = calexp.getWcs()
336
337 if self.config.doApplyExternalGlobalPhotoCalib or self.config.doApplyExternalTractPhotoCalib:
338 self.log.info("Using external photocalib from %s", self.config.externalPhotoCalibName)
339 photoCalib = dataRef.get(self.config.externalPhotoCalibName + "_photoCalib")
340 else:
341 photoCalib = calexp.getPhotoCalib()
342
343 icSourceCat = dataRef.get("icSrc", immediate=True)
344 sfdSourceCat = dataRef.get("src", immediate=True)
345
346 resultStruct = self.run(fakeCat, calexp, wcs=wcs, photoCalib=photoCalib,
347 exposureIdInfo=exposureIdInfo, icSourceCat=icSourceCat,
348 sfdSourceCat=sfdSourceCat)
349
350 dataRef.put(resultStruct.outputExposure, "fakes_calexp")
351 dataRef.put(resultStruct.outputCat, "fakes_src")
352 return resultStruct
353
354 def runQuantum(self, butlerQC, inputRefs, outputRefs):
355 inputs = butlerQC.get(inputRefs)
356 detectorId = inputs["exposure"].getInfo().getDetector().getId()
357
358 if 'exposureIdInfo' not in inputs.keys():
359 expId, expBits = butlerQC.quantum.dataId.pack("visit_detector", returnMaxBits=True)
360 inputs['exposureIdInfo'] = ExposureIdInfo(expId, expBits)
361
362 expWcs = inputs["exposure"].getWcs()
363 tractId = inputs["skyMap"].findTract(
364 expWcs.pixelToSky(inputs["exposure"].getBBox().getCenter())).tract_id
365 if not self.config.doApplyExternalGlobalSkyWcs and not self.config.doApplyExternalTractSkyWcs:
366 inputs["wcs"] = expWcs
367 elif self.config.doApplyExternalGlobalSkyWcs:
368 externalSkyWcsCatalog = inputs["externalSkyWcsGlobalCatalog"]
369 row = externalSkyWcsCatalog.find(detectorId)
370 inputs["wcs"] = row.getWcs()
371 elif self.config.doApplyExternalTractSkyWcs:
372 externalSkyWcsCatalogList = inputs["externalSkyWcsTractCatalog"]
373 externalSkyWcsCatalog = None
374 for externalSkyWcsCatalogRef in externalSkyWcsCatalogList:
375 if externalSkyWcsCatalogRef.dataId["tract"] == tractId:
376 externalSkyWcsCatalog = externalSkyWcsCatalogRef.get(
377 datasetType=self.config.connections.externalSkyWcsTractCatalog)
378 break
379 if externalSkyWcsCatalog is None:
380 usedTract = externalSkyWcsCatalogList[-1].dataId["tract"]
381 self.log.warn(
382 f"Warning, external SkyWcs for tract {tractId} not found. Using tract {usedTract} "
383 "instead.")
384 externalSkyWcsCatalog = externalSkyWcsCatalogList[-1].get(
385 datasetType=self.config.connections.externalSkyWcsTractCatalog)
386 row = externalSkyWcsCatalog.find(detectorId)
387 inputs["wcs"] = row.getWcs()
388
389 if not self.config.doApplyExternalGlobalPhotoCalib and not self.config.doApplyExternalTractPhotoCalib:
390 inputs["photoCalib"] = inputs["exposure"].getPhotoCalib()
391 elif self.config.doApplyExternalGlobalPhotoCalib:
392 externalPhotoCalibCatalog = inputs["externalPhotoCalibGlobalCatalog"]
393 row = externalPhotoCalibCatalog.find(detectorId)
394 inputs["photoCalib"] = row.getPhotoCalib()
395 elif self.config.doApplyExternalTractPhotoCalib:
396 externalPhotoCalibCatalogList = inputs["externalPhotoCalibTractCatalog"]
397 externalPhotoCalibCatalog = None
398 for externalPhotoCalibCatalogRef in externalPhotoCalibCatalogList:
399 if externalPhotoCalibCatalogRef.dataId["tract"] == tractId:
400 externalPhotoCalibCatalog = externalPhotoCalibCatalogRef.get(
401 datasetType=self.config.connections.externalPhotoCalibTractCatalog)
402 break
403 if externalPhotoCalibCatalog is None:
404 usedTract = externalPhotoCalibCatalogList[-1].dataId["tract"]
405 self.log.warn(
406 f"Warning, external PhotoCalib for tract {tractId} not found. Using tract {usedTract} "
407 "instead.")
408 externalPhotoCalibCatalog = externalPhotoCalibCatalogList[-1].get(
409 datasetType=self.config.connections.externalPhotoCalibTractCatalog)
410 row = externalPhotoCalibCatalog.find(detectorId)
411 inputs["photoCalib"] = row.getPhotoCalib()
412
413 outputs = self.run(**inputs)
414 butlerQC.put(outputs, outputRefs)
415
416 @classmethod
417 def _makeArgumentParser(cls):
418 parser = pipeBase.ArgumentParser(name=cls._DefaultName)
419 parser.add_id_argument("--id", "fakes_calexp", help="data ID with raw CCD keys [+ tract optionally], "
420 "e.g. --id visit=12345 ccd=1,2 [tract=0]",
421 ContainerClass=PerTractCcdDataIdContainer)
422 return parser
423
424 def run(self, fakeCats, exposure, skyMap, wcs=None, photoCalib=None, exposureIdInfo=None,
425 icSourceCat=None, sfdSourceCat=None, externalSkyWcsGlobalCatalog=None,
426 externalSkyWcsTractCatalog=None, externalPhotoCalibGlobalCatalog=None,
427 externalPhotoCalibTractCatalog=None):
428 """Add fake sources to a calexp and then run detection, deblending and measurement.
429
430 Parameters
431 ----------
432 fakeCats : `list` of `lsst.daf.butler.DeferredDatasetHandle`
433 Set of tract level fake catalogs that potentially cover
434 this detectorVisit.
435 exposure : `lsst.afw.image.exposure.exposure.ExposureF`
436 The exposure to add the fake sources to
437 skyMap : `lsst.skymap.SkyMap`
438 SkyMap defining the tracts and patches the fakes are stored over.
440 WCS to use to add fake sources
441 photoCalib : `lsst.afw.image.photoCalib.PhotoCalib`
442 Photometric calibration to be used to calibrate the fake sources
443 exposureIdInfo : `lsst.obs.base.ExposureIdInfo`
444 icSourceCat : `lsst.afw.table.SourceCatalog`
445 Default : None
446 Catalog to take the information about which sources were used for calibration from.
447 sfdSourceCat : `lsst.afw.table.SourceCatalog`
448 Default : None
449 Catalog produced by singleFrameDriver, needed to copy some calibration flags from.
450
451 Returns
452 -------
453 resultStruct : `lsst.pipe.base.struct.Struct`
454 contains : outputExposure : `lsst.afw.image.exposure.exposure.ExposureF`
455 outputCat : `lsst.afw.table.source.source.SourceCatalog`
456
457 Notes
458 -----
459 Adds pixel coordinates for each source to the fakeCat and removes objects with bulge or disk half
460 light radius = 0 (if ``config.cleanCat = True``). These columns are called ``x`` and ``y`` and are in
461 pixels.
462
463 Adds the ``Fake`` mask plane to the exposure which is then set by `addFakeSources` to mark where fake
464 sources have been added. Uses the information in the ``fakeCat`` to make fake galaxies (using galsim)
465 and fake stars, using the PSF models from the PSF information for the calexp. These are then added to
466 the calexp and the calexp with fakes included returned.
467
468 The galsim galaxies are made using a double sersic profile, one for the bulge and one for the disk,
469 this is then convolved with the PSF at that point.
470
471 If exposureIdInfo is not provided then the SourceCatalog IDs will not be globally unique.
472 """
473 fakeCat = self.composeFakeCat(fakeCats, skyMap)
474
475 if wcs is None:
476 wcs = exposure.getWcs()
477
478 if photoCalib is None:
479 photoCalib = exposure.getPhotoCalib()
480
481 self.insertFakes.run(fakeCat, exposure, wcs, photoCalib)
482
483 # detect, deblend and measure sources
484 if exposureIdInfo is None:
485 exposureIdInfo = ExposureIdInfo()
486 returnedStruct = self.calibrate.run(exposure, exposureIdInfo=exposureIdInfo)
487 sourceCat = returnedStruct.sourceCat
488
489 sourceCat = self.copyCalibrationFields(sfdSourceCat, sourceCat, self.config.srcFieldsToCopy)
490
491 resultStruct = pipeBase.Struct(outputExposure=exposure, outputCat=sourceCat)
492 return resultStruct
493
494 def composeFakeCat(self, fakeCats, skyMap):
495 """Concatenate the fakeCats from tracts that may cover the exposure.
496
497 Parameters
498 ----------
499 fakeCats : `list` of `lst.daf.butler.DeferredDatasetHandle`
500 Set of fake cats to concatenate.
501 skyMap : `lsst.skymap.SkyMap`
502 SkyMap defining the geometry of the tracts and patches.
503
504 Returns
505 -------
506 combinedFakeCat : `pandas.DataFrame`
507 All fakes that cover the inner polygon of the tracts in this
508 quantum.
509 """
510 if len(fakeCats) == 1:
511 return fakeCats[0].get(
512 datasetType=self.config.connections.fakeCats)
513 outputCat = []
514 for fakeCatRef in fakeCats:
515 cat = fakeCatRef.get(
516 datasetType=self.config.connections.fakeCats)
517 tractId = fakeCatRef.dataId["tract"]
518 # Make sure all data is within the inner part of the tract.
519 outputCat.append(cat[
520 skyMap.findTractIdArray(cat[self.config.insertFakes.ra_col],
521 cat[self.config.insertFakes.dec_col],
522 degrees=False)
523 == tractId])
524
525 return pd.concat(outputCat)
526
527 def copyCalibrationFields(self, calibCat, sourceCat, fieldsToCopy):
528 """Match sources in calibCat and sourceCat and copy the specified fields
529
530 Parameters
531 ----------
533 Catalog from which to copy fields.
534 sourceCat : `lsst.afw.table.SourceCatalog`
535 Catalog to which to copy fields.
536 fieldsToCopy : `lsst.pex.config.listField.List`
537 Fields to copy from calibCat to SoourceCat.
538
539 Returns
540 -------
542 Catalog which includes the copied fields.
543
544 The fields copied are those specified by `fieldsToCopy` that actually exist
545 in the schema of `calibCat`.
546
547 This version was based on and adapted from the one in calibrateTask.
548 """
549
550 # Make a new SourceCatalog with the data from sourceCat so that we can add the new columns to it
551 sourceSchemaMapper = afwTable.SchemaMapper(sourceCat.schema)
552 sourceSchemaMapper.addMinimalSchema(sourceCat.schema, True)
553
554 calibSchemaMapper = afwTable.SchemaMapper(calibCat.schema, sourceCat.schema)
555
556 # Add the desired columns from the option fieldsToCopy
557 missingFieldNames = []
558 for fieldName in fieldsToCopy:
559 if fieldName in calibCat.schema:
560 schemaItem = calibCat.schema.find(fieldName)
561 calibSchemaMapper.editOutputSchema().addField(schemaItem.getField())
562 schema = calibSchemaMapper.editOutputSchema()
563 calibSchemaMapper.addMapping(schemaItem.getKey(), schema.find(fieldName).getField())
564 else:
565 missingFieldNames.append(fieldName)
566 if missingFieldNames:
567 raise RuntimeError(f"calibCat is missing fields {missingFieldNames} specified in "
568 "fieldsToCopy")
569
570 if "calib_detected" not in calibSchemaMapper.getOutputSchema():
571 self.calibSourceKey = calibSchemaMapper.addOutputField(afwTable.Field["Flag"]("calib_detected",
572 "Source was detected as an icSource"))
573 else:
574 self.calibSourceKey = None
575
576 schema = calibSchemaMapper.getOutputSchema()
577 newCat = afwTable.SourceCatalog(schema)
578 newCat.reserve(len(sourceCat))
579 newCat.extend(sourceCat, sourceSchemaMapper)
580
581 # Set the aliases so it doesn't complain.
582 for k, v in sourceCat.schema.getAliasMap().items():
583 newCat.schema.getAliasMap().set(k, v)
584
585 select = newCat["deblend_nChild"] == 0
586 matches = afwTable.matchXy(newCat[select], calibCat, self.config.matchRadiusPix)
587 # Check that no sourceCat sources are listed twice (we already know
588 # that each match has a unique calibCat source ID, due to using
589 # that ID as the key in bestMatches)
590 numMatches = len(matches)
591 numUniqueSources = len(set(m[1].getId() for m in matches))
592 if numUniqueSources != numMatches:
593 self.log.warning("%d calibCat sources matched only %d sourceCat sources", numMatches,
594 numUniqueSources)
595
596 self.log.info("Copying flags from calibCat to sourceCat for %s sources", numMatches)
597
598 # For each match: set the calibSourceKey flag and copy the desired
599 # fields
600 for src, calibSrc, d in matches:
601 if self.calibSourceKey:
602 src.setFlag(self.calibSourceKey, True)
603 # src.assign copies the footprint from calibSrc, which we don't want
604 # (DM-407)
605 # so set calibSrc's footprint to src's footprint before src.assign,
606 # then restore it
607 calibSrcFootprint = calibSrc.getFootprint()
608 try:
609 calibSrc.setFootprint(src.getFootprint())
610 src.assign(calibSrc, calibSchemaMapper)
611 finally:
612 calibSrc.setFootprint(calibSrcFootprint)
613
614 return newCat
615
616
617class ProcessCcdWithVariableFakesConnections(ProcessCcdWithFakesConnections):
618 ccdVisitFakeMagnitudes = cT.Output(
619 doc="Catalog of fakes with magnitudes scattered for this ccdVisit.",
620 name="{fakesType}ccdVisitFakeMagnitudes",
621 storageClass="DataFrame",
622 dimensions=("instrument", "visit", "detector"),
623 )
624
625
626class ProcessCcdWithVariableFakesConfig(ProcessCcdWithFakesConfig,
627 pipelineConnections=ProcessCcdWithVariableFakesConnections):
628 scatterSize = pexConfig.RangeField(
629 dtype=float,
630 default=0.4,
631 min=0,
632 max=100,
633 doc="Amount of scatter to add to the visit magnitude for variable "
634 "sources."
635 )
636
637
638class ProcessCcdWithVariableFakesTask(ProcessCcdWithFakesTask):
639 """As ProcessCcdWithFakes except add variablity to the fakes catalog
640 magnitude in the observed band for this ccdVisit.
641
642 Additionally, write out the modified magnitudes to the Butler.
643 """
644
645 _DefaultName = "processCcdWithVariableFakes"
646 ConfigClass = ProcessCcdWithVariableFakesConfig
647
648 def run(self, fakeCats, exposure, skyMap, wcs=None, photoCalib=None, exposureIdInfo=None,
649 icSourceCat=None, sfdSourceCat=None):
650 """Add fake sources to a calexp and then run detection, deblending and measurement.
651
652 Parameters
653 ----------
654 fakeCat : `pandas.core.frame.DataFrame`
655 The catalog of fake sources to add to the exposure
656 exposure : `lsst.afw.image.exposure.exposure.ExposureF`
657 The exposure to add the fake sources to
658 skyMap : `lsst.skymap.SkyMap`
659 SkyMap defining the tracts and patches the fakes are stored over.
661 WCS to use to add fake sources
662 photoCalib : `lsst.afw.image.photoCalib.PhotoCalib`
663 Photometric calibration to be used to calibrate the fake sources
664 exposureIdInfo : `lsst.obs.base.ExposureIdInfo`
665 icSourceCat : `lsst.afw.table.SourceCatalog`
666 Default : None
667 Catalog to take the information about which sources were used for calibration from.
668 sfdSourceCat : `lsst.afw.table.SourceCatalog`
669 Default : None
670 Catalog produced by singleFrameDriver, needed to copy some calibration flags from.
671
672 Returns
673 -------
674 resultStruct : `lsst.pipe.base.struct.Struct`
675 Results Strcut containing:
676
677 - outputExposure : Exposure with added fakes
678 (`lsst.afw.image.exposure.exposure.ExposureF`)
679 - outputCat : Catalog with detected fakes
680 (`lsst.afw.table.source.source.SourceCatalog`)
681 - ccdVisitFakeMagnitudes : Magnitudes that these fakes were
682 inserted with after being scattered (`pandas.DataFrame`)
683
684 Notes
685 -----
686 Adds pixel coordinates for each source to the fakeCat and removes objects with bulge or disk half
687 light radius = 0 (if ``config.cleanCat = True``). These columns are called ``x`` and ``y`` and are in
688 pixels.
689
690 Adds the ``Fake`` mask plane to the exposure which is then set by `addFakeSources` to mark where fake
691 sources have been added. Uses the information in the ``fakeCat`` to make fake galaxies (using galsim)
692 and fake stars, using the PSF models from the PSF information for the calexp. These are then added to
693 the calexp and the calexp with fakes included returned.
694
695 The galsim galaxies are made using a double sersic profile, one for the bulge and one for the disk,
696 this is then convolved with the PSF at that point.
697
698 If exposureIdInfo is not provided then the SourceCatalog IDs will not be globally unique.
699 """
700 fakeCat = self.composeFakeCat(fakeCats, skyMap)
701
702 if wcs is None:
703 wcs = exposure.getWcs()
704
705 if photoCalib is None:
706 photoCalib = exposure.getPhotoCalib()
707
708 if exposureIdInfo is None:
709 exposureIdInfo = ExposureIdInfo()
710
711 band = exposure.getFilterLabel().bandLabel
712 ccdVisitMagnitudes = self.addVariablity(fakeCat, band, exposure, photoCalib, exposureIdInfo)
713
714 self.insertFakes.run(fakeCat, exposure, wcs, photoCalib)
715
716 # detect, deblend and measure sources
717 returnedStruct = self.calibrate.run(exposure, exposureIdInfo=exposureIdInfo)
718 sourceCat = returnedStruct.sourceCat
719
720 sourceCat = self.copyCalibrationFields(sfdSourceCat, sourceCat, self.config.srcFieldsToCopy)
721
722 resultStruct = pipeBase.Struct(outputExposure=exposure,
723 outputCat=sourceCat,
724 ccdVisitFakeMagnitudes=ccdVisitMagnitudes)
725 return resultStruct
726
727 def addVariablity(self, fakeCat, band, exposure, photoCalib, exposureIdInfo):
728 """Add scatter to the fake catalog visit magnitudes.
729
730 Currently just adds a simple Gaussian scatter around the static fake
731 magnitude. This function could be modified to return any number of
732 fake variability.
733
734 Parameters
735 ----------
736 fakeCat : `pandas.DataFrame`
737 Catalog of fakes to modify magnitudes of.
738 band : `str`
739 Current observing band to modify.
740 exposure : `lsst.afw.image.ExposureF`
741 Exposure fakes will be added to.
742 photoCalib : `lsst.afw.image.PhotoCalib`
743 Photometric calibration object of ``exposure``.
744 exposureIdInfo : `lsst.obs.base.ExposureIdInfo`
745 Exposure id information and metadata.
746
747 Returns
748 -------
749 dataFrame : `pandas.DataFrame`
750 DataFrame containing the values of the magnitudes to that will
751 be inserted into this ccdVisit.
752 """
753 expId = exposureIdInfo.expId
754 rng = np.random.default_rng(expId)
755 magScatter = rng.normal(loc=0,
756 scale=self.config.scatterSize,
757 size=len(fakeCat))
758 visitMagnitudes = fakeCat[self.insertFakes.config.mag_col % band] + magScatter
759 fakeCat.loc[:, self.insertFakes.config.mag_col % band] = visitMagnitudes
760 return pd.DataFrame(data={"variableMag": visitMagnitudes})