lsst.pipe.tasks g11492f7fc6+8335dcbd4d
Loading...
Searching...
No Matches
processCcdWithFakes.py
Go to the documentation of this file.
1# This file is part of pipe tasks
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
21
22"""
23Insert fake sources into calexps
24"""
25from astropy.table import Table
26import numpy as np
27import pandas as pd
28
29import lsst.pex.config as pexConfig
30import lsst.pipe.base as pipeBase
31
32from .insertFakes import InsertFakesTask
33from lsst.meas.base import PerTractCcdDataIdContainer
34from lsst.afw.table import SourceTable
35from lsst.obs.base import ExposureIdInfo
36from lsst.pipe.base import PipelineTask, PipelineTaskConfig, CmdLineTask, PipelineTaskConnections
37import lsst.pipe.base.connectionTypes as cT
38import lsst.afw.table as afwTable
39from lsst.skymap import BaseSkyMap
40from lsst.pipe.tasks.calibrate import CalibrateTask
41
42__all__ = ["ProcessCcdWithFakesConfig", "ProcessCcdWithFakesTask",
43 "ProcessCcdWithVariableFakesConfig", "ProcessCcdWithVariableFakesTask"]
44
45
46class ProcessCcdWithFakesConnections(PipelineTaskConnections,
47 dimensions=("instrument", "visit", "detector"),
48 defaultTemplates={"coaddName": "deep",
49 "wcsName": "jointcal",
50 "photoCalibName": "jointcal",
51 "fakesType": "fakes_"}):
52 skyMap = cT.Input(
53 doc="Input definition of geometry/bbox and projection/wcs for "
54 "template exposures. Needed to test which tract to generate ",
55 name=BaseSkyMap.SKYMAP_DATASET_TYPE_NAME,
56 dimensions=("skymap",),
57 storageClass="SkyMap",
58 )
59
60 exposure = cT.Input(
61 doc="Exposure into which fakes are to be added.",
62 name="calexp",
63 storageClass="ExposureF",
64 dimensions=("instrument", "visit", "detector")
65 )
66
67 fakeCats = cT.Input(
68 doc="Set of catalogs of fake sources to draw inputs from. We "
69 "concatenate the tract catalogs for detectorVisits that cover "
70 "multiple tracts.",
71 name="{fakesType}fakeSourceCat",
72 storageClass="DataFrame",
73 dimensions=("tract", "skymap"),
74 deferLoad=True,
75 multiple=True,
76 )
77
78 externalSkyWcsTractCatalog = cT.Input(
79 doc=("Per-tract, per-visit wcs calibrations. These catalogs use the detector "
80 "id for the catalog id, sorted on id for fast lookup."),
81 name="{wcsName}SkyWcsCatalog",
82 storageClass="ExposureCatalog",
83 dimensions=("instrument", "visit", "tract", "skymap"),
84 deferLoad=True,
85 multiple=True,
86 )
87
88 externalSkyWcsGlobalCatalog = cT.Input(
89 doc=("Per-visit wcs calibrations computed globally (with no tract information). "
90 "These catalogs use the detector id for the catalog id, sorted on id for "
91 "fast lookup."),
92 name="finalVisitSummary",
93 storageClass="ExposureCatalog",
94 dimensions=("instrument", "visit"),
95 )
96
97 externalPhotoCalibTractCatalog = cT.Input(
98 doc=("Per-tract, per-visit photometric calibrations. These catalogs use the "
99 "detector id for the catalog id, sorted on id for fast lookup."),
100 name="{photoCalibName}PhotoCalibCatalog",
101 storageClass="ExposureCatalog",
102 dimensions=("instrument", "visit", "tract"),
103 deferLoad=True,
104 multiple=True,
105 )
106
107 externalPhotoCalibGlobalCatalog = cT.Input(
108 doc=("Per-visit photometric calibrations. These catalogs use the "
109 "detector id for the catalog id, sorted on id for fast lookup."),
110 name="finalVisitSummary",
111 storageClass="ExposureCatalog",
112 dimensions=("instrument", "visit"),
113 )
114
115 icSourceCat = cT.Input(
116 doc="Catalog of calibration sources",
117 name="icSrc",
118 storageClass="SourceCatalog",
119 dimensions=("instrument", "visit", "detector")
120 )
121
122 sfdSourceCat = cT.Input(
123 doc="Catalog of calibration sources",
124 name="src",
125 storageClass="SourceCatalog",
126 dimensions=("instrument", "visit", "detector")
127 )
128
129 outputExposure = cT.Output(
130 doc="Exposure with fake sources added.",
131 name="{fakesType}calexp",
132 storageClass="ExposureF",
133 dimensions=("instrument", "visit", "detector")
134 )
135
136 outputCat = cT.Output(
137 doc="Source catalog produced in calibrate task with fakes also measured.",
138 name="{fakesType}src",
139 storageClass="SourceCatalog",
140 dimensions=("instrument", "visit", "detector"),
141 )
142
143 def __init__(self, *, config=None):
144 super().__init__(config=config)
145
146 if not config.doApplyExternalGlobalPhotoCalib:
147 self.inputs.remove("externalPhotoCalibGlobalCatalog")
148 if not config.doApplyExternalTractPhotoCalib:
149 self.inputs.remove("externalPhotoCalibTractCatalog")
150
151 if not config.doApplyExternalGlobalSkyWcs:
152 self.inputs.remove("externalSkyWcsGlobalCatalog")
153 if not config.doApplyExternalTractSkyWcs:
154 self.inputs.remove("externalSkyWcsTractCatalog")
155
156
157class ProcessCcdWithFakesConfig(PipelineTaskConfig,
158 pipelineConnections=ProcessCcdWithFakesConnections):
159 """Config for inserting fake sources
160
161 Notes
162 -----
163 The default column names are those from the UW sims database.
164 """
165
166 doApplyExternalGlobalPhotoCalib = pexConfig.Field(
167 dtype=bool,
168 default=False,
169 doc="Whether to apply an external photometric calibration via an "
170 "`lsst.afw.image.PhotoCalib` object. Uses the "
171 "`externalPhotoCalibName` config option to determine which "
172 "calibration to use. Uses a global calibration."
173 )
174
175 doApplyExternalTractPhotoCalib = pexConfig.Field(
176 dtype=bool,
177 default=False,
178 doc="Whether to apply an external photometric calibration via an "
179 "`lsst.afw.image.PhotoCalib` object. Uses the "
180 "`externalPhotoCalibName` config option to determine which "
181 "calibration to use. Uses a per tract calibration."
182 )
183
184 externalPhotoCalibName = pexConfig.ChoiceField(
185 doc="What type of external photo calib to use.",
186 dtype=str,
187 default="jointcal",
188 allowed={"jointcal": "Use jointcal_photoCalib",
189 "fgcm": "Use fgcm_photoCalib",
190 "fgcm_tract": "Use fgcm_tract_photoCalib"}
191 )
192
193 doApplyExternalGlobalSkyWcs = pexConfig.Field(
194 dtype=bool,
195 default=False,
196 doc="Whether to apply an external astrometric calibration via an "
197 "`lsst.afw.geom.SkyWcs` object. Uses the "
198 "`externalSkyWcsName` config option to determine which "
199 "calibration to use. Uses a global calibration."
200 )
201
202 doApplyExternalTractSkyWcs = pexConfig.Field(
203 dtype=bool,
204 default=False,
205 doc="Whether to apply an external astrometric calibration via an "
206 "`lsst.afw.geom.SkyWcs` object. Uses the "
207 "`externalSkyWcsName` config option to determine which "
208 "calibration to use. Uses a per tract calibration."
209 )
210
211 externalSkyWcsName = pexConfig.ChoiceField(
212 doc="What type of updated WCS calib to use.",
213 dtype=str,
214 default="jointcal",
215 allowed={"jointcal": "Use jointcal_wcs"}
216 )
217
218 coaddName = pexConfig.Field(
219 doc="The name of the type of coadd used",
220 dtype=str,
221 default="deep",
222 )
223
224 srcFieldsToCopy = pexConfig.ListField(
225 dtype=str,
226 default=("calib_photometry_reserved", "calib_photometry_used", "calib_astrometry_used",
227 "calib_psf_candidate", "calib_psf_used", "calib_psf_reserved"),
228 doc=("Fields to copy from the `src` catalog to the output catalog "
229 "for matching sources Any missing fields will trigger a "
230 "RuntimeError exception.")
231 )
232
233 matchRadiusPix = pexConfig.Field(
234 dtype=float,
235 default=3,
236 doc=("Match radius for matching icSourceCat objects to sourceCat objects (pixels)"),
237 )
238
239 doMatchVisit = pexConfig.Field(
240 dtype=bool,
241 default=False,
242 doc="Match visit to trim the fakeCat"
243 )
244
245 calibrate = pexConfig.ConfigurableField(target=CalibrateTask,
246 doc="The calibration task to use.")
247
248 insertFakes = pexConfig.ConfigurableField(target=InsertFakesTask,
249 doc="Configuration for the fake sources")
250
251 def setDefaults(self):
252 super().setDefaults()
253 self.calibrate.measurement.plugins["base_PixelFlags"].masksFpAnywhere.append("FAKE")
254 self.calibrate.measurement.plugins["base_PixelFlags"].masksFpCenter.append("FAKE")
255 self.calibrate.doAstrometry = False
256 self.calibrate.doWriteMatches = False
257 self.calibrate.doPhotoCal = False
258 self.calibrate.doComputeSummaryStats = False
259 self.calibrate.detection.reEstimateBackground = False
260
261
262class ProcessCcdWithFakesTask(PipelineTask, CmdLineTask):
263 """Insert fake objects into calexps.
264
265 Add fake stars and galaxies to the given calexp, specified in the dataRef. Galaxy parameters are read in
266 from the specified file and then modelled using galsim. Re-runs characterize image and calibrate image to
267 give a new background estimation and measurement of the calexp.
268
269 `ProcessFakeSourcesTask` inherits six functions from insertFakesTask that make images of the fake
270 sources and then add them to the calexp.
271
272 `addPixCoords`
273 Use the WCS information to add the pixel coordinates of each source
274 Adds an ``x`` and ``y`` column to the catalog of fake sources.
275 `trimFakeCat`
276 Trim the fake cat to about the size of the input image.
277 `mkFakeGalsimGalaxies`
278 Use Galsim to make fake double sersic galaxies for each set of galaxy parameters in the input file.
279 `mkFakeStars`
280 Use the PSF information from the calexp to make a fake star using the magnitude information from the
281 input file.
282 `cleanCat`
283 Remove rows of the input fake catalog which have half light radius, of either the bulge or the disk,
284 that are 0.
285 `addFakeSources`
286 Add the fake sources to the calexp.
287
288 Notes
289 -----
290 The ``calexp`` with fake souces added to it is written out as the datatype ``calexp_fakes``.
291 """
292
293 _DefaultName = "processCcdWithFakes"
294 ConfigClass = ProcessCcdWithFakesConfig
295
296 def __init__(self, schema=None, butler=None, **kwargs):
297 """Initalize things! This should go above in the class docstring
298 """
299
300 super().__init__(**kwargs)
301
302 if schema is None:
303 schema = SourceTable.makeMinimalSchema()
304 self.schema = schema
305 self.makeSubtask("insertFakes")
306 self.makeSubtask("calibrate")
307
308 def runDataRef(self, dataRef):
309 """Read in/write out the required data products and add fake sources to the calexp.
310
311 Parameters
312 ----------
314 Data reference defining the ccd to have fakes added to it.
315 Used to access the following data products:
316 calexp
317 jointcal_wcs
318 jointcal_photoCalib
319
320 Notes
321 -----
322 Uses the calibration and WCS information attached to the calexp for the posistioning and calibration
323 of the sources unless the config option config.externalPhotoCalibName or config.externalSkyWcsName
324 are set then it uses the specified outputs. The config defualts for the column names in the catalog
325 of fakes are taken from the University of Washington simulations database.
326 Operates on one ccd at a time.
327 """
328 exposureIdInfo = dataRef.get("expIdInfo")
329
330 if self.config.insertFakes.fakeType == "snapshot":
331 fakeCat = dataRef.get("fakeSourceCat").toDataFrame()
332 elif self.config.insertFakes.fakeType == "static":
333 fakeCat = dataRef.get("deepCoadd_fakeSourceCat").toDataFrame()
334 else:
335 fakeCat = Table.read(self.config.insertFakes.fakeType).to_pandas()
336
337 calexp = dataRef.get("calexp")
338 if self.config.doApplyExternalGlobalSkyWcs or self.config.doApplyExternalTractSkyWcs:
339 self.log.info("Using external wcs from %s", self.config.externalSkyWcsName)
340 wcs = dataRef.get(self.config.externalSkyWcsName + "_wcs")
341 else:
342 wcs = calexp.getWcs()
343
344 if self.config.doApplyExternalGlobalPhotoCalib or self.config.doApplyExternalTractPhotoCalib:
345 self.log.info("Using external photocalib from %s", self.config.externalPhotoCalibName)
346 photoCalib = dataRef.get(self.config.externalPhotoCalibName + "_photoCalib")
347 else:
348 photoCalib = calexp.getPhotoCalib()
349
350 icSourceCat = dataRef.get("icSrc", immediate=True)
351 sfdSourceCat = dataRef.get("src", immediate=True)
352
353 resultStruct = self.run(fakeCat, calexp, wcs=wcs, photoCalib=photoCalib,
354 exposureIdInfo=exposureIdInfo, icSourceCat=icSourceCat,
355 sfdSourceCat=sfdSourceCat)
356
357 dataRef.put(resultStruct.outputExposure, "fakes_calexp")
358 dataRef.put(resultStruct.outputCat, "fakes_src")
359 return resultStruct
360
361 def runQuantum(self, butlerQC, inputRefs, outputRefs):
362 inputs = butlerQC.get(inputRefs)
363 detectorId = inputs["exposure"].getInfo().getDetector().getId()
364
365 if 'exposureIdInfo' not in inputs.keys():
366 expId, expBits = butlerQC.quantum.dataId.pack("visit_detector", returnMaxBits=True)
367 inputs['exposureIdInfo'] = ExposureIdInfo(expId, expBits)
368
369 expWcs = inputs["exposure"].getWcs()
370 tractId = inputs["skyMap"].findTract(
371 expWcs.pixelToSky(inputs["exposure"].getBBox().getCenter())).tract_id
372 if not self.config.doApplyExternalGlobalSkyWcs and not self.config.doApplyExternalTractSkyWcs:
373 inputs["wcs"] = expWcs
374 elif self.config.doApplyExternalGlobalSkyWcs:
375 externalSkyWcsCatalog = inputs["externalSkyWcsGlobalCatalog"]
376 row = externalSkyWcsCatalog.find(detectorId)
377 inputs["wcs"] = row.getWcs()
378 elif self.config.doApplyExternalTractSkyWcs:
379 externalSkyWcsCatalogList = inputs["externalSkyWcsTractCatalog"]
380 externalSkyWcsCatalog = None
381 for externalSkyWcsCatalogRef in externalSkyWcsCatalogList:
382 if externalSkyWcsCatalogRef.dataId["tract"] == tractId:
383 externalSkyWcsCatalog = externalSkyWcsCatalogRef.get(
384 datasetType=self.config.connections.externalSkyWcsTractCatalog)
385 break
386 if externalSkyWcsCatalog is None:
387 usedTract = externalSkyWcsCatalogList[-1].dataId["tract"]
388 self.log.warn(
389 f"Warning, external SkyWcs for tract {tractId} not found. Using tract {usedTract} "
390 "instead.")
391 externalSkyWcsCatalog = externalSkyWcsCatalogList[-1].get(
392 datasetType=self.config.connections.externalSkyWcsTractCatalog)
393 row = externalSkyWcsCatalog.find(detectorId)
394 inputs["wcs"] = row.getWcs()
395
396 if not self.config.doApplyExternalGlobalPhotoCalib and not self.config.doApplyExternalTractPhotoCalib:
397 inputs["photoCalib"] = inputs["exposure"].getPhotoCalib()
398 elif self.config.doApplyExternalGlobalPhotoCalib:
399 externalPhotoCalibCatalog = inputs["externalPhotoCalibGlobalCatalog"]
400 row = externalPhotoCalibCatalog.find(detectorId)
401 inputs["photoCalib"] = row.getPhotoCalib()
402 elif self.config.doApplyExternalTractPhotoCalib:
403 externalPhotoCalibCatalogList = inputs["externalPhotoCalibTractCatalog"]
404 externalPhotoCalibCatalog = None
405 for externalPhotoCalibCatalogRef in externalPhotoCalibCatalogList:
406 if externalPhotoCalibCatalogRef.dataId["tract"] == tractId:
407 externalPhotoCalibCatalog = externalPhotoCalibCatalogRef.get(
408 datasetType=self.config.connections.externalPhotoCalibTractCatalog)
409 break
410 if externalPhotoCalibCatalog is None:
411 usedTract = externalPhotoCalibCatalogList[-1].dataId["tract"]
412 self.log.warn(
413 f"Warning, external PhotoCalib for tract {tractId} not found. Using tract {usedTract} "
414 "instead.")
415 externalPhotoCalibCatalog = externalPhotoCalibCatalogList[-1].get(
416 datasetType=self.config.connections.externalPhotoCalibTractCatalog)
417 row = externalPhotoCalibCatalog.find(detectorId)
418 inputs["photoCalib"] = row.getPhotoCalib()
419
420 outputs = self.run(**inputs)
421 butlerQC.put(outputs, outputRefs)
422
423 @classmethod
424 def _makeArgumentParser(cls):
425 parser = pipeBase.ArgumentParser(name=cls._DefaultName)
426 parser.add_id_argument("--id", "fakes_calexp", help="data ID with raw CCD keys [+ tract optionally], "
427 "e.g. --id visit=12345 ccd=1,2 [tract=0]",
428 ContainerClass=PerTractCcdDataIdContainer)
429 return parser
430
431 def run(self, fakeCats, exposure, skyMap, wcs=None, photoCalib=None, exposureIdInfo=None,
432 icSourceCat=None, sfdSourceCat=None, externalSkyWcsGlobalCatalog=None,
433 externalSkyWcsTractCatalog=None, externalPhotoCalibGlobalCatalog=None,
434 externalPhotoCalibTractCatalog=None):
435 """Add fake sources to a calexp and then run detection, deblending and measurement.
436
437 Parameters
438 ----------
439 fakeCats : `list` of `lsst.daf.butler.DeferredDatasetHandle`
440 Set of tract level fake catalogs that potentially cover
441 this detectorVisit.
442 exposure : `lsst.afw.image.exposure.exposure.ExposureF`
443 The exposure to add the fake sources to
444 skyMap : `lsst.skymap.SkyMap`
445 SkyMap defining the tracts and patches the fakes are stored over.
447 WCS to use to add fake sources
448 photoCalib : `lsst.afw.image.photoCalib.PhotoCalib`
449 Photometric calibration to be used to calibrate the fake sources
450 exposureIdInfo : `lsst.obs.base.ExposureIdInfo`
451 icSourceCat : `lsst.afw.table.SourceCatalog`
452 Default : None
453 Catalog to take the information about which sources were used for calibration from.
454 sfdSourceCat : `lsst.afw.table.SourceCatalog`
455 Default : None
456 Catalog produced by singleFrameDriver, needed to copy some calibration flags from.
457
458 Returns
459 -------
460 resultStruct : `lsst.pipe.base.struct.Struct`
461 contains : outputExposure : `lsst.afw.image.exposure.exposure.ExposureF`
462 outputCat : `lsst.afw.table.source.source.SourceCatalog`
463
464 Notes
465 -----
466 Adds pixel coordinates for each source to the fakeCat and removes objects with bulge or disk half
467 light radius = 0 (if ``config.cleanCat = True``). These columns are called ``x`` and ``y`` and are in
468 pixels.
469
470 Adds the ``Fake`` mask plane to the exposure which is then set by `addFakeSources` to mark where fake
471 sources have been added. Uses the information in the ``fakeCat`` to make fake galaxies (using galsim)
472 and fake stars, using the PSF models from the PSF information for the calexp. These are then added to
473 the calexp and the calexp with fakes included returned.
474
475 The galsim galaxies are made using a double sersic profile, one for the bulge and one for the disk,
476 this is then convolved with the PSF at that point.
477
478 If exposureIdInfo is not provided then the SourceCatalog IDs will not be globally unique.
479 """
480 fakeCat = self.composeFakeCat(fakeCats, skyMap)
481
482 if wcs is None:
483 wcs = exposure.getWcs()
484
485 if photoCalib is None:
486 photoCalib = exposure.getPhotoCalib()
487
488 if self.config.doMatchVisit:
489 fakeCat = self.getVisitMatchedFakeCat(fakeCat, exposure)
490
491 self.insertFakes.run(fakeCat, exposure, wcs, photoCalib)
492
493 # detect, deblend and measure sources
494 if exposureIdInfo is None:
495 exposureIdInfo = ExposureIdInfo()
496 returnedStruct = self.calibrate.run(exposure, exposureIdInfo=exposureIdInfo)
497 sourceCat = returnedStruct.sourceCat
498
499 sourceCat = self.copyCalibrationFields(sfdSourceCat, sourceCat, self.config.srcFieldsToCopy)
500
501 resultStruct = pipeBase.Struct(outputExposure=exposure, outputCat=sourceCat)
502 return resultStruct
503
504 def composeFakeCat(self, fakeCats, skyMap):
505 """Concatenate the fakeCats from tracts that may cover the exposure.
506
507 Parameters
508 ----------
509 fakeCats : `list` of `lst.daf.butler.DeferredDatasetHandle`
510 Set of fake cats to concatenate.
511 skyMap : `lsst.skymap.SkyMap`
512 SkyMap defining the geometry of the tracts and patches.
513
514 Returns
515 -------
516 combinedFakeCat : `pandas.DataFrame`
517 All fakes that cover the inner polygon of the tracts in this
518 quantum.
519 """
520 if len(fakeCats) == 1:
521 return fakeCats[0].get(
522 datasetType=self.config.connections.fakeCats)
523 outputCat = []
524 for fakeCatRef in fakeCats:
525 cat = fakeCatRef.get(
526 datasetType=self.config.connections.fakeCats)
527 tractId = fakeCatRef.dataId["tract"]
528 # Make sure all data is within the inner part of the tract.
529 outputCat.append(cat[
530 skyMap.findTractIdArray(cat[self.config.insertFakes.ra_col],
531 cat[self.config.insertFakes.dec_col],
532 degrees=False)
533 == tractId])
534
535 return pd.concat(outputCat)
536
537 def getVisitMatchedFakeCat(self, fakeCat, exposure):
538 """Trim the fakeCat to select particular visit
539
540 Parameters
541 ----------
542 fakeCat : `pandas.core.frame.DataFrame`
543 The catalog of fake sources to add to the exposure
544 exposure : `lsst.afw.image.exposure.exposure.ExposureF`
545 The exposure to add the fake sources to
546
547 Returns
548 -------
549 movingFakeCat : `pandas.DataFrame`
550 All fakes that belong to the visit
551 """
552 selected = exposure.getInfo().getVisitInfo().getId() == fakeCat["visit"]
553
554 return fakeCat[selected]
555
556 def copyCalibrationFields(self, calibCat, sourceCat, fieldsToCopy):
557 """Match sources in calibCat and sourceCat and copy the specified fields
558
559 Parameters
560 ----------
562 Catalog from which to copy fields.
563 sourceCat : `lsst.afw.table.SourceCatalog`
564 Catalog to which to copy fields.
565 fieldsToCopy : `lsst.pex.config.listField.List`
566 Fields to copy from calibCat to SoourceCat.
567
568 Returns
569 -------
571 Catalog which includes the copied fields.
572
573 The fields copied are those specified by `fieldsToCopy` that actually exist
574 in the schema of `calibCat`.
575
576 This version was based on and adapted from the one in calibrateTask.
577 """
578
579 # Make a new SourceCatalog with the data from sourceCat so that we can add the new columns to it
580 sourceSchemaMapper = afwTable.SchemaMapper(sourceCat.schema)
581 sourceSchemaMapper.addMinimalSchema(sourceCat.schema, True)
582
583 calibSchemaMapper = afwTable.SchemaMapper(calibCat.schema, sourceCat.schema)
584
585 # Add the desired columns from the option fieldsToCopy
586 missingFieldNames = []
587 for fieldName in fieldsToCopy:
588 if fieldName in calibCat.schema:
589 schemaItem = calibCat.schema.find(fieldName)
590 calibSchemaMapper.editOutputSchema().addField(schemaItem.getField())
591 schema = calibSchemaMapper.editOutputSchema()
592 calibSchemaMapper.addMapping(schemaItem.getKey(), schema.find(fieldName).getField())
593 else:
594 missingFieldNames.append(fieldName)
595 if missingFieldNames:
596 raise RuntimeError(f"calibCat is missing fields {missingFieldNames} specified in "
597 "fieldsToCopy")
598
599 if "calib_detected" not in calibSchemaMapper.getOutputSchema():
600 self.calibSourceKey = calibSchemaMapper.addOutputField(afwTable.Field["Flag"]("calib_detected",
601 "Source was detected as an icSource"))
602 else:
603 self.calibSourceKey = None
604
605 schema = calibSchemaMapper.getOutputSchema()
606 newCat = afwTable.SourceCatalog(schema)
607 newCat.reserve(len(sourceCat))
608 newCat.extend(sourceCat, sourceSchemaMapper)
609
610 # Set the aliases so it doesn't complain.
611 for k, v in sourceCat.schema.getAliasMap().items():
612 newCat.schema.getAliasMap().set(k, v)
613
614 select = newCat["deblend_nChild"] == 0
615 matches = afwTable.matchXy(newCat[select], calibCat, self.config.matchRadiusPix)
616 # Check that no sourceCat sources are listed twice (we already know
617 # that each match has a unique calibCat source ID, due to using
618 # that ID as the key in bestMatches)
619 numMatches = len(matches)
620 numUniqueSources = len(set(m[1].getId() for m in matches))
621 if numUniqueSources != numMatches:
622 self.log.warning("%d calibCat sources matched only %d sourceCat sources", numMatches,
623 numUniqueSources)
624
625 self.log.info("Copying flags from calibCat to sourceCat for %s sources", numMatches)
626
627 # For each match: set the calibSourceKey flag and copy the desired
628 # fields
629 for src, calibSrc, d in matches:
630 if self.calibSourceKey:
631 src.setFlag(self.calibSourceKey, True)
632 # src.assign copies the footprint from calibSrc, which we don't want
633 # (DM-407)
634 # so set calibSrc's footprint to src's footprint before src.assign,
635 # then restore it
636 calibSrcFootprint = calibSrc.getFootprint()
637 try:
638 calibSrc.setFootprint(src.getFootprint())
639 src.assign(calibSrc, calibSchemaMapper)
640 finally:
641 calibSrc.setFootprint(calibSrcFootprint)
642
643 return newCat
644
645
646class ProcessCcdWithVariableFakesConnections(ProcessCcdWithFakesConnections):
647 ccdVisitFakeMagnitudes = cT.Output(
648 doc="Catalog of fakes with magnitudes scattered for this ccdVisit.",
649 name="{fakesType}ccdVisitFakeMagnitudes",
650 storageClass="DataFrame",
651 dimensions=("instrument", "visit", "detector"),
652 )
653
654
655class ProcessCcdWithVariableFakesConfig(ProcessCcdWithFakesConfig,
656 pipelineConnections=ProcessCcdWithVariableFakesConnections):
657 scatterSize = pexConfig.RangeField(
658 dtype=float,
659 default=0.4,
660 min=0,
661 max=100,
662 doc="Amount of scatter to add to the visit magnitude for variable "
663 "sources."
664 )
665
666
667class ProcessCcdWithVariableFakesTask(ProcessCcdWithFakesTask):
668 """As ProcessCcdWithFakes except add variablity to the fakes catalog
669 magnitude in the observed band for this ccdVisit.
670
671 Additionally, write out the modified magnitudes to the Butler.
672 """
673
674 _DefaultName = "processCcdWithVariableFakes"
675 ConfigClass = ProcessCcdWithVariableFakesConfig
676
677 def run(self, fakeCats, exposure, skyMap, wcs=None, photoCalib=None, exposureIdInfo=None,
678 icSourceCat=None, sfdSourceCat=None):
679 """Add fake sources to a calexp and then run detection, deblending and measurement.
680
681 Parameters
682 ----------
683 fakeCat : `pandas.core.frame.DataFrame`
684 The catalog of fake sources to add to the exposure
685 exposure : `lsst.afw.image.exposure.exposure.ExposureF`
686 The exposure to add the fake sources to
687 skyMap : `lsst.skymap.SkyMap`
688 SkyMap defining the tracts and patches the fakes are stored over.
690 WCS to use to add fake sources
691 photoCalib : `lsst.afw.image.photoCalib.PhotoCalib`
692 Photometric calibration to be used to calibrate the fake sources
693 exposureIdInfo : `lsst.obs.base.ExposureIdInfo`
694 icSourceCat : `lsst.afw.table.SourceCatalog`
695 Default : None
696 Catalog to take the information about which sources were used for calibration from.
697 sfdSourceCat : `lsst.afw.table.SourceCatalog`
698 Default : None
699 Catalog produced by singleFrameDriver, needed to copy some calibration flags from.
700
701 Returns
702 -------
703 resultStruct : `lsst.pipe.base.struct.Struct`
704 Results Strcut containing:
705
706 - outputExposure : Exposure with added fakes
707 (`lsst.afw.image.exposure.exposure.ExposureF`)
708 - outputCat : Catalog with detected fakes
709 (`lsst.afw.table.source.source.SourceCatalog`)
710 - ccdVisitFakeMagnitudes : Magnitudes that these fakes were
711 inserted with after being scattered (`pandas.DataFrame`)
712
713 Notes
714 -----
715 Adds pixel coordinates for each source to the fakeCat and removes objects with bulge or disk half
716 light radius = 0 (if ``config.cleanCat = True``). These columns are called ``x`` and ``y`` and are in
717 pixels.
718
719 Adds the ``Fake`` mask plane to the exposure which is then set by `addFakeSources` to mark where fake
720 sources have been added. Uses the information in the ``fakeCat`` to make fake galaxies (using galsim)
721 and fake stars, using the PSF models from the PSF information for the calexp. These are then added to
722 the calexp and the calexp with fakes included returned.
723
724 The galsim galaxies are made using a double sersic profile, one for the bulge and one for the disk,
725 this is then convolved with the PSF at that point.
726
727 If exposureIdInfo is not provided then the SourceCatalog IDs will not be globally unique.
728 """
729 fakeCat = self.composeFakeCat(fakeCats, skyMap)
730
731 if wcs is None:
732 wcs = exposure.getWcs()
733
734 if photoCalib is None:
735 photoCalib = exposure.getPhotoCalib()
736
737 if exposureIdInfo is None:
738 exposureIdInfo = ExposureIdInfo()
739
740 band = exposure.getFilter().bandLabel
741 ccdVisitMagnitudes = self.addVariablity(fakeCat, band, exposure, photoCalib, exposureIdInfo)
742
743 self.insertFakes.run(fakeCat, exposure, wcs, photoCalib)
744
745 # detect, deblend and measure sources
746 returnedStruct = self.calibrate.run(exposure, exposureIdInfo=exposureIdInfo)
747 sourceCat = returnedStruct.sourceCat
748
749 sourceCat = self.copyCalibrationFields(sfdSourceCat, sourceCat, self.config.srcFieldsToCopy)
750
751 resultStruct = pipeBase.Struct(outputExposure=exposure,
752 outputCat=sourceCat,
753 ccdVisitFakeMagnitudes=ccdVisitMagnitudes)
754 return resultStruct
755
756 def addVariablity(self, fakeCat, band, exposure, photoCalib, exposureIdInfo):
757 """Add scatter to the fake catalog visit magnitudes.
758
759 Currently just adds a simple Gaussian scatter around the static fake
760 magnitude. This function could be modified to return any number of
761 fake variability.
762
763 Parameters
764 ----------
765 fakeCat : `pandas.DataFrame`
766 Catalog of fakes to modify magnitudes of.
767 band : `str`
768 Current observing band to modify.
769 exposure : `lsst.afw.image.ExposureF`
770 Exposure fakes will be added to.
771 photoCalib : `lsst.afw.image.PhotoCalib`
772 Photometric calibration object of ``exposure``.
773 exposureIdInfo : `lsst.obs.base.ExposureIdInfo`
774 Exposure id information and metadata.
775
776 Returns
777 -------
778 dataFrame : `pandas.DataFrame`
779 DataFrame containing the values of the magnitudes to that will
780 be inserted into this ccdVisit.
781 """
782 expId = exposureIdInfo.expId
783 rng = np.random.default_rng(expId)
784 magScatter = rng.normal(loc=0,
785 scale=self.config.scatterSize,
786 size=len(fakeCat))
787 visitMagnitudes = fakeCat[self.insertFakes.config.mag_col % band] + magScatter
788 fakeCat.loc[:, self.insertFakes.config.mag_col % band] = visitMagnitudes
789 return pd.DataFrame(data={"variableMag": visitMagnitudes})