lsst.pipe.tasks gc750389ca5+5b520963fe
Loading...
Searching...
No Matches
processCcdWithFakes.py
Go to the documentation of this file.
1# This file is part of pipe_tasks.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
21
22"""
23Insert fake sources into calexps
24"""
25
26__all__ = ["ProcessCcdWithFakesConfig", "ProcessCcdWithFakesTask",
27 "ProcessCcdWithVariableFakesConfig", "ProcessCcdWithVariableFakesTask"]
28
29import numpy as np
30import pandas as pd
31
32import lsst.pex.config as pexConfig
33import lsst.pipe.base as pipeBase
34
35from .insertFakes import InsertFakesTask
36from lsst.afw.table import SourceTable
37from lsst.obs.base import ExposureIdInfo
38from lsst.pipe.base import PipelineTask, PipelineTaskConfig, PipelineTaskConnections
39import lsst.pipe.base.connectionTypes as cT
40import lsst.afw.table as afwTable
41from lsst.skymap import BaseSkyMap
42from lsst.pipe.tasks.calibrate import CalibrateTask
43
44
45class ProcessCcdWithFakesConnections(PipelineTaskConnections,
46 dimensions=("instrument", "visit", "detector"),
47 defaultTemplates={"coaddName": "deep",
48 "wcsName": "jointcal",
49 "photoCalibName": "jointcal",
50 "fakesType": "fakes_"}):
51 skyMap = cT.Input(
52 doc="Input definition of geometry/bbox and projection/wcs for "
53 "template exposures. Needed to test which tract to generate ",
54 name=BaseSkyMap.SKYMAP_DATASET_TYPE_NAME,
55 dimensions=("skymap",),
56 storageClass="SkyMap",
57 )
58
59 exposure = cT.Input(
60 doc="Exposure into which fakes are to be added.",
61 name="calexp",
62 storageClass="ExposureF",
63 dimensions=("instrument", "visit", "detector")
64 )
65
66 fakeCats = cT.Input(
67 doc="Set of catalogs of fake sources to draw inputs from. We "
68 "concatenate the tract catalogs for detectorVisits that cover "
69 "multiple tracts.",
70 name="{fakesType}fakeSourceCat",
71 storageClass="DataFrame",
72 dimensions=("tract", "skymap"),
73 deferLoad=True,
74 multiple=True,
75 )
76
77 externalSkyWcsTractCatalog = cT.Input(
78 doc=("Per-tract, per-visit wcs calibrations. These catalogs use the detector "
79 "id for the catalog id, sorted on id for fast lookup."),
80 name="{wcsName}SkyWcsCatalog",
81 storageClass="ExposureCatalog",
82 dimensions=("instrument", "visit", "tract", "skymap"),
83 deferLoad=True,
84 multiple=True,
85 )
86
87 externalSkyWcsGlobalCatalog = cT.Input(
88 doc=("Per-visit wcs calibrations computed globally (with no tract information). "
89 "These catalogs use the detector id for the catalog id, sorted on id for "
90 "fast lookup."),
91 name="finalVisitSummary",
92 storageClass="ExposureCatalog",
93 dimensions=("instrument", "visit"),
94 )
95
96 externalPhotoCalibTractCatalog = cT.Input(
97 doc=("Per-tract, per-visit photometric calibrations. These catalogs use the "
98 "detector id for the catalog id, sorted on id for fast lookup."),
99 name="{photoCalibName}PhotoCalibCatalog",
100 storageClass="ExposureCatalog",
101 dimensions=("instrument", "visit", "tract"),
102 deferLoad=True,
103 multiple=True,
104 )
105
106 externalPhotoCalibGlobalCatalog = cT.Input(
107 doc=("Per-visit photometric calibrations. These catalogs use the "
108 "detector id for the catalog id, sorted on id for fast lookup."),
109 name="finalVisitSummary",
110 storageClass="ExposureCatalog",
111 dimensions=("instrument", "visit"),
112 )
113
114 icSourceCat = cT.Input(
115 doc="Catalog of calibration sources",
116 name="icSrc",
117 storageClass="SourceCatalog",
118 dimensions=("instrument", "visit", "detector")
119 )
120
121 sfdSourceCat = cT.Input(
122 doc="Catalog of calibration sources",
123 name="src",
124 storageClass="SourceCatalog",
125 dimensions=("instrument", "visit", "detector")
126 )
127
128 outputExposure = cT.Output(
129 doc="Exposure with fake sources added.",
130 name="{fakesType}calexp",
131 storageClass="ExposureF",
132 dimensions=("instrument", "visit", "detector")
133 )
134
135 outputCat = cT.Output(
136 doc="Source catalog produced in calibrate task with fakes also measured.",
137 name="{fakesType}src",
138 storageClass="SourceCatalog",
139 dimensions=("instrument", "visit", "detector"),
140 )
141
142 def __init__(self, *, config=None):
143 super().__init__(config=config)
144
145 if not config.doApplyExternalGlobalPhotoCalib:
146 self.inputs.remove("externalPhotoCalibGlobalCatalog")
147 if not config.doApplyExternalTractPhotoCalib:
148 self.inputs.remove("externalPhotoCalibTractCatalog")
149
150 if not config.doApplyExternalGlobalSkyWcs:
151 self.inputs.remove("externalSkyWcsGlobalCatalog")
152 if not config.doApplyExternalTractSkyWcs:
153 self.inputs.remove("externalSkyWcsTractCatalog")
154
155
156class ProcessCcdWithFakesConfig(PipelineTaskConfig,
157 pipelineConnections=ProcessCcdWithFakesConnections):
158 """Config for inserting fake sources
159
160 Notes
161 -----
162 The default column names are those from the UW sims database.
163 """
164
165 doApplyExternalGlobalPhotoCalib = pexConfig.Field(
166 dtype=bool,
167 default=False,
168 doc="Whether to apply an external photometric calibration via an "
169 "`lsst.afw.image.PhotoCalib` object. Uses the "
170 "`externalPhotoCalibName` config option to determine which "
171 "calibration to use. Uses a global calibration."
172 )
173
174 doApplyExternalTractPhotoCalib = pexConfig.Field(
175 dtype=bool,
176 default=False,
177 doc="Whether to apply an external photometric calibration via an "
178 "`lsst.afw.image.PhotoCalib` object. Uses the "
179 "`externalPhotoCalibName` config option to determine which "
180 "calibration to use. Uses a per tract calibration."
181 )
182
183 externalPhotoCalibName = pexConfig.ChoiceField(
184 doc="What type of external photo calib to use.",
185 dtype=str,
186 default="jointcal",
187 allowed={"jointcal": "Use jointcal_photoCalib",
188 "fgcm": "Use fgcm_photoCalib",
189 "fgcm_tract": "Use fgcm_tract_photoCalib"}
190 )
191
192 doApplyExternalGlobalSkyWcs = pexConfig.Field(
193 dtype=bool,
194 default=False,
195 doc="Whether to apply an external astrometric calibration via an "
196 "`lsst.afw.geom.SkyWcs` object. Uses the "
197 "`externalSkyWcsName` config option to determine which "
198 "calibration to use. Uses a global calibration."
199 )
200
201 doApplyExternalTractSkyWcs = pexConfig.Field(
202 dtype=bool,
203 default=False,
204 doc="Whether to apply an external astrometric calibration via an "
205 "`lsst.afw.geom.SkyWcs` object. Uses the "
206 "`externalSkyWcsName` config option to determine which "
207 "calibration to use. Uses a per tract calibration."
208 )
209
210 externalSkyWcsName = pexConfig.ChoiceField(
211 doc="What type of updated WCS calib to use.",
212 dtype=str,
213 default="jointcal",
214 allowed={"jointcal": "Use jointcal_wcs"}
215 )
216
217 coaddName = pexConfig.Field(
218 doc="The name of the type of coadd used",
219 dtype=str,
220 default="deep",
221 )
222
223 srcFieldsToCopy = pexConfig.ListField(
224 dtype=str,
225 default=("calib_photometry_reserved", "calib_photometry_used", "calib_astrometry_used",
226 "calib_psf_candidate", "calib_psf_used", "calib_psf_reserved"),
227 doc=("Fields to copy from the `src` catalog to the output catalog "
228 "for matching sources Any missing fields will trigger a "
229 "RuntimeError exception.")
230 )
231
232 matchRadiusPix = pexConfig.Field(
233 dtype=float,
234 default=3,
235 doc=("Match radius for matching icSourceCat objects to sourceCat objects (pixels)"),
236 )
237
238 doMatchVisit = pexConfig.Field(
239 dtype=bool,
240 default=False,
241 doc="Match visit to trim the fakeCat"
242 )
243
244 calibrate = pexConfig.ConfigurableField(target=CalibrateTask,
245 doc="The calibration task to use.")
246
247 insertFakes = pexConfig.ConfigurableField(target=InsertFakesTask,
248 doc="Configuration for the fake sources")
249
250 def setDefaults(self):
251 super().setDefaults()
252 self.calibrate.measurement.plugins["base_PixelFlags"].masksFpAnywhere.append("FAKE")
253 self.calibrate.measurement.plugins["base_PixelFlags"].masksFpCenter.append("FAKE")
254 self.calibrate.doAstrometry = False
255 self.calibrate.doWriteMatches = False
256 self.calibrate.doPhotoCal = False
257 self.calibrate.doComputeSummaryStats = False
258 self.calibrate.detection.reEstimateBackground = False
259
260
261class ProcessCcdWithFakesTask(PipelineTask):
262 """Insert fake objects into calexps.
263
264 Add fake stars and galaxies to the given calexp, specified in the dataRef. Galaxy parameters are read in
265 from the specified file and then modelled using galsim. Re-runs characterize image and calibrate image to
266 give a new background estimation and measurement of the calexp.
267
268 `ProcessFakeSourcesTask` inherits six functions from insertFakesTask that make images of the fake
269 sources and then add them to the calexp.
270
271 `addPixCoords`
272 Use the WCS information to add the pixel coordinates of each source
273 Adds an ``x`` and ``y`` column to the catalog of fake sources.
274 `trimFakeCat`
275 Trim the fake cat to about the size of the input image.
276 `mkFakeGalsimGalaxies`
277 Use Galsim to make fake double sersic galaxies for each set of galaxy parameters in the input file.
278 `mkFakeStars`
279 Use the PSF information from the calexp to make a fake star using the magnitude information from the
280 input file.
281 `cleanCat`
282 Remove rows of the input fake catalog which have half light radius, of either the bulge or the disk,
283 that are 0.
284 `addFakeSources`
285 Add the fake sources to the calexp.
286
287 Notes
288 -----
289 The ``calexp`` with fake souces added to it is written out as the datatype ``calexp_fakes``.
290 """
291
292 _DefaultName = "processCcdWithFakes"
293 ConfigClass = ProcessCcdWithFakesConfig
294
295 def __init__(self, schema=None, butler=None, **kwargs):
296 """Initalize things! This should go above in the class docstring
297 """
298
299 super().__init__(**kwargs)
300
301 if schema is None:
302 schema = SourceTable.makeMinimalSchema()
303 self.schema = schema
304 self.makeSubtask("insertFakes")
305 self.makeSubtask("calibrate")
306
307 def runQuantum(self, butlerQC, inputRefs, outputRefs):
308 inputs = butlerQC.get(inputRefs)
309 detectorId = inputs["exposure"].getInfo().getDetector().getId()
310
311 if 'exposureIdInfo' not in inputs.keys():
312 expId, expBits = butlerQC.quantum.dataId.pack("visit_detector", returnMaxBits=True)
313 inputs['exposureIdInfo'] = ExposureIdInfo(expId, expBits)
314
315 expWcs = inputs["exposure"].getWcs()
316 tractId = None
317 if not self.config.doApplyExternalGlobalSkyWcs and not self.config.doApplyExternalTractSkyWcs:
318 if expWcs is None:
319 self.log.info("No WCS for exposure %s so cannot insert fake sources. Skipping detector.",
320 butlerQC.quantum.dataId)
321 return None
322 else:
323 inputs["wcs"] = expWcs
324 elif self.config.doApplyExternalGlobalSkyWcs:
325 externalSkyWcsCatalog = inputs["externalSkyWcsGlobalCatalog"]
326 row = externalSkyWcsCatalog.find(detectorId)
327 if row is None:
328 self.log.info("No %s external global sky WCS for exposure %s so cannot insert fake "
329 "sources. Skipping detector.", self.config.externalSkyWcsName,
330 butlerQC.quantum.dataId)
331 return None
332 inputs["wcs"] = row.getWcs()
333 elif self.config.doApplyExternalTractSkyWcs:
334 externalSkyWcsCatalogList = inputs["externalSkyWcsTractCatalog"]
335 if tractId is None:
336 tractId = externalSkyWcsCatalogList[0].dataId["tract"]
337 externalSkyWcsCatalog = None
338 for externalSkyWcsCatalogRef in externalSkyWcsCatalogList:
339 if externalSkyWcsCatalogRef.dataId["tract"] == tractId:
340 externalSkyWcsCatalog = externalSkyWcsCatalogRef.get(
341 datasetType=self.config.connections.externalSkyWcsTractCatalog)
342 break
343 if externalSkyWcsCatalog is None:
344 usedTract = externalSkyWcsCatalogList[-1].dataId["tract"]
345 self.log.warn(
346 f"Warning, external SkyWcs for tract {tractId} not found. Using tract {usedTract} "
347 "instead.")
348 externalSkyWcsCatalog = externalSkyWcsCatalogList[-1].get(
349 datasetType=self.config.connections.externalSkyWcsTractCatalog)
350 row = externalSkyWcsCatalog.find(detectorId)
351 if row is None:
352 self.log.info("No %s external tract sky WCS for exposure %s so cannot insert fake "
353 "sources. Skipping detector.", self.config.externalSkyWcsName,
354 butlerQC.quantum.dataId)
355 return None
356 inputs["wcs"] = row.getWcs()
357
358 if not self.config.doApplyExternalGlobalPhotoCalib and not self.config.doApplyExternalTractPhotoCalib:
359 inputs["photoCalib"] = inputs["exposure"].getPhotoCalib()
360 elif self.config.doApplyExternalGlobalPhotoCalib:
361 externalPhotoCalibCatalog = inputs["externalPhotoCalibGlobalCatalog"]
362 row = externalPhotoCalibCatalog.find(detectorId)
363 if row is None:
364 self.log.info("No %s external global photoCalib for exposure %s so cannot insert fake "
365 "sources. Skipping detector.", self.config.externalPhotoCalibName,
366 butlerQC.quantum.dataId)
367 return None
368 inputs["photoCalib"] = row.getPhotoCalib()
369 elif self.config.doApplyExternalTractPhotoCalib:
370 externalPhotoCalibCatalogList = inputs["externalPhotoCalibTractCatalog"]
371 if tractId is None:
372 tractId = externalPhotoCalibCatalogList[0].dataId["tract"]
373 externalPhotoCalibCatalog = None
374 for externalPhotoCalibCatalogRef in externalPhotoCalibCatalogList:
375 if externalPhotoCalibCatalogRef.dataId["tract"] == tractId:
376 externalPhotoCalibCatalog = externalPhotoCalibCatalogRef.get(
377 datasetType=self.config.connections.externalPhotoCalibTractCatalog)
378 break
379 if externalPhotoCalibCatalog is None:
380 usedTract = externalPhotoCalibCatalogList[-1].dataId["tract"]
381 self.log.warn(
382 f"Warning, external PhotoCalib for tract {tractId} not found. Using tract {usedTract} "
383 "instead.")
384 externalPhotoCalibCatalog = externalPhotoCalibCatalogList[-1].get(
385 datasetType=self.config.connections.externalPhotoCalibTractCatalog)
386 row = externalPhotoCalibCatalog.find(detectorId)
387 if row is None:
388 self.log.info("No %s external tract photoCalib for exposure %s so cannot insert fake "
389 "sources. Skipping detector.", self.config.externalPhotoCalibName,
390 butlerQC.quantum.dataId)
391 return None
392 inputs["photoCalib"] = row.getPhotoCalib()
393
394 outputs = self.run(**inputs)
395 butlerQC.put(outputs, outputRefs)
396
397 def run(self, fakeCats, exposure, skyMap, wcs=None, photoCalib=None, exposureIdInfo=None,
398 icSourceCat=None, sfdSourceCat=None, externalSkyWcsGlobalCatalog=None,
399 externalSkyWcsTractCatalog=None, externalPhotoCalibGlobalCatalog=None,
400 externalPhotoCalibTractCatalog=None):
401 """Add fake sources to a calexp and then run detection, deblending and measurement.
402
403 Parameters
404 ----------
405 fakeCats : `list` of `lsst.daf.butler.DeferredDatasetHandle`
406 Set of tract level fake catalogs that potentially cover
407 this detectorVisit.
408 exposure : `lsst.afw.image.exposure.exposure.ExposureF`
409 The exposure to add the fake sources to
410 skyMap : `lsst.skymap.SkyMap`
411 SkyMap defining the tracts and patches the fakes are stored over.
413 WCS to use to add fake sources
414 photoCalib : `lsst.afw.image.photoCalib.PhotoCalib`
415 Photometric calibration to be used to calibrate the fake sources
416 exposureIdInfo : `lsst.obs.base.ExposureIdInfo`
417 icSourceCat : `lsst.afw.table.SourceCatalog`
418 Default : None
419 Catalog to take the information about which sources were used for calibration from.
420 sfdSourceCat : `lsst.afw.table.SourceCatalog`
421 Default : None
422 Catalog produced by singleFrameDriver, needed to copy some calibration flags from.
423
424 Returns
425 -------
426 resultStruct : `lsst.pipe.base.struct.Struct`
427 contains : outputExposure : `lsst.afw.image.exposure.exposure.ExposureF`
428 outputCat : `lsst.afw.table.source.source.SourceCatalog`
429
430 Notes
431 -----
432 Adds pixel coordinates for each source to the fakeCat and removes objects with bulge or disk half
433 light radius = 0 (if ``config.cleanCat = True``). These columns are called ``x`` and ``y`` and are in
434 pixels.
435
436 Adds the ``Fake`` mask plane to the exposure which is then set by `addFakeSources` to mark where fake
437 sources have been added. Uses the information in the ``fakeCat`` to make fake galaxies (using galsim)
438 and fake stars, using the PSF models from the PSF information for the calexp. These are then added to
439 the calexp and the calexp with fakes included returned.
440
441 The galsim galaxies are made using a double sersic profile, one for the bulge and one for the disk,
442 this is then convolved with the PSF at that point.
443
444 If exposureIdInfo is not provided then the SourceCatalog IDs will not be globally unique.
445 """
446 fakeCat = self.composeFakeCat(fakeCats, skyMap)
447
448 if wcs is None:
449 wcs = exposure.getWcs()
450
451 if photoCalib is None:
452 photoCalib = exposure.getPhotoCalib()
453
454 if self.config.doMatchVisit:
455 fakeCat = self.getVisitMatchedFakeCat(fakeCat, exposure)
456
457 self.insertFakes.run(fakeCat, exposure, wcs, photoCalib)
458
459 # detect, deblend and measure sources
460 if exposureIdInfo is None:
461 exposureIdInfo = ExposureIdInfo()
462 returnedStruct = self.calibrate.run(exposure, exposureIdInfo=exposureIdInfo)
463 sourceCat = returnedStruct.sourceCat
464
465 sourceCat = self.copyCalibrationFields(sfdSourceCat, sourceCat, self.config.srcFieldsToCopy)
466
467 resultStruct = pipeBase.Struct(outputExposure=exposure, outputCat=sourceCat)
468 return resultStruct
469
470 def composeFakeCat(self, fakeCats, skyMap):
471 """Concatenate the fakeCats from tracts that may cover the exposure.
472
473 Parameters
474 ----------
475 fakeCats : `list` of `lst.daf.butler.DeferredDatasetHandle`
476 Set of fake cats to concatenate.
477 skyMap : `lsst.skymap.SkyMap`
478 SkyMap defining the geometry of the tracts and patches.
479
480 Returns
481 -------
482 combinedFakeCat : `pandas.DataFrame`
483 All fakes that cover the inner polygon of the tracts in this
484 quantum.
485 """
486 if len(fakeCats) == 1:
487 return fakeCats[0].get(
488 datasetType=self.config.connections.fakeCats)
489 outputCat = []
490 for fakeCatRef in fakeCats:
491 cat = fakeCatRef.get(
492 datasetType=self.config.connections.fakeCats)
493 tractId = fakeCatRef.dataId["tract"]
494 # Make sure all data is within the inner part of the tract.
495 outputCat.append(cat[
496 skyMap.findTractIdArray(cat[self.config.insertFakes.ra_col],
497 cat[self.config.insertFakes.dec_col],
498 degrees=False)
499 == tractId])
500
501 return pd.concat(outputCat)
502
503 def getVisitMatchedFakeCat(self, fakeCat, exposure):
504 """Trim the fakeCat to select particular visit
505
506 Parameters
507 ----------
508 fakeCat : `pandas.core.frame.DataFrame`
509 The catalog of fake sources to add to the exposure
510 exposure : `lsst.afw.image.exposure.exposure.ExposureF`
511 The exposure to add the fake sources to
512
513 Returns
514 -------
515 movingFakeCat : `pandas.DataFrame`
516 All fakes that belong to the visit
517 """
518 selected = exposure.getInfo().getVisitInfo().getId() == fakeCat["visit"]
519
520 return fakeCat[selected]
521
522 def copyCalibrationFields(self, calibCat, sourceCat, fieldsToCopy):
523 """Match sources in calibCat and sourceCat and copy the specified fields
524
525 Parameters
526 ----------
528 Catalog from which to copy fields.
529 sourceCat : `lsst.afw.table.SourceCatalog`
530 Catalog to which to copy fields.
531 fieldsToCopy : `lsst.pex.config.listField.List`
532 Fields to copy from calibCat to SoourceCat.
533
534 Returns
535 -------
537 Catalog which includes the copied fields.
538
539 The fields copied are those specified by `fieldsToCopy` that actually exist
540 in the schema of `calibCat`.
541
542 This version was based on and adapted from the one in calibrateTask.
543 """
544
545 # Make a new SourceCatalog with the data from sourceCat so that we can add the new columns to it
546 sourceSchemaMapper = afwTable.SchemaMapper(sourceCat.schema)
547 sourceSchemaMapper.addMinimalSchema(sourceCat.schema, True)
548
549 calibSchemaMapper = afwTable.SchemaMapper(calibCat.schema, sourceCat.schema)
550
551 # Add the desired columns from the option fieldsToCopy
552 missingFieldNames = []
553 for fieldName in fieldsToCopy:
554 if fieldName in calibCat.schema:
555 schemaItem = calibCat.schema.find(fieldName)
556 calibSchemaMapper.editOutputSchema().addField(schemaItem.getField())
557 schema = calibSchemaMapper.editOutputSchema()
558 calibSchemaMapper.addMapping(schemaItem.getKey(), schema.find(fieldName).getField())
559 else:
560 missingFieldNames.append(fieldName)
561 if missingFieldNames:
562 raise RuntimeError(f"calibCat is missing fields {missingFieldNames} specified in "
563 "fieldsToCopy")
564
565 if "calib_detected" not in calibSchemaMapper.getOutputSchema():
566 self.calibSourceKey = calibSchemaMapper.addOutputField(afwTable.Field["Flag"]("calib_detected",
567 "Source was detected as an icSource"))
568 else:
569 self.calibSourceKey = None
570
571 schema = calibSchemaMapper.getOutputSchema()
572 newCat = afwTable.SourceCatalog(schema)
573 newCat.reserve(len(sourceCat))
574 newCat.extend(sourceCat, sourceSchemaMapper)
575
576 # Set the aliases so it doesn't complain.
577 for k, v in sourceCat.schema.getAliasMap().items():
578 newCat.schema.getAliasMap().set(k, v)
579
580 select = newCat["deblend_nChild"] == 0
581 matches = afwTable.matchXy(newCat[select], calibCat, self.config.matchRadiusPix)
582 # Check that no sourceCat sources are listed twice (we already know
583 # that each match has a unique calibCat source ID, due to using
584 # that ID as the key in bestMatches)
585 numMatches = len(matches)
586 numUniqueSources = len(set(m[1].getId() for m in matches))
587 if numUniqueSources != numMatches:
588 self.log.warning("%d calibCat sources matched only %d sourceCat sources", numMatches,
589 numUniqueSources)
590
591 self.log.info("Copying flags from calibCat to sourceCat for %s sources", numMatches)
592
593 # For each match: set the calibSourceKey flag and copy the desired
594 # fields
595 for src, calibSrc, d in matches:
596 if self.calibSourceKey:
597 src.setFlag(self.calibSourceKey, True)
598 # src.assign copies the footprint from calibSrc, which we don't want
599 # (DM-407)
600 # so set calibSrc's footprint to src's footprint before src.assign,
601 # then restore it
602 calibSrcFootprint = calibSrc.getFootprint()
603 try:
604 calibSrc.setFootprint(src.getFootprint())
605 src.assign(calibSrc, calibSchemaMapper)
606 finally:
607 calibSrc.setFootprint(calibSrcFootprint)
608
609 return newCat
610
611
612class ProcessCcdWithVariableFakesConnections(ProcessCcdWithFakesConnections):
613 ccdVisitFakeMagnitudes = cT.Output(
614 doc="Catalog of fakes with magnitudes scattered for this ccdVisit.",
615 name="{fakesType}ccdVisitFakeMagnitudes",
616 storageClass="DataFrame",
617 dimensions=("instrument", "visit", "detector"),
618 )
619
620
621class ProcessCcdWithVariableFakesConfig(ProcessCcdWithFakesConfig,
622 pipelineConnections=ProcessCcdWithVariableFakesConnections):
623 scatterSize = pexConfig.RangeField(
624 dtype=float,
625 default=0.4,
626 min=0,
627 max=100,
628 doc="Amount of scatter to add to the visit magnitude for variable "
629 "sources."
630 )
631
632
633class ProcessCcdWithVariableFakesTask(ProcessCcdWithFakesTask):
634 """As ProcessCcdWithFakes except add variablity to the fakes catalog
635 magnitude in the observed band for this ccdVisit.
636
637 Additionally, write out the modified magnitudes to the Butler.
638 """
639
640 _DefaultName = "processCcdWithVariableFakes"
641 ConfigClass = ProcessCcdWithVariableFakesConfig
642
643 def run(self, fakeCats, exposure, skyMap, wcs=None, photoCalib=None, exposureIdInfo=None,
644 icSourceCat=None, sfdSourceCat=None):
645 """Add fake sources to a calexp and then run detection, deblending and measurement.
646
647 Parameters
648 ----------
649 fakeCat : `pandas.core.frame.DataFrame`
650 The catalog of fake sources to add to the exposure
651 exposure : `lsst.afw.image.exposure.exposure.ExposureF`
652 The exposure to add the fake sources to
653 skyMap : `lsst.skymap.SkyMap`
654 SkyMap defining the tracts and patches the fakes are stored over.
656 WCS to use to add fake sources
657 photoCalib : `lsst.afw.image.photoCalib.PhotoCalib`
658 Photometric calibration to be used to calibrate the fake sources
659 exposureIdInfo : `lsst.obs.base.ExposureIdInfo`
660 icSourceCat : `lsst.afw.table.SourceCatalog`
661 Default : None
662 Catalog to take the information about which sources were used for calibration from.
663 sfdSourceCat : `lsst.afw.table.SourceCatalog`
664 Default : None
665 Catalog produced by singleFrameDriver, needed to copy some calibration flags from.
666
667 Returns
668 -------
669 resultStruct : `lsst.pipe.base.struct.Struct`
670 Results Strcut containing:
671
672 - outputExposure : Exposure with added fakes
673 (`lsst.afw.image.exposure.exposure.ExposureF`)
674 - outputCat : Catalog with detected fakes
675 (`lsst.afw.table.source.source.SourceCatalog`)
676 - ccdVisitFakeMagnitudes : Magnitudes that these fakes were
677 inserted with after being scattered (`pandas.DataFrame`)
678
679 Notes
680 -----
681 Adds pixel coordinates for each source to the fakeCat and removes objects with bulge or disk half
682 light radius = 0 (if ``config.cleanCat = True``). These columns are called ``x`` and ``y`` and are in
683 pixels.
684
685 Adds the ``Fake`` mask plane to the exposure which is then set by `addFakeSources` to mark where fake
686 sources have been added. Uses the information in the ``fakeCat`` to make fake galaxies (using galsim)
687 and fake stars, using the PSF models from the PSF information for the calexp. These are then added to
688 the calexp and the calexp with fakes included returned.
689
690 The galsim galaxies are made using a double sersic profile, one for the bulge and one for the disk,
691 this is then convolved with the PSF at that point.
692
693 If exposureIdInfo is not provided then the SourceCatalog IDs will not be globally unique.
694 """
695 fakeCat = self.composeFakeCat(fakeCats, skyMap)
696
697 if wcs is None:
698 wcs = exposure.getWcs()
699
700 if photoCalib is None:
701 photoCalib = exposure.getPhotoCalib()
702
703 if exposureIdInfo is None:
704 exposureIdInfo = ExposureIdInfo()
705
706 band = exposure.getFilter().bandLabel
707 ccdVisitMagnitudes = self.addVariablity(fakeCat, band, exposure, photoCalib, exposureIdInfo)
708
709 self.insertFakes.run(fakeCat, exposure, wcs, photoCalib)
710
711 # detect, deblend and measure sources
712 returnedStruct = self.calibrate.run(exposure, exposureIdInfo=exposureIdInfo)
713 sourceCat = returnedStruct.sourceCat
714
715 sourceCat = self.copyCalibrationFields(sfdSourceCat, sourceCat, self.config.srcFieldsToCopy)
716
717 resultStruct = pipeBase.Struct(outputExposure=exposure,
718 outputCat=sourceCat,
719 ccdVisitFakeMagnitudes=ccdVisitMagnitudes)
720 return resultStruct
721
722 def addVariablity(self, fakeCat, band, exposure, photoCalib, exposureIdInfo):
723 """Add scatter to the fake catalog visit magnitudes.
724
725 Currently just adds a simple Gaussian scatter around the static fake
726 magnitude. This function could be modified to return any number of
727 fake variability.
728
729 Parameters
730 ----------
731 fakeCat : `pandas.DataFrame`
732 Catalog of fakes to modify magnitudes of.
733 band : `str`
734 Current observing band to modify.
735 exposure : `lsst.afw.image.ExposureF`
736 Exposure fakes will be added to.
737 photoCalib : `lsst.afw.image.PhotoCalib`
738 Photometric calibration object of ``exposure``.
739 exposureIdInfo : `lsst.obs.base.ExposureIdInfo`
740 Exposure id information and metadata.
741
742 Returns
743 -------
744 dataFrame : `pandas.DataFrame`
745 DataFrame containing the values of the magnitudes to that will
746 be inserted into this ccdVisit.
747 """
748 expId = exposureIdInfo.expId
749 rng = np.random.default_rng(expId)
750 magScatter = rng.normal(loc=0,
751 scale=self.config.scatterSize,
752 size=len(fakeCat))
753 visitMagnitudes = fakeCat[self.insertFakes.config.mag_col % band] + magScatter
754 fakeCat.loc[:, self.insertFakes.config.mag_col % band] = visitMagnitudes
755 return pd.DataFrame(data={"variableMag": visitMagnitudes})