Coverage for python/lsst/pipe/tasks/processCcdWithFakes.py: 19%
232 statements
« prev ^ index » next coverage.py v7.2.5, created at 2023-05-14 15:47 -0700
« prev ^ index » next coverage.py v7.2.5, created at 2023-05-14 15:47 -0700
1# This file is part of pipe_tasks.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22"""
23Insert fake sources into calexps
24"""
26__all__ = ["ProcessCcdWithFakesConfig", "ProcessCcdWithFakesTask",
27 "ProcessCcdWithVariableFakesConfig", "ProcessCcdWithVariableFakesTask"]
29import numpy as np
30import pandas as pd
32import lsst.pex.config as pexConfig
33import lsst.pipe.base as pipeBase
35from .insertFakes import InsertFakesTask
36from lsst.afw.table import SourceTable
37from lsst.obs.base import ExposureIdInfo
38from lsst.pipe.base import PipelineTask, PipelineTaskConfig, PipelineTaskConnections
39import lsst.pipe.base.connectionTypes as cT
40import lsst.afw.table as afwTable
41from lsst.skymap import BaseSkyMap
42from lsst.pipe.tasks.calibrate import CalibrateTask
45class ProcessCcdWithFakesConnections(PipelineTaskConnections,
46 dimensions=("instrument", "visit", "detector"),
47 defaultTemplates={"coaddName": "deep",
48 "wcsName": "jointcal",
49 "photoCalibName": "jointcal",
50 "fakesType": "fakes_"}):
51 skyMap = cT.Input(
52 doc="Input definition of geometry/bbox and projection/wcs for "
53 "template exposures. Needed to test which tract to generate ",
54 name=BaseSkyMap.SKYMAP_DATASET_TYPE_NAME,
55 dimensions=("skymap",),
56 storageClass="SkyMap",
57 )
59 exposure = cT.Input(
60 doc="Exposure into which fakes are to be added.",
61 name="calexp",
62 storageClass="ExposureF",
63 dimensions=("instrument", "visit", "detector")
64 )
66 fakeCats = cT.Input(
67 doc="Set of catalogs of fake sources to draw inputs from. We "
68 "concatenate the tract catalogs for detectorVisits that cover "
69 "multiple tracts.",
70 name="{fakesType}fakeSourceCat",
71 storageClass="DataFrame",
72 dimensions=("tract", "skymap"),
73 deferLoad=True,
74 multiple=True,
75 )
77 externalSkyWcsTractCatalog = cT.Input(
78 doc=("Per-tract, per-visit wcs calibrations. These catalogs use the detector "
79 "id for the catalog id, sorted on id for fast lookup."),
80 name="{wcsName}SkyWcsCatalog",
81 storageClass="ExposureCatalog",
82 dimensions=("instrument", "visit", "tract", "skymap"),
83 deferLoad=True,
84 multiple=True,
85 )
87 externalSkyWcsGlobalCatalog = cT.Input(
88 doc=("Per-visit wcs calibrations computed globally (with no tract information). "
89 "These catalogs use the detector id for the catalog id, sorted on id for "
90 "fast lookup."),
91 name="finalVisitSummary",
92 storageClass="ExposureCatalog",
93 dimensions=("instrument", "visit"),
94 )
96 externalPhotoCalibTractCatalog = cT.Input(
97 doc=("Per-tract, per-visit photometric calibrations. These catalogs use the "
98 "detector id for the catalog id, sorted on id for fast lookup."),
99 name="{photoCalibName}PhotoCalibCatalog",
100 storageClass="ExposureCatalog",
101 dimensions=("instrument", "visit", "tract"),
102 deferLoad=True,
103 multiple=True,
104 )
106 externalPhotoCalibGlobalCatalog = cT.Input(
107 doc=("Per-visit photometric calibrations. These catalogs use the "
108 "detector id for the catalog id, sorted on id for fast lookup."),
109 name="finalVisitSummary",
110 storageClass="ExposureCatalog",
111 dimensions=("instrument", "visit"),
112 )
114 icSourceCat = cT.Input(
115 doc="Catalog of calibration sources",
116 name="icSrc",
117 storageClass="SourceCatalog",
118 dimensions=("instrument", "visit", "detector")
119 )
121 sfdSourceCat = cT.Input(
122 doc="Catalog of calibration sources",
123 name="src",
124 storageClass="SourceCatalog",
125 dimensions=("instrument", "visit", "detector")
126 )
128 outputExposure = cT.Output(
129 doc="Exposure with fake sources added.",
130 name="{fakesType}calexp",
131 storageClass="ExposureF",
132 dimensions=("instrument", "visit", "detector")
133 )
135 outputCat = cT.Output(
136 doc="Source catalog produced in calibrate task with fakes also measured.",
137 name="{fakesType}src",
138 storageClass="SourceCatalog",
139 dimensions=("instrument", "visit", "detector"),
140 )
142 def __init__(self, *, config=None):
143 super().__init__(config=config)
145 if not config.doApplyExternalGlobalPhotoCalib:
146 self.inputs.remove("externalPhotoCalibGlobalCatalog")
147 if not config.doApplyExternalTractPhotoCalib:
148 self.inputs.remove("externalPhotoCalibTractCatalog")
150 if not config.doApplyExternalGlobalSkyWcs:
151 self.inputs.remove("externalSkyWcsGlobalCatalog")
152 if not config.doApplyExternalTractSkyWcs:
153 self.inputs.remove("externalSkyWcsTractCatalog")
156class ProcessCcdWithFakesConfig(PipelineTaskConfig,
157 pipelineConnections=ProcessCcdWithFakesConnections):
158 """Config for inserting fake sources
160 Notes
161 -----
162 The default column names are those from the UW sims database.
163 """
165 doApplyExternalGlobalPhotoCalib = pexConfig.Field(
166 dtype=bool,
167 default=False,
168 doc="Whether to apply an external photometric calibration via an "
169 "`lsst.afw.image.PhotoCalib` object. Uses the "
170 "`externalPhotoCalibName` config option to determine which "
171 "calibration to use. Uses a global calibration."
172 )
174 doApplyExternalTractPhotoCalib = pexConfig.Field(
175 dtype=bool,
176 default=False,
177 doc="Whether to apply an external photometric calibration via an "
178 "`lsst.afw.image.PhotoCalib` object. Uses the "
179 "`externalPhotoCalibName` config option to determine which "
180 "calibration to use. Uses a per tract calibration."
181 )
183 externalPhotoCalibName = pexConfig.ChoiceField(
184 doc="What type of external photo calib to use.",
185 dtype=str,
186 default="jointcal",
187 allowed={"jointcal": "Use jointcal_photoCalib",
188 "fgcm": "Use fgcm_photoCalib",
189 "fgcm_tract": "Use fgcm_tract_photoCalib"}
190 )
192 doApplyExternalGlobalSkyWcs = pexConfig.Field(
193 dtype=bool,
194 default=False,
195 doc="Whether to apply an external astrometric calibration via an "
196 "`lsst.afw.geom.SkyWcs` object. Uses the "
197 "`externalSkyWcsName` config option to determine which "
198 "calibration to use. Uses a global calibration."
199 )
201 doApplyExternalTractSkyWcs = pexConfig.Field(
202 dtype=bool,
203 default=False,
204 doc="Whether to apply an external astrometric calibration via an "
205 "`lsst.afw.geom.SkyWcs` object. Uses the "
206 "`externalSkyWcsName` config option to determine which "
207 "calibration to use. Uses a per tract calibration."
208 )
210 externalSkyWcsName = pexConfig.ChoiceField(
211 doc="What type of updated WCS calib to use.",
212 dtype=str,
213 default="jointcal",
214 allowed={"jointcal": "Use jointcal_wcs"}
215 )
217 coaddName = pexConfig.Field(
218 doc="The name of the type of coadd used",
219 dtype=str,
220 default="deep",
221 )
223 srcFieldsToCopy = pexConfig.ListField(
224 dtype=str,
225 default=("calib_photometry_reserved", "calib_photometry_used", "calib_astrometry_used",
226 "calib_psf_candidate", "calib_psf_used", "calib_psf_reserved"),
227 doc=("Fields to copy from the `src` catalog to the output catalog "
228 "for matching sources Any missing fields will trigger a "
229 "RuntimeError exception.")
230 )
232 matchRadiusPix = pexConfig.Field(
233 dtype=float,
234 default=3,
235 doc=("Match radius for matching icSourceCat objects to sourceCat objects (pixels)"),
236 )
238 doMatchVisit = pexConfig.Field(
239 dtype=bool,
240 default=False,
241 doc="Match visit to trim the fakeCat"
242 )
244 calibrate = pexConfig.ConfigurableField(target=CalibrateTask,
245 doc="The calibration task to use.")
247 insertFakes = pexConfig.ConfigurableField(target=InsertFakesTask,
248 doc="Configuration for the fake sources")
250 def setDefaults(self):
251 super().setDefaults()
252 self.calibrate.measurement.plugins["base_PixelFlags"].masksFpAnywhere.append("FAKE")
253 self.calibrate.measurement.plugins["base_PixelFlags"].masksFpCenter.append("FAKE")
254 self.calibrate.doAstrometry = False
255 self.calibrate.doWriteMatches = False
256 self.calibrate.doPhotoCal = False
257 self.calibrate.doComputeSummaryStats = False
258 self.calibrate.detection.reEstimateBackground = False
261class ProcessCcdWithFakesTask(PipelineTask):
262 """Insert fake objects into calexps.
264 Add fake stars and galaxies to the given calexp, specified in the dataRef. Galaxy parameters are read in
265 from the specified file and then modelled using galsim. Re-runs characterize image and calibrate image to
266 give a new background estimation and measurement of the calexp.
268 `ProcessFakeSourcesTask` inherits six functions from insertFakesTask that make images of the fake
269 sources and then add them to the calexp.
271 `addPixCoords`
272 Use the WCS information to add the pixel coordinates of each source
273 Adds an ``x`` and ``y`` column to the catalog of fake sources.
274 `trimFakeCat`
275 Trim the fake cat to about the size of the input image.
276 `mkFakeGalsimGalaxies`
277 Use Galsim to make fake double sersic galaxies for each set of galaxy parameters in the input file.
278 `mkFakeStars`
279 Use the PSF information from the calexp to make a fake star using the magnitude information from the
280 input file.
281 `cleanCat`
282 Remove rows of the input fake catalog which have half light radius, of either the bulge or the disk,
283 that are 0.
284 `addFakeSources`
285 Add the fake sources to the calexp.
287 Notes
288 -----
289 The ``calexp`` with fake souces added to it is written out as the datatype ``calexp_fakes``.
290 """
292 _DefaultName = "processCcdWithFakes"
293 ConfigClass = ProcessCcdWithFakesConfig
295 def __init__(self, schema=None, butler=None, **kwargs):
296 """Initalize things! This should go above in the class docstring
297 """
299 super().__init__(**kwargs)
301 if schema is None:
302 schema = SourceTable.makeMinimalSchema()
303 self.schema = schema
304 self.makeSubtask("insertFakes")
305 self.makeSubtask("calibrate")
307 def runQuantum(self, butlerQC, inputRefs, outputRefs):
308 inputs = butlerQC.get(inputRefs)
309 detectorId = inputs["exposure"].getInfo().getDetector().getId()
311 if 'exposureIdInfo' not in inputs.keys():
312 expId, expBits = butlerQC.quantum.dataId.pack("visit_detector", returnMaxBits=True)
313 inputs['exposureIdInfo'] = ExposureIdInfo(expId, expBits)
315 expWcs = inputs["exposure"].getWcs()
316 tractId = None
317 if not self.config.doApplyExternalGlobalSkyWcs and not self.config.doApplyExternalTractSkyWcs:
318 if expWcs is None:
319 self.log.info("No WCS for exposure %s so cannot insert fake sources. Skipping detector.",
320 butlerQC.quantum.dataId)
321 return None
322 else:
323 inputs["wcs"] = expWcs
324 elif self.config.doApplyExternalGlobalSkyWcs:
325 externalSkyWcsCatalog = inputs["externalSkyWcsGlobalCatalog"]
326 row = externalSkyWcsCatalog.find(detectorId)
327 if row is None:
328 self.log.info("No %s external global sky WCS for exposure %s so cannot insert fake "
329 "sources. Skipping detector.", self.config.externalSkyWcsName,
330 butlerQC.quantum.dataId)
331 return None
332 inputs["wcs"] = row.getWcs()
333 elif self.config.doApplyExternalTractSkyWcs:
334 externalSkyWcsCatalogList = inputs["externalSkyWcsTractCatalog"]
335 if tractId is None:
336 tractId = externalSkyWcsCatalogList[0].dataId["tract"]
337 externalSkyWcsCatalog = None
338 for externalSkyWcsCatalogRef in externalSkyWcsCatalogList:
339 if externalSkyWcsCatalogRef.dataId["tract"] == tractId:
340 externalSkyWcsCatalog = externalSkyWcsCatalogRef.get(
341 datasetType=self.config.connections.externalSkyWcsTractCatalog)
342 break
343 if externalSkyWcsCatalog is None:
344 usedTract = externalSkyWcsCatalogList[-1].dataId["tract"]
345 self.log.warn(
346 f"Warning, external SkyWcs for tract {tractId} not found. Using tract {usedTract} "
347 "instead.")
348 externalSkyWcsCatalog = externalSkyWcsCatalogList[-1].get(
349 datasetType=self.config.connections.externalSkyWcsTractCatalog)
350 row = externalSkyWcsCatalog.find(detectorId)
351 if row is None:
352 self.log.info("No %s external tract sky WCS for exposure %s so cannot insert fake "
353 "sources. Skipping detector.", self.config.externalSkyWcsName,
354 butlerQC.quantum.dataId)
355 return None
356 inputs["wcs"] = row.getWcs()
358 if not self.config.doApplyExternalGlobalPhotoCalib and not self.config.doApplyExternalTractPhotoCalib:
359 inputs["photoCalib"] = inputs["exposure"].getPhotoCalib()
360 elif self.config.doApplyExternalGlobalPhotoCalib:
361 externalPhotoCalibCatalog = inputs["externalPhotoCalibGlobalCatalog"]
362 row = externalPhotoCalibCatalog.find(detectorId)
363 if row is None:
364 self.log.info("No %s external global photoCalib for exposure %s so cannot insert fake "
365 "sources. Skipping detector.", self.config.externalPhotoCalibName,
366 butlerQC.quantum.dataId)
367 return None
368 inputs["photoCalib"] = row.getPhotoCalib()
369 elif self.config.doApplyExternalTractPhotoCalib:
370 externalPhotoCalibCatalogList = inputs["externalPhotoCalibTractCatalog"]
371 if tractId is None:
372 tractId = externalPhotoCalibCatalogList[0].dataId["tract"]
373 externalPhotoCalibCatalog = None
374 for externalPhotoCalibCatalogRef in externalPhotoCalibCatalogList:
375 if externalPhotoCalibCatalogRef.dataId["tract"] == tractId:
376 externalPhotoCalibCatalog = externalPhotoCalibCatalogRef.get(
377 datasetType=self.config.connections.externalPhotoCalibTractCatalog)
378 break
379 if externalPhotoCalibCatalog is None:
380 usedTract = externalPhotoCalibCatalogList[-1].dataId["tract"]
381 self.log.warn(
382 f"Warning, external PhotoCalib for tract {tractId} not found. Using tract {usedTract} "
383 "instead.")
384 externalPhotoCalibCatalog = externalPhotoCalibCatalogList[-1].get(
385 datasetType=self.config.connections.externalPhotoCalibTractCatalog)
386 row = externalPhotoCalibCatalog.find(detectorId)
387 if row is None:
388 self.log.info("No %s external tract photoCalib for exposure %s so cannot insert fake "
389 "sources. Skipping detector.", self.config.externalPhotoCalibName,
390 butlerQC.quantum.dataId)
391 return None
392 inputs["photoCalib"] = row.getPhotoCalib()
394 outputs = self.run(**inputs)
395 butlerQC.put(outputs, outputRefs)
397 def run(self, fakeCats, exposure, skyMap, wcs=None, photoCalib=None, exposureIdInfo=None,
398 icSourceCat=None, sfdSourceCat=None, externalSkyWcsGlobalCatalog=None,
399 externalSkyWcsTractCatalog=None, externalPhotoCalibGlobalCatalog=None,
400 externalPhotoCalibTractCatalog=None):
401 """Add fake sources to a calexp and then run detection, deblending and measurement.
403 Parameters
404 ----------
405 fakeCats : `list` of `lsst.daf.butler.DeferredDatasetHandle`
406 Set of tract level fake catalogs that potentially cover
407 this detectorVisit.
408 exposure : `lsst.afw.image.exposure.exposure.ExposureF`
409 The exposure to add the fake sources to
410 skyMap : `lsst.skymap.SkyMap`
411 SkyMap defining the tracts and patches the fakes are stored over.
412 wcs : `lsst.afw.geom.SkyWcs`
413 WCS to use to add fake sources
414 photoCalib : `lsst.afw.image.photoCalib.PhotoCalib`
415 Photometric calibration to be used to calibrate the fake sources
416 exposureIdInfo : `lsst.obs.base.ExposureIdInfo`
417 icSourceCat : `lsst.afw.table.SourceCatalog`
418 Default : None
419 Catalog to take the information about which sources were used for calibration from.
420 sfdSourceCat : `lsst.afw.table.SourceCatalog`
421 Default : None
422 Catalog produced by singleFrameDriver, needed to copy some calibration flags from.
424 Returns
425 -------
426 resultStruct : `lsst.pipe.base.struct.Struct`
427 contains : outputExposure : `lsst.afw.image.exposure.exposure.ExposureF`
428 outputCat : `lsst.afw.table.source.source.SourceCatalog`
430 Notes
431 -----
432 Adds pixel coordinates for each source to the fakeCat and removes objects with bulge or disk half
433 light radius = 0 (if ``config.cleanCat = True``). These columns are called ``x`` and ``y`` and are in
434 pixels.
436 Adds the ``Fake`` mask plane to the exposure which is then set by `addFakeSources` to mark where fake
437 sources have been added. Uses the information in the ``fakeCat`` to make fake galaxies (using galsim)
438 and fake stars, using the PSF models from the PSF information for the calexp. These are then added to
439 the calexp and the calexp with fakes included returned.
441 The galsim galaxies are made using a double sersic profile, one for the bulge and one for the disk,
442 this is then convolved with the PSF at that point.
444 If exposureIdInfo is not provided then the SourceCatalog IDs will not be globally unique.
445 """
446 fakeCat = self.composeFakeCat(fakeCats, skyMap)
448 if wcs is None:
449 wcs = exposure.getWcs()
451 if photoCalib is None:
452 photoCalib = exposure.getPhotoCalib()
454 if self.config.doMatchVisit:
455 fakeCat = self.getVisitMatchedFakeCat(fakeCat, exposure)
457 self.insertFakes.run(fakeCat, exposure, wcs, photoCalib)
459 # detect, deblend and measure sources
460 if exposureIdInfo is None:
461 exposureIdInfo = ExposureIdInfo()
462 returnedStruct = self.calibrate.run(exposure, exposureIdInfo=exposureIdInfo)
463 sourceCat = returnedStruct.sourceCat
465 sourceCat = self.copyCalibrationFields(sfdSourceCat, sourceCat, self.config.srcFieldsToCopy)
467 resultStruct = pipeBase.Struct(outputExposure=exposure, outputCat=sourceCat)
468 return resultStruct
470 def composeFakeCat(self, fakeCats, skyMap):
471 """Concatenate the fakeCats from tracts that may cover the exposure.
473 Parameters
474 ----------
475 fakeCats : `list` of `lst.daf.butler.DeferredDatasetHandle`
476 Set of fake cats to concatenate.
477 skyMap : `lsst.skymap.SkyMap`
478 SkyMap defining the geometry of the tracts and patches.
480 Returns
481 -------
482 combinedFakeCat : `pandas.DataFrame`
483 All fakes that cover the inner polygon of the tracts in this
484 quantum.
485 """
486 if len(fakeCats) == 1:
487 return fakeCats[0].get(
488 datasetType=self.config.connections.fakeCats)
489 outputCat = []
490 for fakeCatRef in fakeCats:
491 cat = fakeCatRef.get(
492 datasetType=self.config.connections.fakeCats)
493 tractId = fakeCatRef.dataId["tract"]
494 # Make sure all data is within the inner part of the tract.
495 outputCat.append(cat[
496 skyMap.findTractIdArray(cat[self.config.insertFakes.ra_col],
497 cat[self.config.insertFakes.dec_col],
498 degrees=False)
499 == tractId])
501 return pd.concat(outputCat)
503 def getVisitMatchedFakeCat(self, fakeCat, exposure):
504 """Trim the fakeCat to select particular visit
506 Parameters
507 ----------
508 fakeCat : `pandas.core.frame.DataFrame`
509 The catalog of fake sources to add to the exposure
510 exposure : `lsst.afw.image.exposure.exposure.ExposureF`
511 The exposure to add the fake sources to
513 Returns
514 -------
515 movingFakeCat : `pandas.DataFrame`
516 All fakes that belong to the visit
517 """
518 selected = exposure.getInfo().getVisitInfo().getId() == fakeCat["visit"]
520 return fakeCat[selected]
522 def copyCalibrationFields(self, calibCat, sourceCat, fieldsToCopy):
523 """Match sources in calibCat and sourceCat and copy the specified fields
525 Parameters
526 ----------
527 calibCat : `lsst.afw.table.SourceCatalog`
528 Catalog from which to copy fields.
529 sourceCat : `lsst.afw.table.SourceCatalog`
530 Catalog to which to copy fields.
531 fieldsToCopy : `lsst.pex.config.listField.List`
532 Fields to copy from calibCat to SoourceCat.
534 Returns
535 -------
536 newCat : `lsst.afw.table.SourceCatalog`
537 Catalog which includes the copied fields.
539 The fields copied are those specified by `fieldsToCopy` that actually exist
540 in the schema of `calibCat`.
542 This version was based on and adapted from the one in calibrateTask.
543 """
545 # Make a new SourceCatalog with the data from sourceCat so that we can add the new columns to it
546 sourceSchemaMapper = afwTable.SchemaMapper(sourceCat.schema)
547 sourceSchemaMapper.addMinimalSchema(sourceCat.schema, True)
549 calibSchemaMapper = afwTable.SchemaMapper(calibCat.schema, sourceCat.schema)
551 # Add the desired columns from the option fieldsToCopy
552 missingFieldNames = []
553 for fieldName in fieldsToCopy:
554 if fieldName in calibCat.schema:
555 schemaItem = calibCat.schema.find(fieldName)
556 calibSchemaMapper.editOutputSchema().addField(schemaItem.getField())
557 schema = calibSchemaMapper.editOutputSchema()
558 calibSchemaMapper.addMapping(schemaItem.getKey(), schema.find(fieldName).getField())
559 else:
560 missingFieldNames.append(fieldName)
561 if missingFieldNames:
562 raise RuntimeError(f"calibCat is missing fields {missingFieldNames} specified in "
563 "fieldsToCopy")
565 if "calib_detected" not in calibSchemaMapper.getOutputSchema():
566 self.calibSourceKey = calibSchemaMapper.addOutputField(afwTable.Field["Flag"]("calib_detected",
567 "Source was detected as an icSource"))
568 else:
569 self.calibSourceKey = None
571 schema = calibSchemaMapper.getOutputSchema()
572 newCat = afwTable.SourceCatalog(schema)
573 newCat.reserve(len(sourceCat))
574 newCat.extend(sourceCat, sourceSchemaMapper)
576 # Set the aliases so it doesn't complain.
577 for k, v in sourceCat.schema.getAliasMap().items():
578 newCat.schema.getAliasMap().set(k, v)
580 select = newCat["deblend_nChild"] == 0
581 matches = afwTable.matchXy(newCat[select], calibCat, self.config.matchRadiusPix)
582 # Check that no sourceCat sources are listed twice (we already know
583 # that each match has a unique calibCat source ID, due to using
584 # that ID as the key in bestMatches)
585 numMatches = len(matches)
586 numUniqueSources = len(set(m[1].getId() for m in matches))
587 if numUniqueSources != numMatches:
588 self.log.warning("%d calibCat sources matched only %d sourceCat sources", numMatches,
589 numUniqueSources)
591 self.log.info("Copying flags from calibCat to sourceCat for %s sources", numMatches)
593 # For each match: set the calibSourceKey flag and copy the desired
594 # fields
595 for src, calibSrc, d in matches:
596 if self.calibSourceKey:
597 src.setFlag(self.calibSourceKey, True)
598 # src.assign copies the footprint from calibSrc, which we don't want
599 # (DM-407)
600 # so set calibSrc's footprint to src's footprint before src.assign,
601 # then restore it
602 calibSrcFootprint = calibSrc.getFootprint()
603 try:
604 calibSrc.setFootprint(src.getFootprint())
605 src.assign(calibSrc, calibSchemaMapper)
606 finally:
607 calibSrc.setFootprint(calibSrcFootprint)
609 return newCat
612class ProcessCcdWithVariableFakesConnections(ProcessCcdWithFakesConnections):
613 ccdVisitFakeMagnitudes = cT.Output(
614 doc="Catalog of fakes with magnitudes scattered for this ccdVisit.",
615 name="{fakesType}ccdVisitFakeMagnitudes",
616 storageClass="DataFrame",
617 dimensions=("instrument", "visit", "detector"),
618 )
621class ProcessCcdWithVariableFakesConfig(ProcessCcdWithFakesConfig,
622 pipelineConnections=ProcessCcdWithVariableFakesConnections):
623 scatterSize = pexConfig.RangeField(
624 dtype=float,
625 default=0.4,
626 min=0,
627 max=100,
628 doc="Amount of scatter to add to the visit magnitude for variable "
629 "sources."
630 )
633class ProcessCcdWithVariableFakesTask(ProcessCcdWithFakesTask):
634 """As ProcessCcdWithFakes except add variablity to the fakes catalog
635 magnitude in the observed band for this ccdVisit.
637 Additionally, write out the modified magnitudes to the Butler.
638 """
640 _DefaultName = "processCcdWithVariableFakes"
641 ConfigClass = ProcessCcdWithVariableFakesConfig
643 def run(self, fakeCats, exposure, skyMap, wcs=None, photoCalib=None, exposureIdInfo=None,
644 icSourceCat=None, sfdSourceCat=None):
645 """Add fake sources to a calexp and then run detection, deblending and measurement.
647 Parameters
648 ----------
649 fakeCat : `pandas.core.frame.DataFrame`
650 The catalog of fake sources to add to the exposure
651 exposure : `lsst.afw.image.exposure.exposure.ExposureF`
652 The exposure to add the fake sources to
653 skyMap : `lsst.skymap.SkyMap`
654 SkyMap defining the tracts and patches the fakes are stored over.
655 wcs : `lsst.afw.geom.SkyWcs`
656 WCS to use to add fake sources
657 photoCalib : `lsst.afw.image.photoCalib.PhotoCalib`
658 Photometric calibration to be used to calibrate the fake sources
659 exposureIdInfo : `lsst.obs.base.ExposureIdInfo`
660 icSourceCat : `lsst.afw.table.SourceCatalog`
661 Default : None
662 Catalog to take the information about which sources were used for calibration from.
663 sfdSourceCat : `lsst.afw.table.SourceCatalog`
664 Default : None
665 Catalog produced by singleFrameDriver, needed to copy some calibration flags from.
667 Returns
668 -------
669 resultStruct : `lsst.pipe.base.struct.Struct`
670 Results Strcut containing:
672 - outputExposure : Exposure with added fakes
673 (`lsst.afw.image.exposure.exposure.ExposureF`)
674 - outputCat : Catalog with detected fakes
675 (`lsst.afw.table.source.source.SourceCatalog`)
676 - ccdVisitFakeMagnitudes : Magnitudes that these fakes were
677 inserted with after being scattered (`pandas.DataFrame`)
679 Notes
680 -----
681 Adds pixel coordinates for each source to the fakeCat and removes objects with bulge or disk half
682 light radius = 0 (if ``config.cleanCat = True``). These columns are called ``x`` and ``y`` and are in
683 pixels.
685 Adds the ``Fake`` mask plane to the exposure which is then set by `addFakeSources` to mark where fake
686 sources have been added. Uses the information in the ``fakeCat`` to make fake galaxies (using galsim)
687 and fake stars, using the PSF models from the PSF information for the calexp. These are then added to
688 the calexp and the calexp with fakes included returned.
690 The galsim galaxies are made using a double sersic profile, one for the bulge and one for the disk,
691 this is then convolved with the PSF at that point.
693 If exposureIdInfo is not provided then the SourceCatalog IDs will not be globally unique.
694 """
695 fakeCat = self.composeFakeCat(fakeCats, skyMap)
697 if wcs is None:
698 wcs = exposure.getWcs()
700 if photoCalib is None:
701 photoCalib = exposure.getPhotoCalib()
703 if exposureIdInfo is None:
704 exposureIdInfo = ExposureIdInfo()
706 band = exposure.getFilter().bandLabel
707 ccdVisitMagnitudes = self.addVariablity(fakeCat, band, exposure, photoCalib, exposureIdInfo)
709 self.insertFakes.run(fakeCat, exposure, wcs, photoCalib)
711 # detect, deblend and measure sources
712 returnedStruct = self.calibrate.run(exposure, exposureIdInfo=exposureIdInfo)
713 sourceCat = returnedStruct.sourceCat
715 sourceCat = self.copyCalibrationFields(sfdSourceCat, sourceCat, self.config.srcFieldsToCopy)
717 resultStruct = pipeBase.Struct(outputExposure=exposure,
718 outputCat=sourceCat,
719 ccdVisitFakeMagnitudes=ccdVisitMagnitudes)
720 return resultStruct
722 def addVariablity(self, fakeCat, band, exposure, photoCalib, exposureIdInfo):
723 """Add scatter to the fake catalog visit magnitudes.
725 Currently just adds a simple Gaussian scatter around the static fake
726 magnitude. This function could be modified to return any number of
727 fake variability.
729 Parameters
730 ----------
731 fakeCat : `pandas.DataFrame`
732 Catalog of fakes to modify magnitudes of.
733 band : `str`
734 Current observing band to modify.
735 exposure : `lsst.afw.image.ExposureF`
736 Exposure fakes will be added to.
737 photoCalib : `lsst.afw.image.PhotoCalib`
738 Photometric calibration object of ``exposure``.
739 exposureIdInfo : `lsst.obs.base.ExposureIdInfo`
740 Exposure id information and metadata.
742 Returns
743 -------
744 dataFrame : `pandas.DataFrame`
745 DataFrame containing the values of the magnitudes to that will
746 be inserted into this ccdVisit.
747 """
748 expId = exposureIdInfo.expId
749 rng = np.random.default_rng(expId)
750 magScatter = rng.normal(loc=0,
751 scale=self.config.scatterSize,
752 size=len(fakeCat))
753 visitMagnitudes = fakeCat[self.insertFakes.config.mag_col % band] + magScatter
754 fakeCat.loc[:, self.insertFakes.config.mag_col % band] = visitMagnitudes
755 return pd.DataFrame(data={"variableMag": visitMagnitudes})