23Insert fake sources into calexps
25from astropy.table
import Table
32from .insertFakes
import InsertFakesTask
35from lsst.obs.base
import ExposureIdInfo
36from lsst.pipe.base import PipelineTask, PipelineTaskConfig, CmdLineTask, PipelineTaskConnections
37import lsst.pipe.base.connectionTypes
as cT
41__all__ = [
"ProcessCcdWithFakesConfig",
"ProcessCcdWithFakesTask",
42 "ProcessCcdWithVariableFakesConfig",
"ProcessCcdWithVariableFakesTask"]
46 dimensions=(
"skymap",
"tract",
"instrument",
"visit",
"detector"),
47 defaultTemplates={
"coaddName":
"deep",
48 "wcsName":
"jointcal",
49 "photoCalibName":
"jointcal",
50 "fakesType":
"fakes_"}):
53 doc=
"Exposure into which fakes are to be added.",
55 storageClass=
"ExposureF",
56 dimensions=(
"instrument",
"visit",
"detector")
60 doc=
"Catalog of fake sources to draw inputs from.",
61 name=
"{fakesType}fakeSourceCat",
62 storageClass=
"DataFrame",
63 dimensions=(
"tract",
"skymap")
67 doc=
"WCS information for the input exposure.",
70 dimensions=(
"tract",
"skymap",
"instrument",
"visit",
"detector")
73 photoCalib = cT.Input(
74 doc=
"Calib information for the input exposure.",
75 name=
"{photoCalibName}_photoCalib",
76 storageClass=
"PhotoCalib",
77 dimensions=(
"tract",
"skymap",
"instrument",
"visit",
"detector")
80 icSourceCat = cT.Input(
81 doc=
"Catalog of calibration sources",
83 storageClass=
"SourceCatalog",
84 dimensions=(
"instrument",
"visit",
"detector")
87 sfdSourceCat = cT.Input(
88 doc=
"Catalog of calibration sources",
90 storageClass=
"SourceCatalog",
91 dimensions=(
"instrument",
"visit",
"detector")
94 outputExposure = cT.Output(
95 doc=
"Exposure with fake sources added.",
96 name=
"{fakesType}calexp",
97 storageClass=
"ExposureF",
98 dimensions=(
"instrument",
"visit",
"detector")
101 outputCat = cT.Output(
102 doc=
"Source catalog produced in calibrate task with fakes also measured.",
103 name=
"{fakesType}src",
104 storageClass=
"SourceCatalog",
105 dimensions=(
"instrument",
"visit",
"detector"),
108 def __init__(self, *, config=None):
109 super().__init__(config=config)
111 if config.doApplyExternalSkyWcs
is False:
112 self.inputs.remove(
"wcs")
113 if config.doApplyExternalPhotoCalib
is False:
114 self.inputs.remove(
"photoCalib")
117class ProcessCcdWithFakesConfig(PipelineTaskConfig,
118 pipelineConnections=ProcessCcdWithFakesConnections):
119 """Config for inserting fake sources
123 The default column names are those from the UW sims database.
126 doApplyExternalPhotoCalib = pexConfig.Field(
129 doc=
"Whether to apply an external photometric calibration via an "
130 "`lsst.afw.image.PhotoCalib` object. Uses the "
131 "`externalPhotoCalibName` config option to determine which "
132 "calibration to use."
135 externalPhotoCalibName = pexConfig.ChoiceField(
136 doc=
"What type of external photo calib to use.",
139 allowed={
"jointcal":
"Use jointcal_photoCalib",
140 "fgcm":
"Use fgcm_photoCalib",
141 "fgcm_tract":
"Use fgcm_tract_photoCalib"}
144 doApplyExternalSkyWcs = pexConfig.Field(
147 doc=
"Whether to apply an external astrometric calibration via an "
148 "`lsst.afw.geom.SkyWcs` object. Uses the "
149 "`externalSkyWcsName` config option to determine which "
150 "calibration to use."
153 externalSkyWcsName = pexConfig.ChoiceField(
154 doc=
"What type of updated WCS calib to use.",
157 allowed={
"jointcal":
"Use jointcal_wcs"}
160 coaddName = pexConfig.Field(
161 doc=
"The name of the type of coadd used",
166 srcFieldsToCopy = pexConfig.ListField(
168 default=(
"calib_photometry_reserved",
"calib_photometry_used",
"calib_astrometry_used",
169 "calib_psf_candidate",
"calib_psf_used",
"calib_psf_reserved"),
170 doc=(
"Fields to copy from the `src` catalog to the output catalog "
171 "for matching sources Any missing fields will trigger a "
172 "RuntimeError exception.")
175 matchRadiusPix = pexConfig.Field(
178 doc=(
"Match radius for matching icSourceCat objects to sourceCat objects (pixels)"),
181 calibrate = pexConfig.ConfigurableField(target=CalibrateTask,
182 doc=
"The calibration task to use.")
184 insertFakes = pexConfig.ConfigurableField(target=InsertFakesTask,
185 doc=
"Configuration for the fake sources")
187 def setDefaults(self):
188 super().setDefaults()
189 self.calibrate.measurement.plugins[
"base_PixelFlags"].masksFpAnywhere.append(
"FAKE")
190 self.calibrate.measurement.plugins[
"base_PixelFlags"].masksFpCenter.append(
"FAKE")
191 self.calibrate.doAstrometry =
False
192 self.calibrate.doWriteMatches =
False
193 self.calibrate.doPhotoCal =
False
194 self.calibrate.detection.reEstimateBackground =
False
197class ProcessCcdWithFakesTask(PipelineTask, CmdLineTask):
198 """Insert fake objects into calexps.
200 Add fake stars and galaxies to the given calexp, specified
in the dataRef. Galaxy parameters are read
in
201 from the specified file
and then modelled using galsim. Re-runs characterize image
and calibrate image to
202 give a new background estimation
and measurement of the calexp.
204 `ProcessFakeSourcesTask` inherits six functions
from insertFakesTask that make images of the fake
205 sources
and then add them to the calexp.
208 Use the WCS information to add the pixel coordinates of each source
209 Adds an ``x``
and ``y`` column to the catalog of fake sources.
211 Trim the fake cat to about the size of the input image.
212 `mkFakeGalsimGalaxies`
213 Use Galsim to make fake double sersic galaxies
for each set of galaxy parameters
in the input file.
215 Use the PSF information
from the calexp to make a fake star using the magnitude information
from the
218 Remove rows of the input fake catalog which have half light radius, of either the bulge
or the disk,
221 Add the fake sources to the calexp.
225 The ``calexp``
with fake souces added to it
is written out
as the datatype ``calexp_fakes``.
228 _DefaultName = "processCcdWithFakes"
229 ConfigClass = ProcessCcdWithFakesConfig
231 def __init__(self, schema=None, butler=None, **kwargs):
232 """Initalize things! This should go above in the class docstring
235 super().__init__(**kwargs)
238 schema = SourceTable.makeMinimalSchema()
240 self.makeSubtask(
"insertFakes")
241 self.makeSubtask(
"calibrate")
243 def runDataRef(self, dataRef):
244 """Read in/write out the required data products and add fake sources to the calexp.
249 Data reference defining the ccd to have fakes added to it.
250 Used to access the following data products:
257 Uses the calibration and WCS information attached to the calexp
for the posistioning
and calibration
258 of the sources unless the config option config.externalPhotoCalibName
or config.externalSkyWcsName
259 are set then it uses the specified outputs. The config defualts
for the column names
in the catalog
260 of fakes are taken
from the University of Washington simulations database.
261 Operates on one ccd at a time.
263 exposureIdInfo = dataRef.get("expIdInfo")
265 if self.config.insertFakes.fakeType ==
"snapshot":
266 fakeCat = dataRef.get(
"fakeSourceCat").toDataFrame()
267 elif self.config.insertFakes.fakeType ==
"static":
268 fakeCat = dataRef.get(
"deepCoadd_fakeSourceCat").toDataFrame()
270 fakeCat = Table.read(self.config.insertFakes.fakeType).to_pandas()
272 calexp = dataRef.get(
"calexp")
273 if self.config.doApplyExternalSkyWcs:
274 self.log.info(
"Using external wcs from %s", self.config.externalSkyWcsName)
275 wcs = dataRef.get(self.config.externalSkyWcsName +
"_wcs")
277 wcs = calexp.getWcs()
279 if self.config.doApplyExternalPhotoCalib:
280 self.log.info(
"Using external photocalib from %s", self.config.externalPhotoCalibName)
281 photoCalib = dataRef.get(self.config.externalPhotoCalibName +
"_photoCalib")
283 photoCalib = calexp.getPhotoCalib()
285 icSourceCat = dataRef.get(
"icSrc", immediate=
True)
286 sfdSourceCat = dataRef.get(
"src", immediate=
True)
288 resultStruct = self.run(fakeCat, calexp, wcs=wcs, photoCalib=photoCalib,
289 exposureIdInfo=exposureIdInfo, icSourceCat=icSourceCat,
290 sfdSourceCat=sfdSourceCat)
292 dataRef.put(resultStruct.outputExposure,
"fakes_calexp")
293 dataRef.put(resultStruct.outputCat,
"fakes_src")
296 def runQuantum(self, butlerQC, inputRefs, outputRefs):
297 inputs = butlerQC.get(inputRefs)
298 if 'exposureIdInfo' not in inputs.keys():
299 expId, expBits = butlerQC.quantum.dataId.pack(
"visit_detector", returnMaxBits=
True)
300 inputs[
'exposureIdInfo'] = ExposureIdInfo(expId, expBits)
302 if not self.config.doApplyExternalSkyWcs:
303 inputs[
"wcs"] = inputs[
"exposure"].getWcs()
305 if not self.config.doApplyExternalPhotoCalib:
306 inputs[
"photoCalib"] = inputs[
"exposure"].getPhotoCalib()
308 outputs = self.run(**inputs)
309 butlerQC.put(outputs, outputRefs)
312 def _makeArgumentParser(cls):
313 parser = pipeBase.ArgumentParser(name=cls._DefaultName)
314 parser.add_id_argument(
"--id",
"fakes_calexp", help=
"data ID with raw CCD keys [+ tract optionally], "
315 "e.g. --id visit=12345 ccd=1,2 [tract=0]",
316 ContainerClass=PerTractCcdDataIdContainer)
319 def run(self, fakeCat, exposure, wcs=None, photoCalib=None, exposureIdInfo=None, icSourceCat=None,
321 """Add fake sources to a calexp and then run detection, deblending and measurement.
325 fakeCat : `pandas.core.frame.DataFrame`
326 The catalog of fake sources to add to the exposure
327 exposure : `lsst.afw.image.exposure.exposure.ExposureF`
328 The exposure to add the fake sources to
330 WCS to use to add fake sources
331 photoCalib : `lsst.afw.image.photoCalib.PhotoCalib`
332 Photometric calibration to be used to calibrate the fake sources
333 exposureIdInfo : `lsst.obs.base.ExposureIdInfo`
336 Catalog to take the information about which sources were used
for calibration
from.
339 Catalog produced by singleFrameDriver, needed to copy some calibration flags
from.
343 resultStruct : `lsst.pipe.base.struct.Struct`
344 contains : outputExposure : `lsst.afw.image.exposure.exposure.ExposureF`
345 outputCat : `lsst.afw.table.source.source.SourceCatalog`
349 Adds pixel coordinates
for each source to the fakeCat
and removes objects
with bulge
or disk half
350 light radius = 0 (
if ``config.cleanCat =
True``). These columns are called ``x``
and ``y``
and are
in
353 Adds the ``Fake`` mask plane to the exposure which
is then set by `addFakeSources` to mark where fake
354 sources have been added. Uses the information
in the ``fakeCat`` to make fake galaxies (using galsim)
355 and fake stars, using the PSF models
from the PSF information
for the calexp. These are then added to
356 the calexp
and the calexp
with fakes included returned.
358 The galsim galaxies are made using a double sersic profile, one
for the bulge
and one
for the disk,
359 this
is then convolved
with the PSF at that point.
361 If exposureIdInfo
is not provided then the SourceCatalog IDs will
not be globally unique.
365 wcs = exposure.getWcs()
367 if photoCalib
is None:
368 photoCalib = exposure.getPhotoCalib()
370 self.insertFakes.run(fakeCat, exposure, wcs, photoCalib)
373 if exposureIdInfo
is None:
374 exposureIdInfo = ExposureIdInfo()
375 returnedStruct = self.calibrate.run(exposure, exposureIdInfo=exposureIdInfo)
376 sourceCat = returnedStruct.sourceCat
378 sourceCat = self.copyCalibrationFields(sfdSourceCat, sourceCat, self.config.srcFieldsToCopy)
380 resultStruct = pipeBase.Struct(outputExposure=exposure, outputCat=sourceCat)
383 def copyCalibrationFields(self, calibCat, sourceCat, fieldsToCopy):
384 """Match sources in calibCat and sourceCat and copy the specified fields
389 Catalog from which to copy fields.
391 Catalog to which to copy fields.
393 Fields to copy
from calibCat to SoourceCat.
398 Catalog which includes the copied fields.
400 The fields copied are those specified by `fieldsToCopy` that actually exist
401 in the schema of `calibCat`.
403 This version was based on
and adapted
from the one
in calibrateTask.
407 sourceSchemaMapper = afwTable.SchemaMapper(sourceCat.schema)
408 sourceSchemaMapper.addMinimalSchema(sourceCat.schema,
True)
410 calibSchemaMapper = afwTable.SchemaMapper(calibCat.schema, sourceCat.schema)
413 missingFieldNames = []
414 for fieldName
in fieldsToCopy:
415 if fieldName
in calibCat.schema:
416 schemaItem = calibCat.schema.find(fieldName)
417 calibSchemaMapper.editOutputSchema().addField(schemaItem.getField())
418 schema = calibSchemaMapper.editOutputSchema()
419 calibSchemaMapper.addMapping(schemaItem.getKey(), schema.find(fieldName).getField())
421 missingFieldNames.append(fieldName)
422 if missingFieldNames:
423 raise RuntimeError(f
"calibCat is missing fields {missingFieldNames} specified in "
426 if "calib_detected" not in calibSchemaMapper.getOutputSchema():
427 self.calibSourceKey = calibSchemaMapper.addOutputField(afwTable.Field[
"Flag"](
"calib_detected",
428 "Source was detected as an icSource"))
430 self.calibSourceKey =
None
432 schema = calibSchemaMapper.getOutputSchema()
433 newCat = afwTable.SourceCatalog(schema)
434 newCat.reserve(len(sourceCat))
435 newCat.extend(sourceCat, sourceSchemaMapper)
438 for k, v
in sourceCat.schema.getAliasMap().items():
439 newCat.schema.getAliasMap().set(k, v)
441 select = newCat[
"deblend_nChild"] == 0
442 matches = afwTable.matchXy(newCat[select], calibCat, self.config.matchRadiusPix)
446 numMatches = len(matches)
447 numUniqueSources = len(set(m[1].getId()
for m
in matches))
448 if numUniqueSources != numMatches:
449 self.log.warning(
"%d calibCat sources matched only %d sourceCat sources", numMatches,
452 self.log.info(
"Copying flags from calibCat to sourceCat for %s sources", numMatches)
456 for src, calibSrc, d
in matches:
457 if self.calibSourceKey:
458 src.setFlag(self.calibSourceKey,
True)
463 calibSrcFootprint = calibSrc.getFootprint()
465 calibSrc.setFootprint(src.getFootprint())
466 src.assign(calibSrc, calibSchemaMapper)
468 calibSrc.setFootprint(calibSrcFootprint)
474 ccdVisitFakeMagnitudes = cT.Output(
475 doc=
"Catalog of fakes with magnitudes scattered for this ccdVisit.",
476 name=
"{fakesType}ccdVisitFakeMagnitudes",
477 storageClass=
"DataFrame",
478 dimensions=(
"instrument",
"visit",
"detector"),
482class ProcessCcdWithVariableFakesConfig(ProcessCcdWithFakesConfig,
483 pipelineConnections=ProcessCcdWithVariableFakesConnections):
484 scatterSize = pexConfig.RangeField(
489 doc=
"Amount of scatter to add to the visit magnitude for variable "
494class ProcessCcdWithVariableFakesTask(ProcessCcdWithFakesTask):
495 """As ProcessCcdWithFakes except add variablity to the fakes catalog
496 magnitude in the observed band
for this ccdVisit.
498 Additionally, write out the modified magnitudes to the Butler.
501 _DefaultName = "processCcdWithVariableFakes"
502 ConfigClass = ProcessCcdWithVariableFakesConfig
504 def run(self, fakeCat, exposure, wcs=None, photoCalib=None, exposureIdInfo=None, icSourceCat=None,
506 """Add fake sources to a calexp and then run detection, deblending and measurement.
510 fakeCat : `pandas.core.frame.DataFrame`
511 The catalog of fake sources to add to the exposure
512 exposure : `lsst.afw.image.exposure.exposure.ExposureF`
513 The exposure to add the fake sources to
515 WCS to use to add fake sources
516 photoCalib : `lsst.afw.image.photoCalib.PhotoCalib`
517 Photometric calibration to be used to calibrate the fake sources
518 exposureIdInfo : `lsst.obs.base.ExposureIdInfo`
521 Catalog to take the information about which sources were used
for calibration
from.
524 Catalog produced by singleFrameDriver, needed to copy some calibration flags
from.
528 resultStruct : `lsst.pipe.base.struct.Struct`
529 Results Strcut containing:
531 - outputExposure : Exposure
with added fakes
532 (`lsst.afw.image.exposure.exposure.ExposureF`)
533 - outputCat : Catalog
with detected fakes
534 (`lsst.afw.table.source.source.SourceCatalog`)
535 - ccdVisitFakeMagnitudes : Magnitudes that these fakes were
536 inserted
with after being scattered (`pandas.DataFrame`)
540 Adds pixel coordinates
for each source to the fakeCat
and removes objects
with bulge
or disk half
541 light radius = 0 (
if ``config.cleanCat =
True``). These columns are called ``x``
and ``y``
and are
in
544 Adds the ``Fake`` mask plane to the exposure which
is then set by `addFakeSources` to mark where fake
545 sources have been added. Uses the information
in the ``fakeCat`` to make fake galaxies (using galsim)
546 and fake stars, using the PSF models
from the PSF information
for the calexp. These are then added to
547 the calexp
and the calexp
with fakes included returned.
549 The galsim galaxies are made using a double sersic profile, one
for the bulge
and one
for the disk,
550 this
is then convolved
with the PSF at that point.
552 If exposureIdInfo
is not provided then the SourceCatalog IDs will
not be globally unique.
555 wcs = exposure.getWcs()
557 if photoCalib
is None:
558 photoCalib = exposure.getPhotoCalib()
560 if exposureIdInfo
is None:
561 exposureIdInfo = ExposureIdInfo()
563 band = exposure.getFilterLabel().bandLabel
564 ccdVisitMagnitudes = self.addVariablity(fakeCat, band, exposure, photoCalib, exposureIdInfo)
566 self.insertFakes.run(fakeCat, exposure, wcs, photoCalib)
569 returnedStruct = self.calibrate.run(exposure, exposureIdInfo=exposureIdInfo)
570 sourceCat = returnedStruct.sourceCat
572 sourceCat = self.copyCalibrationFields(sfdSourceCat, sourceCat, self.config.srcFieldsToCopy)
574 resultStruct = pipeBase.Struct(outputExposure=exposure,
576 ccdVisitFakeMagnitudes=ccdVisitMagnitudes)
579 def addVariablity(self, fakeCat, band, exposure, photoCalib, exposureIdInfo):
580 """Add scatter to the fake catalog visit magnitudes.
582 Currently just adds a simple Gaussian scatter around the static fake
583 magnitude. This function could be modified to return any number of
588 fakeCat : `pandas.DataFrame`
589 Catalog of fakes to modify magnitudes of.
591 Current observing band to modify.
592 exposure : `lsst.afw.image.ExposureF`
593 Exposure fakes will be added to.
595 Photometric calibration object of ``exposure``.
596 exposureIdInfo : `lsst.obs.base.ExposureIdInfo`
597 Exposure id information
and metadata.
601 dataFrame : `pandas.DataFrame`
602 DataFrame containing the values of the magnitudes to that will
603 be inserted into this ccdVisit.
605 expId = exposureIdInfo.expId
606 rng = np.random.default_rng(expId)
607 magScatter = rng.normal(loc=0,
608 scale=self.config.scatterSize,
610 visitMagnitudes = fakeCat[self.insertFakes.config.mag_col % band] + magScatter
611 fakeCat.loc[:, self.insertFakes.config.mag_col % band] = visitMagnitudes
612 return pd.DataFrame(data={
"variableMag": visitMagnitudes})