23 Insert fake sources into calexps
25 from astropy.table
import Table
27 import lsst.pex.config
as pexConfig
31 from .insertFakes
import InsertFakesTask
33 from lsst.meas.base import (SingleFrameMeasurementTask, ApplyApCorrTask, CatalogCalculationTask,
34 PerTractCcdDataIdContainer)
38 from lsst.pipe.base import PipelineTask, PipelineTaskConfig, CmdLineTask, PipelineTaskConnections
43 __all__ = [
"ProcessCcdWithFakesConfig",
"ProcessCcdWithFakesTask"]
47 defaultTemplates={
"CoaddName":
"deep"}):
50 doc=
"Exposure into which fakes are to be added.",
52 storageClass=
"ExposureF",
53 dimensions=(
"instrument",
"visit",
"detector")
57 doc=
"Catalog of fake sources to draw inputs from.",
58 name=
"{CoaddName}Coadd_fakeSourceCat",
59 storageClass=
"Parquet",
60 dimensions=(
"tract",
"skymap")
64 doc=
"WCS information for the input exposure.",
67 dimensions=(
"Tract",
"SkyMap",
"Instrument",
"Visit",
"Detector")
70 photoCalib = cT.Input(
71 doc=
"Calib information for the input exposure.",
72 name=
"jointcal_photoCalib",
73 storageClass=
"PhotoCalib",
74 dimensions=(
"Tract",
"SkyMap",
"Instrument",
"Visit",
"Detector")
77 outputExposure = cT.Output(
78 doc=
"Exposure with fake sources added.",
80 storageClass=
"ExposureF",
81 dimensions=(
"instrument",
"visit",
"detector")
84 outputCat = cT.Output(
85 doc=
"Source catalog produced in calibrate task with fakes also measured.",
87 storageClass=
"SourceCatalog",
88 dimensions=(
"instrument",
"visit",
"detector"),
92 class ProcessCcdWithFakesConfig(PipelineTaskConfig,
93 pipelineConnections=ProcessCcdWithFakesConnections):
94 """Config for inserting fake sources
98 The default column names are those from the UW sims database.
101 useUpdatedCalibs = pexConfig.Field(
102 doc=
"Use updated calibs and wcs from jointcal?",
107 coaddName = pexConfig.Field(
108 doc=
"The name of the type of coadd used",
113 calibrationFieldsToCopy = pexConfig.ListField(
115 default=(
"calib_psf_candidate",
"calib_psf_used",
"calib_psf_reserved"),
116 doc=(
"Fields to copy from the icSource catalog to the output catalog "
117 "for matching sources Any missing fields will trigger a "
118 "RuntimeError exception.")
121 matchRadiusPix = pexConfig.Field(
124 doc=(
"Match radius for matching icSourceCat objects to sourceCat objects (pixels)"),
127 insertFakes = pexConfig.ConfigurableField(target=InsertFakesTask,
128 doc=
"Configuration for the fake sources")
130 detection = pexConfig.ConfigurableField(target=SourceDetectionTask,
131 doc=
"The detection task to use.")
133 deblend = pexConfig.ConfigurableField(target=SourceDeblendTask, doc=
"The deblending task to use.")
135 measurement = pexConfig.ConfigurableField(target=SingleFrameMeasurementTask,
136 doc=
"The measurement task to use")
138 applyApCorr = pexConfig.ConfigurableField(target=ApplyApCorrTask,
139 doc=
"The apply aperture correction task to use.")
141 catalogCalculation = pexConfig.ConfigurableField(target=CatalogCalculationTask,
142 doc=
"The catalog calculation task to use.")
144 def setDefaults(self):
145 self.detection.reEstimateBackground =
False
146 super().setDefaults()
147 self.measurement.plugins[
"base_PixelFlags"].masksFpAnywhere.append(
"FAKE")
148 self.measurement.plugins[
"base_PixelFlags"].masksFpCenter.append(
"FAKE")
151 class ProcessCcdWithFakesTask(PipelineTask, CmdLineTask):
152 """Insert fake objects into calexps.
154 Add fake stars and galaxies to the given calexp, specified in the dataRef. Galaxy parameters are read in
155 from the specified file and then modelled using galsim. Re-runs characterize image and calibrate image to
156 give a new background estimation and measurement of the calexp.
158 `ProcessFakeSourcesTask` inherits six functions from insertFakesTask that make images of the fake
159 sources and then add them to the calexp.
162 Use the WCS information to add the pixel coordinates of each source
163 Adds an ``x`` and ``y`` column to the catalog of fake sources.
165 Trim the fake cat to about the size of the input image.
166 `mkFakeGalsimGalaxies`
167 Use Galsim to make fake double sersic galaxies for each set of galaxy parameters in the input file.
169 Use the PSF information from the calexp to make a fake star using the magnitude information from the
172 Remove rows of the input fake catalog which have half light radius, of either the bulge or the disk,
175 Add the fake sources to the calexp.
179 The ``calexp`` with fake souces added to it is written out as the datatype ``calexp_fakes``.
182 _DefaultName =
"processCcdWithFakes"
183 ConfigClass = ProcessCcdWithFakesConfig
185 def __init__(self, schema=None, butler=None, **kwargs):
186 """Initalize things! This should go above in the class docstring
189 super().__init__(**kwargs)
192 schema = SourceTable.makeMinimalSchema()
194 self.makeSubtask(
"insertFakes")
195 self.algMetadata = dafBase.PropertyList()
196 self.makeSubtask(
"detection", schema=self.schema)
197 self.makeSubtask(
"deblend", schema=self.schema)
198 self.makeSubtask(
"measurement", schema=self.schema, algMetadata=self.algMetadata)
199 self.makeSubtask(
"applyApCorr", schema=self.schema)
200 self.makeSubtask(
"catalogCalculation", schema=self.schema)
202 def runDataRef(self, dataRef):
203 """Read in/write out the required data products and add fake sources to the calexp.
207 dataRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef`
208 Data reference defining the ccd to have fakes added to it.
209 Used to access the following data products:
216 Uses the calibration and WCS information attached to the calexp for the posistioning and calibration
217 of the sources unless the config option config.useUpdatedCalibs is set then it uses the
218 meas_mosaic/jointCal outputs. The config defualts for the column names in the catalog of fakes are
219 taken from the University of Washington simulations database. Operates on one ccd at a time.
221 exposureIdInfo = dataRef.get(
"expIdInfo")
223 if self.config.insertFakes.fakeType ==
"snapshot":
224 fakeCat = dataRef.get(
"fakeSourceCat").toDataFrame()
225 elif self.config.insertFakes.fakeType ==
"static":
226 fakeCat = dataRef.get(
"deepCoadd_fakeSourceCat").toDataFrame()
228 fakeCat = Table.read(self.config.insertFakes.fakeType).to_pandas()
230 calexp = dataRef.get(
"calexp")
231 if self.config.useUpdatedCalibs:
232 self.log.info(
"Using updated calibs from meas_mosaic/jointCal")
233 wcs = dataRef.get(
"jointcal_wcs")
234 photoCalib = dataRef.get(
"jointcal_photoCalib")
236 wcs = calexp.getWcs()
237 photoCalib = calexp.getPhotoCalib()
239 icSourceCat = dataRef.get(
"icSrc", immediate=
True)
241 resultStruct = self.run(fakeCat, calexp, wcs=wcs, photoCalib=photoCalib,
242 exposureIdInfo=exposureIdInfo, icSourceCat=icSourceCat)
244 dataRef.put(resultStruct.outputExposure,
"fakes_calexp")
245 dataRef.put(resultStruct.outputCat,
"fakes_src")
247 def runQuantum(self, butlerQC, inputRefs, outputRefs):
248 inputs = butlerQC.get(inputRefs)
249 if 'exposureIdInfo' not in inputs.keys():
250 expId, expBits = butlerQC.quantum.dataId.pack(
"visit_detector", returnMaxBits=
True)
251 inputs[
'exposureIdInfo'] = ExposureIdInfo(expId, expBits)
253 if inputs[
"wcs"]
is None:
254 inputs[
"wcs"] = inputs[
"image"].getWcs()
255 if inputs[
"photoCalib"]
is None:
256 inputs[
"photoCalib"] = inputs[
"image"].getPhotoCalib()
258 outputs = self.run(**inputs)
259 butlerQC.put(outputs, outputRefs)
262 def _makeArgumentParser(cls):
263 parser = pipeBase.ArgumentParser(name=cls._DefaultName)
264 parser.add_id_argument(
"--id",
"fakes_calexp", help=
"data ID with raw CCD keys [+ tract optionally], "
265 "e.g. --id visit=12345 ccd=1,2 [tract=0]",
266 ContainerClass=PerTractCcdDataIdContainer)
269 def run(self, fakeCat, exposure, wcs=None, photoCalib=None, exposureIdInfo=None, icSourceCat=None):
270 """Add fake sources to a calexp and then run detection, deblending and measurement.
274 fakeCat : `pandas.core.frame.DataFrame`
275 The catalog of fake sources to add to the exposure
276 exposure : `lsst.afw.image.exposure.exposure.ExposureF`
277 The exposure to add the fake sources to
278 wcs : `lsst.afw.geom.SkyWcs`
279 WCS to use to add fake sources
280 photoCalib : `lsst.afw.image.photoCalib.PhotoCalib`
281 Photometric calibration to be used to calibrate the fake sources
282 exposureIdInfo : `lsst.obs.base.ExposureIdInfo`
286 resultStruct : `lsst.pipe.base.struct.Struct`
287 contains : outputExposure : `lsst.afw.image.exposure.exposure.ExposureF`
288 outputCat : `lsst.afw.table.source.source.SourceCatalog`
292 Adds pixel coordinates for each source to the fakeCat and removes objects with bulge or disk half
293 light radius = 0 (if ``config.cleanCat = True``). These columns are called ``x`` and ``y`` and are in
296 Adds the ``Fake`` mask plane to the exposure which is then set by `addFakeSources` to mark where fake
297 sources have been added. Uses the information in the ``fakeCat`` to make fake galaxies (using galsim)
298 and fake stars, using the PSF models from the PSF information for the calexp. These are then added to
299 the calexp and the calexp with fakes included returned.
301 The galsim galaxies are made using a double sersic profile, one for the bulge and one for the disk,
302 this is then convolved with the PSF at that point.
304 If exposureIdInfo is not provided then the SourceCatalog IDs will not be globally unique.
308 wcs = exposure.getWcs()
310 if photoCalib
is None:
311 photoCalib = exposure.getPhotoCalib()
313 self.insertFakes.
run(fakeCat, exposure, wcs, photoCalib)
316 if exposureIdInfo
is None:
317 exposureIdInfo = ExposureIdInfo()
319 sourceIdFactory = IdFactory.makeSource(exposureIdInfo.expId, exposureIdInfo.unusedBits)
320 table = SourceTable.make(self.schema, sourceIdFactory)
321 table.setMetadata(self.algMetadata)
323 detRes = self.detection.
run(table=table, exposure=exposure, doSmooth=
True)
324 sourceCat = detRes.sources
325 self.deblend.
run(exposure=exposure, sources=sourceCat)
326 self.measurement.
run(measCat=sourceCat, exposure=exposure, exposureId=exposureIdInfo.expId)
327 self.applyApCorr.
run(catalog=sourceCat, apCorrMap=exposure.getInfo().getApCorrMap())
328 self.catalogCalculation.
run(sourceCat)
329 sourceCat = self.copyCalibrationFields(icSourceCat, sourceCat)
331 resultStruct = pipeBase.Struct(outputExposure=exposure, outputCat=sourceCat)
334 def copyCalibrationFields(self, calibCat, sourceCat):
335 """Match sources in calibCat and sourceCat and copy the specified fields
339 calibCat : `lsst.afw.table.SourceCatalog`
340 Catalog from which to copy fields.
341 sourceCat : `lsst.afw.table.SourceCatalog`
342 Catalog to which to copy fields.
346 newCat : `lsst.afw.table.SourceCatalog`
347 Catalog which includes the copied fields.
349 The fields copied are those specified by `config.calibrationFieldsToCopy` that actually exist
350 in the schema of `calibCat`.
352 This version was based on and adapted from the one in calibrateTask.
356 sourceSchemaMapper = afwTable.SchemaMapper(sourceCat.schema)
357 sourceSchemaMapper.addMinimalSchema(sourceCat.schema,
True)
359 calibSchemaMapper = afwTable.SchemaMapper(calibCat.schema, sourceCat.schema)
362 missingFieldNames = []
363 for fieldName
in self.config.calibrationFieldsToCopy:
364 if fieldName
in calibCat.schema:
365 schemaItem = calibCat.schema.find(fieldName)
366 calibSchemaMapper.editOutputSchema().addField(schemaItem.getField())
367 schema = calibSchemaMapper.editOutputSchema()
368 calibSchemaMapper.addMapping(schemaItem.getKey(), schema.find(fieldName).getField())
370 missingFieldNames.append(fieldName)
371 if missingFieldNames:
372 raise RuntimeError(f
"calibCat is missing fields {missingFieldNames} specified in "
373 "calibrationFieldsToCopy")
375 self.calibSourceKey = calibSchemaMapper.addOutputField(afwTable.Field[
"Flag"](
"calib_detected",
376 "Source was detected as an icSource"))
378 schema = calibSchemaMapper.getOutputSchema()
379 newCat = afwTable.SourceCatalog(schema)
380 newCat.reserve(len(sourceCat))
381 newCat.extend(sourceCat, sourceSchemaMapper)
384 for k, v
in sourceCat.schema.getAliasMap().items():
385 newCat.schema.getAliasMap().set(k, v)
387 select = newCat[
"deblend_nChild"] == 0
388 matches = afwTable.matchXy(newCat[select], calibCat, self.config.matchRadiusPix)
392 numMatches = len(matches)
393 numUniqueSources = len(set(m[1].getId()
for m
in matches))
394 if numUniqueSources != numMatches:
395 self.log.warn(
"%d calibCat sources matched only %d sourceCat sources", numMatches,
398 self.log.info(
"Copying flags from calibCat to sourceCat for %s sources", numMatches)
402 for src, calibSrc, d
in matches:
403 src.setFlag(self.calibSourceKey,
True)
408 calibSrcFootprint = calibSrc.getFootprint()
410 calibSrc.setFootprint(src.getFootprint())
411 src.assign(calibSrc, calibSchemaMapper)
413 calibSrc.setFootprint(calibSrcFootprint)