23 Insert fake sources into calexps
25 from astropy.table
import Table
27 import lsst.pex.config
as pexConfig
31 from .insertFakes
import InsertFakesTask
33 from lsst.meas.base import (SingleFrameMeasurementTask, ApplyApCorrTask, CatalogCalculationTask,
34 PerTractCcdDataIdContainer)
38 from lsst.pipe.base import PipelineTask, PipelineTaskConfig, CmdLineTask, PipelineTaskConnections
43 __all__ = [
"ProcessCcdWithFakesConfig",
"ProcessCcdWithFakesTask"]
47 defaultTemplates={
"CoaddName":
"deep"}):
50 doc=
"Exposure into which fakes are to be added.",
52 storageClass=
"ExposureF",
53 dimensions=(
"instrument",
"visit",
"detector")
57 doc=
"Catalog of fake sources to draw inputs from.",
58 name=
"{CoaddName}Coadd_fakeSourceCat",
59 storageClass=
"Parquet",
60 dimensions=(
"tract",
"skymap")
64 doc=
"WCS information for the input exposure.",
67 dimensions=(
"Tract",
"SkyMap",
"Instrument",
"Visit",
"Detector")
70 photoCalib = cT.Input(
71 doc=
"Calib information for the input exposure.",
72 name=
"jointcal_photoCalib",
73 storageClass=
"PhotoCalib",
74 dimensions=(
"Tract",
"SkyMap",
"Instrument",
"Visit",
"Detector")
77 icSourceCat = cT.Input(
78 doc=
"Catalog of calibration sources",
80 storageClass=
"sourceCatalog",
81 dimensions=(
"tract",
"skymap",
"instrument",
"visit",
"detector")
84 sfdSourceCat = cT.Input(
85 doc=
"Catalog of calibration sources",
87 storageClass=
"sourceCatalog",
88 dimensions=(
"tract",
"skymap",
"instrument",
"visit",
"detector")
91 outputExposure = cT.Output(
92 doc=
"Exposure with fake sources added.",
94 storageClass=
"ExposureF",
95 dimensions=(
"instrument",
"visit",
"detector")
98 outputCat = cT.Output(
99 doc=
"Source catalog produced in calibrate task with fakes also measured.",
101 storageClass=
"SourceCatalog",
102 dimensions=(
"instrument",
"visit",
"detector"),
106 class ProcessCcdWithFakesConfig(PipelineTaskConfig,
107 pipelineConnections=ProcessCcdWithFakesConnections):
108 """Config for inserting fake sources
112 The default column names are those from the UW sims database.
115 useUpdatedCalibs = pexConfig.Field(
116 doc=
"Use updated calibs and wcs from jointcal?",
121 coaddName = pexConfig.Field(
122 doc=
"The name of the type of coadd used",
127 calibrationFieldsToCopy = pexConfig.ListField(
129 default=(
"calib_psf_candidate",
"calib_psf_used",
"calib_psf_reserved"),
130 doc=(
"Fields to copy from the icSource catalog to the output catalog "
131 "for matching sources Any missing fields will trigger a "
132 "RuntimeError exception.")
135 srcFieldsToCopy = pexConfig.ListField(
137 default=(
"calib_photometry_reserved",
"calib_photometry_used",
"calib_astrometry_used"),
138 doc=(
"Fields to copy from the `src` catalog to the output catalog "
139 "for matching sources Any missing fields will trigger a "
140 "RuntimeError exception.")
143 matchRadiusPix = pexConfig.Field(
146 doc=(
"Match radius for matching icSourceCat objects to sourceCat objects (pixels)"),
149 insertFakes = pexConfig.ConfigurableField(target=InsertFakesTask,
150 doc=
"Configuration for the fake sources")
152 detection = pexConfig.ConfigurableField(target=SourceDetectionTask,
153 doc=
"The detection task to use.")
155 deblend = pexConfig.ConfigurableField(target=SourceDeblendTask, doc=
"The deblending task to use.")
157 measurement = pexConfig.ConfigurableField(target=SingleFrameMeasurementTask,
158 doc=
"The measurement task to use")
160 applyApCorr = pexConfig.ConfigurableField(target=ApplyApCorrTask,
161 doc=
"The apply aperture correction task to use.")
163 catalogCalculation = pexConfig.ConfigurableField(target=CatalogCalculationTask,
164 doc=
"The catalog calculation task to use.")
166 def setDefaults(self):
167 self.detection.reEstimateBackground =
False
168 super().setDefaults()
169 self.measurement.plugins[
"base_PixelFlags"].masksFpAnywhere.append(
"FAKE")
170 self.measurement.plugins[
"base_PixelFlags"].masksFpCenter.append(
"FAKE")
171 self.deblend.maxFootprintSize = 2000
172 self.measurement.plugins.names |= [
"base_LocalPhotoCalib",
"base_LocalWcs"]
175 class ProcessCcdWithFakesTask(PipelineTask, CmdLineTask):
176 """Insert fake objects into calexps.
178 Add fake stars and galaxies to the given calexp, specified in the dataRef. Galaxy parameters are read in
179 from the specified file and then modelled using galsim. Re-runs characterize image and calibrate image to
180 give a new background estimation and measurement of the calexp.
182 `ProcessFakeSourcesTask` inherits six functions from insertFakesTask that make images of the fake
183 sources and then add them to the calexp.
186 Use the WCS information to add the pixel coordinates of each source
187 Adds an ``x`` and ``y`` column to the catalog of fake sources.
189 Trim the fake cat to about the size of the input image.
190 `mkFakeGalsimGalaxies`
191 Use Galsim to make fake double sersic galaxies for each set of galaxy parameters in the input file.
193 Use the PSF information from the calexp to make a fake star using the magnitude information from the
196 Remove rows of the input fake catalog which have half light radius, of either the bulge or the disk,
199 Add the fake sources to the calexp.
203 The ``calexp`` with fake souces added to it is written out as the datatype ``calexp_fakes``.
206 _DefaultName =
"processCcdWithFakes"
207 ConfigClass = ProcessCcdWithFakesConfig
209 def __init__(self, schema=None, butler=None, **kwargs):
210 """Initalize things! This should go above in the class docstring
213 super().__init__(**kwargs)
216 schema = SourceTable.makeMinimalSchema()
218 self.makeSubtask(
"insertFakes")
219 self.algMetadata = dafBase.PropertyList()
220 self.makeSubtask(
"detection", schema=self.schema)
221 self.makeSubtask(
"deblend", schema=self.schema)
222 self.makeSubtask(
"measurement", schema=self.schema, algMetadata=self.algMetadata)
223 self.makeSubtask(
"applyApCorr", schema=self.schema)
224 self.makeSubtask(
"catalogCalculation", schema=self.schema)
226 def runDataRef(self, dataRef):
227 """Read in/write out the required data products and add fake sources to the calexp.
231 dataRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef`
232 Data reference defining the ccd to have fakes added to it.
233 Used to access the following data products:
240 Uses the calibration and WCS information attached to the calexp for the posistioning and calibration
241 of the sources unless the config option config.useUpdatedCalibs is set then it uses the
242 meas_mosaic/jointCal outputs. The config defualts for the column names in the catalog of fakes are
243 taken from the University of Washington simulations database. Operates on one ccd at a time.
245 exposureIdInfo = dataRef.get(
"expIdInfo")
247 if self.config.insertFakes.fakeType ==
"snapshot":
248 fakeCat = dataRef.get(
"fakeSourceCat").toDataFrame()
249 elif self.config.insertFakes.fakeType ==
"static":
250 fakeCat = dataRef.get(
"deepCoadd_fakeSourceCat").toDataFrame()
252 fakeCat = Table.read(self.config.insertFakes.fakeType).to_pandas()
254 calexp = dataRef.get(
"calexp")
255 if self.config.useUpdatedCalibs:
256 self.log.info(
"Using updated calibs from meas_mosaic/jointCal")
257 wcs = dataRef.get(
"jointcal_wcs")
258 photoCalib = dataRef.get(
"jointcal_photoCalib")
260 wcs = calexp.getWcs()
261 photoCalib = calexp.getPhotoCalib()
263 icSourceCat = dataRef.get(
"icSrc", immediate=
True)
264 sfdSourceCat = dataRef.get(
"src", immediate=
True)
266 resultStruct = self.run(fakeCat, calexp, wcs=wcs, photoCalib=photoCalib,
267 exposureIdInfo=exposureIdInfo, icSourceCat=icSourceCat,
268 sfdSourceCat=sfdSourceCat)
270 dataRef.put(resultStruct.outputExposure,
"fakes_calexp")
271 dataRef.put(resultStruct.outputCat,
"fakes_src")
274 def runQuantum(self, butlerQC, inputRefs, outputRefs):
275 inputs = butlerQC.get(inputRefs)
276 if 'exposureIdInfo' not in inputs.keys():
277 expId, expBits = butlerQC.quantum.dataId.pack(
"visit_detector", returnMaxBits=
True)
278 inputs[
'exposureIdInfo'] = ExposureIdInfo(expId, expBits)
280 if inputs[
"wcs"]
is None:
281 inputs[
"wcs"] = inputs[
"image"].getWcs()
282 if inputs[
"photoCalib"]
is None:
283 inputs[
"photoCalib"] = inputs[
"image"].getPhotoCalib()
285 outputs = self.run(**inputs)
286 butlerQC.put(outputs, outputRefs)
289 def _makeArgumentParser(cls):
290 parser = pipeBase.ArgumentParser(name=cls._DefaultName)
291 parser.add_id_argument(
"--id",
"fakes_calexp", help=
"data ID with raw CCD keys [+ tract optionally], "
292 "e.g. --id visit=12345 ccd=1,2 [tract=0]",
293 ContainerClass=PerTractCcdDataIdContainer)
296 def run(self, fakeCat, exposure, wcs=None, photoCalib=None, exposureIdInfo=None, icSourceCat=None,
298 """Add fake sources to a calexp and then run detection, deblending and measurement.
302 fakeCat : `pandas.core.frame.DataFrame`
303 The catalog of fake sources to add to the exposure
304 exposure : `lsst.afw.image.exposure.exposure.ExposureF`
305 The exposure to add the fake sources to
306 wcs : `lsst.afw.geom.SkyWcs`
307 WCS to use to add fake sources
308 photoCalib : `lsst.afw.image.photoCalib.PhotoCalib`
309 Photometric calibration to be used to calibrate the fake sources
310 exposureIdInfo : `lsst.obs.base.ExposureIdInfo`
311 icSourceCat : `lsst.afw.table.SourceCatalog`
313 Catalog to take the information about which sources were used for calibration from.
314 sfdSourceCat : `lsst.afw.table.SourceCatalog`
316 Catalog produced by singleFrameDriver, needed to copy some calibration flags from.
320 resultStruct : `lsst.pipe.base.struct.Struct`
321 contains : outputExposure : `lsst.afw.image.exposure.exposure.ExposureF`
322 outputCat : `lsst.afw.table.source.source.SourceCatalog`
326 Adds pixel coordinates for each source to the fakeCat and removes objects with bulge or disk half
327 light radius = 0 (if ``config.cleanCat = True``). These columns are called ``x`` and ``y`` and are in
330 Adds the ``Fake`` mask plane to the exposure which is then set by `addFakeSources` to mark where fake
331 sources have been added. Uses the information in the ``fakeCat`` to make fake galaxies (using galsim)
332 and fake stars, using the PSF models from the PSF information for the calexp. These are then added to
333 the calexp and the calexp with fakes included returned.
335 The galsim galaxies are made using a double sersic profile, one for the bulge and one for the disk,
336 this is then convolved with the PSF at that point.
338 If exposureIdInfo is not provided then the SourceCatalog IDs will not be globally unique.
342 wcs = exposure.getWcs()
344 if photoCalib
is None:
345 photoCalib = exposure.getPhotoCalib()
347 self.insertFakes.
run(fakeCat, exposure, wcs, photoCalib)
350 if exposureIdInfo
is None:
351 exposureIdInfo = ExposureIdInfo()
353 sourceIdFactory = IdFactory.makeSource(exposureIdInfo.expId, exposureIdInfo.unusedBits)
354 table = SourceTable.make(self.schema, sourceIdFactory)
355 table.setMetadata(self.algMetadata)
357 detRes = self.detection.
run(table=table, exposure=exposure, doSmooth=
True)
358 sourceCat = detRes.sources
359 self.deblend.
run(exposure=exposure, sources=sourceCat)
360 self.measurement.
run(measCat=sourceCat, exposure=exposure, exposureId=exposureIdInfo.expId)
361 self.applyApCorr.
run(catalog=sourceCat, apCorrMap=exposure.getInfo().getApCorrMap())
362 self.catalogCalculation.
run(sourceCat)
363 sourceCat = self.copyCalibrationFields(icSourceCat, sourceCat, self.config.calibrationFieldsToCopy)
364 sourceCat = self.copyCalibrationFields(sfdSourceCat, sourceCat, self.config.srcFieldsToCopy)
366 resultStruct = pipeBase.Struct(outputExposure=exposure, outputCat=sourceCat)
369 def copyCalibrationFields(self, calibCat, sourceCat, fieldsToCopy):
370 """Match sources in calibCat and sourceCat and copy the specified fields
374 calibCat : `lsst.afw.table.SourceCatalog`
375 Catalog from which to copy fields.
376 sourceCat : `lsst.afw.table.SourceCatalog`
377 Catalog to which to copy fields.
378 fieldsToCopy : `lsst.pex.config.listField.List`
379 Fields to copy from calibCat to SoourceCat.
383 newCat : `lsst.afw.table.SourceCatalog`
384 Catalog which includes the copied fields.
386 The fields copied are those specified by `fieldsToCopy` that actually exist
387 in the schema of `calibCat`.
389 This version was based on and adapted from the one in calibrateTask.
393 sourceSchemaMapper = afwTable.SchemaMapper(sourceCat.schema)
394 sourceSchemaMapper.addMinimalSchema(sourceCat.schema,
True)
396 calibSchemaMapper = afwTable.SchemaMapper(calibCat.schema, sourceCat.schema)
399 missingFieldNames = []
400 for fieldName
in fieldsToCopy:
401 if fieldName
in calibCat.schema:
402 schemaItem = calibCat.schema.find(fieldName)
403 calibSchemaMapper.editOutputSchema().addField(schemaItem.getField())
404 schema = calibSchemaMapper.editOutputSchema()
405 calibSchemaMapper.addMapping(schemaItem.getKey(), schema.find(fieldName).getField())
407 missingFieldNames.append(fieldName)
408 if missingFieldNames:
409 raise RuntimeError(f
"calibCat is missing fields {missingFieldNames} specified in "
412 if "calib_detected" not in calibSchemaMapper.getOutputSchema():
413 self.calibSourceKey = calibSchemaMapper.addOutputField(afwTable.Field[
"Flag"](
"calib_detected",
414 "Source was detected as an icSource"))
416 self.calibSourceKey =
None
418 schema = calibSchemaMapper.getOutputSchema()
419 newCat = afwTable.SourceCatalog(schema)
420 newCat.reserve(len(sourceCat))
421 newCat.extend(sourceCat, sourceSchemaMapper)
424 for k, v
in sourceCat.schema.getAliasMap().items():
425 newCat.schema.getAliasMap().set(k, v)
427 select = newCat[
"deblend_nChild"] == 0
428 matches = afwTable.matchXy(newCat[select], calibCat, self.config.matchRadiusPix)
432 numMatches = len(matches)
433 numUniqueSources = len(set(m[1].getId()
for m
in matches))
434 if numUniqueSources != numMatches:
435 self.log.warn(
"%d calibCat sources matched only %d sourceCat sources", numMatches,
438 self.log.info(
"Copying flags from calibCat to sourceCat for %s sources", numMatches)
442 for src, calibSrc, d
in matches:
443 if self.calibSourceKey:
444 src.setFlag(self.calibSourceKey,
True)
449 calibSrcFootprint = calibSrc.getFootprint()
451 calibSrc.setFootprint(src.getFootprint())
452 src.assign(calibSrc, calibSchemaMapper)
454 calibSrc.setFootprint(calibSrcFootprint)