lsst.pipe.tasks  19.0.0-54-g1bde8684
processCcdWithFakes.py
Go to the documentation of this file.
1 # This file is part of pipe tasks
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (http://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <http://www.gnu.org/licenses/>.
21 
22 """
23 Insert fake sources into calexps
24 """
25 from astropy.table import Table
26 
27 import lsst.pex.config as pexConfig
28 import lsst.pipe.base as pipeBase
29 import lsst.daf.base as dafBase
30 
31 from .insertFakes import InsertFakesTask
32 from lsst.meas.algorithms import SourceDetectionTask
33 from lsst.meas.base import (SingleFrameMeasurementTask, ApplyApCorrTask, CatalogCalculationTask,
34  PerTractCcdDataIdContainer)
35 from lsst.meas.deblender import SourceDeblendTask
36 from lsst.afw.table import SourceTable, IdFactory
37 from lsst.obs.base import ExposureIdInfo
38 from lsst.pipe.base import PipelineTask, PipelineTaskConfig, CmdLineTask, PipelineTaskConnections
40 import lsst.afw.table as afwTable
41 
42 
43 __all__ = ["ProcessCcdWithFakesConfig", "ProcessCcdWithFakesTask"]
44 
45 
46 class ProcessCcdWithFakesConnections(PipelineTaskConnections, dimensions=("instrument", "visit", "detector"),
47  defaultTemplates={"CoaddName": "deep"}):
48 
49  exposure = cT.Input(
50  doc="Exposure into which fakes are to be added.",
51  name="calexp",
52  storageClass="ExposureF",
53  dimensions=("instrument", "visit", "detector")
54  )
55 
56  fakeCat = cT.Input(
57  doc="Catalog of fake sources to draw inputs from.",
58  name="{CoaddName}Coadd_fakeSourceCat",
59  storageClass="Parquet",
60  dimensions=("tract", "skymap")
61  )
62 
63  wcs = cT.Input(
64  doc="WCS information for the input exposure.",
65  name="jointcal_wcs",
66  storageClass="Wcs",
67  dimensions=("Tract", "SkyMap", "Instrument", "Visit", "Detector")
68  )
69 
70  photoCalib = cT.Input(
71  doc="Calib information for the input exposure.",
72  name="jointcal_photoCalib",
73  storageClass="PhotoCalib",
74  dimensions=("Tract", "SkyMap", "Instrument", "Visit", "Detector")
75  )
76 
77  outputExposure = cT.Output(
78  doc="Exposure with fake sources added.",
79  name="fakes_calexp",
80  storageClass="ExposureF",
81  dimensions=("instrument", "visit", "detector")
82  )
83 
84  outputCat = cT.Output(
85  doc="Source catalog produced in calibrate task with fakes also measured.",
86  name="fakes_src",
87  storageClass="SourceCatalog",
88  dimensions=("instrument", "visit", "detector"),
89  )
90 
91 
92 class ProcessCcdWithFakesConfig(PipelineTaskConfig,
93  pipelineConnections=ProcessCcdWithFakesConnections):
94  """Config for inserting fake sources
95 
96  Notes
97  -----
98  The default column names are those from the UW sims database.
99  """
100 
101  useUpdatedCalibs = pexConfig.Field(
102  doc="Use updated calibs and wcs from jointcal?",
103  dtype=bool,
104  default=False,
105  )
106 
107  coaddName = pexConfig.Field(
108  doc="The name of the type of coadd used",
109  dtype=str,
110  default="deep",
111  )
112 
113  calibrationFieldsToCopy = pexConfig.ListField(
114  dtype=str,
115  default=("calib_psf_candidate", "calib_psf_used", "calib_psf_reserved"),
116  doc=("Fields to copy from the icSource catalog to the output catalog "
117  "for matching sources Any missing fields will trigger a "
118  "RuntimeError exception.")
119  )
120 
121  matchRadiusPix = pexConfig.Field(
122  dtype=float,
123  default=3,
124  doc=("Match radius for matching icSourceCat objects to sourceCat objects (pixels)"),
125  )
126 
127  insertFakes = pexConfig.ConfigurableField(target=InsertFakesTask,
128  doc="Configuration for the fake sources")
129 
130  detection = pexConfig.ConfigurableField(target=SourceDetectionTask,
131  doc="The detection task to use.")
132 
133  deblend = pexConfig.ConfigurableField(target=SourceDeblendTask, doc="The deblending task to use.")
134 
135  measurement = pexConfig.ConfigurableField(target=SingleFrameMeasurementTask,
136  doc="The measurement task to use")
137 
138  applyApCorr = pexConfig.ConfigurableField(target=ApplyApCorrTask,
139  doc="The apply aperture correction task to use.")
140 
141  catalogCalculation = pexConfig.ConfigurableField(target=CatalogCalculationTask,
142  doc="The catalog calculation task to use.")
143 
144  def setDefaults(self):
145  self.detection.reEstimateBackground = False
146  super().setDefaults()
147  self.measurement.plugins["base_PixelFlags"].masksFpAnywhere.append("FAKE")
148  self.measurement.plugins["base_PixelFlags"].masksFpCenter.append("FAKE")
149 
150 
151 class ProcessCcdWithFakesTask(PipelineTask, CmdLineTask):
152  """Insert fake objects into calexps.
153 
154  Add fake stars and galaxies to the given calexp, specified in the dataRef. Galaxy parameters are read in
155  from the specified file and then modelled using galsim. Re-runs characterize image and calibrate image to
156  give a new background estimation and measurement of the calexp.
157 
158  `ProcessFakeSourcesTask` inherits six functions from insertFakesTask that make images of the fake
159  sources and then add them to the calexp.
160 
161  `addPixCoords`
162  Use the WCS information to add the pixel coordinates of each source
163  Adds an ``x`` and ``y`` column to the catalog of fake sources.
164  `trimFakeCat`
165  Trim the fake cat to about the size of the input image.
166  `mkFakeGalsimGalaxies`
167  Use Galsim to make fake double sersic galaxies for each set of galaxy parameters in the input file.
168  `mkFakeStars`
169  Use the PSF information from the calexp to make a fake star using the magnitude information from the
170  input file.
171  `cleanCat`
172  Remove rows of the input fake catalog which have half light radius, of either the bulge or the disk,
173  that are 0.
174  `addFakeSources`
175  Add the fake sources to the calexp.
176 
177  Notes
178  -----
179  The ``calexp`` with fake souces added to it is written out as the datatype ``calexp_fakes``.
180  """
181 
182  _DefaultName = "processCcdWithFakes"
183  ConfigClass = ProcessCcdWithFakesConfig
184 
185  def __init__(self, schema=None, butler=None, **kwargs):
186  """Initalize things! This should go above in the class docstring
187  """
188 
189  super().__init__(**kwargs)
190 
191  if schema is None:
192  schema = SourceTable.makeMinimalSchema()
193  self.schema = schema
194  self.makeSubtask("insertFakes")
195  self.algMetadata = dafBase.PropertyList()
196  self.makeSubtask("detection", schema=self.schema)
197  self.makeSubtask("deblend", schema=self.schema)
198  self.makeSubtask("measurement", schema=self.schema, algMetadata=self.algMetadata)
199  self.makeSubtask("applyApCorr", schema=self.schema)
200  self.makeSubtask("catalogCalculation", schema=self.schema)
201 
202  def runDataRef(self, dataRef):
203  """Read in/write out the required data products and add fake sources to the calexp.
204 
205  Parameters
206  ----------
207  dataRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef`
208  Data reference defining the ccd to have fakes added to it.
209  Used to access the following data products:
210  calexp
211  jointcal_wcs
212  jointcal_photoCalib
213 
214  Notes
215  -----
216  Uses the calibration and WCS information attached to the calexp for the posistioning and calibration
217  of the sources unless the config option config.useUpdatedCalibs is set then it uses the
218  meas_mosaic/jointCal outputs. The config defualts for the column names in the catalog of fakes are
219  taken from the University of Washington simulations database. Operates on one ccd at a time.
220  """
221  exposureIdInfo = dataRef.get("expIdInfo")
222 
223  if self.config.insertFakes.fakeType == "snapshot":
224  fakeCat = dataRef.get("fakeSourceCat").toDataFrame()
225  elif self.config.insertFakes.fakeType == "static":
226  fakeCat = dataRef.get("deepCoadd_fakeSourceCat").toDataFrame()
227  else:
228  fakeCat = Table.read(self.config.insertFakes.fakeType).to_pandas()
229 
230  calexp = dataRef.get("calexp")
231  if self.config.useUpdatedCalibs:
232  self.log.info("Using updated calibs from meas_mosaic/jointCal")
233  wcs = dataRef.get("jointcal_wcs")
234  photoCalib = dataRef.get("jointcal_photoCalib")
235  else:
236  wcs = calexp.getWcs()
237  photoCalib = calexp.getPhotoCalib()
238 
239  icSourceCat = dataRef.get("icSrc", immediate=True)
240 
241  resultStruct = self.run(fakeCat, calexp, wcs=wcs, photoCalib=photoCalib,
242  exposureIdInfo=exposureIdInfo, icSourceCat=icSourceCat)
243 
244  dataRef.put(resultStruct.outputExposure, "fakes_calexp")
245  dataRef.put(resultStruct.outputCat, "fakes_src")
246 
247  def runQuantum(self, butlerQC, inputRefs, outputRefs):
248  inputs = butlerQC.get(inputRefs)
249  if 'exposureIdInfo' not in inputs.keys():
250  expId, expBits = butlerQC.quantum.dataId.pack("visit_detector", returnMaxBits=True)
251  inputs['exposureIdInfo'] = ExposureIdInfo(expId, expBits)
252 
253  if inputs["wcs"] is None:
254  inputs["wcs"] = inputs["image"].getWcs()
255  if inputs["photoCalib"] is None:
256  inputs["photoCalib"] = inputs["image"].getPhotoCalib()
257 
258  outputs = self.run(**inputs)
259  butlerQC.put(outputs, outputRefs)
260 
261  @classmethod
262  def _makeArgumentParser(cls):
263  parser = pipeBase.ArgumentParser(name=cls._DefaultName)
264  parser.add_id_argument("--id", "fakes_calexp", help="data ID with raw CCD keys [+ tract optionally], "
265  "e.g. --id visit=12345 ccd=1,2 [tract=0]",
266  ContainerClass=PerTractCcdDataIdContainer)
267  return parser
268 
269  def run(self, fakeCat, exposure, wcs=None, photoCalib=None, exposureIdInfo=None, icSourceCat=None):
270  """Add fake sources to a calexp and then run detection, deblending and measurement.
271 
272  Parameters
273  ----------
274  fakeCat : `pandas.core.frame.DataFrame`
275  The catalog of fake sources to add to the exposure
276  exposure : `lsst.afw.image.exposure.exposure.ExposureF`
277  The exposure to add the fake sources to
278  wcs : `lsst.afw.geom.SkyWcs`
279  WCS to use to add fake sources
280  photoCalib : `lsst.afw.image.photoCalib.PhotoCalib`
281  Photometric calibration to be used to calibrate the fake sources
282  exposureIdInfo : `lsst.obs.base.ExposureIdInfo`
283 
284  Returns
285  -------
286  resultStruct : `lsst.pipe.base.struct.Struct`
287  contains : outputExposure : `lsst.afw.image.exposure.exposure.ExposureF`
288  outputCat : `lsst.afw.table.source.source.SourceCatalog`
289 
290  Notes
291  -----
292  Adds pixel coordinates for each source to the fakeCat and removes objects with bulge or disk half
293  light radius = 0 (if ``config.cleanCat = True``). These columns are called ``x`` and ``y`` and are in
294  pixels.
295 
296  Adds the ``Fake`` mask plane to the exposure which is then set by `addFakeSources` to mark where fake
297  sources have been added. Uses the information in the ``fakeCat`` to make fake galaxies (using galsim)
298  and fake stars, using the PSF models from the PSF information for the calexp. These are then added to
299  the calexp and the calexp with fakes included returned.
300 
301  The galsim galaxies are made using a double sersic profile, one for the bulge and one for the disk,
302  this is then convolved with the PSF at that point.
303 
304  If exposureIdInfo is not provided then the SourceCatalog IDs will not be globally unique.
305  """
306 
307  if wcs is None:
308  wcs = exposure.getWcs()
309 
310  if photoCalib is None:
311  photoCalib = exposure.getPhotoCalib()
312 
313  self.insertFakes.run(fakeCat, exposure, wcs, photoCalib)
314 
315  # detect, deblend and measure sources
316  if exposureIdInfo is None:
317  exposureIdInfo = ExposureIdInfo()
318 
319  sourceIdFactory = IdFactory.makeSource(exposureIdInfo.expId, exposureIdInfo.unusedBits)
320  table = SourceTable.make(self.schema, sourceIdFactory)
321  table.setMetadata(self.algMetadata)
322 
323  detRes = self.detection.run(table=table, exposure=exposure, doSmooth=True)
324  sourceCat = detRes.sources
325  self.deblend.run(exposure=exposure, sources=sourceCat)
326  self.measurement.run(measCat=sourceCat, exposure=exposure, exposureId=exposureIdInfo.expId)
327  self.applyApCorr.run(catalog=sourceCat, apCorrMap=exposure.getInfo().getApCorrMap())
328  self.catalogCalculation.run(sourceCat)
329  sourceCat = self.copyCalibrationFields(icSourceCat, sourceCat)
330 
331  resultStruct = pipeBase.Struct(outputExposure=exposure, outputCat=sourceCat)
332  return resultStruct
333 
334  def copyCalibrationFields(self, calibCat, sourceCat):
335  """Match sources in calibCat and sourceCat and copy the specified fields
336 
337  Parameters
338  ----------
339  calibCat : `lsst.afw.table.SourceCatalog`
340  Catalog from which to copy fields.
341  sourceCat : `lsst.afw.table.SourceCatalog`
342  Catalog to which to copy fields.
343 
344  Returns
345  -------
346  newCat : `lsst.afw.table.SourceCatalog`
347  Catalog which includes the copied fields.
348 
349  The fields copied are those specified by `config.calibrationFieldsToCopy` that actually exist
350  in the schema of `calibCat`.
351 
352  This version was based on and adapted from the one in calibrateTask.
353  """
354 
355  # Make a new SourceCatalog with the data from sourceCat so that we can add the new columns to it
356  sourceSchemaMapper = afwTable.SchemaMapper(sourceCat.schema)
357  sourceSchemaMapper.addMinimalSchema(sourceCat.schema, True)
358 
359  calibSchemaMapper = afwTable.SchemaMapper(calibCat.schema, sourceCat.schema)
360 
361  # Add the desired columns from the config option calibrationFieldsToCopy
362  missingFieldNames = []
363  for fieldName in self.config.calibrationFieldsToCopy:
364  if fieldName in calibCat.schema:
365  schemaItem = calibCat.schema.find(fieldName)
366  calibSchemaMapper.editOutputSchema().addField(schemaItem.getField())
367  schema = calibSchemaMapper.editOutputSchema()
368  calibSchemaMapper.addMapping(schemaItem.getKey(), schema.find(fieldName).getField())
369  else:
370  missingFieldNames.append(fieldName)
371  if missingFieldNames:
372  raise RuntimeError(f"calibCat is missing fields {missingFieldNames} specified in "
373  "calibrationFieldsToCopy")
374 
375  self.calibSourceKey = calibSchemaMapper.addOutputField(afwTable.Field["Flag"]("calib_detected",
376  "Source was detected as an icSource"))
377 
378  schema = calibSchemaMapper.getOutputSchema()
379  newCat = afwTable.SourceCatalog(schema)
380  newCat.reserve(len(sourceCat))
381  newCat.extend(sourceCat, sourceSchemaMapper)
382 
383  # Set the aliases so it doesn't complain.
384  for k, v in sourceCat.schema.getAliasMap().items():
385  newCat.schema.getAliasMap().set(k, v)
386 
387  select = newCat["deblend_nChild"] == 0
388  matches = afwTable.matchXy(newCat[select], calibCat, self.config.matchRadiusPix)
389  # Check that no sourceCat sources are listed twice (we already know
390  # that each match has a unique calibCat source ID, due to using
391  # that ID as the key in bestMatches)
392  numMatches = len(matches)
393  numUniqueSources = len(set(m[1].getId() for m in matches))
394  if numUniqueSources != numMatches:
395  self.log.warn("%d calibCat sources matched only %d sourceCat sources", numMatches,
396  numUniqueSources)
397 
398  self.log.info("Copying flags from calibCat to sourceCat for %s sources", numMatches)
399 
400  # For each match: set the calibSourceKey flag and copy the desired
401  # fields
402  for src, calibSrc, d in matches:
403  src.setFlag(self.calibSourceKey, True)
404  # src.assign copies the footprint from calibSrc, which we don't want
405  # (DM-407)
406  # so set calibSrc's footprint to src's footprint before src.assign,
407  # then restore it
408  calibSrcFootprint = calibSrc.getFootprint()
409  try:
410  calibSrc.setFootprint(src.getFootprint())
411  src.assign(calibSrc, calibSchemaMapper)
412  finally:
413  calibSrc.setFootprint(calibSrcFootprint)
414 
415  return newCat
lsst.pipe.tasks.assembleCoadd.run
def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, altMaskList=None, mask=None, supplementaryData=None)
Definition: assembleCoadd.py:712
lsst::meas::base
lsst::afw::table
lsst::meas::deblender
lsst.pipe.tasks.processCcdWithFakes.ProcessCcdWithFakesConnections
Definition: processCcdWithFakes.py:47
lsst::daf::base
lsst.pipe::base
lsst::meas::algorithms
lsst.pipe::base::connectionTypes
lsst::obs::base