lsst.pipe.tasks  20.0.0-28-g282f9e7e+feda6aebd8
processCcdWithFakes.py
Go to the documentation of this file.
1 # This file is part of pipe tasks
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (http://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <http://www.gnu.org/licenses/>.
21 
22 """
23 Insert fake sources into calexps
24 """
25 from astropy.table import Table
26 
27 import lsst.pex.config as pexConfig
28 import lsst.pipe.base as pipeBase
29 
30 from .insertFakes import InsertFakesTask
31 from lsst.meas.base import PerTractCcdDataIdContainer
32 from lsst.afw.table import SourceTable
33 from lsst.obs.base import ExposureIdInfo
34 from lsst.pipe.base import PipelineTask, PipelineTaskConfig, CmdLineTask, PipelineTaskConnections
36 import lsst.afw.table as afwTable
37 from lsst.pipe.tasks.calibrate import CalibrateTask
38 
39 __all__ = ["ProcessCcdWithFakesConfig", "ProcessCcdWithFakesTask"]
40 
41 
42 class ProcessCcdWithFakesConnections(PipelineTaskConnections,
43  dimensions=("skymap", "tract", "instrument", "visit", "detector"),
44  defaultTemplates={"CoaddName": "deep"}):
45 
46  exposure = cT.Input(
47  doc="Exposure into which fakes are to be added.",
48  name="calexp",
49  storageClass="ExposureF",
50  dimensions=("instrument", "visit", "detector")
51  )
52 
53  fakeCat = cT.Input(
54  doc="Catalog of fake sources to draw inputs from.",
55  name="{CoaddName}Coadd_fakeSourceCat",
56  storageClass="DataFrame",
57  dimensions=("tract", "skymap")
58  )
59 
60  wcs = cT.Input(
61  doc="WCS information for the input exposure.",
62  name="jointcal_wcs",
63  storageClass="Wcs",
64  dimensions=("tract", "skymap", "instrument", "visit", "detector")
65  )
66 
67  photoCalib = cT.Input(
68  doc="Calib information for the input exposure.",
69  name="jointcal_photoCalib",
70  storageClass="PhotoCalib",
71  dimensions=("tract", "skymap", "instrument", "visit", "detector")
72  )
73 
74  icSourceCat = cT.Input(
75  doc="Catalog of calibration sources",
76  name="icSrc",
77  storageClass="SourceCatalog",
78  dimensions=("instrument", "visit", "detector")
79  )
80 
81  sfdSourceCat = cT.Input(
82  doc="Catalog of calibration sources",
83  name="src",
84  storageClass="SourceCatalog",
85  dimensions=("instrument", "visit", "detector")
86  )
87 
88  outputExposure = cT.Output(
89  doc="Exposure with fake sources added.",
90  name="fakes_calexp",
91  storageClass="ExposureF",
92  dimensions=("instrument", "visit", "detector")
93  )
94 
95  outputCat = cT.Output(
96  doc="Source catalog produced in calibrate task with fakes also measured.",
97  name="fakes_src",
98  storageClass="SourceCatalog",
99  dimensions=("instrument", "visit", "detector"),
100  )
101 
102  def __init__(self, *, config=None):
103  super().__init__(config=config)
104 
105  if not config.useUpdatedCalibs:
106  self.inputs.remove("wcs")
107  self.inputs.remove("photoCalib")
108 
109 
110 class ProcessCcdWithFakesConfig(PipelineTaskConfig,
111  pipelineConnections=ProcessCcdWithFakesConnections):
112  """Config for inserting fake sources
113 
114  Notes
115  -----
116  The default column names are those from the UW sims database.
117  """
118 
119  useUpdatedCalibs = pexConfig.Field(
120  doc="Use updated calibs and wcs from jointcal?",
121  dtype=bool,
122  default=False,
123  )
124 
125  coaddName = pexConfig.Field(
126  doc="The name of the type of coadd used",
127  dtype=str,
128  default="deep",
129  )
130 
131  srcFieldsToCopy = pexConfig.ListField(
132  dtype=str,
133  default=("calib_photometry_reserved", "calib_photometry_used", "calib_astrometry_used",
134  "calib_psf_candidate", "calib_psf_used", "calib_psf_reserved"),
135  doc=("Fields to copy from the `src` catalog to the output catalog "
136  "for matching sources Any missing fields will trigger a "
137  "RuntimeError exception.")
138  )
139 
140  matchRadiusPix = pexConfig.Field(
141  dtype=float,
142  default=3,
143  doc=("Match radius for matching icSourceCat objects to sourceCat objects (pixels)"),
144  )
145 
146  calibrate = pexConfig.ConfigurableField(target=CalibrateTask,
147  doc="The calibration task to use.")
148 
149  insertFakes = pexConfig.ConfigurableField(target=InsertFakesTask,
150  doc="Configuration for the fake sources")
151 
152  def setDefaults(self):
153  super().setDefaults()
154  self.calibrate.measurement.plugins["base_PixelFlags"].masksFpAnywhere.append("FAKE")
155  self.calibrate.measurement.plugins["base_PixelFlags"].masksFpCenter.append("FAKE")
156  self.calibrate.doAstrometry = False
157  self.calibrate.doWriteMatches = False
158  self.calibrate.doPhotoCal = False
159  self.calibrate.detection.reEstimateBackground = False
160 
161 
162 class ProcessCcdWithFakesTask(PipelineTask, CmdLineTask):
163  """Insert fake objects into calexps.
164 
165  Add fake stars and galaxies to the given calexp, specified in the dataRef. Galaxy parameters are read in
166  from the specified file and then modelled using galsim. Re-runs characterize image and calibrate image to
167  give a new background estimation and measurement of the calexp.
168 
169  `ProcessFakeSourcesTask` inherits six functions from insertFakesTask that make images of the fake
170  sources and then add them to the calexp.
171 
172  `addPixCoords`
173  Use the WCS information to add the pixel coordinates of each source
174  Adds an ``x`` and ``y`` column to the catalog of fake sources.
175  `trimFakeCat`
176  Trim the fake cat to about the size of the input image.
177  `mkFakeGalsimGalaxies`
178  Use Galsim to make fake double sersic galaxies for each set of galaxy parameters in the input file.
179  `mkFakeStars`
180  Use the PSF information from the calexp to make a fake star using the magnitude information from the
181  input file.
182  `cleanCat`
183  Remove rows of the input fake catalog which have half light radius, of either the bulge or the disk,
184  that are 0.
185  `addFakeSources`
186  Add the fake sources to the calexp.
187 
188  Notes
189  -----
190  The ``calexp`` with fake souces added to it is written out as the datatype ``calexp_fakes``.
191  """
192 
193  _DefaultName = "processCcdWithFakes"
194  ConfigClass = ProcessCcdWithFakesConfig
195 
196  def __init__(self, schema=None, butler=None, **kwargs):
197  """Initalize things! This should go above in the class docstring
198  """
199 
200  super().__init__(**kwargs)
201 
202  if schema is None:
203  schema = SourceTable.makeMinimalSchema()
204  self.schema = schema
205  self.makeSubtask("insertFakes")
206  self.makeSubtask("calibrate")
207 
208  def runDataRef(self, dataRef):
209  """Read in/write out the required data products and add fake sources to the calexp.
210 
211  Parameters
212  ----------
213  dataRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef`
214  Data reference defining the ccd to have fakes added to it.
215  Used to access the following data products:
216  calexp
217  jointcal_wcs
218  jointcal_photoCalib
219 
220  Notes
221  -----
222  Uses the calibration and WCS information attached to the calexp for the posistioning and calibration
223  of the sources unless the config option config.useUpdatedCalibs is set then it uses the
224  meas_mosaic/jointCal outputs. The config defualts for the column names in the catalog of fakes are
225  taken from the University of Washington simulations database. Operates on one ccd at a time.
226  """
227  exposureIdInfo = dataRef.get("expIdInfo")
228 
229  if self.config.insertFakes.fakeType == "snapshot":
230  fakeCat = dataRef.get("fakeSourceCat").toDataFrame()
231  elif self.config.insertFakes.fakeType == "static":
232  fakeCat = dataRef.get("deepCoadd_fakeSourceCat").toDataFrame()
233  else:
234  fakeCat = Table.read(self.config.insertFakes.fakeType).to_pandas()
235 
236  calexp = dataRef.get("calexp")
237  if self.config.useUpdatedCalibs:
238  self.log.info("Using updated calibs from meas_mosaic/jointCal")
239  wcs = dataRef.get("jointcal_wcs")
240  photoCalib = dataRef.get("jointcal_photoCalib")
241  else:
242  wcs = calexp.getWcs()
243  photoCalib = calexp.getPhotoCalib()
244 
245  icSourceCat = dataRef.get("icSrc", immediate=True)
246  sfdSourceCat = dataRef.get("src", immediate=True)
247 
248  resultStruct = self.run(fakeCat, calexp, wcs=wcs, photoCalib=photoCalib,
249  exposureIdInfo=exposureIdInfo, icSourceCat=icSourceCat,
250  sfdSourceCat=sfdSourceCat)
251 
252  dataRef.put(resultStruct.outputExposure, "fakes_calexp")
253  dataRef.put(resultStruct.outputCat, "fakes_src")
254  return resultStruct
255 
256  def runQuantum(self, butlerQC, inputRefs, outputRefs):
257  inputs = butlerQC.get(inputRefs)
258  if 'exposureIdInfo' not in inputs.keys():
259  expId, expBits = butlerQC.quantum.dataId.pack("visit_detector", returnMaxBits=True)
260  inputs['exposureIdInfo'] = ExposureIdInfo(expId, expBits)
261 
262  if self.config.useUpdatedCalibs:
263  inputs["wcs"] = inputs["image"].getWcs()
264  inputs["photoCalib"] = inputs["image"].getPhotoCalib()
265 
266  outputs = self.run(**inputs)
267  butlerQC.put(outputs, outputRefs)
268 
269  @classmethod
270  def _makeArgumentParser(cls):
271  parser = pipeBase.ArgumentParser(name=cls._DefaultName)
272  parser.add_id_argument("--id", "fakes_calexp", help="data ID with raw CCD keys [+ tract optionally], "
273  "e.g. --id visit=12345 ccd=1,2 [tract=0]",
274  ContainerClass=PerTractCcdDataIdContainer)
275  return parser
276 
277  def run(self, fakeCat, exposure, wcs=None, photoCalib=None, exposureIdInfo=None, icSourceCat=None,
278  sfdSourceCat=None):
279  """Add fake sources to a calexp and then run detection, deblending and measurement.
280 
281  Parameters
282  ----------
283  fakeCat : `pandas.core.frame.DataFrame`
284  The catalog of fake sources to add to the exposure
285  exposure : `lsst.afw.image.exposure.exposure.ExposureF`
286  The exposure to add the fake sources to
287  wcs : `lsst.afw.geom.SkyWcs`
288  WCS to use to add fake sources
289  photoCalib : `lsst.afw.image.photoCalib.PhotoCalib`
290  Photometric calibration to be used to calibrate the fake sources
291  exposureIdInfo : `lsst.obs.base.ExposureIdInfo`
292  icSourceCat : `lsst.afw.table.SourceCatalog`
293  Default : None
294  Catalog to take the information about which sources were used for calibration from.
295  sfdSourceCat : `lsst.afw.table.SourceCatalog`
296  Default : None
297  Catalog produced by singleFrameDriver, needed to copy some calibration flags from.
298 
299  Returns
300  -------
301  resultStruct : `lsst.pipe.base.struct.Struct`
302  contains : outputExposure : `lsst.afw.image.exposure.exposure.ExposureF`
303  outputCat : `lsst.afw.table.source.source.SourceCatalog`
304 
305  Notes
306  -----
307  Adds pixel coordinates for each source to the fakeCat and removes objects with bulge or disk half
308  light radius = 0 (if ``config.cleanCat = True``). These columns are called ``x`` and ``y`` and are in
309  pixels.
310 
311  Adds the ``Fake`` mask plane to the exposure which is then set by `addFakeSources` to mark where fake
312  sources have been added. Uses the information in the ``fakeCat`` to make fake galaxies (using galsim)
313  and fake stars, using the PSF models from the PSF information for the calexp. These are then added to
314  the calexp and the calexp with fakes included returned.
315 
316  The galsim galaxies are made using a double sersic profile, one for the bulge and one for the disk,
317  this is then convolved with the PSF at that point.
318 
319  If exposureIdInfo is not provided then the SourceCatalog IDs will not be globally unique.
320  """
321 
322  if wcs is None:
323  wcs = exposure.getWcs()
324 
325  if photoCalib is None:
326  photoCalib = exposure.getPhotoCalib()
327 
328  self.insertFakes.run(fakeCat, exposure, wcs, photoCalib)
329 
330  # detect, deblend and measure sources
331  if exposureIdInfo is None:
332  exposureIdInfo = ExposureIdInfo()
333  returnedStruct = self.calibrate.run(exposure, exposureIdInfo=exposureIdInfo)
334  sourceCat = returnedStruct.sourceCat
335 
336  sourceCat = self.copyCalibrationFields(sfdSourceCat, sourceCat, self.config.srcFieldsToCopy)
337 
338  resultStruct = pipeBase.Struct(outputExposure=exposure, outputCat=sourceCat)
339  return resultStruct
340 
341  def copyCalibrationFields(self, calibCat, sourceCat, fieldsToCopy):
342  """Match sources in calibCat and sourceCat and copy the specified fields
343 
344  Parameters
345  ----------
346  calibCat : `lsst.afw.table.SourceCatalog`
347  Catalog from which to copy fields.
348  sourceCat : `lsst.afw.table.SourceCatalog`
349  Catalog to which to copy fields.
350  fieldsToCopy : `lsst.pex.config.listField.List`
351  Fields to copy from calibCat to SoourceCat.
352 
353  Returns
354  -------
355  newCat : `lsst.afw.table.SourceCatalog`
356  Catalog which includes the copied fields.
357 
358  The fields copied are those specified by `fieldsToCopy` that actually exist
359  in the schema of `calibCat`.
360 
361  This version was based on and adapted from the one in calibrateTask.
362  """
363 
364  # Make a new SourceCatalog with the data from sourceCat so that we can add the new columns to it
365  sourceSchemaMapper = afwTable.SchemaMapper(sourceCat.schema)
366  sourceSchemaMapper.addMinimalSchema(sourceCat.schema, True)
367 
368  calibSchemaMapper = afwTable.SchemaMapper(calibCat.schema, sourceCat.schema)
369 
370  # Add the desired columns from the option fieldsToCopy
371  missingFieldNames = []
372  for fieldName in fieldsToCopy:
373  if fieldName in calibCat.schema:
374  schemaItem = calibCat.schema.find(fieldName)
375  calibSchemaMapper.editOutputSchema().addField(schemaItem.getField())
376  schema = calibSchemaMapper.editOutputSchema()
377  calibSchemaMapper.addMapping(schemaItem.getKey(), schema.find(fieldName).getField())
378  else:
379  missingFieldNames.append(fieldName)
380  if missingFieldNames:
381  raise RuntimeError(f"calibCat is missing fields {missingFieldNames} specified in "
382  "fieldsToCopy")
383 
384  if "calib_detected" not in calibSchemaMapper.getOutputSchema():
385  self.calibSourceKey = calibSchemaMapper.addOutputField(afwTable.Field["Flag"]("calib_detected",
386  "Source was detected as an icSource"))
387  else:
388  self.calibSourceKey = None
389 
390  schema = calibSchemaMapper.getOutputSchema()
391  newCat = afwTable.SourceCatalog(schema)
392  newCat.reserve(len(sourceCat))
393  newCat.extend(sourceCat, sourceSchemaMapper)
394 
395  # Set the aliases so it doesn't complain.
396  for k, v in sourceCat.schema.getAliasMap().items():
397  newCat.schema.getAliasMap().set(k, v)
398 
399  select = newCat["deblend_nChild"] == 0
400  matches = afwTable.matchXy(newCat[select], calibCat, self.config.matchRadiusPix)
401  # Check that no sourceCat sources are listed twice (we already know
402  # that each match has a unique calibCat source ID, due to using
403  # that ID as the key in bestMatches)
404  numMatches = len(matches)
405  numUniqueSources = len(set(m[1].getId() for m in matches))
406  if numUniqueSources != numMatches:
407  self.log.warn("%d calibCat sources matched only %d sourceCat sources", numMatches,
408  numUniqueSources)
409 
410  self.log.info("Copying flags from calibCat to sourceCat for %s sources", numMatches)
411 
412  # For each match: set the calibSourceKey flag and copy the desired
413  # fields
414  for src, calibSrc, d in matches:
415  if self.calibSourceKey:
416  src.setFlag(self.calibSourceKey, True)
417  # src.assign copies the footprint from calibSrc, which we don't want
418  # (DM-407)
419  # so set calibSrc's footprint to src's footprint before src.assign,
420  # then restore it
421  calibSrcFootprint = calibSrc.getFootprint()
422  try:
423  calibSrc.setFootprint(src.getFootprint())
424  src.assign(calibSrc, calibSchemaMapper)
425  finally:
426  calibSrc.setFootprint(calibSrcFootprint)
427 
428  return newCat
lsst.pipe.tasks.assembleCoadd.run
def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, altMaskList=None, mask=None, supplementaryData=None)
Definition: assembleCoadd.py:720
lsst::meas::base
lsst::afw::table
lsst::pex::config
lsst.pipe.tasks.processCcdWithFakes.ProcessCcdWithFakesConnections
Definition: processCcdWithFakes.py:44
lsst.pipe.tasks.calibrate
Definition: calibrate.py:1
lsst.pipe::base
lsst.pipe::base::connectionTypes
lsst::obs::base