lsst.pipe.tasks  20.0.0-14-g7cb2d310
processCcdWithFakes.py
Go to the documentation of this file.
1 # This file is part of pipe tasks
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (http://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <http://www.gnu.org/licenses/>.
21 
22 """
23 Insert fake sources into calexps
24 """
25 from astropy.table import Table
26 
27 import lsst.pex.config as pexConfig
28 import lsst.pipe.base as pipeBase
29 
30 from .insertFakes import InsertFakesTask
31 from lsst.meas.base import PerTractCcdDataIdContainer
32 from lsst.afw.table import SourceTable
33 from lsst.obs.base import ExposureIdInfo
34 from lsst.pipe.base import PipelineTask, PipelineTaskConfig, CmdLineTask, PipelineTaskConnections
36 import lsst.afw.table as afwTable
37 from lsst.pipe.tasks.calibrate import CalibrateTask
38 
39 __all__ = ["ProcessCcdWithFakesConfig", "ProcessCcdWithFakesTask"]
40 
41 
42 class ProcessCcdWithFakesConnections(PipelineTaskConnections, dimensions=("instrument", "visit", "detector"),
43  defaultTemplates={"CoaddName": "deep"}):
44 
45  exposure = cT.Input(
46  doc="Exposure into which fakes are to be added.",
47  name="calexp",
48  storageClass="ExposureF",
49  dimensions=("instrument", "visit", "detector")
50  )
51 
52  fakeCat = cT.Input(
53  doc="Catalog of fake sources to draw inputs from.",
54  name="{CoaddName}Coadd_fakeSourceCat",
55  storageClass="Parquet",
56  dimensions=("tract", "skymap")
57  )
58 
59  wcs = cT.Input(
60  doc="WCS information for the input exposure.",
61  name="jointcal_wcs",
62  storageClass="Wcs",
63  dimensions=("Tract", "SkyMap", "Instrument", "Visit", "Detector")
64  )
65 
66  photoCalib = cT.Input(
67  doc="Calib information for the input exposure.",
68  name="jointcal_photoCalib",
69  storageClass="PhotoCalib",
70  dimensions=("Tract", "SkyMap", "Instrument", "Visit", "Detector")
71  )
72 
73  icSourceCat = cT.Input(
74  doc="Catalog of calibration sources",
75  name="icSrc",
76  storageClass="sourceCatalog",
77  dimensions=("tract", "skymap", "instrument", "visit", "detector")
78  )
79 
80  sfdSourceCat = cT.Input(
81  doc="Catalog of calibration sources",
82  name="src",
83  storageClass="sourceCatalog",
84  dimensions=("tract", "skymap", "instrument", "visit", "detector")
85  )
86 
87  outputExposure = cT.Output(
88  doc="Exposure with fake sources added.",
89  name="fakes_calexp",
90  storageClass="ExposureF",
91  dimensions=("instrument", "visit", "detector")
92  )
93 
94  outputCat = cT.Output(
95  doc="Source catalog produced in calibrate task with fakes also measured.",
96  name="fakes_src",
97  storageClass="SourceCatalog",
98  dimensions=("instrument", "visit", "detector"),
99  )
100 
101 
102 class ProcessCcdWithFakesConfig(PipelineTaskConfig,
103  pipelineConnections=ProcessCcdWithFakesConnections):
104  """Config for inserting fake sources
105 
106  Notes
107  -----
108  The default column names are those from the UW sims database.
109  """
110 
111  useUpdatedCalibs = pexConfig.Field(
112  doc="Use updated calibs and wcs from jointcal?",
113  dtype=bool,
114  default=False,
115  )
116 
117  coaddName = pexConfig.Field(
118  doc="The name of the type of coadd used",
119  dtype=str,
120  default="deep",
121  )
122 
123  srcFieldsToCopy = pexConfig.ListField(
124  dtype=str,
125  default=("calib_photometry_reserved", "calib_photometry_used", "calib_astrometry_used",
126  "calib_psf_candidate", "calib_psf_used", "calib_psf_reserved"),
127  doc=("Fields to copy from the `src` catalog to the output catalog "
128  "for matching sources Any missing fields will trigger a "
129  "RuntimeError exception.")
130  )
131 
132  matchRadiusPix = pexConfig.Field(
133  dtype=float,
134  default=3,
135  doc=("Match radius for matching icSourceCat objects to sourceCat objects (pixels)"),
136  )
137 
138  calibrate = pexConfig.ConfigurableField(target=CalibrateTask,
139  doc="The calibration task to use.")
140 
141  insertFakes = pexConfig.ConfigurableField(target=InsertFakesTask,
142  doc="Configuration for the fake sources")
143 
144  def setDefaults(self):
145  super().setDefaults()
146  self.calibrate.measurement.plugins["base_PixelFlags"].masksFpAnywhere.append("FAKE")
147  self.calibrate.measurement.plugins["base_PixelFlags"].masksFpCenter.append("FAKE")
148  self.calibrate.doAstrometry = False
149  self.calibrate.doWriteMatches = False
150  self.calibrate.doPhotoCal = False
151  self.calibrate.detection.reEstimateBackground = False
152 
153 
154 class ProcessCcdWithFakesTask(PipelineTask, CmdLineTask):
155  """Insert fake objects into calexps.
156 
157  Add fake stars and galaxies to the given calexp, specified in the dataRef. Galaxy parameters are read in
158  from the specified file and then modelled using galsim. Re-runs characterize image and calibrate image to
159  give a new background estimation and measurement of the calexp.
160 
161  `ProcessFakeSourcesTask` inherits six functions from insertFakesTask that make images of the fake
162  sources and then add them to the calexp.
163 
164  `addPixCoords`
165  Use the WCS information to add the pixel coordinates of each source
166  Adds an ``x`` and ``y`` column to the catalog of fake sources.
167  `trimFakeCat`
168  Trim the fake cat to about the size of the input image.
169  `mkFakeGalsimGalaxies`
170  Use Galsim to make fake double sersic galaxies for each set of galaxy parameters in the input file.
171  `mkFakeStars`
172  Use the PSF information from the calexp to make a fake star using the magnitude information from the
173  input file.
174  `cleanCat`
175  Remove rows of the input fake catalog which have half light radius, of either the bulge or the disk,
176  that are 0.
177  `addFakeSources`
178  Add the fake sources to the calexp.
179 
180  Notes
181  -----
182  The ``calexp`` with fake souces added to it is written out as the datatype ``calexp_fakes``.
183  """
184 
185  _DefaultName = "processCcdWithFakes"
186  ConfigClass = ProcessCcdWithFakesConfig
187 
188  def __init__(self, schema=None, butler=None, **kwargs):
189  """Initalize things! This should go above in the class docstring
190  """
191 
192  super().__init__(**kwargs)
193 
194  if schema is None:
195  schema = SourceTable.makeMinimalSchema()
196  self.schema = schema
197  self.makeSubtask("insertFakes")
198  self.makeSubtask("calibrate")
199 
200  def runDataRef(self, dataRef):
201  """Read in/write out the required data products and add fake sources to the calexp.
202 
203  Parameters
204  ----------
205  dataRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef`
206  Data reference defining the ccd to have fakes added to it.
207  Used to access the following data products:
208  calexp
209  jointcal_wcs
210  jointcal_photoCalib
211 
212  Notes
213  -----
214  Uses the calibration and WCS information attached to the calexp for the posistioning and calibration
215  of the sources unless the config option config.useUpdatedCalibs is set then it uses the
216  meas_mosaic/jointCal outputs. The config defualts for the column names in the catalog of fakes are
217  taken from the University of Washington simulations database. Operates on one ccd at a time.
218  """
219  exposureIdInfo = dataRef.get("expIdInfo")
220 
221  if self.config.insertFakes.fakeType == "snapshot":
222  fakeCat = dataRef.get("fakeSourceCat").toDataFrame()
223  elif self.config.insertFakes.fakeType == "static":
224  fakeCat = dataRef.get("deepCoadd_fakeSourceCat").toDataFrame()
225  else:
226  fakeCat = Table.read(self.config.insertFakes.fakeType).to_pandas()
227 
228  calexp = dataRef.get("calexp")
229  if self.config.useUpdatedCalibs:
230  self.log.info("Using updated calibs from meas_mosaic/jointCal")
231  wcs = dataRef.get("jointcal_wcs")
232  photoCalib = dataRef.get("jointcal_photoCalib")
233  else:
234  wcs = calexp.getWcs()
235  photoCalib = calexp.getPhotoCalib()
236 
237  icSourceCat = dataRef.get("icSrc", immediate=True)
238  sfdSourceCat = dataRef.get("src", immediate=True)
239 
240  resultStruct = self.run(fakeCat, calexp, wcs=wcs, photoCalib=photoCalib,
241  exposureIdInfo=exposureIdInfo, icSourceCat=icSourceCat,
242  sfdSourceCat=sfdSourceCat)
243 
244  dataRef.put(resultStruct.outputExposure, "fakes_calexp")
245  dataRef.put(resultStruct.outputCat, "fakes_src")
246  return resultStruct
247 
248  def runQuantum(self, butlerQC, inputRefs, outputRefs):
249  inputs = butlerQC.get(inputRefs)
250  if 'exposureIdInfo' not in inputs.keys():
251  expId, expBits = butlerQC.quantum.dataId.pack("visit_detector", returnMaxBits=True)
252  inputs['exposureIdInfo'] = ExposureIdInfo(expId, expBits)
253 
254  if inputs["wcs"] is None:
255  inputs["wcs"] = inputs["image"].getWcs()
256  if inputs["photoCalib"] is None:
257  inputs["photoCalib"] = inputs["image"].getPhotoCalib()
258 
259  outputs = self.run(**inputs)
260  butlerQC.put(outputs, outputRefs)
261 
262  @classmethod
263  def _makeArgumentParser(cls):
264  parser = pipeBase.ArgumentParser(name=cls._DefaultName)
265  parser.add_id_argument("--id", "fakes_calexp", help="data ID with raw CCD keys [+ tract optionally], "
266  "e.g. --id visit=12345 ccd=1,2 [tract=0]",
267  ContainerClass=PerTractCcdDataIdContainer)
268  return parser
269 
270  def run(self, fakeCat, exposure, wcs=None, photoCalib=None, exposureIdInfo=None, icSourceCat=None,
271  sfdSourceCat=None):
272  """Add fake sources to a calexp and then run detection, deblending and measurement.
273 
274  Parameters
275  ----------
276  fakeCat : `pandas.core.frame.DataFrame`
277  The catalog of fake sources to add to the exposure
278  exposure : `lsst.afw.image.exposure.exposure.ExposureF`
279  The exposure to add the fake sources to
280  wcs : `lsst.afw.geom.SkyWcs`
281  WCS to use to add fake sources
282  photoCalib : `lsst.afw.image.photoCalib.PhotoCalib`
283  Photometric calibration to be used to calibrate the fake sources
284  exposureIdInfo : `lsst.obs.base.ExposureIdInfo`
285  icSourceCat : `lsst.afw.table.SourceCatalog`
286  Default : None
287  Catalog to take the information about which sources were used for calibration from.
288  sfdSourceCat : `lsst.afw.table.SourceCatalog`
289  Default : None
290  Catalog produced by singleFrameDriver, needed to copy some calibration flags from.
291 
292  Returns
293  -------
294  resultStruct : `lsst.pipe.base.struct.Struct`
295  contains : outputExposure : `lsst.afw.image.exposure.exposure.ExposureF`
296  outputCat : `lsst.afw.table.source.source.SourceCatalog`
297 
298  Notes
299  -----
300  Adds pixel coordinates for each source to the fakeCat and removes objects with bulge or disk half
301  light radius = 0 (if ``config.cleanCat = True``). These columns are called ``x`` and ``y`` and are in
302  pixels.
303 
304  Adds the ``Fake`` mask plane to the exposure which is then set by `addFakeSources` to mark where fake
305  sources have been added. Uses the information in the ``fakeCat`` to make fake galaxies (using galsim)
306  and fake stars, using the PSF models from the PSF information for the calexp. These are then added to
307  the calexp and the calexp with fakes included returned.
308 
309  The galsim galaxies are made using a double sersic profile, one for the bulge and one for the disk,
310  this is then convolved with the PSF at that point.
311 
312  If exposureIdInfo is not provided then the SourceCatalog IDs will not be globally unique.
313  """
314 
315  if wcs is None:
316  wcs = exposure.getWcs()
317 
318  if photoCalib is None:
319  photoCalib = exposure.getPhotoCalib()
320 
321  self.insertFakes.run(fakeCat, exposure, wcs, photoCalib)
322 
323  # detect, deblend and measure sources
324  if exposureIdInfo is None:
325  exposureIdInfo = ExposureIdInfo()
326  returnedStruct = self.calibrate.run(exposure, exposureIdInfo=exposureIdInfo)
327  sourceCat = returnedStruct.sourceCat
328 
329  sourceCat = self.copyCalibrationFields(sfdSourceCat, sourceCat, self.config.srcFieldsToCopy)
330 
331  resultStruct = pipeBase.Struct(outputExposure=exposure, outputCat=sourceCat)
332  return resultStruct
333 
334  def copyCalibrationFields(self, calibCat, sourceCat, fieldsToCopy):
335  """Match sources in calibCat and sourceCat and copy the specified fields
336 
337  Parameters
338  ----------
339  calibCat : `lsst.afw.table.SourceCatalog`
340  Catalog from which to copy fields.
341  sourceCat : `lsst.afw.table.SourceCatalog`
342  Catalog to which to copy fields.
343  fieldsToCopy : `lsst.pex.config.listField.List`
344  Fields to copy from calibCat to SoourceCat.
345 
346  Returns
347  -------
348  newCat : `lsst.afw.table.SourceCatalog`
349  Catalog which includes the copied fields.
350 
351  The fields copied are those specified by `fieldsToCopy` that actually exist
352  in the schema of `calibCat`.
353 
354  This version was based on and adapted from the one in calibrateTask.
355  """
356 
357  # Make a new SourceCatalog with the data from sourceCat so that we can add the new columns to it
358  sourceSchemaMapper = afwTable.SchemaMapper(sourceCat.schema)
359  sourceSchemaMapper.addMinimalSchema(sourceCat.schema, True)
360 
361  calibSchemaMapper = afwTable.SchemaMapper(calibCat.schema, sourceCat.schema)
362 
363  # Add the desired columns from the option fieldsToCopy
364  missingFieldNames = []
365  for fieldName in fieldsToCopy:
366  if fieldName in calibCat.schema:
367  schemaItem = calibCat.schema.find(fieldName)
368  calibSchemaMapper.editOutputSchema().addField(schemaItem.getField())
369  schema = calibSchemaMapper.editOutputSchema()
370  calibSchemaMapper.addMapping(schemaItem.getKey(), schema.find(fieldName).getField())
371  else:
372  missingFieldNames.append(fieldName)
373  if missingFieldNames:
374  raise RuntimeError(f"calibCat is missing fields {missingFieldNames} specified in "
375  "fieldsToCopy")
376 
377  if "calib_detected" not in calibSchemaMapper.getOutputSchema():
378  self.calibSourceKey = calibSchemaMapper.addOutputField(afwTable.Field["Flag"]("calib_detected",
379  "Source was detected as an icSource"))
380  else:
381  self.calibSourceKey = None
382 
383  schema = calibSchemaMapper.getOutputSchema()
384  newCat = afwTable.SourceCatalog(schema)
385  newCat.reserve(len(sourceCat))
386  newCat.extend(sourceCat, sourceSchemaMapper)
387 
388  # Set the aliases so it doesn't complain.
389  for k, v in sourceCat.schema.getAliasMap().items():
390  newCat.schema.getAliasMap().set(k, v)
391 
392  select = newCat["deblend_nChild"] == 0
393  matches = afwTable.matchXy(newCat[select], calibCat, self.config.matchRadiusPix)
394  # Check that no sourceCat sources are listed twice (we already know
395  # that each match has a unique calibCat source ID, due to using
396  # that ID as the key in bestMatches)
397  numMatches = len(matches)
398  numUniqueSources = len(set(m[1].getId() for m in matches))
399  if numUniqueSources != numMatches:
400  self.log.warn("%d calibCat sources matched only %d sourceCat sources", numMatches,
401  numUniqueSources)
402 
403  self.log.info("Copying flags from calibCat to sourceCat for %s sources", numMatches)
404 
405  # For each match: set the calibSourceKey flag and copy the desired
406  # fields
407  for src, calibSrc, d in matches:
408  if self.calibSourceKey:
409  src.setFlag(self.calibSourceKey, True)
410  # src.assign copies the footprint from calibSrc, which we don't want
411  # (DM-407)
412  # so set calibSrc's footprint to src's footprint before src.assign,
413  # then restore it
414  calibSrcFootprint = calibSrc.getFootprint()
415  try:
416  calibSrc.setFootprint(src.getFootprint())
417  src.assign(calibSrc, calibSchemaMapper)
418  finally:
419  calibSrc.setFootprint(calibSrcFootprint)
420 
421  return newCat
lsst.pipe.tasks.assembleCoadd.run
def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, altMaskList=None, mask=None, supplementaryData=None)
Definition: assembleCoadd.py:712
lsst::meas::base
lsst::afw::table
lsst.pipe.tasks.processCcdWithFakes.ProcessCcdWithFakesConnections
Definition: processCcdWithFakes.py:43
lsst.pipe.tasks.calibrate
Definition: calibrate.py:1
lsst.pipe::base
lsst.pipe::base::connectionTypes
lsst::obs::base