lsst.pipe.tasks  19.0.0-60-gafafd468+10
processCcdWithFakes.py
Go to the documentation of this file.
1 # This file is part of pipe tasks
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (http://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <http://www.gnu.org/licenses/>.
21 
22 """
23 Insert fake sources into calexps
24 """
25 from astropy.table import Table
26 
27 import lsst.pex.config as pexConfig
28 import lsst.pipe.base as pipeBase
29 import lsst.daf.base as dafBase
30 
31 from .insertFakes import InsertFakesTask
32 from lsst.meas.algorithms import SourceDetectionTask
33 from lsst.meas.base import (SingleFrameMeasurementTask, ApplyApCorrTask, CatalogCalculationTask,
34  PerTractCcdDataIdContainer)
35 from lsst.meas.deblender import SourceDeblendTask
36 from lsst.afw.table import SourceTable, IdFactory
37 from lsst.obs.base import ExposureIdInfo
38 from lsst.pipe.base import PipelineTask, PipelineTaskConfig, CmdLineTask, PipelineTaskConnections
40 import lsst.afw.table as afwTable
41 
42 
43 __all__ = ["ProcessCcdWithFakesConfig", "ProcessCcdWithFakesTask"]
44 
45 
46 class ProcessCcdWithFakesConnections(PipelineTaskConnections, dimensions=("instrument", "visit", "detector"),
47  defaultTemplates={"CoaddName": "deep"}):
48 
49  exposure = cT.Input(
50  doc="Exposure into which fakes are to be added.",
51  name="calexp",
52  storageClass="ExposureF",
53  dimensions=("instrument", "visit", "detector")
54  )
55 
56  fakeCat = cT.Input(
57  doc="Catalog of fake sources to draw inputs from.",
58  name="{CoaddName}Coadd_fakeSourceCat",
59  storageClass="Parquet",
60  dimensions=("tract", "skymap")
61  )
62 
63  wcs = cT.Input(
64  doc="WCS information for the input exposure.",
65  name="jointcal_wcs",
66  storageClass="Wcs",
67  dimensions=("Tract", "SkyMap", "Instrument", "Visit", "Detector")
68  )
69 
70  photoCalib = cT.Input(
71  doc="Calib information for the input exposure.",
72  name="jointcal_photoCalib",
73  storageClass="PhotoCalib",
74  dimensions=("Tract", "SkyMap", "Instrument", "Visit", "Detector")
75  )
76 
77  icSourceCat = cT.Input(
78  doc="Catalog of calibration sources",
79  name="icSrc",
80  storageClass="sourceCatalog",
81  dimensions=("tract", "skymap", "instrument", "visit", "detector")
82  )
83 
84  sfdSourceCat = cT.Input(
85  doc="Catalog of calibration sources",
86  name="src",
87  storageClass="sourceCatalog",
88  dimensions=("tract", "skymap", "instrument", "visit", "detector")
89  )
90 
91  outputExposure = cT.Output(
92  doc="Exposure with fake sources added.",
93  name="fakes_calexp",
94  storageClass="ExposureF",
95  dimensions=("instrument", "visit", "detector")
96  )
97 
98  outputCat = cT.Output(
99  doc="Source catalog produced in calibrate task with fakes also measured.",
100  name="fakes_src",
101  storageClass="SourceCatalog",
102  dimensions=("instrument", "visit", "detector"),
103  )
104 
105 
106 class ProcessCcdWithFakesConfig(PipelineTaskConfig,
107  pipelineConnections=ProcessCcdWithFakesConnections):
108  """Config for inserting fake sources
109 
110  Notes
111  -----
112  The default column names are those from the UW sims database.
113  """
114 
115  useUpdatedCalibs = pexConfig.Field(
116  doc="Use updated calibs and wcs from jointcal?",
117  dtype=bool,
118  default=False,
119  )
120 
121  coaddName = pexConfig.Field(
122  doc="The name of the type of coadd used",
123  dtype=str,
124  default="deep",
125  )
126 
127  calibrationFieldsToCopy = pexConfig.ListField(
128  dtype=str,
129  default=("calib_psf_candidate", "calib_psf_used", "calib_psf_reserved"),
130  doc=("Fields to copy from the icSource catalog to the output catalog "
131  "for matching sources Any missing fields will trigger a "
132  "RuntimeError exception.")
133  )
134 
135  srcFieldsToCopy = pexConfig.ListField(
136  dtype=str,
137  default=("calib_photometry_reserved", "calib_photometry_used", "calib_astrometry_used"),
138  doc=("Fields to copy from the `src` catalog to the output catalog "
139  "for matching sources Any missing fields will trigger a "
140  "RuntimeError exception.")
141  )
142 
143  matchRadiusPix = pexConfig.Field(
144  dtype=float,
145  default=3,
146  doc=("Match radius for matching icSourceCat objects to sourceCat objects (pixels)"),
147  )
148 
149  insertFakes = pexConfig.ConfigurableField(target=InsertFakesTask,
150  doc="Configuration for the fake sources")
151 
152  detection = pexConfig.ConfigurableField(target=SourceDetectionTask,
153  doc="The detection task to use.")
154 
155  deblend = pexConfig.ConfigurableField(target=SourceDeblendTask, doc="The deblending task to use.")
156 
157  measurement = pexConfig.ConfigurableField(target=SingleFrameMeasurementTask,
158  doc="The measurement task to use")
159 
160  applyApCorr = pexConfig.ConfigurableField(target=ApplyApCorrTask,
161  doc="The apply aperture correction task to use.")
162 
163  catalogCalculation = pexConfig.ConfigurableField(target=CatalogCalculationTask,
164  doc="The catalog calculation task to use.")
165 
166  def setDefaults(self):
167  self.detection.reEstimateBackground = False
168  super().setDefaults()
169  self.measurement.plugins["base_PixelFlags"].masksFpAnywhere.append("FAKE")
170  self.measurement.plugins["base_PixelFlags"].masksFpCenter.append("FAKE")
171  self.deblend.maxFootprintSize = 2000
172  self.measurement.plugins.names |= ["base_LocalPhotoCalib", "base_LocalWcs"]
173 
174 
175 class ProcessCcdWithFakesTask(PipelineTask, CmdLineTask):
176  """Insert fake objects into calexps.
177 
178  Add fake stars and galaxies to the given calexp, specified in the dataRef. Galaxy parameters are read in
179  from the specified file and then modelled using galsim. Re-runs characterize image and calibrate image to
180  give a new background estimation and measurement of the calexp.
181 
182  `ProcessFakeSourcesTask` inherits six functions from insertFakesTask that make images of the fake
183  sources and then add them to the calexp.
184 
185  `addPixCoords`
186  Use the WCS information to add the pixel coordinates of each source
187  Adds an ``x`` and ``y`` column to the catalog of fake sources.
188  `trimFakeCat`
189  Trim the fake cat to about the size of the input image.
190  `mkFakeGalsimGalaxies`
191  Use Galsim to make fake double sersic galaxies for each set of galaxy parameters in the input file.
192  `mkFakeStars`
193  Use the PSF information from the calexp to make a fake star using the magnitude information from the
194  input file.
195  `cleanCat`
196  Remove rows of the input fake catalog which have half light radius, of either the bulge or the disk,
197  that are 0.
198  `addFakeSources`
199  Add the fake sources to the calexp.
200 
201  Notes
202  -----
203  The ``calexp`` with fake souces added to it is written out as the datatype ``calexp_fakes``.
204  """
205 
206  _DefaultName = "processCcdWithFakes"
207  ConfigClass = ProcessCcdWithFakesConfig
208 
209  def __init__(self, schema=None, butler=None, **kwargs):
210  """Initalize things! This should go above in the class docstring
211  """
212 
213  super().__init__(**kwargs)
214 
215  if schema is None:
216  schema = SourceTable.makeMinimalSchema()
217  self.schema = schema
218  self.makeSubtask("insertFakes")
219  self.algMetadata = dafBase.PropertyList()
220  self.makeSubtask("detection", schema=self.schema)
221  self.makeSubtask("deblend", schema=self.schema)
222  self.makeSubtask("measurement", schema=self.schema, algMetadata=self.algMetadata)
223  self.makeSubtask("applyApCorr", schema=self.schema)
224  self.makeSubtask("catalogCalculation", schema=self.schema)
225 
226  def runDataRef(self, dataRef):
227  """Read in/write out the required data products and add fake sources to the calexp.
228 
229  Parameters
230  ----------
231  dataRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef`
232  Data reference defining the ccd to have fakes added to it.
233  Used to access the following data products:
234  calexp
235  jointcal_wcs
236  jointcal_photoCalib
237 
238  Notes
239  -----
240  Uses the calibration and WCS information attached to the calexp for the posistioning and calibration
241  of the sources unless the config option config.useUpdatedCalibs is set then it uses the
242  meas_mosaic/jointCal outputs. The config defualts for the column names in the catalog of fakes are
243  taken from the University of Washington simulations database. Operates on one ccd at a time.
244  """
245  exposureIdInfo = dataRef.get("expIdInfo")
246 
247  if self.config.insertFakes.fakeType == "snapshot":
248  fakeCat = dataRef.get("fakeSourceCat").toDataFrame()
249  elif self.config.insertFakes.fakeType == "static":
250  fakeCat = dataRef.get("deepCoadd_fakeSourceCat").toDataFrame()
251  else:
252  fakeCat = Table.read(self.config.insertFakes.fakeType).to_pandas()
253 
254  calexp = dataRef.get("calexp")
255  if self.config.useUpdatedCalibs:
256  self.log.info("Using updated calibs from meas_mosaic/jointCal")
257  wcs = dataRef.get("jointcal_wcs")
258  photoCalib = dataRef.get("jointcal_photoCalib")
259  else:
260  wcs = calexp.getWcs()
261  photoCalib = calexp.getPhotoCalib()
262 
263  icSourceCat = dataRef.get("icSrc", immediate=True)
264  sfdSourceCat = dataRef.get("src", immediate=True)
265 
266  resultStruct = self.run(fakeCat, calexp, wcs=wcs, photoCalib=photoCalib,
267  exposureIdInfo=exposureIdInfo, icSourceCat=icSourceCat,
268  sfdSourceCat=sfdSourceCat)
269 
270  dataRef.put(resultStruct.outputExposure, "fakes_calexp")
271  dataRef.put(resultStruct.outputCat, "fakes_src")
272  return resultStruct
273 
274  def runQuantum(self, butlerQC, inputRefs, outputRefs):
275  inputs = butlerQC.get(inputRefs)
276  if 'exposureIdInfo' not in inputs.keys():
277  expId, expBits = butlerQC.quantum.dataId.pack("visit_detector", returnMaxBits=True)
278  inputs['exposureIdInfo'] = ExposureIdInfo(expId, expBits)
279 
280  if inputs["wcs"] is None:
281  inputs["wcs"] = inputs["image"].getWcs()
282  if inputs["photoCalib"] is None:
283  inputs["photoCalib"] = inputs["image"].getPhotoCalib()
284 
285  outputs = self.run(**inputs)
286  butlerQC.put(outputs, outputRefs)
287 
288  @classmethod
289  def _makeArgumentParser(cls):
290  parser = pipeBase.ArgumentParser(name=cls._DefaultName)
291  parser.add_id_argument("--id", "fakes_calexp", help="data ID with raw CCD keys [+ tract optionally], "
292  "e.g. --id visit=12345 ccd=1,2 [tract=0]",
293  ContainerClass=PerTractCcdDataIdContainer)
294  return parser
295 
296  def run(self, fakeCat, exposure, wcs=None, photoCalib=None, exposureIdInfo=None, icSourceCat=None,
297  sfdSourceCat=None):
298  """Add fake sources to a calexp and then run detection, deblending and measurement.
299 
300  Parameters
301  ----------
302  fakeCat : `pandas.core.frame.DataFrame`
303  The catalog of fake sources to add to the exposure
304  exposure : `lsst.afw.image.exposure.exposure.ExposureF`
305  The exposure to add the fake sources to
306  wcs : `lsst.afw.geom.SkyWcs`
307  WCS to use to add fake sources
308  photoCalib : `lsst.afw.image.photoCalib.PhotoCalib`
309  Photometric calibration to be used to calibrate the fake sources
310  exposureIdInfo : `lsst.obs.base.ExposureIdInfo`
311  icSourceCat : `lsst.afw.table.SourceCatalog`
312  Default : None
313  Catalog to take the information about which sources were used for calibration from.
314  sfdSourceCat : `lsst.afw.table.SourceCatalog`
315  Default : None
316  Catalog produced by singleFrameDriver, needed to copy some calibration flags from.
317 
318  Returns
319  -------
320  resultStruct : `lsst.pipe.base.struct.Struct`
321  contains : outputExposure : `lsst.afw.image.exposure.exposure.ExposureF`
322  outputCat : `lsst.afw.table.source.source.SourceCatalog`
323 
324  Notes
325  -----
326  Adds pixel coordinates for each source to the fakeCat and removes objects with bulge or disk half
327  light radius = 0 (if ``config.cleanCat = True``). These columns are called ``x`` and ``y`` and are in
328  pixels.
329 
330  Adds the ``Fake`` mask plane to the exposure which is then set by `addFakeSources` to mark where fake
331  sources have been added. Uses the information in the ``fakeCat`` to make fake galaxies (using galsim)
332  and fake stars, using the PSF models from the PSF information for the calexp. These are then added to
333  the calexp and the calexp with fakes included returned.
334 
335  The galsim galaxies are made using a double sersic profile, one for the bulge and one for the disk,
336  this is then convolved with the PSF at that point.
337 
338  If exposureIdInfo is not provided then the SourceCatalog IDs will not be globally unique.
339  """
340 
341  if wcs is None:
342  wcs = exposure.getWcs()
343 
344  if photoCalib is None:
345  photoCalib = exposure.getPhotoCalib()
346 
347  self.insertFakes.run(fakeCat, exposure, wcs, photoCalib)
348 
349  # detect, deblend and measure sources
350  if exposureIdInfo is None:
351  exposureIdInfo = ExposureIdInfo()
352 
353  sourceIdFactory = IdFactory.makeSource(exposureIdInfo.expId, exposureIdInfo.unusedBits)
354  table = SourceTable.make(self.schema, sourceIdFactory)
355  table.setMetadata(self.algMetadata)
356 
357  detRes = self.detection.run(table=table, exposure=exposure, doSmooth=True)
358  sourceCat = detRes.sources
359  self.deblend.run(exposure=exposure, sources=sourceCat)
360  self.measurement.run(measCat=sourceCat, exposure=exposure, exposureId=exposureIdInfo.expId)
361  self.applyApCorr.run(catalog=sourceCat, apCorrMap=exposure.getInfo().getApCorrMap())
362  self.catalogCalculation.run(sourceCat)
363  sourceCat = self.copyCalibrationFields(icSourceCat, sourceCat, self.config.calibrationFieldsToCopy)
364  sourceCat = self.copyCalibrationFields(sfdSourceCat, sourceCat, self.config.srcFieldsToCopy)
365 
366  resultStruct = pipeBase.Struct(outputExposure=exposure, outputCat=sourceCat)
367  return resultStruct
368 
369  def copyCalibrationFields(self, calibCat, sourceCat, fieldsToCopy):
370  """Match sources in calibCat and sourceCat and copy the specified fields
371 
372  Parameters
373  ----------
374  calibCat : `lsst.afw.table.SourceCatalog`
375  Catalog from which to copy fields.
376  sourceCat : `lsst.afw.table.SourceCatalog`
377  Catalog to which to copy fields.
378  fieldsToCopy : `lsst.pex.config.listField.List`
379  Fields to copy from calibCat to SoourceCat.
380 
381  Returns
382  -------
383  newCat : `lsst.afw.table.SourceCatalog`
384  Catalog which includes the copied fields.
385 
386  The fields copied are those specified by `fieldsToCopy` that actually exist
387  in the schema of `calibCat`.
388 
389  This version was based on and adapted from the one in calibrateTask.
390  """
391 
392  # Make a new SourceCatalog with the data from sourceCat so that we can add the new columns to it
393  sourceSchemaMapper = afwTable.SchemaMapper(sourceCat.schema)
394  sourceSchemaMapper.addMinimalSchema(sourceCat.schema, True)
395 
396  calibSchemaMapper = afwTable.SchemaMapper(calibCat.schema, sourceCat.schema)
397 
398  # Add the desired columns from the option fieldsToCopy
399  missingFieldNames = []
400  for fieldName in fieldsToCopy:
401  if fieldName in calibCat.schema:
402  schemaItem = calibCat.schema.find(fieldName)
403  calibSchemaMapper.editOutputSchema().addField(schemaItem.getField())
404  schema = calibSchemaMapper.editOutputSchema()
405  calibSchemaMapper.addMapping(schemaItem.getKey(), schema.find(fieldName).getField())
406  else:
407  missingFieldNames.append(fieldName)
408  if missingFieldNames:
409  raise RuntimeError(f"calibCat is missing fields {missingFieldNames} specified in "
410  "fieldsToCopy")
411 
412  if "calib_detected" not in calibSchemaMapper.getOutputSchema():
413  self.calibSourceKey = calibSchemaMapper.addOutputField(afwTable.Field["Flag"]("calib_detected",
414  "Source was detected as an icSource"))
415  else:
416  self.calibSourceKey = None
417 
418  schema = calibSchemaMapper.getOutputSchema()
419  newCat = afwTable.SourceCatalog(schema)
420  newCat.reserve(len(sourceCat))
421  newCat.extend(sourceCat, sourceSchemaMapper)
422 
423  # Set the aliases so it doesn't complain.
424  for k, v in sourceCat.schema.getAliasMap().items():
425  newCat.schema.getAliasMap().set(k, v)
426 
427  select = newCat["deblend_nChild"] == 0
428  matches = afwTable.matchXy(newCat[select], calibCat, self.config.matchRadiusPix)
429  # Check that no sourceCat sources are listed twice (we already know
430  # that each match has a unique calibCat source ID, due to using
431  # that ID as the key in bestMatches)
432  numMatches = len(matches)
433  numUniqueSources = len(set(m[1].getId() for m in matches))
434  if numUniqueSources != numMatches:
435  self.log.warn("%d calibCat sources matched only %d sourceCat sources", numMatches,
436  numUniqueSources)
437 
438  self.log.info("Copying flags from calibCat to sourceCat for %s sources", numMatches)
439 
440  # For each match: set the calibSourceKey flag and copy the desired
441  # fields
442  for src, calibSrc, d in matches:
443  if self.calibSourceKey:
444  src.setFlag(self.calibSourceKey, True)
445  # src.assign copies the footprint from calibSrc, which we don't want
446  # (DM-407)
447  # so set calibSrc's footprint to src's footprint before src.assign,
448  # then restore it
449  calibSrcFootprint = calibSrc.getFootprint()
450  try:
451  calibSrc.setFootprint(src.getFootprint())
452  src.assign(calibSrc, calibSchemaMapper)
453  finally:
454  calibSrc.setFootprint(calibSrcFootprint)
455 
456  return newCat
lsst.pipe.tasks.assembleCoadd.run
def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, altMaskList=None, mask=None, supplementaryData=None)
Definition: assembleCoadd.py:712
lsst::meas::base
lsst::afw::table
lsst::meas::deblender
lsst.pipe.tasks.processCcdWithFakes.ProcessCcdWithFakesConnections
Definition: processCcdWithFakes.py:47
lsst::daf::base
lsst.pipe::base
lsst::meas::algorithms
lsst.pipe::base::connectionTypes
lsst::obs::base