lsst.pipe.tasks  18.1.0-15-gc153667b
processCcdWithFakes.py
Go to the documentation of this file.
1 # This file is part of pipe tasks
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (http://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <http://www.gnu.org/licenses/>.
21 
22 """
23 Insert fake sources into calexps
24 """
25 from astropy.table import Table
26 
27 import lsst.pex.config as pexConfig
28 import lsst.pipe.base as pipeBase
29 import lsst.daf.base as dafBase
30 
31 from .insertFakes import InsertFakesTask
32 from lsst.meas.algorithms import SourceDetectionTask
33 from lsst.meas.base import (SingleFrameMeasurementTask, ApplyApCorrTask, CatalogCalculationTask,
34  PerTractCcdDataIdContainer)
35 from lsst.meas.deblender import SourceDeblendTask
36 from lsst.afw.table import SourceTable, IdFactory
37 from lsst.obs.base import ExposureIdInfo
38 from lsst.pipe.base import PipelineTask, PipelineTaskConfig, CmdLineTask, PipelineTaskConnections
40 
41 
42 __all__ = ["ProcessCcdWithFakesConfig", "ProcessCcdWithFakesTask"]
43 
44 
45 class ProcessCcdWithFakesConnections(PipelineTaskConnections, dimensions=("instrument", "visit", "detector"),
46  defaultTemplates={}):
47 
48  exposure = cT.Input(
49  doc="Exposure into which fakes are to be added.",
50  name="calexp",
51  storageClass="ExposureF",
52  dimensions=("instrument", "visit", "detector")
53  )
54 
55  fakeCat = cT.Input(
56  doc="Catalog of fake sources to draw inputs from.",
57  nameTemplate="{CoaddName}Coadd_fakeSourceCat",
58  storageClass="Parquet",
59  dimensions=("tract", "skymap")
60  )
61 
62  wcs = cT.Input(
63  doc="WCS information for the input exposure.",
64  name="jointcal_wcs",
65  storageClass="Wcs",
66  dimensions=("Tract", "SkyMap", "Instrument", "Visit", "Detector")
67  )
68 
69  photoCalib = cT.Input(
70  doc="Calib information for the input exposure.",
71  name="jointcal_photoCalib",
72  storageClass="PhotoCalib",
73  dimensions=("Tract", "SkyMap", "Instrument", "Visit", "Detector")
74  )
75 
76  outputExposure = cT.Output(
77  doc="Exposure with fake sources added.",
78  name="fakes_calexp",
79  storageClass="ExposureF",
80  dimensions=("instrument", "visit", "detector")
81  )
82 
83  outputCat = cT.Output(
84  doc="Source catalog produced in calibrate task with fakes also measured.",
85  name="src",
86  storageClass="SourceCatalog",
87  dimensions=("instrument", "visit", "detector"),
88  )
89 
90 
91 class ProcessCcdWithFakesConfig(PipelineTaskConfig):
92  """Config for inserting fake sources
93 
94  Notes
95  -----
96  The default column names are those from the UW sims database.
97  """
98 
99  useUpdatedCalibs = pexConfig.Field(
100  doc="Use updated calibs and wcs from jointcal?",
101  dtype=bool,
102  default=False,
103  )
104 
105  coaddName = pexConfig.Field(
106  doc="The name of the type of coadd used",
107  dtype=str,
108  default="deep",
109  )
110 
111  insertFakes = pexConfig.ConfigurableField(target=InsertFakesTask,
112  doc="Configuration for the fake sources")
113 
114  detection = pexConfig.ConfigurableField(target=SourceDetectionTask,
115  doc="The detection task to use.")
116 
117  deblend = pexConfig.ConfigurableField(target=SourceDeblendTask, doc="The deblending task to use.")
118 
119  measurement = pexConfig.ConfigurableField(target=SingleFrameMeasurementTask,
120  doc="The measurement task to use")
121 
122  applyApCorr = pexConfig.ConfigurableField(target=ApplyApCorrTask,
123  doc="The apply aperture correction task to use.")
124 
125  catalogCalculation = pexConfig.ConfigurableField(target=CatalogCalculationTask,
126  doc="The catalog calculation ask to use.")
127 
128  def setDefaults(self):
129  self.detection.reEstimateBackground = False
130  super().setDefaults()
131  self.measurement.plugins["base_PixelFlags"].masksFpAnywhere.append("FAKE")
132  self.measurement.plugins["base_PixelFlags"].masksFpCenter.append("FAKE")
133 
134 
135 class ProcessCcdWithFakesTask(PipelineTask, CmdLineTask):
136  """Insert fake objects into calexps.
137 
138  Add fake stars and galaxies to the given calexp, specified in the dataRef. Galaxy parameters are read in
139  from the specified file and then modelled using galsim. Re-runs characterize image and calibrate image to
140  give a new background estimation and measurement of the calexp.
141 
142  `ProcessFakeSourcesTask` inherits six functions from insertFakesTask that make images of the fake
143  sources and then add them to the calexp.
144 
145  `addPixCoords`
146  Use the WCS information to add the pixel coordinates of each source
147  Adds an ``x`` and ``y`` column to the catalog of fake sources.
148  `trimFakeCat`
149  Trim the fake cat to about the size of the input image.
150  `mkFakeGalsimGalaxies`
151  Use Galsim to make fake double sersic galaxies for each set of galaxy parameters in the input file.
152  `mkFakeStars`
153  Use the PSF information from the calexp to make a fake star using the magnitude information from the
154  input file.
155  `cleanCat`
156  Remove rows of the input fake catalog which have half light radius, of either the bulge or the disk,
157  that are 0.
158  `addFakeSources`
159  Add the fake sources to the calexp.
160 
161  Notes
162  -----
163  The ``calexp`` with fake souces added to it is written out as the datatype ``calexp_fakes``.
164  """
165 
166  _DefaultName = "processCcdWithFakes"
167  ConfigClass = ProcessCcdWithFakesConfig
168 
169  def __init__(self, schema=None, **kwargs):
170  """Initalize tings! This should go above in the class docstring
171  """
172 
173  super().__init__(**kwargs)
174 
175  if schema is None:
176  schema = SourceTable.makeMinimalSchema()
177  self.schema = schema
178  self.makeSubtask("insertFakes")
179  self.algMetadata = dafBase.PropertyList()
180  self.makeSubtask("detection", schema=self.schema)
181  self.makeSubtask("deblend", schema=self.schema)
182  self.makeSubtask("measurement", schema=self.schema, algMetadata=self.algMetadata)
183  self.makeSubtask("applyApCorr", schema=self.schema)
184  self.makeSubtask("catalogCalculation", schema=self.schema)
185 
186  def runDataRef(self, dataRef):
187  """Read in/write out the required data products and add fake sources to the calexp.
188 
189  Parameters
190  ----------
191  dataRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef`
192  Data reference defining the ccd to have fakes added to it.
193  Used to access the following data products:
194  calexp
195  jointcal_wcs
196  jointcal_photoCalib
197 
198  Notes
199  -----
200  Uses the calibration and WCS information attached to the calexp for the posistioning and calibration
201  of the sources unless the config option config.useUpdatedCalibs is set then it uses the
202  meas_mosaic/jointCal outputs. The config defualts for the column names in the catalog of fakes are
203  taken from the University of Washington simulations database. Operates on one ccd at a time.
204  """
205  exposureIdInfo = dataRef.get("expIdInfo")
206 
207  if self.config.insertFakes.fakeType == "snapshot":
208  fakeCat = dataRef.get("fakeSourceCat").toDataFrame()
209  elif self.config.insertFakes.fakeType == "static":
210  fakeCat = dataRef.get("deepCoadd_fakeSourceCat").toDataFrame()
211  else:
212  fakeCat = Table.read(self.config.insertFakes.fakeType).to_pandas()
213 
214  calexp = dataRef.get("calexp")
215  if self.config.useUpdatedCalibs:
216  self.log.info("Using updated calibs from meas_mosaic/jointCal")
217  wcs = dataRef.get("jointcal_wcs")
218  photoCalib = dataRef.get("jointcal_photoCalib")
219  else:
220  wcs = calexp.getWcs()
221  photoCalib = calexp.getCalib()
222 
223  resultStruct = self.run(fakeCat, calexp, wcs=wcs, photoCalib=photoCalib,
224  exposureIdInfo=exposureIdInfo)
225 
226  dataRef.put(resultStruct.outputExposure, "fakes_calexp")
227  dataRef.put(resultStruct.outputCat, "fakes_src")
228 
229  def runQuantum(self, butlerQC, inputRefs, outputRefs):
230  inputs = butlerQC.get(inputRefs)
231  if 'exposureIdInfo' not in inputs.keys():
232  expId, expBits = butlerQC.registry.packDataId("visit_detector", butlerQC.quantum.dataId,
233  returnMaxBits=True)
234  inputs['exposureIdInfo'] = ExposureIdInfo(expId, expBits)
235 
236  if inputs["wcs"] is None:
237  inputs["wcs"] = inputs["image"].getWcs()
238  if inputs["photoCalib"] is None:
239  inputs["photoCalib"] = inputs["image"].getCalib()
240 
241  outputs = self.run(**inputs)
242  butlerQC.put(outputs, outputRefs)
243 
244  @classmethod
245  def _makeArgumentParser(cls):
246  parser = pipeBase.ArgumentParser(name=cls._DefaultName)
247  parser.add_id_argument("--id", "fakes_calexp", help="data ID with raw CCD keys [+ tract optionally], "
248  "e.g. --id visit=12345 ccd=1,2 [tract=0]",
249  ContainerClass=PerTractCcdDataIdContainer)
250  return parser
251 
252  def run(self, fakeCat, exposure, wcs=None, photoCalib=None, exposureIdInfo=None):
253  """Add fake sources to a calexp and then run detection, deblending and measurement.
254 
255  Parameters
256  ----------
257  fakeCat : `pandas.core.frame.DataFrame`
258  The catalog of fake sources to add to the exposure
259  exposure : `lsst.afw.image.exposure.exposure.ExposureF`
260  The exposure to add the fake sources to
261  wcs : `lsst.afw.geom.SkyWcs`
262  WCS to use to add fake sources
263  photoCalib : `lsst.afw.image.photoCalib.PhotoCalib`
264  Photometric calibration to be used to calibrate the fake sources
265  exposureIdInfo : `lsst.obs.base.ExposureIdInfo`
266 
267  Returns
268  -------
269  resultStruct : `lsst.pipe.base.struct.Struct`
270  contains : outputExposure : `lsst.afw.image.exposure.exposure.ExposureF`
271  outputCat : `lsst.afw.table.source.source.SourceCatalog`
272 
273  Notes
274  -----
275  Adds pixel coordinates for each source to the fakeCat and removes objects with bulge or disk half
276  light radius = 0 (if ``config.cleanCat = True``). These columns are called ``x`` and ``y`` and are in
277  pixels.
278 
279  Adds the ``Fake`` mask plane to the exposure which is then set by `addFakeSources` to mark where fake
280  sources have been added. Uses the information in the ``fakeCat`` to make fake galaxies (using galsim)
281  and fake stars, using the PSF models from the PSF information for the calexp. These are then added to
282  the calexp and the calexp with fakes included returned.
283 
284  The galsim galaxies are made using a double sersic profile, one for the bulge and one for the disk,
285  this is then convolved with the PSF at that point.
286 
287  If exposureIdInfo is not provided then the SourceCatalog IDs will not be globally unique.
288  """
289 
290  if wcs is None:
291  wcs = exposure.getWcs()
292 
293  if photoCalib is None:
294  photoCalib = exposure.getCalib()
295 
296  self.insertFakes.run(fakeCat, exposure, wcs, photoCalib)
297 
298  # detect, deblend and measure sources
299  if exposureIdInfo is None:
300  exposureIdInfo = ExposureIdInfo()
301 
302  sourceIdFactory = IdFactory.makeSource(exposureIdInfo.expId, exposureIdInfo.unusedBits)
303  table = SourceTable.make(self.schema, sourceIdFactory)
304  table.setMetadata(self.algMetadata)
305 
306  detRes = self.detection.run(table=table, exposure=exposure, doSmooth=True)
307  sourceCat = detRes.sources
308  self.deblend.run(exposure=exposure, sources=sourceCat)
309  self.measurement.run(measCat=sourceCat, exposure=exposure, exposureId=exposureIdInfo.expId)
310  self.applyApCorr.run(catalog=sourceCat, apCorrMap=exposure.getInfo().getApCorrMap())
311  self.catalogCalculation.run(sourceCat)
312 
313  resultStruct = pipeBase.Struct(outputExposure=exposure, outputCat=sourceCat)
314  return resultStruct
def run(self, fakeCat, exposure, wcs=None, photoCalib=None, exposureIdInfo=None)
def runQuantum(self, butlerQC, inputRefs, outputRefs)