lsst.pipe.tasks  19.0.0-53-g31c4d99c
insertFakes.py
Go to the documentation of this file.
1 # This file is part of pipe tasks
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (http://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <http://www.gnu.org/licenses/>.
21 
22 """
23 Insert fakes into deepCoadds
24 """
25 import galsim
26 from astropy.table import Table
27 
28 import lsst.geom as geom
29 import lsst.afw.image as afwImage
30 import lsst.afw.math as afwMath
31 import lsst.pex.config as pexConfig
32 import lsst.pipe.base as pipeBase
33 
34 from lsst.pipe.base import CmdLineTask, PipelineTask, PipelineTaskConfig, PipelineTaskConnections
36 from lsst.pex.exceptions import LogicError, InvalidParameterError
37 from lsst.coadd.utils.coaddDataIdContainer import ExistingCoaddDataIdContainer
38 from lsst.geom import SpherePoint, radians, Box2D
39 from lsst.sphgeom import ConvexPolygon
40 
41 __all__ = ["InsertFakesConfig", "InsertFakesTask"]
42 
43 
44 class InsertFakesConnections(PipelineTaskConnections, defaultTemplates={"CoaddName": "deep"},
45  dimensions=("tract", "patch", "abstract_filter", "skymap")):
46 
47  image = cT.Input(
48  doc="Image into which fakes are to be added.",
49  name="{CoaddName}Coadd",
50  storageClass="ExposureF",
51  dimensions=("tract", "patch", "abstract_filter", "skymap")
52  )
53 
54  fakeCat = cT.Input(
55  doc="Catalog of fake sources to draw inputs from.",
56  name="{CoaddName}Coadd_fakeSourceCat",
57  storageClass="Parquet",
58  dimensions=("tract", "skymap")
59  )
60 
61  imageWithFakes = cT.Output(
62  doc="Image with fake sources added.",
63  name="fakes_{CoaddName}Coadd",
64  storageClass="ExposureF",
65  dimensions=("tract", "patch", "abstract_filter", "skymap")
66  )
67 
68 
69 class InsertFakesConfig(PipelineTaskConfig,
70  pipelineConnections=InsertFakesConnections):
71  """Config for inserting fake sources
72 
73  Notes
74  -----
75  The default column names are those from the University of Washington sims database.
76  """
77 
78  raColName = pexConfig.Field(
79  doc="RA column name used in the fake source catalog.",
80  dtype=str,
81  default="raJ2000",
82  )
83 
84  decColName = pexConfig.Field(
85  doc="Dec. column name used in the fake source catalog.",
86  dtype=str,
87  default="decJ2000",
88  )
89 
90  doCleanCat = pexConfig.Field(
91  doc="If true removes bad sources from the catalog.",
92  dtype=bool,
93  default=True,
94  )
95 
96  diskHLR = pexConfig.Field(
97  doc="Column name for the disk half light radius used in the fake source catalog.",
98  dtype=str,
99  default="DiskHalfLightRadius",
100  )
101 
102  bulgeHLR = pexConfig.Field(
103  doc="Column name for the bulge half light radius used in the fake source catalog.",
104  dtype=str,
105  default="BulgeHalfLightRadius",
106  )
107 
108  magVar = pexConfig.Field(
109  doc="The column name for the magnitude calculated taking variability into account. In the format "
110  "``filter name``magVar, e.g. imagVar for the magnitude in the i band.",
111  dtype=str,
112  default="%smagVar",
113  )
114 
115  nDisk = pexConfig.Field(
116  doc="The column name for the sersic index of the disk component used in the fake source catalog.",
117  dtype=str,
118  default="disk_n",
119  )
120 
121  nBulge = pexConfig.Field(
122  doc="The column name for the sersic index of the bulge component used in the fake source catalog.",
123  dtype=str,
124  default="bulge_n",
125  )
126 
127  aDisk = pexConfig.Field(
128  doc="The column name for the semi major axis length of the disk component used in the fake source"
129  "catalog.",
130  dtype=str,
131  default="a_d",
132  )
133 
134  aBulge = pexConfig.Field(
135  doc="The column name for the semi major axis length of the bulge component.",
136  dtype=str,
137  default="a_b",
138  )
139 
140  bDisk = pexConfig.Field(
141  doc="The column name for the semi minor axis length of the disk component.",
142  dtype=str,
143  default="b_d",
144  )
145 
146  bBulge = pexConfig.Field(
147  doc="The column name for the semi minor axis length of the bulge component used in the fake source "
148  "catalog.",
149  dtype=str,
150  default="b_b",
151  )
152 
153  paDisk = pexConfig.Field(
154  doc="The column name for the PA of the disk component used in the fake source catalog.",
155  dtype=str,
156  default="pa_disk",
157  )
158 
159  paBulge = pexConfig.Field(
160  doc="The column name for the PA of the bulge component used in the fake source catalog.",
161  dtype=str,
162  default="pa_bulge",
163  )
164 
165  sourceType = pexConfig.Field(
166  doc="The column name for the source type used in the fake source catalog.",
167  dtype=str,
168  default="sourceType",
169  )
170 
171  fakeType = pexConfig.Field(
172  doc="What type of fake catalog to use, snapshot (includes variability in the magnitudes calculated "
173  "from the MJD of the image), static (no variability) or filename for a user defined fits"
174  "catalog.",
175  dtype=str,
176  default="static",
177  )
178 
179  calibFluxRadius = pexConfig.Field(
180  doc="Radius for the calib flux (in pixels).",
181  dtype=float,
182  default=12.0,
183  )
184 
185  coaddName = pexConfig.Field(
186  doc="The name of the type of coadd used",
187  dtype=str,
188  default="deep",
189  )
190 
191 
192 class InsertFakesTask(PipelineTask, CmdLineTask):
193  """Insert fake objects into images.
194 
195  Add fake stars and galaxies to the given image, read in through the dataRef. Galaxy parameters are read in
196  from the specified file and then modelled using galsim.
197 
198  `InsertFakesTask` has five functions that make images of the fake sources and then add them to the
199  image.
200 
201  `addPixCoords`
202  Use the WCS information to add the pixel coordinates of each source.
203  `mkFakeGalsimGalaxies`
204  Use Galsim to make fake double sersic galaxies for each set of galaxy parameters in the input file.
205  `mkFakeStars`
206  Use the PSF information from the image to make a fake star using the magnitude information from the
207  input file.
208  `cleanCat`
209  Remove rows of the input fake catalog which have half light radius, of either the bulge or the disk,
210  that are 0.
211  `addFakeSources`
212  Add the fake sources to the image.
213 
214  """
215 
216  _DefaultName = "insertFakes"
217  ConfigClass = InsertFakesConfig
218 
219  def runDataRef(self, dataRef):
220  """Read in/write out the required data products and add fake sources to the deepCoadd.
221 
222  Parameters
223  ----------
224  dataRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef`
225  Data reference defining the image to have fakes added to it
226  Used to access the following data products:
227  deepCoadd
228  """
229 
230  # To do: should it warn when asked to insert variable sources into the coadd
231 
232  if self.config.fakeType == "static":
233  fakeCat = dataRef.get("deepCoadd_fakeSourceCat").toDataFrame()
234  # To do: DM-16254, the read and write of the fake catalogs will be changed once the new pipeline
235  # task structure for ref cats is in place.
236  self.fakeSourceCatType = "deepCoadd_fakeSourceCat"
237  else:
238  fakeCat = Table.read(self.config.fakeType).to_pandas()
239 
240  coadd = dataRef.get("deepCoadd")
241  wcs = coadd.getWcs()
242  photoCalib = coadd.getPhotoCalib()
243 
244  imageWithFakes = self.run(fakeCat, coadd, wcs, photoCalib)
245 
246  dataRef.put(imageWithFakes.imageWithFakes, "fakes_deepCoadd")
247 
248  def runQuantum(self, butlerQC, inputRefs, outputRefs):
249  inputs = butlerQC.get(inputRefs)
250  inputs["wcs"] = inputs["image"].getWcs()
251  inputs["photoCalib"] = inputs["image"].getPhotoCalib()
252 
253  outputs = self.run(**inputs)
254  butlerQC.put(outputs, outputRefs)
255 
256  @classmethod
257  def _makeArgumentParser(cls):
258  parser = pipeBase.ArgumentParser(name=cls._DefaultName)
259  parser.add_id_argument(name="--id", datasetType="deepCoadd",
260  help="data IDs for the deepCoadd, e.g. --id tract=12345 patch=1,2 filter=r",
261  ContainerClass=ExistingCoaddDataIdContainer)
262  return parser
263 
264  def run(self, fakeCat, image, wcs, photoCalib):
265  """Add fake sources to an image.
266 
267  Parameters
268  ----------
269  fakeCat : `pandas.core.frame.DataFrame`
270  The catalog of fake sources to be input
271  image : `lsst.afw.image.exposure.exposure.ExposureF`
272  The image into which the fake sources should be added
273  wcs : `lsst.afw.geom.SkyWcs`
274  WCS to use to add fake sources
275  photoCalib : `lsst.afw.image.photoCalib.PhotoCalib`
276  Photometric calibration to be used to calibrate the fake sources
277 
278  Returns
279  -------
280  resultStruct : `lsst.pipe.base.struct.Struct`
281  contains : image : `lsst.afw.image.exposure.exposure.ExposureF`
282 
283  Notes
284  -----
285  Adds pixel coordinates for each source to the fakeCat and removes objects with bulge or disk half
286  light radius = 0 (if ``config.doCleanCat = True``).
287 
288  Adds the ``Fake`` mask plane to the image which is then set by `addFakeSources` to mark where fake
289  sources have been added. Uses the information in the ``fakeCat`` to make fake galaxies (using galsim)
290  and fake stars, using the PSF models from the PSF information for the image. These are then added to
291  the image and the image with fakes included returned.
292 
293  The galsim galaxies are made using a double sersic profile, one for the bulge and one for the disk,
294  this is then convolved with the PSF at that point.
295  """
296 
297  image.mask.addMaskPlane("FAKE")
298  self.bitmask = image.mask.getPlaneBitMask("FAKE")
299  self.log.info("Adding mask plane with bitmask %d" % self.bitmask)
300 
301  fakeCat = self.addPixCoords(fakeCat, wcs)
302  if self.config.doCleanCat:
303  fakeCat = self.cleanCat(fakeCat)
304  fakeCat = self.trimFakeCat(fakeCat, image, wcs)
305 
306  band = image.getFilter().getName()
307  pixelScale = wcs.getPixelScale().asArcseconds()
308  psf = image.getPsf()
309 
310  galaxies = (fakeCat[self.config.sourceType] == "galaxy")
311  galImages = self.mkFakeGalsimGalaxies(fakeCat[galaxies], band, photoCalib, pixelScale, psf, image)
312  image = self.addFakeSources(image, galImages, "galaxy")
313 
314  stars = (fakeCat[self.config.sourceType] == "star")
315  starImages = self.mkFakeStars(fakeCat[stars], band, photoCalib, psf, image)
316  image = self.addFakeSources(image, starImages, "star")
317  resultStruct = pipeBase.Struct(imageWithFakes=image)
318 
319  return resultStruct
320 
321  def addPixCoords(self, fakeCat, wcs):
322 
323  """Add pixel coordinates to the catalog of fakes.
324 
325  Parameters
326  ----------
327  fakeCat : `pandas.core.frame.DataFrame`
328  The catalog of fake sources to be input
329  wcs : `lsst.afw.geom.SkyWcs`
330  WCS to use to add fake sources
331 
332  Returns
333  -------
334  fakeCat : `pandas.core.frame.DataFrame`
335 
336  Notes
337  -----
338  The default option is to use the WCS information from the image. If the ``useUpdatedCalibs`` config
339  option is set then it will use the updated WCS from jointCal.
340  """
341 
342  ras = fakeCat[self.config.raColName].values
343  decs = fakeCat[self.config.decColName].values
344  skyCoords = [SpherePoint(ra, dec, radians) for (ra, dec) in zip(ras, decs)]
345  pixCoords = wcs.skyToPixel(skyCoords)
346  xs = [coord.getX() for coord in pixCoords]
347  ys = [coord.getY() for coord in pixCoords]
348  fakeCat["x"] = xs
349  fakeCat["y"] = ys
350 
351  return fakeCat
352 
353  def trimFakeCat(self, fakeCat, image, wcs):
354  """Trim the fake cat to about the size of the input image.
355 
356  Parameters
357  ----------
358  fakeCat : `pandas.core.frame.DataFrame`
359  The catalog of fake sources to be input
360  image : `lsst.afw.image.exposure.exposure.ExposureF`
361  The image into which the fake sources should be added
362  wcs : `lsst.afw.geom.SkyWcs`
363  WCS to use to add fake sources
364 
365  Returns
366  -------
367  fakeCat : `pandas.core.frame.DataFrame`
368  The original fakeCat trimmed to the area of the image
369  """
370 
371  bbox = Box2D(image.getBBox())
372  corners = bbox.getCorners()
373 
374  skyCorners = wcs.pixelToSky(corners)
375  region = ConvexPolygon([s.getVector() for s in skyCorners])
376 
377  def trim(row):
378  coord = SpherePoint(row[self.config.raColName], row[self.config.decColName], radians)
379  return region.contains(coord.getVector())
380 
381  return fakeCat[fakeCat.apply(trim, axis=1)]
382 
383  def mkFakeGalsimGalaxies(self, fakeCat, band, photoCalib, pixelScale, psf, image):
384  """Make images of fake galaxies using GalSim.
385 
386  Parameters
387  ----------
388  band : `str`
389  pixelScale : `float`
390  psf : `lsst.meas.extensions.psfex.psfexPsf.PsfexPsf`
391  The PSF information to use to make the PSF images
392  fakeCat : `pandas.core.frame.DataFrame`
393  The catalog of fake sources to be input
394  photoCalib : `lsst.afw.image.photoCalib.PhotoCalib`
395  Photometric calibration to be used to calibrate the fake sources
396 
397  Yields
398  -------
399  galImages : `generator`
400  A generator of tuples of `lsst.afw.image.exposure.exposure.ExposureF` and
401  `lsst.geom.Point2D` of their locations.
402 
403  Notes
404  -----
405 
406  Fake galaxies are made by combining two sersic profiles, one for the bulge and one for the disk. Each
407  component has an individual sersic index (n), a, b and position angle (PA). The combined profile is
408  then convolved with the PSF at the specified x, y position on the image.
409 
410  The names of the columns in the ``fakeCat`` are configurable and are the column names from the
411  University of Washington simulations database as default. For more information see the doc strings
412  attached to the config options.
413  """
414 
415  self.log.info("Making %d fake galaxy images" % len(fakeCat))
416 
417  for (index, row) in fakeCat.iterrows():
418  xy = geom.Point2D(row["x"], row["y"])
419 
420  try:
421  # Due to the different radii used for calibration and measurement a correction factor is
422  # needed to prevent there being an offset in the final processed output.
423  correctedFlux = psf.computeApertureFlux(self.config.calibFluxRadius, xy)
424  psfKernel = psf.computeKernelImage(xy).getArray()
425  psfKernel /= correctedFlux
426 
427  except InvalidParameterError:
428  self.log.info("Galaxy at %0.4f, %0.4f outside of image" % (row["x"], row["y"]))
429  continue
430 
431  try:
432  flux = photoCalib.magnitudeToInstFlux(row[self.config.magVar % band], xy)
433  except LogicError:
434  flux = 0
435 
436  bulge = galsim.Sersic(row[self.config.nBulge], half_light_radius=row[self.config.bulgeHLR])
437  axisRatioBulge = row[self.config.bBulge]/row[self.config.aBulge]
438  bulge = bulge.shear(q=axisRatioBulge, beta=((90 - row[self.config.paBulge])*galsim.degrees))
439 
440  disk = galsim.Sersic(row[self.config.nDisk], half_light_radius=row[self.config.diskHLR])
441  axisRatioDisk = row[self.config.bDisk]/row[self.config.aDisk]
442  disk = disk.shear(q=axisRatioDisk, beta=((90 - row[self.config.paDisk])*galsim.degrees))
443 
444  gal = disk + bulge
445  gal = gal.withFlux(flux)
446 
447  psfIm = galsim.InterpolatedImage(galsim.Image(psfKernel), scale=pixelScale)
448  gal = galsim.Convolve([gal, psfIm])
449  try:
450  galIm = gal.drawImage(scale=pixelScale, method="real_space").array
451  except (galsim.errors.GalSimFFTSizeError, MemoryError):
452  continue
453 
454  yield (afwImage.ImageF(galIm), xy)
455 
456  def mkFakeStars(self, fakeCat, band, photoCalib, psf, image):
457 
458  """Make fake stars based off the properties in the fakeCat.
459 
460  Parameters
461  ----------
462  band : `str`
463  psf : `lsst.meas.extensions.psfex.psfexPsf.PsfexPsf`
464  The PSF information to use to make the PSF images
465  fakeCat : `pandas.core.frame.DataFrame`
466  The catalog of fake sources to be input
467  image : `lsst.afw.image.exposure.exposure.ExposureF`
468  The image into which the fake sources should be added
469  photoCalib : `lsst.afw.image.photoCalib.PhotoCalib`
470  Photometric calibration to be used to calibrate the fake sources
471 
472  Yields
473  -------
474  starImages : `generator`
475  A generator of tuples of `lsst.afw.image.ImageF` of fake stars and
476  `lsst.geom.Point2D` of their locations.
477  """
478 
479  self.log.info("Making %d fake star images" % len(fakeCat))
480 
481  for (index, row) in fakeCat.iterrows():
482  xy = geom.Point2D(row["x"], row["y"])
483 
484  try:
485  # Due to the different radii used for calibration and measurement a correction factor is
486  # needed to prevent there being an offset in the final processed output.
487  correctedFlux = psf.computeApertureFlux(self.config.calibFluxRadius, xy)
488  starIm = psf.computeImage(xy)
489  starIm /= correctedFlux
490 
491  except InvalidParameterError:
492  self.log.info("Star at %0.4f, %0.4f outside of image" % (row["x"], row["y"]))
493  continue
494 
495  try:
496  flux = photoCalib.magnitudeToInstFlux(row[self.config.magVar % band], xy)
497  except LogicError:
498  flux = 0
499 
500  starIm *= flux
501  yield ((starIm.convertF(), xy))
502 
503  def cleanCat(self, fakeCat):
504  """Remove rows from the fakes catalog which have HLR = 0 for either the buldge or disk component
505 
506  Parameters
507  ----------
508  fakeCat : `pandas.core.frame.DataFrame`
509  The catalog of fake sources to be input
510 
511  Returns
512  -------
513  fakeCat : `pandas.core.frame.DataFrame`
514  The input catalog of fake sources but with the bad objects removed
515  """
516 
517  goodRows = ((fakeCat[self.config.bulgeHLR] != 0.0) & (fakeCat[self.config.diskHLR] != 0.0))
518 
519  badRows = len(fakeCat) - len(goodRows)
520  self.log.info("Removing %d rows with HLR = 0 for either the bulge or disk" % badRows)
521 
522  return fakeCat[goodRows]
523 
524  def addFakeSources(self, image, fakeImages, sourceType):
525  """Add the fake sources to the given image
526 
527  Parameters
528  ----------
529  image : `lsst.afw.image.exposure.exposure.ExposureF`
530  The image into which the fake sources should be added
531  fakeImages : `typing.Iterator` [`tuple` ['lsst.afw.image.ImageF`, `lsst.geom.Point2d`]]
532  An iterator of tuples that contains (or generates) images of fake sources,
533  and the locations they are to be inserted at.
534  sourceType : `str`
535  The type (star/galaxy) of fake sources input
536 
537  Returns
538  -------
539  image : `lsst.afw.image.exposure.exposure.ExposureF`
540 
541  Notes
542  -----
543  Uses the x, y information in the ``fakeCat`` to position an image of the fake interpolated onto the
544  pixel grid of the image. Sets the ``FAKE`` mask plane for the pixels added with the fake source.
545  """
546 
547  imageBBox = image.getBBox()
548  imageMI = image.maskedImage
549 
550  for (fakeImage, xy) in fakeImages:
551  X0 = xy.getX() - fakeImage.getWidth()/2 + 0.5
552  Y0 = xy.getY() - fakeImage.getHeight()/2 + 0.5
553  self.log.debug("Adding fake source at %d, %d" % (xy.getX(), xy.getY()))
554  if sourceType == "galaxy":
555  interpFakeImage = afwMath.offsetImage(fakeImage, X0, Y0, "lanczos3")
556  interpFakeImBBox = interpFakeImage.getBBox()
557  else:
558  interpFakeImage = fakeImage
559  interpFakeImBBox = fakeImage.getBBox()
560 
561  interpFakeImBBox.clip(imageBBox)
562  imageMIView = imageMI.Factory(imageMI, interpFakeImBBox)
563 
564  if interpFakeImBBox.getArea() > 0:
565  clippedFakeImage = interpFakeImage.Factory(interpFakeImage, interpFakeImBBox)
566  clippedFakeImageMI = afwImage.MaskedImageF(clippedFakeImage)
567  clippedFakeImageMI.mask.set(self.bitmask)
568  imageMIView += clippedFakeImageMI
569 
570  return image
571 
572  def _getMetadataName(self):
573  """Disable metadata writing"""
574  return None
lsst::afw::image
lsst::sphgeom::ConvexPolygon
lsst.pipe.tasks.insertFakes.InsertFakesConnections
Definition: insertFakes.py:44
lsst.pipe.tasks.assembleCoadd.run
def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, altMaskList=None, mask=None, supplementaryData=None)
Definition: assembleCoadd.py:712
lsst::sphgeom
lsst::geom
lsst::afw::math
Point< double, 2 >
lsst::pex::exceptions
lsst::geom::SpherePoint
lsst::coadd::utils::coaddDataIdContainer
lsst::geom::Box2D
lsst.pipe::base
lsst.pipe::base::connectionTypes