lsst.pipe.tasks  20.0.0-10-g1b4d8e16+2
insertFakes.py
Go to the documentation of this file.
1 # This file is part of pipe tasks
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (http://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <http://www.gnu.org/licenses/>.
21 
22 """
23 Insert fakes into deepCoadds
24 """
25 import galsim
26 from astropy.table import Table
27 
28 import lsst.geom as geom
29 import lsst.afw.image as afwImage
30 import lsst.afw.math as afwMath
31 import lsst.pex.config as pexConfig
32 import lsst.pipe.base as pipeBase
33 
34 from lsst.pipe.base import CmdLineTask, PipelineTask, PipelineTaskConfig, PipelineTaskConnections
36 from lsst.pex.exceptions import LogicError, InvalidParameterError
37 from lsst.coadd.utils.coaddDataIdContainer import ExistingCoaddDataIdContainer
38 from lsst.geom import SpherePoint, radians, Box2D
39 from lsst.sphgeom import ConvexPolygon
40 
41 __all__ = ["InsertFakesConfig", "InsertFakesTask"]
42 
43 
44 class InsertFakesConnections(PipelineTaskConnections, defaultTemplates={"CoaddName": "deep"},
45  dimensions=("tract", "patch", "abstract_filter", "skymap")):
46 
47  image = cT.Input(
48  doc="Image into which fakes are to be added.",
49  name="{CoaddName}Coadd",
50  storageClass="ExposureF",
51  dimensions=("tract", "patch", "abstract_filter", "skymap")
52  )
53 
54  fakeCat = cT.Input(
55  doc="Catalog of fake sources to draw inputs from.",
56  name="{CoaddName}Coadd_fakeSourceCat",
57  storageClass="Parquet",
58  dimensions=("tract", "skymap")
59  )
60 
61  imageWithFakes = cT.Output(
62  doc="Image with fake sources added.",
63  name="fakes_{CoaddName}Coadd",
64  storageClass="ExposureF",
65  dimensions=("tract", "patch", "abstract_filter", "skymap")
66  )
67 
68 
69 class InsertFakesConfig(PipelineTaskConfig,
70  pipelineConnections=InsertFakesConnections):
71  """Config for inserting fake sources
72 
73  Notes
74  -----
75  The default column names are those from the University of Washington sims database.
76  """
77 
78  raColName = pexConfig.Field(
79  doc="RA column name used in the fake source catalog.",
80  dtype=str,
81  default="raJ2000",
82  )
83 
84  decColName = pexConfig.Field(
85  doc="Dec. column name used in the fake source catalog.",
86  dtype=str,
87  default="decJ2000",
88  )
89 
90  doCleanCat = pexConfig.Field(
91  doc="If true removes bad sources from the catalog.",
92  dtype=bool,
93  default=True,
94  )
95 
96  diskHLR = pexConfig.Field(
97  doc="Column name for the disk half light radius used in the fake source catalog.",
98  dtype=str,
99  default="DiskHalfLightRadius",
100  )
101 
102  bulgeHLR = pexConfig.Field(
103  doc="Column name for the bulge half light radius used in the fake source catalog.",
104  dtype=str,
105  default="BulgeHalfLightRadius",
106  )
107 
108  magVar = pexConfig.Field(
109  doc="The column name for the magnitude calculated taking variability into account. In the format "
110  "``filter name``magVar, e.g. imagVar for the magnitude in the i band.",
111  dtype=str,
112  default="%smagVar",
113  )
114 
115  nDisk = pexConfig.Field(
116  doc="The column name for the sersic index of the disk component used in the fake source catalog.",
117  dtype=str,
118  default="disk_n",
119  )
120 
121  nBulge = pexConfig.Field(
122  doc="The column name for the sersic index of the bulge component used in the fake source catalog.",
123  dtype=str,
124  default="bulge_n",
125  )
126 
127  aDisk = pexConfig.Field(
128  doc="The column name for the semi major axis length of the disk component used in the fake source"
129  "catalog.",
130  dtype=str,
131  default="a_d",
132  )
133 
134  aBulge = pexConfig.Field(
135  doc="The column name for the semi major axis length of the bulge component.",
136  dtype=str,
137  default="a_b",
138  )
139 
140  bDisk = pexConfig.Field(
141  doc="The column name for the semi minor axis length of the disk component.",
142  dtype=str,
143  default="b_d",
144  )
145 
146  bBulge = pexConfig.Field(
147  doc="The column name for the semi minor axis length of the bulge component used in the fake source "
148  "catalog.",
149  dtype=str,
150  default="b_b",
151  )
152 
153  paDisk = pexConfig.Field(
154  doc="The column name for the PA of the disk component used in the fake source catalog.",
155  dtype=str,
156  default="pa_disk",
157  )
158 
159  paBulge = pexConfig.Field(
160  doc="The column name for the PA of the bulge component used in the fake source catalog.",
161  dtype=str,
162  default="pa_bulge",
163  )
164 
165  sourceType = pexConfig.Field(
166  doc="The column name for the source type used in the fake source catalog.",
167  dtype=str,
168  default="sourceType",
169  )
170 
171  fakeType = pexConfig.Field(
172  doc="What type of fake catalog to use, snapshot (includes variability in the magnitudes calculated "
173  "from the MJD of the image), static (no variability) or filename for a user defined fits"
174  "catalog.",
175  dtype=str,
176  default="static",
177  )
178 
179  calibFluxRadius = pexConfig.Field(
180  doc="Aperture radius (in pixels) that was used to define the calibration for this image+catalog. "
181  "This will be used to produce the correct instrumental fluxes within the radius. "
182  "This value should match that of the field defined in slot_CalibFlux_instFlux.",
183  dtype=float,
184  default=12.0,
185  )
186 
187  coaddName = pexConfig.Field(
188  doc="The name of the type of coadd used",
189  dtype=str,
190  default="deep",
191  )
192 
193 
194 class InsertFakesTask(PipelineTask, CmdLineTask):
195  """Insert fake objects into images.
196 
197  Add fake stars and galaxies to the given image, read in through the dataRef. Galaxy parameters are read in
198  from the specified file and then modelled using galsim.
199 
200  `InsertFakesTask` has five functions that make images of the fake sources and then add them to the
201  image.
202 
203  `addPixCoords`
204  Use the WCS information to add the pixel coordinates of each source.
205  `mkFakeGalsimGalaxies`
206  Use Galsim to make fake double sersic galaxies for each set of galaxy parameters in the input file.
207  `mkFakeStars`
208  Use the PSF information from the image to make a fake star using the magnitude information from the
209  input file.
210  `cleanCat`
211  Remove rows of the input fake catalog which have half light radius, of either the bulge or the disk,
212  that are 0.
213  `addFakeSources`
214  Add the fake sources to the image.
215 
216  """
217 
218  _DefaultName = "insertFakes"
219  ConfigClass = InsertFakesConfig
220 
221  def runDataRef(self, dataRef):
222  """Read in/write out the required data products and add fake sources to the deepCoadd.
223 
224  Parameters
225  ----------
226  dataRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef`
227  Data reference defining the image to have fakes added to it
228  Used to access the following data products:
229  deepCoadd
230  """
231 
232  infoStr = "Adding fakes to: tract: %d, patch: %s, filter: %s" % (dataRef.dataId["tract"],
233  dataRef.dataId["patch"],
234  dataRef.dataId["filter"])
235  self.log.info(infoStr)
236 
237  # To do: should it warn when asked to insert variable sources into the coadd
238 
239  if self.config.fakeType == "static":
240  fakeCat = dataRef.get("deepCoadd_fakeSourceCat").toDataFrame()
241  # To do: DM-16254, the read and write of the fake catalogs will be changed once the new pipeline
242  # task structure for ref cats is in place.
243  self.fakeSourceCatType = "deepCoadd_fakeSourceCat"
244  else:
245  fakeCat = Table.read(self.config.fakeType).to_pandas()
246 
247  coadd = dataRef.get("deepCoadd")
248  wcs = coadd.getWcs()
249  photoCalib = coadd.getPhotoCalib()
250 
251  imageWithFakes = self.run(fakeCat, coadd, wcs, photoCalib)
252 
253  dataRef.put(imageWithFakes.imageWithFakes, "fakes_deepCoadd")
254 
255  def runQuantum(self, butlerQC, inputRefs, outputRefs):
256  inputs = butlerQC.get(inputRefs)
257  inputs["wcs"] = inputs["image"].getWcs()
258  inputs["photoCalib"] = inputs["image"].getPhotoCalib()
259 
260  outputs = self.run(**inputs)
261  butlerQC.put(outputs, outputRefs)
262 
263  @classmethod
264  def _makeArgumentParser(cls):
265  parser = pipeBase.ArgumentParser(name=cls._DefaultName)
266  parser.add_id_argument(name="--id", datasetType="deepCoadd",
267  help="data IDs for the deepCoadd, e.g. --id tract=12345 patch=1,2 filter=r",
268  ContainerClass=ExistingCoaddDataIdContainer)
269  return parser
270 
271  def run(self, fakeCat, image, wcs, photoCalib):
272  """Add fake sources to an image.
273 
274  Parameters
275  ----------
276  fakeCat : `pandas.core.frame.DataFrame`
277  The catalog of fake sources to be input
278  image : `lsst.afw.image.exposure.exposure.ExposureF`
279  The image into which the fake sources should be added
280  wcs : `lsst.afw.geom.SkyWcs`
281  WCS to use to add fake sources
282  photoCalib : `lsst.afw.image.photoCalib.PhotoCalib`
283  Photometric calibration to be used to calibrate the fake sources
284 
285  Returns
286  -------
287  resultStruct : `lsst.pipe.base.struct.Struct`
288  contains : image : `lsst.afw.image.exposure.exposure.ExposureF`
289 
290  Notes
291  -----
292  Adds pixel coordinates for each source to the fakeCat and removes objects with bulge or disk half
293  light radius = 0 (if ``config.doCleanCat = True``).
294 
295  Adds the ``Fake`` mask plane to the image which is then set by `addFakeSources` to mark where fake
296  sources have been added. Uses the information in the ``fakeCat`` to make fake galaxies (using galsim)
297  and fake stars, using the PSF models from the PSF information for the image. These are then added to
298  the image and the image with fakes included returned.
299 
300  The galsim galaxies are made using a double sersic profile, one for the bulge and one for the disk,
301  this is then convolved with the PSF at that point.
302  """
303 
304  image.mask.addMaskPlane("FAKE")
305  self.bitmask = image.mask.getPlaneBitMask("FAKE")
306  self.log.info("Adding mask plane with bitmask %d" % self.bitmask)
307 
308  fakeCat = self.addPixCoords(fakeCat, wcs)
309  if self.config.doCleanCat:
310  fakeCat = self.cleanCat(fakeCat)
311  fakeCat = self.trimFakeCat(fakeCat, image, wcs)
312 
313  band = image.getFilter().getName()
314  pixelScale = wcs.getPixelScale().asArcseconds()
315  psf = image.getPsf()
316 
317  if isinstance(fakeCat[self.config.sourceType].iloc[0], str):
318  galCheckVal = "galaxy"
319  starCheckVal = "star"
320  elif isinstance(fakeCat[self.config.sourceType].iloc[0], bytes):
321  galCheckVal = b"galaxy"
322  starCheckVal = b"star"
323  elif isinstance(fakeCat[self.config.sourceType].iloc[0], (int, float)):
324  galCheckVal = 1
325  starCheckVal = 0
326  else:
327  raise TypeError("sourceType column does not have required type, should be str, bytes or int")
328 
329  galaxies = (fakeCat[self.config.sourceType] == galCheckVal)
330  galImages = self.mkFakeGalsimGalaxies(fakeCat[galaxies], band, photoCalib, pixelScale, psf, image)
331  image = self.addFakeSources(image, galImages, "galaxy")
332 
333  stars = (fakeCat[self.config.sourceType] == starCheckVal)
334  starImages = self.mkFakeStars(fakeCat[stars], band, photoCalib, psf, image)
335  image = self.addFakeSources(image, starImages, "star")
336  resultStruct = pipeBase.Struct(imageWithFakes=image)
337 
338  return resultStruct
339 
340  def addPixCoords(self, fakeCat, wcs):
341 
342  """Add pixel coordinates to the catalog of fakes.
343 
344  Parameters
345  ----------
346  fakeCat : `pandas.core.frame.DataFrame`
347  The catalog of fake sources to be input
348  wcs : `lsst.afw.geom.SkyWcs`
349  WCS to use to add fake sources
350 
351  Returns
352  -------
353  fakeCat : `pandas.core.frame.DataFrame`
354 
355  Notes
356  -----
357  The default option is to use the WCS information from the image. If the ``useUpdatedCalibs`` config
358  option is set then it will use the updated WCS from jointCal.
359  """
360 
361  ras = fakeCat[self.config.raColName].values
362  decs = fakeCat[self.config.decColName].values
363  skyCoords = [SpherePoint(ra, dec, radians) for (ra, dec) in zip(ras, decs)]
364  pixCoords = wcs.skyToPixel(skyCoords)
365  xs = [coord.getX() for coord in pixCoords]
366  ys = [coord.getY() for coord in pixCoords]
367  fakeCat["x"] = xs
368  fakeCat["y"] = ys
369 
370  return fakeCat
371 
372  def trimFakeCat(self, fakeCat, image, wcs):
373  """Trim the fake cat to about the size of the input image.
374 
375  Parameters
376  ----------
377  fakeCat : `pandas.core.frame.DataFrame`
378  The catalog of fake sources to be input
379  image : `lsst.afw.image.exposure.exposure.ExposureF`
380  The image into which the fake sources should be added
381  wcs : `lsst.afw.geom.SkyWcs`
382  WCS to use to add fake sources
383 
384  Returns
385  -------
386  fakeCat : `pandas.core.frame.DataFrame`
387  The original fakeCat trimmed to the area of the image
388  """
389 
390  bbox = Box2D(image.getBBox())
391  corners = bbox.getCorners()
392 
393  skyCorners = wcs.pixelToSky(corners)
394  region = ConvexPolygon([s.getVector() for s in skyCorners])
395 
396  def trim(row):
397  coord = SpherePoint(row[self.config.raColName], row[self.config.decColName], radians)
398  return region.contains(coord.getVector())
399 
400  return fakeCat[fakeCat.apply(trim, axis=1)]
401 
402  def mkFakeGalsimGalaxies(self, fakeCat, band, photoCalib, pixelScale, psf, image):
403  """Make images of fake galaxies using GalSim.
404 
405  Parameters
406  ----------
407  band : `str`
408  pixelScale : `float`
409  psf : `lsst.meas.extensions.psfex.psfexPsf.PsfexPsf`
410  The PSF information to use to make the PSF images
411  fakeCat : `pandas.core.frame.DataFrame`
412  The catalog of fake sources to be input
413  photoCalib : `lsst.afw.image.photoCalib.PhotoCalib`
414  Photometric calibration to be used to calibrate the fake sources
415 
416  Yields
417  -------
418  galImages : `generator`
419  A generator of tuples of `lsst.afw.image.exposure.exposure.ExposureF` and
420  `lsst.geom.Point2D` of their locations.
421 
422  Notes
423  -----
424 
425  Fake galaxies are made by combining two sersic profiles, one for the bulge and one for the disk. Each
426  component has an individual sersic index (n), a, b and position angle (PA). The combined profile is
427  then convolved with the PSF at the specified x, y position on the image.
428 
429  The names of the columns in the ``fakeCat`` are configurable and are the column names from the
430  University of Washington simulations database as default. For more information see the doc strings
431  attached to the config options.
432 
433  See mkFakeStars doc string for an explanation of calibration to instrumental flux.
434  """
435 
436  self.log.info("Making %d fake galaxy images" % len(fakeCat))
437 
438  for (index, row) in fakeCat.iterrows():
439  xy = geom.Point2D(row["x"], row["y"])
440 
441  try:
442  correctedFlux = psf.computeApertureFlux(self.config.calibFluxRadius, xy)
443  psfKernel = psf.computeKernelImage(xy).getArray()
444  psfKernel /= correctedFlux
445 
446  except InvalidParameterError:
447  self.log.info("Galaxy at %0.4f, %0.4f outside of image" % (row["x"], row["y"]))
448  continue
449 
450  try:
451  flux = photoCalib.magnitudeToInstFlux(row[self.config.magVar % band], xy)
452  except LogicError:
453  flux = 0
454 
455  bulge = galsim.Sersic(row[self.config.nBulge], half_light_radius=row[self.config.bulgeHLR])
456  axisRatioBulge = row[self.config.bBulge]/row[self.config.aBulge]
457  bulge = bulge.shear(q=axisRatioBulge, beta=((90 - row[self.config.paBulge])*galsim.degrees))
458 
459  disk = galsim.Sersic(row[self.config.nDisk], half_light_radius=row[self.config.diskHLR])
460  axisRatioDisk = row[self.config.bDisk]/row[self.config.aDisk]
461  disk = disk.shear(q=axisRatioDisk, beta=((90 - row[self.config.paDisk])*galsim.degrees))
462 
463  gal = disk + bulge
464  gal = gal.withFlux(flux)
465 
466  psfIm = galsim.InterpolatedImage(galsim.Image(psfKernel), scale=pixelScale)
467  gal = galsim.Convolve([gal, psfIm])
468  try:
469  galIm = gal.drawImage(scale=pixelScale, method="real_space").array
470  except (galsim.errors.GalSimFFTSizeError, MemoryError):
471  continue
472 
473  yield (afwImage.ImageF(galIm), xy)
474 
475  def mkFakeStars(self, fakeCat, band, photoCalib, psf, image):
476 
477  """Make fake stars based off the properties in the fakeCat.
478 
479  Parameters
480  ----------
481  band : `str`
482  psf : `lsst.meas.extensions.psfex.psfexPsf.PsfexPsf`
483  The PSF information to use to make the PSF images
484  fakeCat : `pandas.core.frame.DataFrame`
485  The catalog of fake sources to be input
486  image : `lsst.afw.image.exposure.exposure.ExposureF`
487  The image into which the fake sources should be added
488  photoCalib : `lsst.afw.image.photoCalib.PhotoCalib`
489  Photometric calibration to be used to calibrate the fake sources
490 
491  Yields
492  -------
493  starImages : `generator`
494  A generator of tuples of `lsst.afw.image.ImageF` of fake stars and
495  `lsst.geom.Point2D` of their locations.
496 
497  Notes
498  -----
499  To take a given magnitude and translate to the number of counts in the image
500  we use photoCalib.magnitudeToInstFlux, which returns the instrumental flux for the
501  given calibration radius used in the photometric calibration step.
502  Thus `calibFluxRadius` should be set to this same radius so that we can normalize
503  the PSF model to the correct instrumental flux within calibFluxRadius.
504  """
505 
506  self.log.info("Making %d fake star images" % len(fakeCat))
507 
508  for (index, row) in fakeCat.iterrows():
509  xy = geom.Point2D(row["x"], row["y"])
510 
511  # We put these two PSF calculations within this same try block so that we catch cases
512  # where the object's position is outside of the image.
513  try:
514  correctedFlux = psf.computeApertureFlux(self.config.calibFluxRadius, xy)
515  starIm = psf.computeImage(xy)
516  starIm /= correctedFlux
517 
518  except InvalidParameterError:
519  self.log.info("Star at %0.4f, %0.4f outside of image" % (row["x"], row["y"]))
520  continue
521 
522  try:
523  flux = photoCalib.magnitudeToInstFlux(row[self.config.magVar % band], xy)
524  except LogicError:
525  flux = 0
526 
527  starIm *= flux
528  yield ((starIm.convertF(), xy))
529 
530  def cleanCat(self, fakeCat):
531  """Remove rows from the fakes catalog which have HLR = 0 for either the buldge or disk component,
532  also remove rows that have Sersic index outside the galsim min and max allowed. (0.3 <= n <= 6.2)
533 
534  Parameters
535  ----------
536  fakeCat : `pandas.core.frame.DataFrame`
537  The catalog of fake sources to be input
538 
539  Returns
540  -------
541  fakeCat : `pandas.core.frame.DataFrame`
542  The input catalog of fake sources but with the bad objects removed
543  """
544 
545  goodRows = ((fakeCat[self.config.bulgeHLR] != 0.0) & (fakeCat[self.config.diskHLR] != 0.0))
546  badRows = len(fakeCat) - len(goodRows)
547  self.log.info("Removing %d rows with HLR = 0 for either the bulge or disk" % badRows)
548  fakeCat = fakeCat[goodRows]
549 
550  minN = galsim.Sersic._minimum_n
551  maxN = galsim.Sersic._maximum_n
552  goodRows = ((fakeCat[self.config.nBulge] >= minN) & (fakeCat[self.config.nBulge] <= maxN)
553  & (fakeCat[self.config.nDisk] >= minN) & (fakeCat[self.config.nDisk] <= maxN))
554  badRows = len(fakeCat) - len(goodRows)
555  self.log.info("Removing %d rows with nBulge or nDisk outside of %0.2f <= n <= %0.2f" %
556  (badRows, minN, maxN))
557 
558  return fakeCat[goodRows]
559 
560  def addFakeSources(self, image, fakeImages, sourceType):
561  """Add the fake sources to the given image
562 
563  Parameters
564  ----------
565  image : `lsst.afw.image.exposure.exposure.ExposureF`
566  The image into which the fake sources should be added
567  fakeImages : `typing.Iterator` [`tuple` ['lsst.afw.image.ImageF`, `lsst.geom.Point2d`]]
568  An iterator of tuples that contains (or generates) images of fake sources,
569  and the locations they are to be inserted at.
570  sourceType : `str`
571  The type (star/galaxy) of fake sources input
572 
573  Returns
574  -------
575  image : `lsst.afw.image.exposure.exposure.ExposureF`
576 
577  Notes
578  -----
579  Uses the x, y information in the ``fakeCat`` to position an image of the fake interpolated onto the
580  pixel grid of the image. Sets the ``FAKE`` mask plane for the pixels added with the fake source.
581  """
582 
583  imageBBox = image.getBBox()
584  imageMI = image.maskedImage
585 
586  for (fakeImage, xy) in fakeImages:
587  X0 = xy.getX() - fakeImage.getWidth()/2 + 0.5
588  Y0 = xy.getY() - fakeImage.getHeight()/2 + 0.5
589  self.log.debug("Adding fake source at %d, %d" % (xy.getX(), xy.getY()))
590  if sourceType == "galaxy":
591  interpFakeImage = afwMath.offsetImage(fakeImage, X0, Y0, "lanczos3")
592  interpFakeImBBox = interpFakeImage.getBBox()
593  else:
594  interpFakeImage = fakeImage
595  interpFakeImBBox = fakeImage.getBBox()
596 
597  interpFakeImBBox.clip(imageBBox)
598  imageMIView = imageMI.Factory(imageMI, interpFakeImBBox)
599 
600  if interpFakeImBBox.getArea() > 0:
601  clippedFakeImage = interpFakeImage.Factory(interpFakeImage, interpFakeImBBox)
602  clippedFakeImageMI = afwImage.MaskedImageF(clippedFakeImage)
603  clippedFakeImageMI.mask.set(self.bitmask)
604  imageMIView += clippedFakeImageMI
605 
606  return image
607 
608  def _getMetadataName(self):
609  """Disable metadata writing"""
610  return None
lsst::afw::image
lsst::sphgeom::ConvexPolygon
lsst.pipe.tasks.insertFakes.InsertFakesConnections
Definition: insertFakes.py:44
lsst.pipe.tasks.assembleCoadd.run
def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, altMaskList=None, mask=None, supplementaryData=None)
Definition: assembleCoadd.py:712
lsst::sphgeom
lsst::geom
lsst::afw::math
Point< double, 2 >
lsst::pex::exceptions
lsst::geom::SpherePoint
lsst::coadd::utils::coaddDataIdContainer
lsst::geom::Box2D
lsst.pipe::base
lsst.pipe::base::connectionTypes