23 Insert fakes into deepCoadds
26 from astropy.table
import Table
32 import lsst.pex.config
as pexConfig
35 from lsst.pipe.base import CmdLineTask, PipelineTask, PipelineTaskConfig, PipelineTaskConnections
39 from lsst.geom import SpherePoint, radians, Box2D
42 __all__ = [
"InsertFakesConfig",
"InsertFakesTask"]
46 dimensions=(
"tract",
"patch",
"abstract_filter",
"skymap")):
49 doc=
"Image into which fakes are to be added.",
50 name=
"{CoaddName}Coadd",
51 storageClass=
"ExposureF",
52 dimensions=(
"tract",
"patch",
"abstract_filter",
"skymap")
56 doc=
"Catalog of fake sources to draw inputs from.",
57 name=
"{CoaddName}Coadd_fakeSourceCat",
58 storageClass=
"Parquet",
59 dimensions=(
"tract",
"skymap")
62 imageWithFakes = cT.Output(
63 doc=
"Image with fake sources added.",
64 name=
"fakes_{CoaddName}Coadd",
65 storageClass=
"ExposureF",
66 dimensions=(
"tract",
"patch",
"abstract_filter",
"skymap")
70 class InsertFakesConfig(PipelineTaskConfig,
71 pipelineConnections=InsertFakesConnections):
72 """Config for inserting fake sources
76 The default column names are those from the University of Washington sims database.
79 raColName = pexConfig.Field(
80 doc=
"RA column name used in the fake source catalog.",
85 decColName = pexConfig.Field(
86 doc=
"Dec. column name used in the fake source catalog.",
91 doCleanCat = pexConfig.Field(
92 doc=
"If true removes bad sources from the catalog.",
97 diskHLR = pexConfig.Field(
98 doc=
"Column name for the disk half light radius used in the fake source catalog.",
100 default=
"DiskHalfLightRadius",
103 bulgeHLR = pexConfig.Field(
104 doc=
"Column name for the bulge half light radius used in the fake source catalog.",
106 default=
"BulgeHalfLightRadius",
109 magVar = pexConfig.Field(
110 doc=
"The column name for the magnitude calculated taking variability into account. In the format "
111 "``filter name``magVar, e.g. imagVar for the magnitude in the i band.",
116 nDisk = pexConfig.Field(
117 doc=
"The column name for the sersic index of the disk component used in the fake source catalog.",
122 nBulge = pexConfig.Field(
123 doc=
"The column name for the sersic index of the bulge component used in the fake source catalog.",
128 aDisk = pexConfig.Field(
129 doc=
"The column name for the semi major axis length of the disk component used in the fake source"
135 aBulge = pexConfig.Field(
136 doc=
"The column name for the semi major axis length of the bulge component.",
141 bDisk = pexConfig.Field(
142 doc=
"The column name for the semi minor axis length of the disk component.",
147 bBulge = pexConfig.Field(
148 doc=
"The column name for the semi minor axis length of the bulge component used in the fake source "
154 paDisk = pexConfig.Field(
155 doc=
"The column name for the PA of the disk component used in the fake source catalog.",
160 paBulge = pexConfig.Field(
161 doc=
"The column name for the PA of the bulge component used in the fake source catalog.",
166 sourceType = pexConfig.Field(
167 doc=
"The column name for the source type used in the fake source catalog.",
169 default=
"sourceType",
172 fakeType = pexConfig.Field(
173 doc=
"What type of fake catalog to use, snapshot (includes variability in the magnitudes calculated "
174 "from the MJD of the image), static (no variability) or filename for a user defined fits"
180 calibFluxRadius = pexConfig.Field(
181 doc=
"Aperture radius (in pixels) that was used to define the calibration for this image+catalog. "
182 "This will be used to produce the correct instrumental fluxes within the radius. "
183 "This value should match that of the field defined in slot_CalibFlux_instFlux.",
188 coaddName = pexConfig.Field(
189 doc=
"The name of the type of coadd used",
194 doSubSelectSources = pexConfig.Field(
195 doc=
"Set to True if you wish to sub select sources to be input based on the value in the column"
196 "set in the sourceSelectionColName config option.",
201 sourceSelectionColName = pexConfig.Field(
202 doc=
"The name of the column in the input fakes catalogue to be used to determine which sources to"
203 "add, default is none and when this is used all sources are added.",
205 default=
"templateSource"
208 insertImages = pexConfig.Field(
209 doc=
"Insert images directly? True or False.",
215 class InsertFakesTask(PipelineTask, CmdLineTask):
216 """Insert fake objects into images.
218 Add fake stars and galaxies to the given image, read in through the dataRef. Galaxy parameters are read in
219 from the specified file and then modelled using galsim.
221 `InsertFakesTask` has five functions that make images of the fake sources and then add them to the
225 Use the WCS information to add the pixel coordinates of each source.
226 `mkFakeGalsimGalaxies`
227 Use Galsim to make fake double sersic galaxies for each set of galaxy parameters in the input file.
229 Use the PSF information from the image to make a fake star using the magnitude information from the
232 Remove rows of the input fake catalog which have half light radius, of either the bulge or the disk,
233 that are 0. Also removes rows that have Sersic index outside of galsim's allowed paramters. If
234 the config option sourceSelectionColName is set then this function limits the catalog of input fakes
235 to only those which are True in this column.
237 Add the fake sources to the image.
241 _DefaultName =
"insertFakes"
242 ConfigClass = InsertFakesConfig
244 def runDataRef(self, dataRef):
245 """Read in/write out the required data products and add fake sources to the deepCoadd.
249 dataRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef`
250 Data reference defining the image to have fakes added to it
251 Used to access the following data products:
255 infoStr =
"Adding fakes to: tract: %d, patch: %s, filter: %s" % (dataRef.dataId[
"tract"],
256 dataRef.dataId[
"patch"],
257 dataRef.dataId[
"filter"])
258 self.log.info(infoStr)
262 if self.config.fakeType ==
"static":
263 fakeCat = dataRef.get(
"deepCoadd_fakeSourceCat").toDataFrame()
266 self.fakeSourceCatType =
"deepCoadd_fakeSourceCat"
268 fakeCat = Table.read(self.config.fakeType).to_pandas()
270 coadd = dataRef.get(
"deepCoadd")
272 photoCalib = coadd.getPhotoCalib()
274 imageWithFakes = self.run(fakeCat, coadd, wcs, photoCalib)
276 dataRef.put(imageWithFakes.imageWithFakes,
"fakes_deepCoadd")
278 def runQuantum(self, butlerQC, inputRefs, outputRefs):
279 inputs = butlerQC.get(inputRefs)
280 inputs[
"wcs"] = inputs[
"image"].getWcs()
281 inputs[
"photoCalib"] = inputs[
"image"].getPhotoCalib()
283 outputs = self.run(**inputs)
284 butlerQC.put(outputs, outputRefs)
287 def _makeArgumentParser(cls):
288 parser = pipeBase.ArgumentParser(name=cls._DefaultName)
289 parser.add_id_argument(name=
"--id", datasetType=
"deepCoadd",
290 help=
"data IDs for the deepCoadd, e.g. --id tract=12345 patch=1,2 filter=r",
291 ContainerClass=ExistingCoaddDataIdContainer)
294 def run(self, fakeCat, image, wcs, photoCalib):
295 """Add fake sources to an image.
299 fakeCat : `pandas.core.frame.DataFrame`
300 The catalog of fake sources to be input
301 image : `lsst.afw.image.exposure.exposure.ExposureF`
302 The image into which the fake sources should be added
303 wcs : `lsst.afw.geom.SkyWcs`
304 WCS to use to add fake sources
305 photoCalib : `lsst.afw.image.photoCalib.PhotoCalib`
306 Photometric calibration to be used to calibrate the fake sources
310 resultStruct : `lsst.pipe.base.struct.Struct`
311 contains : image : `lsst.afw.image.exposure.exposure.ExposureF`
315 Adds pixel coordinates for each source to the fakeCat and removes objects with bulge or disk half
316 light radius = 0 (if ``config.doCleanCat = True``).
318 Adds the ``Fake`` mask plane to the image which is then set by `addFakeSources` to mark where fake
319 sources have been added. Uses the information in the ``fakeCat`` to make fake galaxies (using galsim)
320 and fake stars, using the PSF models from the PSF information for the image. These are then added to
321 the image and the image with fakes included returned.
323 The galsim galaxies are made using a double sersic profile, one for the bulge and one for the disk,
324 this is then convolved with the PSF at that point.
327 image.mask.addMaskPlane(
"FAKE")
328 self.bitmask = image.mask.getPlaneBitMask(
"FAKE")
329 self.log.info(
"Adding mask plane with bitmask %d" % self.bitmask)
331 fakeCat = self.addPixCoords(fakeCat, wcs)
332 fakeCat = self.trimFakeCat(fakeCat, image, wcs)
333 band = image.getFilter().getName()
335 pixelScale = wcs.getPixelScale().asArcseconds()
337 if len(fakeCat) == 0:
338 errMsg =
"No fakes found for this dataRef."
339 raise RuntimeError(errMsg)
341 if isinstance(fakeCat[self.config.sourceType].iloc[0], str):
342 galCheckVal =
"galaxy"
343 starCheckVal =
"star"
344 elif isinstance(fakeCat[self.config.sourceType].iloc[0], bytes):
345 galCheckVal = b
"galaxy"
346 starCheckVal = b
"star"
347 elif isinstance(fakeCat[self.config.sourceType].iloc[0], (int, float)):
351 raise TypeError(
"sourceType column does not have required type, should be str, bytes or int")
353 if not self.config.insertImages:
354 if self.config.doCleanCat:
355 fakeCat = self.cleanCat(fakeCat)
357 galaxies = (fakeCat[self.config.sourceType] == galCheckVal)
358 galImages = self.mkFakeGalsimGalaxies(fakeCat[galaxies], band, photoCalib, pixelScale, psf, image)
360 stars = (fakeCat[self.config.sourceType] == starCheckVal)
361 starImages = self.mkFakeStars(fakeCat[stars], band, photoCalib, psf, image)
364 galImages, starImages = self.processImagesForInsertion(fakeCat, wcs, psf, photoCalib, band,
367 image = self.addFakeSources(image, galImages,
"galaxy")
368 image = self.addFakeSources(image, starImages,
"star")
369 resultStruct = pipeBase.Struct(imageWithFakes=image)
373 def processImagesForInsertion(self, fakeCat, wcs, psf, photoCalib, band, pixelScale):
374 """Process images from files into the format needed for insertion.
378 fakeCat : `pandas.core.frame.DataFrame`
379 The catalog of fake sources to be input
380 wcs : `lsst.afw.geom.skyWcs.skyWcs.SkyWc`
381 WCS to use to add fake sources
382 psf : `lsst.meas.algorithms.coaddPsf.coaddPsf.CoaddPsf` or
383 `lsst.meas.extensions.psfex.psfexPsf.PsfexPsf`
384 The PSF information to use to make the PSF images
385 photoCalib : `lsst.afw.image.photoCalib.PhotoCalib`
386 Photometric calibration to be used to calibrate the fake sources
388 The filter band that the observation was taken in.
390 The pixel scale of the image the sources are to be added to.
395 A list of tuples of `lsst.afw.image.exposure.exposure.ExposureF` and
396 `lsst.geom.Point2D` of their locations.
397 For sources labelled as galaxy.
399 A list of tuples of `lsst.afw.image.exposure.exposure.ExposureF` and
400 `lsst.geom.Point2D` of their locations.
401 For sources labelled as star.
405 The input fakes catalog needs to contain the absolute path to the image in the
406 band that is being used to add images to. It also needs to have the R.A. and
407 declination of the fake source in radians and the sourceType of the object.
412 self.log.info(
"Processing %d fake images" % len(fakeCat))
414 for (imFile, sourceType, mag, x, y)
in zip(fakeCat[band +
"imFilename"].array,
415 fakeCat[
"sourceType"].array,
416 fakeCat[self.config.magVar % band].array,
417 fakeCat[
"x"].array, fakeCat[
"y"].array):
419 im = afwImage.ImageF.readFits(imFile)
426 correctedFlux = psf.computeApertureFlux(self.config.calibFluxRadius, xy)
427 psfKernel = psf.computeKernelImage(xy).getArray()
428 psfKernel /= correctedFlux
430 except InvalidParameterError:
431 self.log.info(
"%s at %0.4f, %0.4f outside of image" % (sourceType, x, y))
434 psfIm = galsim.InterpolatedImage(galsim.Image(psfKernel), scale=pixelScale)
435 galsimIm = galsim.InterpolatedImage(galsim.Image(im.array), scale=pixelScale)
436 convIm = galsim.Convolve([galsimIm, psfIm])
439 outIm = convIm.drawImage(scale=pixelScale, method=
"real_space").array
440 except (galsim.errors.GalSimFFTSizeError, MemoryError):
443 imSum = np.sum(outIm)
447 flux = photoCalib.magnitudeToInstFlux(mag, xy)
451 imWithFlux = flux*divIm
453 if sourceType == b
"galaxy":
454 galImages.append((afwImage.ImageF(imWithFlux), xy))
455 if sourceType == b
"star":
456 starImages.append((afwImage.ImageF(imWithFlux), xy))
458 return galImages, starImages
462 """Add pixel coordinates to the catalog of fakes.
466 fakeCat : `pandas.core.frame.DataFrame`
467 The catalog of fake sources to be input
468 wcs : `lsst.afw.geom.SkyWcs`
469 WCS to use to add fake sources
473 fakeCat : `pandas.core.frame.DataFrame`
477 The default option is to use the WCS information from the image. If the ``useUpdatedCalibs`` config
478 option is set then it will use the updated WCS from jointCal.
481 ras = fakeCat[self.config.raColName].values
482 decs = fakeCat[self.config.decColName].values
483 skyCoords = [
SpherePoint(ra, dec, radians)
for (ra, dec)
in zip(ras, decs)]
484 pixCoords = wcs.skyToPixel(skyCoords)
485 xs = [coord.getX()
for coord
in pixCoords]
486 ys = [coord.getY()
for coord
in pixCoords]
493 """Trim the fake cat to about the size of the input image.
497 fakeCat : `pandas.core.frame.DataFrame`
498 The catalog of fake sources to be input
499 image : `lsst.afw.image.exposure.exposure.ExposureF`
500 The image into which the fake sources should be added
501 wcs : `lsst.afw.geom.SkyWcs`
502 WCS to use to add fake sources
506 fakeCat : `pandas.core.frame.DataFrame`
507 The original fakeCat trimmed to the area of the image
510 bbox =
Box2D(image.getBBox())
511 corners = bbox.getCorners()
513 skyCorners = wcs.pixelToSky(corners)
517 coord =
SpherePoint(row[self.config.raColName], row[self.config.decColName], radians)
518 return region.contains(coord.getVector())
520 return fakeCat[fakeCat.apply(trim, axis=1)]
523 """Make images of fake galaxies using GalSim.
529 psf : `lsst.meas.extensions.psfex.psfexPsf.PsfexPsf`
530 The PSF information to use to make the PSF images
531 fakeCat : `pandas.core.frame.DataFrame`
532 The catalog of fake sources to be input
533 photoCalib : `lsst.afw.image.photoCalib.PhotoCalib`
534 Photometric calibration to be used to calibrate the fake sources
538 galImages : `generator`
539 A generator of tuples of `lsst.afw.image.exposure.exposure.ExposureF` and
540 `lsst.geom.Point2D` of their locations.
545 Fake galaxies are made by combining two sersic profiles, one for the bulge and one for the disk. Each
546 component has an individual sersic index (n), a, b and position angle (PA). The combined profile is
547 then convolved with the PSF at the specified x, y position on the image.
549 The names of the columns in the ``fakeCat`` are configurable and are the column names from the
550 University of Washington simulations database as default. For more information see the doc strings
551 attached to the config options.
553 See mkFakeStars doc string for an explanation of calibration to instrumental flux.
556 self.log.info(
"Making %d fake galaxy images" % len(fakeCat))
558 for (index, row)
in fakeCat.iterrows():
564 correctedFlux = psf.computeApertureFlux(self.config.calibFluxRadius, xy)
565 psfKernel = psf.computeKernelImage(xy).getArray()
566 psfKernel /= correctedFlux
568 except InvalidParameterError:
569 self.log.info(
"Galaxy at %0.4f, %0.4f outside of image" % (row[
"x"], row[
"y"]))
573 flux = photoCalib.magnitudeToInstFlux(row[self.config.magVar % band], xy)
577 bulge = galsim.Sersic(row[self.config.nBulge], half_light_radius=row[self.config.bulgeHLR])
578 axisRatioBulge = row[self.config.bBulge]/row[self.config.aBulge]
579 bulge = bulge.shear(q=axisRatioBulge, beta=((90 - row[self.config.paBulge])*galsim.degrees))
581 disk = galsim.Sersic(row[self.config.nDisk], half_light_radius=row[self.config.diskHLR])
582 axisRatioDisk = row[self.config.bDisk]/row[self.config.aDisk]
583 disk = disk.shear(q=axisRatioDisk, beta=((90 - row[self.config.paDisk])*galsim.degrees))
586 gal = gal.withFlux(flux)
588 psfIm = galsim.InterpolatedImage(galsim.Image(psfKernel), scale=pixelScale)
589 gal = galsim.Convolve([gal, psfIm])
591 galIm = gal.drawImage(scale=pixelScale, method=
"real_space").array
592 except (galsim.errors.GalSimFFTSizeError, MemoryError):
595 yield (afwImage.ImageF(galIm), xy)
599 """Make fake stars based off the properties in the fakeCat.
604 psf : `lsst.meas.extensions.psfex.psfexPsf.PsfexPsf`
605 The PSF information to use to make the PSF images
606 fakeCat : `pandas.core.frame.DataFrame`
607 The catalog of fake sources to be input
608 image : `lsst.afw.image.exposure.exposure.ExposureF`
609 The image into which the fake sources should be added
610 photoCalib : `lsst.afw.image.photoCalib.PhotoCalib`
611 Photometric calibration to be used to calibrate the fake sources
615 starImages : `generator`
616 A generator of tuples of `lsst.afw.image.ImageF` of fake stars and
617 `lsst.geom.Point2D` of their locations.
621 To take a given magnitude and translate to the number of counts in the image
622 we use photoCalib.magnitudeToInstFlux, which returns the instrumental flux for the
623 given calibration radius used in the photometric calibration step.
624 Thus `calibFluxRadius` should be set to this same radius so that we can normalize
625 the PSF model to the correct instrumental flux within calibFluxRadius.
628 self.log.info(
"Making %d fake star images" % len(fakeCat))
630 for (index, row)
in fakeCat.iterrows():
636 correctedFlux = psf.computeApertureFlux(self.config.calibFluxRadius, xy)
637 starIm = psf.computeImage(xy)
638 starIm /= correctedFlux
640 except InvalidParameterError:
641 self.log.info(
"Star at %0.4f, %0.4f outside of image" % (row[
"x"], row[
"y"]))
645 flux = photoCalib.magnitudeToInstFlux(row[self.config.magVar % band], xy)
650 yield ((starIm.convertF(), xy))
653 """Remove rows from the fakes catalog which have HLR = 0 for either the buldge or disk component,
654 also remove rows that have Sersic index outside the galsim min and max allowed. (0.3 <= n <= 6.2)
658 fakeCat : `pandas.core.frame.DataFrame`
659 The catalog of fake sources to be input
663 fakeCat : `pandas.core.frame.DataFrame`
664 The input catalog of fake sources but with the bad objects removed
668 If the config option sourceSelectionColName is set then only objects with this column set to True
672 rowsToKeep = ((fakeCat[self.config.bulgeHLR] != 0.0) & (fakeCat[self.config.diskHLR] != 0.0))
673 numRowsNotUsed = len(fakeCat) - len(rowsToKeep)
674 self.log.info(
"Removing %d rows with HLR = 0 for either the bulge or disk" % numRowsNotUsed)
675 fakeCat = fakeCat[rowsToKeep]
677 minN = galsim.Sersic._minimum_n
678 maxN = galsim.Sersic._maximum_n
679 rowsToKeep = ((fakeCat[self.config.nBulge] >= minN) & (fakeCat[self.config.nBulge] <= maxN)
680 & (fakeCat[self.config.nDisk] >= minN) & (fakeCat[self.config.nDisk] <= maxN))
681 numRowsNotUsed = len(fakeCat) - len(rowsToKeep)
682 self.log.info(
"Removing %d rows with nBulge or nDisk outside of %0.2f <= n <= %0.2f" %
683 (numRowsNotUsed, minN, maxN))
685 if self.config.doSubSelectSources:
687 rowsToKeep = (fakeCat[self.config.sourceSelectionColName])
689 raise KeyError(
"Given column, %s, for source selection not found." %
690 self.config.sourceSelectionColName)
691 numRowsNotUsed = len(fakeCat) - len(rowsToKeep)
692 self.log.info(
"Removing %d rows which were not designated as template sources" % numRowsNotUsed)
694 return fakeCat[rowsToKeep]
697 """Add the fake sources to the given image
701 image : `lsst.afw.image.exposure.exposure.ExposureF`
702 The image into which the fake sources should be added
703 fakeImages : `typing.Iterator` [`tuple` ['lsst.afw.image.ImageF`, `lsst.geom.Point2d`]]
704 An iterator of tuples that contains (or generates) images of fake sources,
705 and the locations they are to be inserted at.
707 The type (star/galaxy) of fake sources input
711 image : `lsst.afw.image.exposure.exposure.ExposureF`
715 Uses the x, y information in the ``fakeCat`` to position an image of the fake interpolated onto the
716 pixel grid of the image. Sets the ``FAKE`` mask plane for the pixels added with the fake source.
719 imageBBox = image.getBBox()
720 imageMI = image.maskedImage
722 for (fakeImage, xy)
in fakeImages:
723 X0 = xy.getX() - fakeImage.getWidth()/2 + 0.5
724 Y0 = xy.getY() - fakeImage.getHeight()/2 + 0.5
725 self.log.debug(
"Adding fake source at %d, %d" % (xy.getX(), xy.getY()))
726 if sourceType ==
"galaxy":
727 interpFakeImage = afwMath.offsetImage(fakeImage, X0, Y0,
"lanczos3")
728 interpFakeImBBox = interpFakeImage.getBBox()
730 interpFakeImage = fakeImage
731 interpFakeImBBox = fakeImage.getBBox()
733 interpFakeImBBox.clip(imageBBox)
734 imageMIView = imageMI.Factory(imageMI, interpFakeImBBox)
736 if interpFakeImBBox.getArea() > 0:
737 clippedFakeImage = interpFakeImage.Factory(interpFakeImage, interpFakeImBBox)
738 clippedFakeImageMI = afwImage.MaskedImageF(clippedFakeImage)
739 clippedFakeImageMI.mask.set(self.bitmask)
740 imageMIView += clippedFakeImageMI
744 def _getMetadataName(self):
745 """Disable metadata writing"""