23 Insert fakes into deepCoadds
26 from astropy.table
import Table
35 from lsst.pipe.base import CmdLineTask, PipelineTask, PipelineTaskConfig, PipelineTaskConnections
39 from lsst.geom import SpherePoint, radians, Box2D
41 __all__ = [
"InsertFakesConfig",
"InsertFakesTask"]
45 defaultTemplates={
"coaddName":
"deep",
46 "fakesType":
"fakes_"},
47 dimensions=(
"tract",
"patch",
"band",
"skymap")):
50 doc=
"Image into which fakes are to be added.",
51 name=
"{coaddName}Coadd",
52 storageClass=
"ExposureF",
53 dimensions=(
"tract",
"patch",
"band",
"skymap")
57 doc=
"Catalog of fake sources to draw inputs from.",
58 name=
"{fakesType}fakeSourceCat",
59 storageClass=
"DataFrame",
60 dimensions=(
"tract",
"skymap")
63 imageWithFakes = cT.Output(
64 doc=
"Image with fake sources added.",
65 name=
"{fakesType}{coaddName}Coadd",
66 storageClass=
"ExposureF",
67 dimensions=(
"tract",
"patch",
"band",
"skymap")
71 class InsertFakesConfig(PipelineTaskConfig,
72 pipelineConnections=InsertFakesConnections):
73 """Config for inserting fake sources
77 The default column names are those from the University of Washington sims database.
80 raColName = pexConfig.Field(
81 doc=
"RA column name used in the fake source catalog.",
86 decColName = pexConfig.Field(
87 doc=
"Dec. column name used in the fake source catalog.",
92 doCleanCat = pexConfig.Field(
93 doc=
"If true removes bad sources from the catalog.",
98 diskHLR = pexConfig.Field(
99 doc=
"Column name for the disk half light radius used in the fake source catalog.",
101 default=
"DiskHalfLightRadius",
104 bulgeHLR = pexConfig.Field(
105 doc=
"Column name for the bulge half light radius used in the fake source catalog.",
107 default=
"BulgeHalfLightRadius",
110 magVar = pexConfig.Field(
111 doc=
"The column name for the magnitude calculated taking variability into account. In the format "
112 "``filter name``magVar, e.g. imagVar for the magnitude in the i band.",
117 nDisk = pexConfig.Field(
118 doc=
"The column name for the sersic index of the disk component used in the fake source catalog.",
123 nBulge = pexConfig.Field(
124 doc=
"The column name for the sersic index of the bulge component used in the fake source catalog.",
129 aDisk = pexConfig.Field(
130 doc=
"The column name for the semi major axis length of the disk component used in the fake source"
136 aBulge = pexConfig.Field(
137 doc=
"The column name for the semi major axis length of the bulge component.",
142 bDisk = pexConfig.Field(
143 doc=
"The column name for the semi minor axis length of the disk component.",
148 bBulge = pexConfig.Field(
149 doc=
"The column name for the semi minor axis length of the bulge component used in the fake source "
155 paDisk = pexConfig.Field(
156 doc=
"The column name for the PA of the disk component used in the fake source catalog.",
161 paBulge = pexConfig.Field(
162 doc=
"The column name for the PA of the bulge component used in the fake source catalog.",
167 sourceType = pexConfig.Field(
168 doc=
"The column name for the source type used in the fake source catalog.",
170 default=
"sourceType",
173 fakeType = pexConfig.Field(
174 doc=
"What type of fake catalog to use, snapshot (includes variability in the magnitudes calculated "
175 "from the MJD of the image), static (no variability) or filename for a user defined fits"
181 calibFluxRadius = pexConfig.Field(
182 doc=
"Aperture radius (in pixels) that was used to define the calibration for this image+catalog. "
183 "This will be used to produce the correct instrumental fluxes within the radius. "
184 "This value should match that of the field defined in slot_CalibFlux_instFlux.",
189 coaddName = pexConfig.Field(
190 doc=
"The name of the type of coadd used",
195 doSubSelectSources = pexConfig.Field(
196 doc=
"Set to True if you wish to sub select sources to be input based on the value in the column"
197 "set in the sourceSelectionColName config option.",
202 sourceSelectionColName = pexConfig.Field(
203 doc=
"The name of the column in the input fakes catalogue to be used to determine which sources to"
204 "add, default is none and when this is used all sources are added.",
206 default=
"templateSource"
209 insertImages = pexConfig.Field(
210 doc=
"Insert images directly? True or False.",
215 doProcessAllDataIds = pexConfig.Field(
216 doc=
"If True, all input data IDs will be processed, even those containing no fake sources.",
222 class InsertFakesTask(PipelineTask, CmdLineTask):
223 """Insert fake objects into images.
225 Add fake stars and galaxies to the given image, read in through the dataRef. Galaxy parameters are read in
226 from the specified file and then modelled using galsim.
228 `InsertFakesTask` has five functions that make images of the fake sources and then add them to the
232 Use the WCS information to add the pixel coordinates of each source.
233 `mkFakeGalsimGalaxies`
234 Use Galsim to make fake double sersic galaxies for each set of galaxy parameters in the input file.
236 Use the PSF information from the image to make a fake star using the magnitude information from the
239 Remove rows of the input fake catalog which have half light radius, of either the bulge or the disk,
240 that are 0. Also removes rows that have Sersic index outside of galsim's allowed paramters. If
241 the config option sourceSelectionColName is set then this function limits the catalog of input fakes
242 to only those which are True in this column.
244 Add the fake sources to the image.
248 _DefaultName =
"insertFakes"
249 ConfigClass = InsertFakesConfig
251 def runDataRef(self, dataRef):
252 """Read in/write out the required data products and add fake sources to the deepCoadd.
256 dataRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef`
257 Data reference defining the image to have fakes added to it
258 Used to access the following data products:
262 infoStr =
"Adding fakes to: tract: %d, patch: %s, filter: %s" % (dataRef.dataId[
"tract"],
263 dataRef.dataId[
"patch"],
264 dataRef.dataId[
"filter"])
265 self.log.info(infoStr)
269 if self.config.fakeType ==
"static":
270 fakeCat = dataRef.get(
"deepCoadd_fakeSourceCat").toDataFrame()
273 self.fakeSourceCatType =
"deepCoadd_fakeSourceCat"
275 fakeCat = Table.read(self.config.fakeType).to_pandas()
277 coadd = dataRef.get(
"deepCoadd")
279 photoCalib = coadd.getPhotoCalib()
281 imageWithFakes = self.run(fakeCat, coadd, wcs, photoCalib)
283 dataRef.put(imageWithFakes.imageWithFakes,
"fakes_deepCoadd")
285 def runQuantum(self, butlerQC, inputRefs, outputRefs):
286 inputs = butlerQC.get(inputRefs)
287 inputs[
"wcs"] = inputs[
"image"].getWcs()
288 inputs[
"photoCalib"] = inputs[
"image"].getPhotoCalib()
290 outputs = self.run(**inputs)
291 butlerQC.put(outputs, outputRefs)
294 def _makeArgumentParser(cls):
295 parser = pipeBase.ArgumentParser(name=cls._DefaultName)
296 parser.add_id_argument(name=
"--id", datasetType=
"deepCoadd",
297 help=
"data IDs for the deepCoadd, e.g. --id tract=12345 patch=1,2 filter=r",
298 ContainerClass=ExistingCoaddDataIdContainer)
301 def run(self, fakeCat, image, wcs, photoCalib):
302 """Add fake sources to an image.
306 fakeCat : `pandas.core.frame.DataFrame`
307 The catalog of fake sources to be input
308 image : `lsst.afw.image.exposure.exposure.ExposureF`
309 The image into which the fake sources should be added
310 wcs : `lsst.afw.geom.SkyWcs`
311 WCS to use to add fake sources
312 photoCalib : `lsst.afw.image.photoCalib.PhotoCalib`
313 Photometric calibration to be used to calibrate the fake sources
317 resultStruct : `lsst.pipe.base.struct.Struct`
318 contains : image : `lsst.afw.image.exposure.exposure.ExposureF`
322 Adds pixel coordinates for each source to the fakeCat and removes objects with bulge or disk half
323 light radius = 0 (if ``config.doCleanCat = True``).
325 Adds the ``Fake`` mask plane to the image which is then set by `addFakeSources` to mark where fake
326 sources have been added. Uses the information in the ``fakeCat`` to make fake galaxies (using galsim)
327 and fake stars, using the PSF models from the PSF information for the image. These are then added to
328 the image and the image with fakes included returned.
330 The galsim galaxies are made using a double sersic profile, one for the bulge and one for the disk,
331 this is then convolved with the PSF at that point.
334 image.mask.addMaskPlane(
"FAKE")
335 self.bitmask = image.mask.getPlaneBitMask(
"FAKE")
336 self.log.info(
"Adding mask plane with bitmask %d" % self.bitmask)
338 fakeCat = self.addPixCoords(fakeCat, wcs)
339 fakeCat = self.trimFakeCat(fakeCat, image, wcs)
340 band = image.getFilter().getCanonicalName()[0]
342 pixelScale = wcs.getPixelScale().asArcseconds()
345 if isinstance(fakeCat[self.config.sourceType].iloc[0], str):
346 galCheckVal =
"galaxy"
347 starCheckVal =
"star"
348 elif isinstance(fakeCat[self.config.sourceType].iloc[0], bytes):
349 galCheckVal = b
"galaxy"
350 starCheckVal = b
"star"
351 elif isinstance(fakeCat[self.config.sourceType].iloc[0], (int, float)):
355 raise TypeError(
"sourceType column does not have required type, should be str, bytes or int")
357 if not self.config.insertImages:
358 if self.config.doCleanCat:
359 fakeCat = self.cleanCat(fakeCat, starCheckVal)
361 galaxies = (fakeCat[self.config.sourceType] == galCheckVal)
362 galImages = self.mkFakeGalsimGalaxies(fakeCat[galaxies], band, photoCalib, pixelScale, psf,
365 stars = (fakeCat[self.config.sourceType] == starCheckVal)
366 starImages = self.mkFakeStars(fakeCat[stars], band, photoCalib, psf, image)
368 galImages, starImages = self.processImagesForInsertion(fakeCat, wcs, psf, photoCalib, band,
371 image = self.addFakeSources(image, galImages,
"galaxy")
372 image = self.addFakeSources(image, starImages,
"star")
373 elif len(fakeCat) == 0
and self.config.doProcessAllDataIds:
374 self.log.warn(
"No fakes found for this dataRef; processing anyway.")
376 raise RuntimeError(
"No fakes found for this dataRef.")
378 resultStruct = pipeBase.Struct(imageWithFakes=image)
382 def processImagesForInsertion(self, fakeCat, wcs, psf, photoCalib, band, pixelScale):
383 """Process images from files into the format needed for insertion.
387 fakeCat : `pandas.core.frame.DataFrame`
388 The catalog of fake sources to be input
389 wcs : `lsst.afw.geom.skyWcs.skyWcs.SkyWc`
390 WCS to use to add fake sources
391 psf : `lsst.meas.algorithms.coaddPsf.coaddPsf.CoaddPsf` or
392 `lsst.meas.extensions.psfex.psfexPsf.PsfexPsf`
393 The PSF information to use to make the PSF images
394 photoCalib : `lsst.afw.image.photoCalib.PhotoCalib`
395 Photometric calibration to be used to calibrate the fake sources
397 The filter band that the observation was taken in.
399 The pixel scale of the image the sources are to be added to.
404 A list of tuples of `lsst.afw.image.exposure.exposure.ExposureF` and
405 `lsst.geom.Point2D` of their locations.
406 For sources labelled as galaxy.
408 A list of tuples of `lsst.afw.image.exposure.exposure.ExposureF` and
409 `lsst.geom.Point2D` of their locations.
410 For sources labelled as star.
414 The input fakes catalog needs to contain the absolute path to the image in the
415 band that is being used to add images to. It also needs to have the R.A. and
416 declination of the fake source in radians and the sourceType of the object.
421 self.log.info(
"Processing %d fake images" % len(fakeCat))
423 for (imFile, sourceType, mag, x, y)
in zip(fakeCat[band +
"imFilename"].array,
424 fakeCat[
"sourceType"].array,
425 fakeCat[self.config.magVar % band].array,
426 fakeCat[
"x"].array, fakeCat[
"y"].array):
428 im = afwImage.ImageF.readFits(imFile)
435 correctedFlux = psf.computeApertureFlux(self.config.calibFluxRadius, xy)
436 psfKernel = psf.computeKernelImage(xy).getArray()
437 psfKernel /= correctedFlux
439 except InvalidParameterError:
440 self.log.info(
"%s at %0.4f, %0.4f outside of image" % (sourceType, x, y))
443 psfIm = galsim.InterpolatedImage(galsim.Image(psfKernel), scale=pixelScale)
444 galsimIm = galsim.InterpolatedImage(galsim.Image(im.array), scale=pixelScale)
445 convIm = galsim.Convolve([galsimIm, psfIm])
448 outIm = convIm.drawImage(scale=pixelScale, method=
"real_space").array
449 except (galsim.errors.GalSimFFTSizeError, MemoryError):
452 imSum = np.sum(outIm)
456 flux = photoCalib.magnitudeToInstFlux(mag, xy)
460 imWithFlux = flux*divIm
462 if sourceType == b
"galaxy":
463 galImages.append((afwImage.ImageF(imWithFlux), xy))
464 if sourceType == b
"star":
465 starImages.append((afwImage.ImageF(imWithFlux), xy))
467 return galImages, starImages
471 """Add pixel coordinates to the catalog of fakes.
475 fakeCat : `pandas.core.frame.DataFrame`
476 The catalog of fake sources to be input
477 wcs : `lsst.afw.geom.SkyWcs`
478 WCS to use to add fake sources
482 fakeCat : `pandas.core.frame.DataFrame`
486 The default option is to use the WCS information from the image. If the ``useUpdatedCalibs`` config
487 option is set then it will use the updated WCS from jointCal.
490 ras = fakeCat[self.config.raColName].values
491 decs = fakeCat[self.config.decColName].values
492 skyCoords = [
SpherePoint(ra, dec, radians)
for (ra, dec)
in zip(ras, decs)]
493 pixCoords = wcs.skyToPixel(skyCoords)
494 xs = [coord.getX()
for coord
in pixCoords]
495 ys = [coord.getY()
for coord
in pixCoords]
502 """Trim the fake cat to about the size of the input image.
504 `fakeCat` must be processed with addPixCoords before using this method.
508 fakeCat : `pandas.core.frame.DataFrame`
509 The catalog of fake sources to be input
510 image : `lsst.afw.image.exposure.exposure.ExposureF`
511 The image into which the fake sources should be added
512 wcs : `lsst.afw.geom.SkyWcs`
513 WCS to use to add fake sources
517 fakeCat : `pandas.core.frame.DataFrame`
518 The original fakeCat trimmed to the area of the image
521 bbox =
Box2D(image.getBBox())
524 return bbox.contains(row[
"x"], row[
"y"])
526 return fakeCat[fakeCat.apply(trim, axis=1)]
529 """Make images of fake galaxies using GalSim.
535 psf : `lsst.meas.extensions.psfex.psfexPsf.PsfexPsf`
536 The PSF information to use to make the PSF images
537 fakeCat : `pandas.core.frame.DataFrame`
538 The catalog of fake sources to be input
539 photoCalib : `lsst.afw.image.photoCalib.PhotoCalib`
540 Photometric calibration to be used to calibrate the fake sources
544 galImages : `generator`
545 A generator of tuples of `lsst.afw.image.exposure.exposure.ExposureF` and
546 `lsst.geom.Point2D` of their locations.
551 Fake galaxies are made by combining two sersic profiles, one for the bulge and one for the disk. Each
552 component has an individual sersic index (n), a, b and position angle (PA). The combined profile is
553 then convolved with the PSF at the specified x, y position on the image.
555 The names of the columns in the ``fakeCat`` are configurable and are the column names from the
556 University of Washington simulations database as default. For more information see the doc strings
557 attached to the config options.
559 See mkFakeStars doc string for an explanation of calibration to instrumental flux.
562 self.log.info(
"Making %d fake galaxy images" % len(fakeCat))
564 for (index, row)
in fakeCat.iterrows():
570 correctedFlux = psf.computeApertureFlux(self.config.calibFluxRadius, xy)
571 psfKernel = psf.computeKernelImage(xy).getArray()
572 psfKernel /= correctedFlux
574 except InvalidParameterError:
575 self.log.info(
"Galaxy at %0.4f, %0.4f outside of image" % (row[
"x"], row[
"y"]))
579 flux = photoCalib.magnitudeToInstFlux(row[self.config.magVar % band], xy)
583 bulge = galsim.Sersic(row[self.config.nBulge], half_light_radius=row[self.config.bulgeHLR])
584 axisRatioBulge = row[self.config.bBulge]/row[self.config.aBulge]
585 bulge = bulge.shear(q=axisRatioBulge, beta=((90 - row[self.config.paBulge])*galsim.degrees))
587 disk = galsim.Sersic(row[self.config.nDisk], half_light_radius=row[self.config.diskHLR])
588 axisRatioDisk = row[self.config.bDisk]/row[self.config.aDisk]
589 disk = disk.shear(q=axisRatioDisk, beta=((90 - row[self.config.paDisk])*galsim.degrees))
592 gal = gal.withFlux(flux)
594 psfIm = galsim.InterpolatedImage(galsim.Image(psfKernel), scale=pixelScale)
595 gal = galsim.Convolve([gal, psfIm])
597 galIm = gal.drawImage(scale=pixelScale, method=
"real_space").array
598 except (galsim.errors.GalSimFFTSizeError, MemoryError):
601 yield (afwImage.ImageF(galIm), xy)
605 """Make fake stars based off the properties in the fakeCat.
610 psf : `lsst.meas.extensions.psfex.psfexPsf.PsfexPsf`
611 The PSF information to use to make the PSF images
612 fakeCat : `pandas.core.frame.DataFrame`
613 The catalog of fake sources to be input
614 image : `lsst.afw.image.exposure.exposure.ExposureF`
615 The image into which the fake sources should be added
616 photoCalib : `lsst.afw.image.photoCalib.PhotoCalib`
617 Photometric calibration to be used to calibrate the fake sources
621 starImages : `generator`
622 A generator of tuples of `lsst.afw.image.ImageF` of fake stars and
623 `lsst.geom.Point2D` of their locations.
627 To take a given magnitude and translate to the number of counts in the image
628 we use photoCalib.magnitudeToInstFlux, which returns the instrumental flux for the
629 given calibration radius used in the photometric calibration step.
630 Thus `calibFluxRadius` should be set to this same radius so that we can normalize
631 the PSF model to the correct instrumental flux within calibFluxRadius.
634 self.log.info(
"Making %d fake star images" % len(fakeCat))
636 for (index, row)
in fakeCat.iterrows():
642 correctedFlux = psf.computeApertureFlux(self.config.calibFluxRadius, xy)
643 starIm = psf.computeImage(xy)
644 starIm /= correctedFlux
646 except InvalidParameterError:
647 self.log.info(
"Star at %0.4f, %0.4f outside of image" % (row[
"x"], row[
"y"]))
651 flux = photoCalib.magnitudeToInstFlux(row[self.config.magVar % band], xy)
656 yield ((starIm.convertF(), xy))
659 """Remove rows from the fakes catalog which have HLR = 0 for either the buldge or disk component,
660 also remove galaxies that have Sersic index outside the galsim min and max
661 allowed (0.3 <= n <= 6.2).
665 fakeCat : `pandas.core.frame.DataFrame`
666 The catalog of fake sources to be input
667 starCheckVal : `str`, `bytes` or `int`
668 The value that is set in the sourceType column to specifiy an object is a star.
672 fakeCat : `pandas.core.frame.DataFrame`
673 The input catalog of fake sources but with the bad objects removed
677 If the config option sourceSelectionColName is set then only objects with this column set to True
681 rowsToKeep = (((fakeCat[self.config.bulgeHLR] != 0.0) & (fakeCat[self.config.diskHLR] != 0.0))
682 | (fakeCat[self.config.sourceType] == starCheckVal))
683 numRowsNotUsed = len(fakeCat) - len(np.where(rowsToKeep)[0])
684 self.log.info(
"Removing %d rows with HLR = 0 for either the bulge or disk" % numRowsNotUsed)
685 fakeCat = fakeCat[rowsToKeep]
687 minN = galsim.Sersic._minimum_n
688 maxN = galsim.Sersic._maximum_n
689 rowsWithGoodSersic = (((fakeCat[self.config.nBulge] >= minN) & (fakeCat[self.config.nBulge] <= maxN)
690 & (fakeCat[self.config.nDisk] >= minN) & (fakeCat[self.config.nDisk] <= maxN))
691 | (fakeCat[self.config.sourceType] == starCheckVal))
692 numRowsNotUsed = len(fakeCat) - len(np.where(rowsWithGoodSersic)[0])
693 self.log.info(
"Removing %d rows of galaxies with nBulge or nDisk outside of %0.2f <= n <= %0.2f" %
694 (numRowsNotUsed, minN, maxN))
695 fakeCat = fakeCat[rowsWithGoodSersic]
697 if self.config.doSubSelectSources:
699 rowsSelected = (fakeCat[self.config.sourceSelectionColName])
701 raise KeyError(
"Given column, %s, for source selection not found." %
702 self.config.sourceSelectionColName)
703 numRowsNotUsed = len(fakeCat) - len(rowsSelected)
704 self.log.info(
"Removing %d rows which were not designated as template sources" % numRowsNotUsed)
705 fakeCat = fakeCat[rowsSelected]
710 """Add the fake sources to the given image
714 image : `lsst.afw.image.exposure.exposure.ExposureF`
715 The image into which the fake sources should be added
716 fakeImages : `typing.Iterator` [`tuple` ['lsst.afw.image.ImageF`, `lsst.geom.Point2d`]]
717 An iterator of tuples that contains (or generates) images of fake sources,
718 and the locations they are to be inserted at.
720 The type (star/galaxy) of fake sources input
724 image : `lsst.afw.image.exposure.exposure.ExposureF`
728 Uses the x, y information in the ``fakeCat`` to position an image of the fake interpolated onto the
729 pixel grid of the image. Sets the ``FAKE`` mask plane for the pixels added with the fake source.
732 imageBBox = image.getBBox()
733 imageMI = image.maskedImage
735 for (fakeImage, xy)
in fakeImages:
736 X0 = xy.getX() - fakeImage.getWidth()/2 + 0.5
737 Y0 = xy.getY() - fakeImage.getHeight()/2 + 0.5
738 self.log.debug(
"Adding fake source at %d, %d" % (xy.getX(), xy.getY()))
739 if sourceType ==
"galaxy":
740 interpFakeImage = afwMath.offsetImage(fakeImage, X0, Y0,
"lanczos3")
741 interpFakeImBBox = interpFakeImage.getBBox()
743 interpFakeImage = fakeImage
744 interpFakeImBBox = fakeImage.getBBox()
746 interpFakeImBBox.clip(imageBBox)
747 imageMIView = imageMI.Factory(imageMI, interpFakeImBBox)
749 if interpFakeImBBox.getArea() > 0:
750 clippedFakeImage = interpFakeImage.Factory(interpFakeImage, interpFakeImBBox)
751 clippedFakeImageMI = afwImage.MaskedImageF(clippedFakeImage)
752 clippedFakeImageMI.mask.set(self.bitmask)
753 imageMIView += clippedFakeImageMI
757 def _getMetadataName(self):
758 """Disable metadata writing"""
def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, altMaskList=None, mask=None, supplementaryData=None)
def mkFakeStars(self, fakeCat, band, photoCalib, psf, image)
def trimFakeCat(self, fakeCat, image, wcs)
def addPixCoords(self, fakeCat, wcs)
def mkFakeGalsimGalaxies(self, fakeCat, band, photoCalib, pixelScale, psf, image)
def cleanCat(self, fakeCat, starCheckVal)
def addFakeSources(self, image, fakeImages, sourceType)