23 Insert fakes into deepCoadds
26 from astropy.table
import Table
35 from lsst.pipe.base import CmdLineTask, PipelineTask, PipelineTaskConfig, PipelineTaskConnections
39 from lsst.geom import SpherePoint, radians, Box2D
42 __all__ = [
"InsertFakesConfig",
"InsertFakesTask"]
46 dimensions=(
"tract",
"patch",
"abstract_filter",
"skymap")):
49 doc=
"Image into which fakes are to be added.",
50 name=
"{CoaddName}Coadd",
51 storageClass=
"ExposureF",
52 dimensions=(
"tract",
"patch",
"abstract_filter",
"skymap")
56 doc=
"Catalog of fake sources to draw inputs from.",
57 name=
"{CoaddName}Coadd_fakeSourceCat",
58 storageClass=
"Parquet",
59 dimensions=(
"tract",
"skymap")
62 imageWithFakes = cT.Output(
63 doc=
"Image with fake sources added.",
64 name=
"fakes_{CoaddName}Coadd",
65 storageClass=
"ExposureF",
66 dimensions=(
"tract",
"patch",
"abstract_filter",
"skymap")
70 class InsertFakesConfig(PipelineTaskConfig,
71 pipelineConnections=InsertFakesConnections):
72 """Config for inserting fake sources
76 The default column names are those from the University of Washington sims database.
79 raColName = pexConfig.Field(
80 doc=
"RA column name used in the fake source catalog.",
85 decColName = pexConfig.Field(
86 doc=
"Dec. column name used in the fake source catalog.",
91 doCleanCat = pexConfig.Field(
92 doc=
"If true removes bad sources from the catalog.",
97 diskHLR = pexConfig.Field(
98 doc=
"Column name for the disk half light radius used in the fake source catalog.",
100 default=
"DiskHalfLightRadius",
103 bulgeHLR = pexConfig.Field(
104 doc=
"Column name for the bulge half light radius used in the fake source catalog.",
106 default=
"BulgeHalfLightRadius",
109 magVar = pexConfig.Field(
110 doc=
"The column name for the magnitude calculated taking variability into account. In the format "
111 "``filter name``magVar, e.g. imagVar for the magnitude in the i band.",
116 nDisk = pexConfig.Field(
117 doc=
"The column name for the sersic index of the disk component used in the fake source catalog.",
122 nBulge = pexConfig.Field(
123 doc=
"The column name for the sersic index of the bulge component used in the fake source catalog.",
128 aDisk = pexConfig.Field(
129 doc=
"The column name for the semi major axis length of the disk component used in the fake source"
135 aBulge = pexConfig.Field(
136 doc=
"The column name for the semi major axis length of the bulge component.",
141 bDisk = pexConfig.Field(
142 doc=
"The column name for the semi minor axis length of the disk component.",
147 bBulge = pexConfig.Field(
148 doc=
"The column name for the semi minor axis length of the bulge component used in the fake source "
154 paDisk = pexConfig.Field(
155 doc=
"The column name for the PA of the disk component used in the fake source catalog.",
160 paBulge = pexConfig.Field(
161 doc=
"The column name for the PA of the bulge component used in the fake source catalog.",
166 sourceType = pexConfig.Field(
167 doc=
"The column name for the source type used in the fake source catalog.",
169 default=
"sourceType",
172 fakeType = pexConfig.Field(
173 doc=
"What type of fake catalog to use, snapshot (includes variability in the magnitudes calculated "
174 "from the MJD of the image), static (no variability) or filename for a user defined fits"
180 calibFluxRadius = pexConfig.Field(
181 doc=
"Aperture radius (in pixels) that was used to define the calibration for this image+catalog. "
182 "This will be used to produce the correct instrumental fluxes within the radius. "
183 "This value should match that of the field defined in slot_CalibFlux_instFlux.",
188 coaddName = pexConfig.Field(
189 doc=
"The name of the type of coadd used",
194 doSubSelectSources = pexConfig.Field(
195 doc=
"Set to True if you wish to sub select sources to be input based on the value in the column"
196 "set in the sourceSelectionColName config option.",
201 sourceSelectionColName = pexConfig.Field(
202 doc=
"The name of the column in the input fakes catalogue to be used to determine which sources to"
203 "add, default is none and when this is used all sources are added.",
205 default=
"templateSource"
208 insertImages = pexConfig.Field(
209 doc=
"Insert images directly? True or False.",
215 class InsertFakesTask(PipelineTask, CmdLineTask):
216 """Insert fake objects into images.
218 Add fake stars and galaxies to the given image, read in through the dataRef. Galaxy parameters are read in
219 from the specified file and then modelled using galsim.
221 `InsertFakesTask` has five functions that make images of the fake sources and then add them to the
225 Use the WCS information to add the pixel coordinates of each source.
226 `mkFakeGalsimGalaxies`
227 Use Galsim to make fake double sersic galaxies for each set of galaxy parameters in the input file.
229 Use the PSF information from the image to make a fake star using the magnitude information from the
232 Remove rows of the input fake catalog which have half light radius, of either the bulge or the disk,
233 that are 0. Also removes rows that have Sersic index outside of galsim's allowed paramters. If
234 the config option sourceSelectionColName is set then this function limits the catalog of input fakes
235 to only those which are True in this column.
237 Add the fake sources to the image.
241 _DefaultName =
"insertFakes"
242 ConfigClass = InsertFakesConfig
244 def runDataRef(self, dataRef):
245 """Read in/write out the required data products and add fake sources to the deepCoadd.
249 dataRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef`
250 Data reference defining the image to have fakes added to it
251 Used to access the following data products:
255 infoStr =
"Adding fakes to: tract: %d, patch: %s, filter: %s" % (dataRef.dataId[
"tract"],
256 dataRef.dataId[
"patch"],
257 dataRef.dataId[
"filter"])
258 self.log.info(infoStr)
262 if self.config.fakeType ==
"static":
263 fakeCat = dataRef.get(
"deepCoadd_fakeSourceCat").toDataFrame()
266 self.fakeSourceCatType =
"deepCoadd_fakeSourceCat"
268 fakeCat = Table.read(self.config.fakeType).to_pandas()
270 coadd = dataRef.get(
"deepCoadd")
272 photoCalib = coadd.getPhotoCalib()
274 imageWithFakes = self.run(fakeCat, coadd, wcs, photoCalib)
276 dataRef.put(imageWithFakes.imageWithFakes,
"fakes_deepCoadd")
278 def runQuantum(self, butlerQC, inputRefs, outputRefs):
279 inputs = butlerQC.get(inputRefs)
280 inputs[
"wcs"] = inputs[
"image"].getWcs()
281 inputs[
"photoCalib"] = inputs[
"image"].getPhotoCalib()
283 outputs = self.run(**inputs)
284 butlerQC.put(outputs, outputRefs)
287 def _makeArgumentParser(cls):
288 parser = pipeBase.ArgumentParser(name=cls._DefaultName)
289 parser.add_id_argument(name=
"--id", datasetType=
"deepCoadd",
290 help=
"data IDs for the deepCoadd, e.g. --id tract=12345 patch=1,2 filter=r",
291 ContainerClass=ExistingCoaddDataIdContainer)
294 def run(self, fakeCat, image, wcs, photoCalib):
295 """Add fake sources to an image.
299 fakeCat : `pandas.core.frame.DataFrame`
300 The catalog of fake sources to be input
301 image : `lsst.afw.image.exposure.exposure.ExposureF`
302 The image into which the fake sources should be added
303 wcs : `lsst.afw.geom.SkyWcs`
304 WCS to use to add fake sources
305 photoCalib : `lsst.afw.image.photoCalib.PhotoCalib`
306 Photometric calibration to be used to calibrate the fake sources
310 resultStruct : `lsst.pipe.base.struct.Struct`
311 contains : image : `lsst.afw.image.exposure.exposure.ExposureF`
315 Adds pixel coordinates for each source to the fakeCat and removes objects with bulge or disk half
316 light radius = 0 (if ``config.doCleanCat = True``).
318 Adds the ``Fake`` mask plane to the image which is then set by `addFakeSources` to mark where fake
319 sources have been added. Uses the information in the ``fakeCat`` to make fake galaxies (using galsim)
320 and fake stars, using the PSF models from the PSF information for the image. These are then added to
321 the image and the image with fakes included returned.
323 The galsim galaxies are made using a double sersic profile, one for the bulge and one for the disk,
324 this is then convolved with the PSF at that point.
327 image.mask.addMaskPlane(
"FAKE")
328 self.bitmask = image.mask.getPlaneBitMask(
"FAKE")
329 self.log.info(
"Adding mask plane with bitmask %d" % self.bitmask)
331 fakeCat = self.addPixCoords(fakeCat, wcs)
332 fakeCat = self.trimFakeCat(fakeCat, image, wcs)
333 band = image.getFilter().getName()
335 pixelScale = wcs.getPixelScale().asArcseconds()
337 if len(fakeCat) == 0:
338 errMsg =
"No fakes found for this dataRef."
339 raise RuntimeError(errMsg)
341 if isinstance(fakeCat[self.config.sourceType].iloc[0], str):
342 galCheckVal =
"galaxy"
343 starCheckVal =
"star"
344 elif isinstance(fakeCat[self.config.sourceType].iloc[0], bytes):
345 galCheckVal = b
"galaxy"
346 starCheckVal = b
"star"
347 elif isinstance(fakeCat[self.config.sourceType].iloc[0], (int, float)):
351 raise TypeError(
"sourceType column does not have required type, should be str, bytes or int")
353 if not self.config.insertImages:
354 if self.config.doCleanCat:
355 fakeCat = self.cleanCat(fakeCat, starCheckVal)
357 galaxies = (fakeCat[self.config.sourceType] == galCheckVal)
358 galImages = self.mkFakeGalsimGalaxies(fakeCat[galaxies], band, photoCalib, pixelScale, psf, image)
360 stars = (fakeCat[self.config.sourceType] == starCheckVal)
361 starImages = self.mkFakeStars(fakeCat[stars], band, photoCalib, psf, image)
363 galImages, starImages = self.processImagesForInsertion(fakeCat, wcs, psf, photoCalib, band,
366 image = self.addFakeSources(image, galImages,
"galaxy")
367 image = self.addFakeSources(image, starImages,
"star")
368 resultStruct = pipeBase.Struct(imageWithFakes=image)
372 def processImagesForInsertion(self, fakeCat, wcs, psf, photoCalib, band, pixelScale):
373 """Process images from files into the format needed for insertion.
377 fakeCat : `pandas.core.frame.DataFrame`
378 The catalog of fake sources to be input
379 wcs : `lsst.afw.geom.skyWcs.skyWcs.SkyWc`
380 WCS to use to add fake sources
381 psf : `lsst.meas.algorithms.coaddPsf.coaddPsf.CoaddPsf` or
382 `lsst.meas.extensions.psfex.psfexPsf.PsfexPsf`
383 The PSF information to use to make the PSF images
384 photoCalib : `lsst.afw.image.photoCalib.PhotoCalib`
385 Photometric calibration to be used to calibrate the fake sources
387 The filter band that the observation was taken in.
389 The pixel scale of the image the sources are to be added to.
394 A list of tuples of `lsst.afw.image.exposure.exposure.ExposureF` and
395 `lsst.geom.Point2D` of their locations.
396 For sources labelled as galaxy.
398 A list of tuples of `lsst.afw.image.exposure.exposure.ExposureF` and
399 `lsst.geom.Point2D` of their locations.
400 For sources labelled as star.
404 The input fakes catalog needs to contain the absolute path to the image in the
405 band that is being used to add images to. It also needs to have the R.A. and
406 declination of the fake source in radians and the sourceType of the object.
411 self.log.info(
"Processing %d fake images" % len(fakeCat))
413 for (imFile, sourceType, mag, x, y)
in zip(fakeCat[band +
"imFilename"].array,
414 fakeCat[
"sourceType"].array,
415 fakeCat[self.config.magVar % band].array,
416 fakeCat[
"x"].array, fakeCat[
"y"].array):
418 im = afwImage.ImageF.readFits(imFile)
425 correctedFlux = psf.computeApertureFlux(self.config.calibFluxRadius, xy)
426 psfKernel = psf.computeKernelImage(xy).getArray()
427 psfKernel /= correctedFlux
429 except InvalidParameterError:
430 self.log.info(
"%s at %0.4f, %0.4f outside of image" % (sourceType, x, y))
433 psfIm = galsim.InterpolatedImage(galsim.Image(psfKernel), scale=pixelScale)
434 galsimIm = galsim.InterpolatedImage(galsim.Image(im.array), scale=pixelScale)
435 convIm = galsim.Convolve([galsimIm, psfIm])
438 outIm = convIm.drawImage(scale=pixelScale, method=
"real_space").array
439 except (galsim.errors.GalSimFFTSizeError, MemoryError):
442 imSum = np.sum(outIm)
446 flux = photoCalib.magnitudeToInstFlux(mag, xy)
450 imWithFlux = flux*divIm
452 if sourceType == b
"galaxy":
453 galImages.append((afwImage.ImageF(imWithFlux), xy))
454 if sourceType == b
"star":
455 starImages.append((afwImage.ImageF(imWithFlux), xy))
457 return galImages, starImages
461 """Add pixel coordinates to the catalog of fakes.
465 fakeCat : `pandas.core.frame.DataFrame`
466 The catalog of fake sources to be input
467 wcs : `lsst.afw.geom.SkyWcs`
468 WCS to use to add fake sources
472 fakeCat : `pandas.core.frame.DataFrame`
476 The default option is to use the WCS information from the image. If the ``useUpdatedCalibs`` config
477 option is set then it will use the updated WCS from jointCal.
480 ras = fakeCat[self.config.raColName].values
481 decs = fakeCat[self.config.decColName].values
482 skyCoords = [
SpherePoint(ra, dec, radians)
for (ra, dec)
in zip(ras, decs)]
483 pixCoords = wcs.skyToPixel(skyCoords)
484 xs = [coord.getX()
for coord
in pixCoords]
485 ys = [coord.getY()
for coord
in pixCoords]
492 """Trim the fake cat to about the size of the input image.
496 fakeCat : `pandas.core.frame.DataFrame`
497 The catalog of fake sources to be input
498 image : `lsst.afw.image.exposure.exposure.ExposureF`
499 The image into which the fake sources should be added
500 wcs : `lsst.afw.geom.SkyWcs`
501 WCS to use to add fake sources
505 fakeCat : `pandas.core.frame.DataFrame`
506 The original fakeCat trimmed to the area of the image
509 bbox =
Box2D(image.getBBox())
510 corners = bbox.getCorners()
512 skyCorners = wcs.pixelToSky(corners)
516 coord =
SpherePoint(row[self.config.raColName], row[self.config.decColName], radians)
517 return region.contains(coord.getVector())
519 return fakeCat[fakeCat.apply(trim, axis=1)]
522 """Make images of fake galaxies using GalSim.
528 psf : `lsst.meas.extensions.psfex.psfexPsf.PsfexPsf`
529 The PSF information to use to make the PSF images
530 fakeCat : `pandas.core.frame.DataFrame`
531 The catalog of fake sources to be input
532 photoCalib : `lsst.afw.image.photoCalib.PhotoCalib`
533 Photometric calibration to be used to calibrate the fake sources
537 galImages : `generator`
538 A generator of tuples of `lsst.afw.image.exposure.exposure.ExposureF` and
539 `lsst.geom.Point2D` of their locations.
544 Fake galaxies are made by combining two sersic profiles, one for the bulge and one for the disk. Each
545 component has an individual sersic index (n), a, b and position angle (PA). The combined profile is
546 then convolved with the PSF at the specified x, y position on the image.
548 The names of the columns in the ``fakeCat`` are configurable and are the column names from the
549 University of Washington simulations database as default. For more information see the doc strings
550 attached to the config options.
552 See mkFakeStars doc string for an explanation of calibration to instrumental flux.
555 self.log.info(
"Making %d fake galaxy images" % len(fakeCat))
557 for (index, row)
in fakeCat.iterrows():
563 correctedFlux = psf.computeApertureFlux(self.config.calibFluxRadius, xy)
564 psfKernel = psf.computeKernelImage(xy).getArray()
565 psfKernel /= correctedFlux
567 except InvalidParameterError:
568 self.log.info(
"Galaxy at %0.4f, %0.4f outside of image" % (row[
"x"], row[
"y"]))
572 flux = photoCalib.magnitudeToInstFlux(row[self.config.magVar % band], xy)
576 bulge = galsim.Sersic(row[self.config.nBulge], half_light_radius=row[self.config.bulgeHLR])
577 axisRatioBulge = row[self.config.bBulge]/row[self.config.aBulge]
578 bulge = bulge.shear(q=axisRatioBulge, beta=((90 - row[self.config.paBulge])*galsim.degrees))
580 disk = galsim.Sersic(row[self.config.nDisk], half_light_radius=row[self.config.diskHLR])
581 axisRatioDisk = row[self.config.bDisk]/row[self.config.aDisk]
582 disk = disk.shear(q=axisRatioDisk, beta=((90 - row[self.config.paDisk])*galsim.degrees))
585 gal = gal.withFlux(flux)
587 psfIm = galsim.InterpolatedImage(galsim.Image(psfKernel), scale=pixelScale)
588 gal = galsim.Convolve([gal, psfIm])
590 galIm = gal.drawImage(scale=pixelScale, method=
"real_space").array
591 except (galsim.errors.GalSimFFTSizeError, MemoryError):
594 yield (afwImage.ImageF(galIm), xy)
598 """Make fake stars based off the properties in the fakeCat.
603 psf : `lsst.meas.extensions.psfex.psfexPsf.PsfexPsf`
604 The PSF information to use to make the PSF images
605 fakeCat : `pandas.core.frame.DataFrame`
606 The catalog of fake sources to be input
607 image : `lsst.afw.image.exposure.exposure.ExposureF`
608 The image into which the fake sources should be added
609 photoCalib : `lsst.afw.image.photoCalib.PhotoCalib`
610 Photometric calibration to be used to calibrate the fake sources
614 starImages : `generator`
615 A generator of tuples of `lsst.afw.image.ImageF` of fake stars and
616 `lsst.geom.Point2D` of their locations.
620 To take a given magnitude and translate to the number of counts in the image
621 we use photoCalib.magnitudeToInstFlux, which returns the instrumental flux for the
622 given calibration radius used in the photometric calibration step.
623 Thus `calibFluxRadius` should be set to this same radius so that we can normalize
624 the PSF model to the correct instrumental flux within calibFluxRadius.
627 self.log.info(
"Making %d fake star images" % len(fakeCat))
629 for (index, row)
in fakeCat.iterrows():
635 correctedFlux = psf.computeApertureFlux(self.config.calibFluxRadius, xy)
636 starIm = psf.computeImage(xy)
637 starIm /= correctedFlux
639 except InvalidParameterError:
640 self.log.info(
"Star at %0.4f, %0.4f outside of image" % (row[
"x"], row[
"y"]))
644 flux = photoCalib.magnitudeToInstFlux(row[self.config.magVar % band], xy)
649 yield ((starIm.convertF(), xy))
652 """Remove rows from the fakes catalog which have HLR = 0 for either the buldge or disk component,
653 also remove galaxies that have Sersic index outside the galsim min and max
654 allowed (0.3 <= n <= 6.2).
658 fakeCat : `pandas.core.frame.DataFrame`
659 The catalog of fake sources to be input
660 starCheckVal : `str`, `bytes` or `int`
661 The value that is set in the sourceType column to specifiy an object is a star.
665 fakeCat : `pandas.core.frame.DataFrame`
666 The input catalog of fake sources but with the bad objects removed
670 If the config option sourceSelectionColName is set then only objects with this column set to True
674 rowsToKeep = ((fakeCat[self.config.bulgeHLR] != 0.0) & (fakeCat[self.config.diskHLR] != 0.0))
675 numRowsNotUsed = len(fakeCat) - len(np.where(rowsToKeep)[0])
676 self.log.info(
"Removing %d rows with HLR = 0 for either the bulge or disk" % numRowsNotUsed)
677 fakeCat = fakeCat[rowsToKeep]
679 minN = galsim.Sersic._minimum_n
680 maxN = galsim.Sersic._maximum_n
681 rowsWithGoodSersic = (((fakeCat[self.config.nBulge] >= minN) & (fakeCat[self.config.nBulge] <= maxN)
682 & (fakeCat[self.config.nDisk] >= minN) & (fakeCat[self.config.nDisk] <= maxN))
683 | (fakeCat[self.config.sourceType] == starCheckVal))
684 numRowsNotUsed = len(fakeCat) - len(np.where(rowsWithGoodSersic)[0])
685 self.log.info(
"Removing %d rows of galaxies with nBulge or nDisk outside of %0.2f <= n <= %0.2f" %
686 (numRowsNotUsed, minN, maxN))
687 fakeCat = fakeCat[rowsWithGoodSersic]
689 if self.config.doSubSelectSources:
691 rowsSelected = (fakeCat[self.config.sourceSelectionColName])
693 raise KeyError(
"Given column, %s, for source selection not found." %
694 self.config.sourceSelectionColName)
695 numRowsNotUsed = len(fakeCat) - len(rowsSelected)
696 self.log.info(
"Removing %d rows which were not designated as template sources" % numRowsNotUsed)
697 fakeCat = fakeCat[rowsSelected]
702 """Add the fake sources to the given image
706 image : `lsst.afw.image.exposure.exposure.ExposureF`
707 The image into which the fake sources should be added
708 fakeImages : `typing.Iterator` [`tuple` ['lsst.afw.image.ImageF`, `lsst.geom.Point2d`]]
709 An iterator of tuples that contains (or generates) images of fake sources,
710 and the locations they are to be inserted at.
712 The type (star/galaxy) of fake sources input
716 image : `lsst.afw.image.exposure.exposure.ExposureF`
720 Uses the x, y information in the ``fakeCat`` to position an image of the fake interpolated onto the
721 pixel grid of the image. Sets the ``FAKE`` mask plane for the pixels added with the fake source.
724 imageBBox = image.getBBox()
725 imageMI = image.maskedImage
727 for (fakeImage, xy)
in fakeImages:
728 X0 = xy.getX() - fakeImage.getWidth()/2 + 0.5
729 Y0 = xy.getY() - fakeImage.getHeight()/2 + 0.5
730 self.log.debug(
"Adding fake source at %d, %d" % (xy.getX(), xy.getY()))
731 if sourceType ==
"galaxy":
732 interpFakeImage = afwMath.offsetImage(fakeImage, X0, Y0,
"lanczos3")
733 interpFakeImBBox = interpFakeImage.getBBox()
735 interpFakeImage = fakeImage
736 interpFakeImBBox = fakeImage.getBBox()
738 interpFakeImBBox.clip(imageBBox)
739 imageMIView = imageMI.Factory(imageMI, interpFakeImBBox)
741 if interpFakeImBBox.getArea() > 0:
742 clippedFakeImage = interpFakeImage.Factory(interpFakeImage, interpFakeImBBox)
743 clippedFakeImageMI = afwImage.MaskedImageF(clippedFakeImage)
744 clippedFakeImageMI.mask.set(self.bitmask)
745 imageMIView += clippedFakeImageMI
749 def _getMetadataName(self):
750 """Disable metadata writing"""