23 Insert fakes into deepCoadds
26 from astropy.table
import Table
35 from lsst.pipe.base import CmdLineTask, PipelineTask, PipelineTaskConfig, PipelineTaskConnections
39 from lsst.geom import SpherePoint, radians, Box2D
42 __all__ = [
"InsertFakesConfig",
"InsertFakesTask"]
46 dimensions=(
"tract",
"patch",
"abstract_filter",
"skymap")):
49 doc=
"Image into which fakes are to be added.",
50 name=
"{CoaddName}Coadd",
51 storageClass=
"ExposureF",
52 dimensions=(
"tract",
"patch",
"abstract_filter",
"skymap")
56 doc=
"Catalog of fake sources to draw inputs from.",
57 name=
"{CoaddName}Coadd_fakeSourceCat",
58 storageClass=
"Parquet",
59 dimensions=(
"tract",
"skymap")
62 imageWithFakes = cT.Output(
63 doc=
"Image with fake sources added.",
64 name=
"fakes_{CoaddName}Coadd",
65 storageClass=
"ExposureF",
66 dimensions=(
"tract",
"patch",
"abstract_filter",
"skymap")
70 class InsertFakesConfig(PipelineTaskConfig,
71 pipelineConnections=InsertFakesConnections):
72 """Config for inserting fake sources
76 The default column names are those from the University of Washington sims database.
79 raColName = pexConfig.Field(
80 doc=
"RA column name used in the fake source catalog.",
85 decColName = pexConfig.Field(
86 doc=
"Dec. column name used in the fake source catalog.",
91 doCleanCat = pexConfig.Field(
92 doc=
"If true removes bad sources from the catalog.",
97 diskHLR = pexConfig.Field(
98 doc=
"Column name for the disk half light radius used in the fake source catalog.",
100 default=
"DiskHalfLightRadius",
103 bulgeHLR = pexConfig.Field(
104 doc=
"Column name for the bulge half light radius used in the fake source catalog.",
106 default=
"BulgeHalfLightRadius",
109 magVar = pexConfig.Field(
110 doc=
"The column name for the magnitude calculated taking variability into account. In the format "
111 "``filter name``magVar, e.g. imagVar for the magnitude in the i band.",
116 nDisk = pexConfig.Field(
117 doc=
"The column name for the sersic index of the disk component used in the fake source catalog.",
122 nBulge = pexConfig.Field(
123 doc=
"The column name for the sersic index of the bulge component used in the fake source catalog.",
128 aDisk = pexConfig.Field(
129 doc=
"The column name for the semi major axis length of the disk component used in the fake source"
135 aBulge = pexConfig.Field(
136 doc=
"The column name for the semi major axis length of the bulge component.",
141 bDisk = pexConfig.Field(
142 doc=
"The column name for the semi minor axis length of the disk component.",
147 bBulge = pexConfig.Field(
148 doc=
"The column name for the semi minor axis length of the bulge component used in the fake source "
154 paDisk = pexConfig.Field(
155 doc=
"The column name for the PA of the disk component used in the fake source catalog.",
160 paBulge = pexConfig.Field(
161 doc=
"The column name for the PA of the bulge component used in the fake source catalog.",
166 sourceType = pexConfig.Field(
167 doc=
"The column name for the source type used in the fake source catalog.",
169 default=
"sourceType",
172 fakeType = pexConfig.Field(
173 doc=
"What type of fake catalog to use, snapshot (includes variability in the magnitudes calculated "
174 "from the MJD of the image), static (no variability) or filename for a user defined fits"
180 calibFluxRadius = pexConfig.Field(
181 doc=
"Aperture radius (in pixels) that was used to define the calibration for this image+catalog. "
182 "This will be used to produce the correct instrumental fluxes within the radius. "
183 "This value should match that of the field defined in slot_CalibFlux_instFlux.",
188 coaddName = pexConfig.Field(
189 doc=
"The name of the type of coadd used",
194 doSubSelectSources = pexConfig.Field(
195 doc=
"Set to True if you wish to sub select sources to be input based on the value in the column"
196 "set in the sourceSelectionColName config option.",
201 sourceSelectionColName = pexConfig.Field(
202 doc=
"The name of the column in the input fakes catalogue to be used to determine which sources to"
203 "add, default is none and when this is used all sources are added.",
205 default=
"templateSource"
208 insertImages = pexConfig.Field(
209 doc=
"Insert images directly? True or False.",
214 doProcessAllDataIds = pexConfig.Field(
215 doc=
"If True, all input data IDs will be processed, even those containing no fake sources.",
221 class InsertFakesTask(PipelineTask, CmdLineTask):
222 """Insert fake objects into images.
224 Add fake stars and galaxies to the given image, read in through the dataRef. Galaxy parameters are read in
225 from the specified file and then modelled using galsim.
227 `InsertFakesTask` has five functions that make images of the fake sources and then add them to the
231 Use the WCS information to add the pixel coordinates of each source.
232 `mkFakeGalsimGalaxies`
233 Use Galsim to make fake double sersic galaxies for each set of galaxy parameters in the input file.
235 Use the PSF information from the image to make a fake star using the magnitude information from the
238 Remove rows of the input fake catalog which have half light radius, of either the bulge or the disk,
239 that are 0. Also removes rows that have Sersic index outside of galsim's allowed paramters. If
240 the config option sourceSelectionColName is set then this function limits the catalog of input fakes
241 to only those which are True in this column.
243 Add the fake sources to the image.
247 _DefaultName =
"insertFakes"
248 ConfigClass = InsertFakesConfig
250 def runDataRef(self, dataRef):
251 """Read in/write out the required data products and add fake sources to the deepCoadd.
255 dataRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef`
256 Data reference defining the image to have fakes added to it
257 Used to access the following data products:
261 infoStr =
"Adding fakes to: tract: %d, patch: %s, filter: %s" % (dataRef.dataId[
"tract"],
262 dataRef.dataId[
"patch"],
263 dataRef.dataId[
"filter"])
264 self.log.info(infoStr)
268 if self.config.fakeType ==
"static":
269 fakeCat = dataRef.get(
"deepCoadd_fakeSourceCat").toDataFrame()
272 self.fakeSourceCatType =
"deepCoadd_fakeSourceCat"
274 fakeCat = Table.read(self.config.fakeType).to_pandas()
276 coadd = dataRef.get(
"deepCoadd")
278 photoCalib = coadd.getPhotoCalib()
280 imageWithFakes = self.run(fakeCat, coadd, wcs, photoCalib)
282 dataRef.put(imageWithFakes.imageWithFakes,
"fakes_deepCoadd")
284 def runQuantum(self, butlerQC, inputRefs, outputRefs):
285 inputs = butlerQC.get(inputRefs)
286 inputs[
"wcs"] = inputs[
"image"].getWcs()
287 inputs[
"photoCalib"] = inputs[
"image"].getPhotoCalib()
289 outputs = self.run(**inputs)
290 butlerQC.put(outputs, outputRefs)
293 def _makeArgumentParser(cls):
294 parser = pipeBase.ArgumentParser(name=cls._DefaultName)
295 parser.add_id_argument(name=
"--id", datasetType=
"deepCoadd",
296 help=
"data IDs for the deepCoadd, e.g. --id tract=12345 patch=1,2 filter=r",
297 ContainerClass=ExistingCoaddDataIdContainer)
300 def run(self, fakeCat, image, wcs, photoCalib):
301 """Add fake sources to an image.
305 fakeCat : `pandas.core.frame.DataFrame`
306 The catalog of fake sources to be input
307 image : `lsst.afw.image.exposure.exposure.ExposureF`
308 The image into which the fake sources should be added
309 wcs : `lsst.afw.geom.SkyWcs`
310 WCS to use to add fake sources
311 photoCalib : `lsst.afw.image.photoCalib.PhotoCalib`
312 Photometric calibration to be used to calibrate the fake sources
316 resultStruct : `lsst.pipe.base.struct.Struct`
317 contains : image : `lsst.afw.image.exposure.exposure.ExposureF`
321 Adds pixel coordinates for each source to the fakeCat and removes objects with bulge or disk half
322 light radius = 0 (if ``config.doCleanCat = True``).
324 Adds the ``Fake`` mask plane to the image which is then set by `addFakeSources` to mark where fake
325 sources have been added. Uses the information in the ``fakeCat`` to make fake galaxies (using galsim)
326 and fake stars, using the PSF models from the PSF information for the image. These are then added to
327 the image and the image with fakes included returned.
329 The galsim galaxies are made using a double sersic profile, one for the bulge and one for the disk,
330 this is then convolved with the PSF at that point.
333 image.mask.addMaskPlane(
"FAKE")
334 self.bitmask = image.mask.getPlaneBitMask(
"FAKE")
335 self.log.info(
"Adding mask plane with bitmask %d" % self.bitmask)
337 fakeCat = self.addPixCoords(fakeCat, wcs)
338 fakeCat = self.trimFakeCat(fakeCat, image, wcs)
339 band = image.getFilter().getName()
341 pixelScale = wcs.getPixelScale().asArcseconds()
344 if isinstance(fakeCat[self.config.sourceType].iloc[0], str):
345 galCheckVal =
"galaxy"
346 starCheckVal =
"star"
347 elif isinstance(fakeCat[self.config.sourceType].iloc[0], bytes):
348 galCheckVal = b
"galaxy"
349 starCheckVal = b
"star"
350 elif isinstance(fakeCat[self.config.sourceType].iloc[0], (int, float)):
354 raise TypeError(
"sourceType column does not have required type, should be str, bytes or int")
356 if not self.config.insertImages:
357 if self.config.doCleanCat:
358 fakeCat = self.cleanCat(fakeCat, starCheckVal)
360 galaxies = (fakeCat[self.config.sourceType] == galCheckVal)
361 galImages = self.mkFakeGalsimGalaxies(fakeCat[galaxies], band, photoCalib, pixelScale, psf,
364 stars = (fakeCat[self.config.sourceType] == starCheckVal)
365 starImages = self.mkFakeStars(fakeCat[stars], band, photoCalib, psf, image)
367 galImages, starImages = self.processImagesForInsertion(fakeCat, wcs, psf, photoCalib, band,
370 image = self.addFakeSources(image, galImages,
"galaxy")
371 image = self.addFakeSources(image, starImages,
"star")
373 elif len(fakeCat) == 0
and self.config.doProcessAllDataIds:
374 self.log.warn(
"No fakes found for this dataRef; processing anyway.")
376 raise RuntimeError(
"No fakes found for this dataRef.")
378 resultStruct = pipeBase.Struct(imageWithFakes=image)
382 def processImagesForInsertion(self, fakeCat, wcs, psf, photoCalib, band, pixelScale):
383 """Process images from files into the format needed for insertion.
387 fakeCat : `pandas.core.frame.DataFrame`
388 The catalog of fake sources to be input
389 wcs : `lsst.afw.geom.skyWcs.skyWcs.SkyWc`
390 WCS to use to add fake sources
391 psf : `lsst.meas.algorithms.coaddPsf.coaddPsf.CoaddPsf` or
392 `lsst.meas.extensions.psfex.psfexPsf.PsfexPsf`
393 The PSF information to use to make the PSF images
394 photoCalib : `lsst.afw.image.photoCalib.PhotoCalib`
395 Photometric calibration to be used to calibrate the fake sources
397 The filter band that the observation was taken in.
399 The pixel scale of the image the sources are to be added to.
404 A list of tuples of `lsst.afw.image.exposure.exposure.ExposureF` and
405 `lsst.geom.Point2D` of their locations.
406 For sources labelled as galaxy.
408 A list of tuples of `lsst.afw.image.exposure.exposure.ExposureF` and
409 `lsst.geom.Point2D` of their locations.
410 For sources labelled as star.
414 The input fakes catalog needs to contain the absolute path to the image in the
415 band that is being used to add images to. It also needs to have the R.A. and
416 declination of the fake source in radians and the sourceType of the object.
421 self.log.info(
"Processing %d fake images" % len(fakeCat))
423 for (imFile, sourceType, mag, x, y)
in zip(fakeCat[band +
"imFilename"].array,
424 fakeCat[
"sourceType"].array,
425 fakeCat[self.config.magVar % band].array,
426 fakeCat[
"x"].array, fakeCat[
"y"].array):
428 im = afwImage.ImageF.readFits(imFile)
435 correctedFlux = psf.computeApertureFlux(self.config.calibFluxRadius, xy)
436 psfKernel = psf.computeKernelImage(xy).getArray()
437 psfKernel /= correctedFlux
439 except InvalidParameterError:
440 self.log.info(
"%s at %0.4f, %0.4f outside of image" % (sourceType, x, y))
443 psfIm = galsim.InterpolatedImage(galsim.Image(psfKernel), scale=pixelScale)
444 galsimIm = galsim.InterpolatedImage(galsim.Image(im.array), scale=pixelScale)
445 convIm = galsim.Convolve([galsimIm, psfIm])
448 outIm = convIm.drawImage(scale=pixelScale, method=
"real_space").array
449 except (galsim.errors.GalSimFFTSizeError, MemoryError):
452 imSum = np.sum(outIm)
456 flux = photoCalib.magnitudeToInstFlux(mag, xy)
460 imWithFlux = flux*divIm
462 if sourceType == b
"galaxy":
463 galImages.append((afwImage.ImageF(imWithFlux), xy))
464 if sourceType == b
"star":
465 starImages.append((afwImage.ImageF(imWithFlux), xy))
467 return galImages, starImages
471 """Add pixel coordinates to the catalog of fakes.
475 fakeCat : `pandas.core.frame.DataFrame`
476 The catalog of fake sources to be input
477 wcs : `lsst.afw.geom.SkyWcs`
478 WCS to use to add fake sources
482 fakeCat : `pandas.core.frame.DataFrame`
486 The default option is to use the WCS information from the image. If the ``useUpdatedCalibs`` config
487 option is set then it will use the updated WCS from jointCal.
490 ras = fakeCat[self.config.raColName].values
491 decs = fakeCat[self.config.decColName].values
492 skyCoords = [
SpherePoint(ra, dec, radians)
for (ra, dec)
in zip(ras, decs)]
493 pixCoords = wcs.skyToPixel(skyCoords)
494 xs = [coord.getX()
for coord
in pixCoords]
495 ys = [coord.getY()
for coord
in pixCoords]
502 """Trim the fake cat to about the size of the input image.
506 fakeCat : `pandas.core.frame.DataFrame`
507 The catalog of fake sources to be input
508 image : `lsst.afw.image.exposure.exposure.ExposureF`
509 The image into which the fake sources should be added
510 wcs : `lsst.afw.geom.SkyWcs`
511 WCS to use to add fake sources
515 fakeCat : `pandas.core.frame.DataFrame`
516 The original fakeCat trimmed to the area of the image
519 bbox =
Box2D(image.getBBox())
520 corners = bbox.getCorners()
522 skyCorners = wcs.pixelToSky(corners)
526 coord =
SpherePoint(row[self.config.raColName], row[self.config.decColName], radians)
527 return region.contains(coord.getVector())
529 return fakeCat[fakeCat.apply(trim, axis=1)]
532 """Make images of fake galaxies using GalSim.
538 psf : `lsst.meas.extensions.psfex.psfexPsf.PsfexPsf`
539 The PSF information to use to make the PSF images
540 fakeCat : `pandas.core.frame.DataFrame`
541 The catalog of fake sources to be input
542 photoCalib : `lsst.afw.image.photoCalib.PhotoCalib`
543 Photometric calibration to be used to calibrate the fake sources
547 galImages : `generator`
548 A generator of tuples of `lsst.afw.image.exposure.exposure.ExposureF` and
549 `lsst.geom.Point2D` of their locations.
554 Fake galaxies are made by combining two sersic profiles, one for the bulge and one for the disk. Each
555 component has an individual sersic index (n), a, b and position angle (PA). The combined profile is
556 then convolved with the PSF at the specified x, y position on the image.
558 The names of the columns in the ``fakeCat`` are configurable and are the column names from the
559 University of Washington simulations database as default. For more information see the doc strings
560 attached to the config options.
562 See mkFakeStars doc string for an explanation of calibration to instrumental flux.
565 self.log.info(
"Making %d fake galaxy images" % len(fakeCat))
567 for (index, row)
in fakeCat.iterrows():
573 correctedFlux = psf.computeApertureFlux(self.config.calibFluxRadius, xy)
574 psfKernel = psf.computeKernelImage(xy).getArray()
575 psfKernel /= correctedFlux
577 except InvalidParameterError:
578 self.log.info(
"Galaxy at %0.4f, %0.4f outside of image" % (row[
"x"], row[
"y"]))
582 flux = photoCalib.magnitudeToInstFlux(row[self.config.magVar % band], xy)
586 bulge = galsim.Sersic(row[self.config.nBulge], half_light_radius=row[self.config.bulgeHLR])
587 axisRatioBulge = row[self.config.bBulge]/row[self.config.aBulge]
588 bulge = bulge.shear(q=axisRatioBulge, beta=((90 - row[self.config.paBulge])*galsim.degrees))
590 disk = galsim.Sersic(row[self.config.nDisk], half_light_radius=row[self.config.diskHLR])
591 axisRatioDisk = row[self.config.bDisk]/row[self.config.aDisk]
592 disk = disk.shear(q=axisRatioDisk, beta=((90 - row[self.config.paDisk])*galsim.degrees))
595 gal = gal.withFlux(flux)
597 psfIm = galsim.InterpolatedImage(galsim.Image(psfKernel), scale=pixelScale)
598 gal = galsim.Convolve([gal, psfIm])
600 galIm = gal.drawImage(scale=pixelScale, method=
"real_space").array
601 except (galsim.errors.GalSimFFTSizeError, MemoryError):
604 yield (afwImage.ImageF(galIm), xy)
608 """Make fake stars based off the properties in the fakeCat.
613 psf : `lsst.meas.extensions.psfex.psfexPsf.PsfexPsf`
614 The PSF information to use to make the PSF images
615 fakeCat : `pandas.core.frame.DataFrame`
616 The catalog of fake sources to be input
617 image : `lsst.afw.image.exposure.exposure.ExposureF`
618 The image into which the fake sources should be added
619 photoCalib : `lsst.afw.image.photoCalib.PhotoCalib`
620 Photometric calibration to be used to calibrate the fake sources
624 starImages : `generator`
625 A generator of tuples of `lsst.afw.image.ImageF` of fake stars and
626 `lsst.geom.Point2D` of their locations.
630 To take a given magnitude and translate to the number of counts in the image
631 we use photoCalib.magnitudeToInstFlux, which returns the instrumental flux for the
632 given calibration radius used in the photometric calibration step.
633 Thus `calibFluxRadius` should be set to this same radius so that we can normalize
634 the PSF model to the correct instrumental flux within calibFluxRadius.
637 self.log.info(
"Making %d fake star images" % len(fakeCat))
639 for (index, row)
in fakeCat.iterrows():
645 correctedFlux = psf.computeApertureFlux(self.config.calibFluxRadius, xy)
646 starIm = psf.computeImage(xy)
647 starIm /= correctedFlux
649 except InvalidParameterError:
650 self.log.info(
"Star at %0.4f, %0.4f outside of image" % (row[
"x"], row[
"y"]))
654 flux = photoCalib.magnitudeToInstFlux(row[self.config.magVar % band], xy)
659 yield ((starIm.convertF(), xy))
662 """Remove rows from the fakes catalog which have HLR = 0 for either the buldge or disk component,
663 also remove galaxies that have Sersic index outside the galsim min and max
664 allowed (0.3 <= n <= 6.2).
668 fakeCat : `pandas.core.frame.DataFrame`
669 The catalog of fake sources to be input
670 starCheckVal : `str`, `bytes` or `int`
671 The value that is set in the sourceType column to specifiy an object is a star.
675 fakeCat : `pandas.core.frame.DataFrame`
676 The input catalog of fake sources but with the bad objects removed
680 If the config option sourceSelectionColName is set then only objects with this column set to True
684 rowsToKeep = (((fakeCat[self.config.bulgeHLR] != 0.0) & (fakeCat[self.config.diskHLR] != 0.0))
685 | (fakeCat[self.config.sourceType] == starCheckVal))
686 numRowsNotUsed = len(fakeCat) - len(np.where(rowsToKeep)[0])
687 self.log.info(
"Removing %d rows with HLR = 0 for either the bulge or disk" % numRowsNotUsed)
688 fakeCat = fakeCat[rowsToKeep]
690 minN = galsim.Sersic._minimum_n
691 maxN = galsim.Sersic._maximum_n
692 rowsWithGoodSersic = (((fakeCat[self.config.nBulge] >= minN) & (fakeCat[self.config.nBulge] <= maxN)
693 & (fakeCat[self.config.nDisk] >= minN) & (fakeCat[self.config.nDisk] <= maxN))
694 | (fakeCat[self.config.sourceType] == starCheckVal))
695 numRowsNotUsed = len(fakeCat) - len(np.where(rowsWithGoodSersic)[0])
696 self.log.info(
"Removing %d rows of galaxies with nBulge or nDisk outside of %0.2f <= n <= %0.2f" %
697 (numRowsNotUsed, minN, maxN))
698 fakeCat = fakeCat[rowsWithGoodSersic]
700 if self.config.doSubSelectSources:
702 rowsSelected = (fakeCat[self.config.sourceSelectionColName])
704 raise KeyError(
"Given column, %s, for source selection not found." %
705 self.config.sourceSelectionColName)
706 numRowsNotUsed = len(fakeCat) - len(rowsSelected)
707 self.log.info(
"Removing %d rows which were not designated as template sources" % numRowsNotUsed)
708 fakeCat = fakeCat[rowsSelected]
713 """Add the fake sources to the given image
717 image : `lsst.afw.image.exposure.exposure.ExposureF`
718 The image into which the fake sources should be added
719 fakeImages : `typing.Iterator` [`tuple` ['lsst.afw.image.ImageF`, `lsst.geom.Point2d`]]
720 An iterator of tuples that contains (or generates) images of fake sources,
721 and the locations they are to be inserted at.
723 The type (star/galaxy) of fake sources input
727 image : `lsst.afw.image.exposure.exposure.ExposureF`
731 Uses the x, y information in the ``fakeCat`` to position an image of the fake interpolated onto the
732 pixel grid of the image. Sets the ``FAKE`` mask plane for the pixels added with the fake source.
735 imageBBox = image.getBBox()
736 imageMI = image.maskedImage
738 for (fakeImage, xy)
in fakeImages:
739 X0 = xy.getX() - fakeImage.getWidth()/2 + 0.5
740 Y0 = xy.getY() - fakeImage.getHeight()/2 + 0.5
741 self.log.debug(
"Adding fake source at %d, %d" % (xy.getX(), xy.getY()))
742 if sourceType ==
"galaxy":
743 interpFakeImage = afwMath.offsetImage(fakeImage, X0, Y0,
"lanczos3")
744 interpFakeImBBox = interpFakeImage.getBBox()
746 interpFakeImage = fakeImage
747 interpFakeImBBox = fakeImage.getBBox()
749 interpFakeImBBox.clip(imageBBox)
750 imageMIView = imageMI.Factory(imageMI, interpFakeImBBox)
752 if interpFakeImBBox.getArea() > 0:
753 clippedFakeImage = interpFakeImage.Factory(interpFakeImage, interpFakeImBBox)
754 clippedFakeImageMI = afwImage.MaskedImageF(clippedFakeImage)
755 clippedFakeImageMI.mask.set(self.bitmask)
756 imageMIView += clippedFakeImageMI
760 def _getMetadataName(self):
761 """Disable metadata writing"""