Coverage for python/lsst/pipe/tasks/processBrightStars.py: 24%
180 statements
« prev ^ index » next coverage.py v6.4.4, created at 2022-09-15 03:34 -0700
« prev ^ index » next coverage.py v6.4.4, created at 2022-09-15 03:34 -0700
1# This file is part of pipe_tasks.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
21#
22"""Extract small cutouts around bright stars, normalize and warp them to the
23same arbitrary pixel grid.
24"""
26__all__ = ["ProcessBrightStarsTask"]
28import numpy as np
29import astropy.units as u
31from lsst import geom
32from lsst.afw import math as afwMath
33from lsst.afw import image as afwImage
34from lsst.afw import detection as afwDetect
35from lsst.afw import cameraGeom as cg
36from lsst.afw.geom import transformFactory as tFactory
37import lsst.pex.config as pexConfig
38from lsst.pipe import base as pipeBase
39from lsst.pipe.base import connectionTypes as cT
40from lsst.pex.exceptions import InvalidParameterError
41from lsst.meas.algorithms import LoadReferenceObjectsConfig
42from lsst.meas.algorithms import ReferenceObjectLoader
43from lsst.meas.algorithms import brightStarStamps as bSS
44from lsst.utils.timer import timeMethod
47class ProcessBrightStarsConnections(pipeBase.PipelineTaskConnections,
48 dimensions=("instrument", "visit", "detector")):
49 inputExposure = cT.Input(
50 doc="Input exposure from which to extract bright star stamps",
51 name="calexp",
52 storageClass="ExposureF",
53 dimensions=("visit", "detector")
54 )
55 skyCorr = cT.Input(
56 doc="Input Sky Correction to be subtracted from the calexp if doApplySkyCorr=True",
57 name="skyCorr",
58 storageClass="Background",
59 dimensions=("instrument", "visit", "detector")
60 )
61 refCat = cT.PrerequisiteInput(
62 doc="Reference catalog that contains bright star positions",
63 name="gaia_dr2_20200414",
64 storageClass="SimpleCatalog",
65 dimensions=("skypix",),
66 multiple=True,
67 deferLoad=True
68 )
69 brightStarStamps = cT.Output(
70 doc="Set of preprocessed postage stamps, each centered on a single bright star.",
71 name="brightStarStamps",
72 storageClass="BrightStarStamps",
73 dimensions=("visit", "detector")
74 )
76 def __init__(self, *, config=None):
77 super().__init__(config=config)
78 if not config.doApplySkyCorr:
79 self.inputs.remove("skyCorr")
82class ProcessBrightStarsConfig(pipeBase.PipelineTaskConfig,
83 pipelineConnections=ProcessBrightStarsConnections):
84 """Configuration parameters for ProcessBrightStarsTask
85 """
86 magLimit = pexConfig.Field(
87 dtype=float,
88 doc="Magnitude limit, in Gaia G; all stars brighter than this value will be processed",
89 default=18
90 )
91 stampSize = pexConfig.ListField(
92 dtype=int,
93 doc="Size of the stamps to be extracted, in pixels",
94 default=(250, 250)
95 )
96 modelStampBuffer = pexConfig.Field(
97 dtype=float,
98 doc="'Buffer' factor to be applied to determine the size of the stamp the processed stars will "
99 "be saved in. This will also be the size of the extended PSF model.",
100 default=1.1
101 )
102 doRemoveDetected = pexConfig.Field(
103 dtype=bool,
104 doc="Whether DETECTION footprints, other than that for the central object, should be changed to "
105 "BAD",
106 default=True
107 )
108 doApplyTransform = pexConfig.Field(
109 dtype=bool,
110 doc="Apply transform to bright star stamps to correct for optical distortions?",
111 default=True
112 )
113 warpingKernelName = pexConfig.ChoiceField(
114 dtype=str,
115 doc="Warping kernel",
116 default="lanczos5",
117 allowed={
118 "bilinear": "bilinear interpolation",
119 "lanczos3": "Lanczos kernel of order 3",
120 "lanczos4": "Lanczos kernel of order 4",
121 "lanczos5": "Lanczos kernel of order 5",
122 }
123 )
124 annularFluxRadii = pexConfig.ListField(
125 dtype=int,
126 doc="Inner and outer radii of the annulus used to compute the AnnularFlux for normalization, "
127 "in pixels.",
128 default=(40, 50)
129 )
130 annularFluxStatistic = pexConfig.ChoiceField(
131 dtype=str,
132 doc="Type of statistic to use to compute annular flux.",
133 default="MEANCLIP",
134 allowed={
135 "MEAN": "mean",
136 "MEDIAN": "median",
137 "MEANCLIP": "clipped mean",
138 }
139 )
140 numSigmaClip = pexConfig.Field(
141 dtype=float,
142 doc="Sigma for outlier rejection; ignored if annularFluxStatistic != 'MEANCLIP'.",
143 default=4
144 )
145 numIter = pexConfig.Field(
146 dtype=int,
147 doc="Number of iterations of outlier rejection; ignored if annularFluxStatistic != 'MEANCLIP'.",
148 default=3
149 )
150 badMaskPlanes = pexConfig.ListField(
151 dtype=str,
152 doc="Mask planes that, if set, lead to associated pixels not being included in the computation of the"
153 " annular flux.",
154 default=('BAD', 'CR', 'CROSSTALK', 'EDGE', 'NO_DATA', 'SAT', 'SUSPECT', 'UNMASKEDNAN')
155 )
156 minPixelsWithinFrame = pexConfig.Field(
157 dtype=int,
158 doc="Minimum number of pixels that must fall within the stamp boundary for the bright star to be"
159 " saved when its center is beyond the exposure boundary.",
160 default=50
161 )
162 doApplySkyCorr = pexConfig.Field(
163 dtype=bool,
164 doc="Apply full focal plane sky correction before extracting stars?",
165 default=True
166 )
167 discardNanFluxStars = pexConfig.Field(
168 dtype=bool,
169 doc="Should stars with NaN annular flux be discarded?",
170 default=False
171 )
172 refObjLoader = pexConfig.ConfigField(
173 dtype=LoadReferenceObjectsConfig,
174 doc="Reference object loader for astrometric calibration.",
175 )
178class ProcessBrightStarsTask(pipeBase.PipelineTask):
179 """The description of the parameters for this Task are detailed in
180 :lsst-task:`~lsst.pipe.base.PipelineTask`.
182 Notes
183 -----
184 `ProcessBrightStarsTask` is used to extract, process, and store small
185 image cut-outs (or "postage stamps") around bright stars. It relies on
186 three methods, called in succession:
188 `extractStamps`
189 Find bright stars within the exposure using a reference catalog and
190 extract a stamp centered on each.
191 `warpStamps`
192 Shift and warp each stamp to remove optical distortions and sample all
193 stars on the same pixel grid.
194 `measureAndNormalize`
195 Compute the flux of an object in an annulus and normalize it. This is
196 required to normalize each bright star stamp as their central pixels
197 are likely saturated and/or contain ghosts, and cannot be used.
198 """
199 ConfigClass = ProcessBrightStarsConfig
200 _DefaultName = "processBrightStars"
202 def __init__(self, butler=None, initInputs=None, *args, **kwargs):
203 super().__init__(*args, **kwargs)
204 # Compute (model) stamp size depending on provided "buffer" value
205 self.modelStampSize = [int(self.config.stampSize[0]*self.config.modelStampBuffer),
206 int(self.config.stampSize[1]*self.config.modelStampBuffer)]
207 # force it to be odd-sized so we have a central pixel
208 if not self.modelStampSize[0] % 2:
209 self.modelStampSize[0] += 1
210 if not self.modelStampSize[1] % 2:
211 self.modelStampSize[1] += 1
212 # central pixel
213 self.modelCenter = self.modelStampSize[0]//2, self.modelStampSize[1]//2
214 # configure Gaia refcat
215 if butler is not None:
216 self.makeSubtask('refObjLoader', butler=butler)
218 def applySkyCorr(self, calexp, skyCorr):
219 """Apply correction to the sky background level.
221 Sky corrections can be generated with the 'skyCorrection.py'
222 executable in pipe_drivers. Because the sky model used by that
223 code extends over the entire focal plane, this can produce
224 better sky subtraction.
225 The calexp is updated in-place.
227 Parameters
228 ----------
229 calexp : `lsst.afw.image.Exposure` or `lsst.afw.image.MaskedImage`
230 Calibrated exposure.
231 skyCorr : `lsst.afw.math.backgroundList.BackgroundList` or None,
232 optional
233 Full focal plane sky correction, obtained by running
234 `lsst.pipe.drivers.skyCorrection.SkyCorrectionTask`.
235 """
236 if isinstance(calexp, afwImage.Exposure):
237 calexp = calexp.getMaskedImage()
238 calexp -= skyCorr.getImage()
240 def extractStamps(self, inputExposure, refObjLoader=None):
241 """ Read position of bright stars within `inputExposure` from refCat
242 and extract them.
244 Parameters
245 ----------
246 inputExposure : `afwImage.exposure.exposure.ExposureF`
247 The image from which bright star stamps should be extracted.
248 refObjLoader : `lsst.meas.algorithms.ReferenceObjectLoader`, optional
249 Loader to find objects within a reference catalog.
251 Returns
252 -------
253 result : `lsst.pipe.base.Struct`
254 Result struct with components:
256 - ``starIms``: `list` of stamps
257 - ``pixCenters``: `list` of corresponding coordinates to each
258 star's center, in pixels.
259 - ``GMags``: `list` of corresponding (Gaia) G magnitudes.
260 - ``gaiaIds``: `np.ndarray` of corresponding unique Gaia
261 identifiers.
262 """
263 if refObjLoader is None:
264 refObjLoader = self.refObjLoader
265 starIms = []
266 pixCenters = []
267 GMags = []
268 ids = []
269 wcs = inputExposure.getWcs()
270 # select stars within, or close enough to input exposure from refcat
271 inputIm = inputExposure.maskedImage
272 inputExpBBox = inputExposure.getBBox()
273 dilatationExtent = geom.Extent2I(np.array(self.config.stampSize) - self.config.minPixelsWithinFrame)
274 # TODO (DM-25894): handle catalog with stars missing from Gaia
275 withinCalexp = refObjLoader.loadPixelBox(inputExpBBox.dilatedBy(dilatationExtent), wcs,
276 filterName="phot_g_mean")
277 refCat = withinCalexp.refCat
278 # keep bright objects
279 fluxLimit = ((self.config.magLimit*u.ABmag).to(u.nJy)).to_value()
280 GFluxes = np.array(refCat['phot_g_mean_flux'])
281 bright = GFluxes > fluxLimit
282 # convert to AB magnitudes
283 allGMags = [((gFlux*u.nJy).to(u.ABmag)).to_value() for gFlux in GFluxes[bright]]
284 allIds = refCat.columns.extract("id", where=bright)["id"]
285 selectedColumns = refCat.columns.extract('coord_ra', 'coord_dec', where=bright)
286 for j, (ra, dec) in enumerate(zip(selectedColumns["coord_ra"], selectedColumns["coord_dec"])):
287 sp = geom.SpherePoint(ra, dec, geom.radians)
288 cpix = wcs.skyToPixel(sp)
289 try:
290 starIm = inputExposure.getCutout(sp, geom.Extent2I(self.config.stampSize))
291 except InvalidParameterError:
292 # star is beyond boundary
293 bboxCorner = np.array(cpix) - np.array(self.config.stampSize)/2
294 # compute bbox as it would be otherwise
295 idealBBox = geom.Box2I(geom.Point2I(bboxCorner), geom.Extent2I(self.config.stampSize))
296 clippedStarBBox = geom.Box2I(idealBBox)
297 clippedStarBBox.clip(inputExpBBox)
298 if clippedStarBBox.getArea() > 0:
299 # create full-sized stamp with all pixels
300 # flagged as NO_DATA
301 starIm = afwImage.ExposureF(bbox=idealBBox)
302 starIm.image[:] = np.nan
303 starIm.mask.set(inputExposure.mask.getPlaneBitMask("NO_DATA"))
304 # recover pixels from intersection with the exposure
305 clippedIm = inputIm.Factory(inputIm, clippedStarBBox)
306 starIm.maskedImage[clippedStarBBox] = clippedIm
307 # set detector and wcs, used in warpStars
308 starIm.setDetector(inputExposure.getDetector())
309 starIm.setWcs(inputExposure.getWcs())
310 else:
311 continue
312 if self.config.doRemoveDetected:
313 # give detection footprint of other objects the BAD flag
314 detThreshold = afwDetect.Threshold(starIm.mask.getPlaneBitMask("DETECTED"),
315 afwDetect.Threshold.BITMASK)
316 omask = afwDetect.FootprintSet(starIm.mask, detThreshold)
317 allFootprints = omask.getFootprints()
318 otherFootprints = []
319 for fs in allFootprints:
320 if not fs.contains(geom.Point2I(cpix)):
321 otherFootprints.append(fs)
322 nbMatchingFootprints = len(allFootprints) - len(otherFootprints)
323 if not nbMatchingFootprints == 1:
324 self.log.warning("Failed to uniquely identify central DETECTION footprint for star "
325 "%s; found %d footprints instead.",
326 allIds[j], nbMatchingFootprints)
327 omask.setFootprints(otherFootprints)
328 omask.setMask(starIm.mask, "BAD")
329 starIms.append(starIm)
330 pixCenters.append(cpix)
331 GMags.append(allGMags[j])
332 ids.append(allIds[j])
333 return pipeBase.Struct(starIms=starIms,
334 pixCenters=pixCenters,
335 GMags=GMags,
336 gaiaIds=ids)
338 def warpStamps(self, stamps, pixCenters):
339 """Warps and shifts all given stamps so they are sampled on the same
340 pixel grid and centered on the central pixel. This includes rotating
341 the stamp depending on detector orientation.
343 Parameters
344 ----------
345 stamps : `collections.abc.Sequence`
346 [`afwImage.exposure.exposure.ExposureF`]
347 Image cutouts centered on a single object.
348 pixCenters : `collections.abc.Sequence` [`geom.Point2D`]
349 Positions of each object's center (as obtained from the refCat),
350 in pixels.
352 Returns
353 -------
354 result : `lsst.pipe.base.Struct`
355 Result struct with components:
357 - ``warpedStars``:
358 `list` [`afwImage.maskedImage.maskedImage.MaskedImage`] of
359 stamps of warped stars
360 - ``warpTransforms``:
361 `list` [`afwGeom.TransformPoint2ToPoint2`] of
362 the corresponding Transform from the initial star stamp to
363 the common model grid
364 - ``xy0s``:
365 `list` [`geom.Point2I`] of coordinates of the bottom-left
366 pixels of each stamp, before rotation
367 - ``nb90Rots``: `int`, the number of 90 degrees rotations required
368 to compensate for detector orientation
369 """
370 # warping control; only contains shiftingALg provided in config
371 warpCont = afwMath.WarpingControl(self.config.warpingKernelName)
372 # Compare model to star stamp sizes
373 bufferPix = (self.modelStampSize[0] - self.config.stampSize[0],
374 self.modelStampSize[1] - self.config.stampSize[1])
375 # Initialize detector instance (note all stars were extracted from an
376 # exposure from the same detector)
377 det = stamps[0].getDetector()
378 # Define correction for optical distortions
379 if self.config.doApplyTransform:
380 pixToTan = det.getTransform(cg.PIXELS, cg.TAN_PIXELS)
381 else:
382 pixToTan = tFactory.makeIdentityTransform()
383 # Array of all possible rotations for detector orientation:
384 possibleRots = np.array([k*np.pi/2 for k in range(4)])
385 # determine how many, if any, rotations are required
386 yaw = det.getOrientation().getYaw()
387 nb90Rots = np.argmin(np.abs(possibleRots - float(yaw)))
389 # apply transformation to each star
390 warpedStars, warpTransforms, xy0s = [], [], []
391 for star, cent in zip(stamps, pixCenters):
392 # (re)create empty destination image
393 destImage = afwImage.MaskedImageF(*self.modelStampSize)
394 bottomLeft = geom.Point2D(star.image.getXY0())
395 newBottomLeft = pixToTan.applyForward(bottomLeft)
396 newBottomLeft.setX(newBottomLeft.getX() - bufferPix[0]/2)
397 newBottomLeft.setY(newBottomLeft.getY() - bufferPix[1]/2)
398 # Convert to int
399 newBottomLeft = geom.Point2I(newBottomLeft)
400 # Set origin and save it
401 destImage.setXY0(newBottomLeft)
402 xy0s.append(newBottomLeft)
404 # Define linear shifting to recenter stamps
405 newCenter = pixToTan.applyForward(cent) # center of warped star
406 shift = self.modelCenter[0] + newBottomLeft[0] - newCenter[0],\
407 self.modelCenter[1] + newBottomLeft[1] - newCenter[1]
408 affineShift = geom.AffineTransform(shift)
409 shiftTransform = tFactory.makeTransform(affineShift)
411 # Define full transform (warp and shift)
412 starWarper = pixToTan.then(shiftTransform)
414 # Apply it
415 goodPix = afwMath.warpImage(destImage, star.getMaskedImage(),
416 starWarper, warpCont)
417 if not goodPix:
418 self.log.debug("Warping of a star failed: no good pixel in output")
420 # Arbitrarily set origin of shifted star to 0
421 destImage.setXY0(0, 0)
423 # Apply rotation if appropriate
424 if nb90Rots:
425 destImage = afwMath.rotateImageBy90(destImage, nb90Rots)
426 warpedStars.append(destImage.clone())
427 warpTransforms.append(starWarper)
428 return pipeBase.Struct(warpedStars=warpedStars, warpTransforms=warpTransforms, xy0s=xy0s,
429 nb90Rots=nb90Rots)
431 @timeMethod
432 def run(self, inputExposure, refObjLoader=None, dataId=None, skyCorr=None):
433 """Identify bright stars within an exposure using a reference catalog,
434 extract stamps around each, then preprocess them. The preprocessing
435 steps are: shifting, warping and potentially rotating them to the same
436 pixel grid; computing their annular flux and normalizing them.
438 Parameters
439 ----------
440 inputExposure : `afwImage.exposure.exposure.ExposureF`
441 The image from which bright star stamps should be extracted.
442 refObjLoader : `lsst.meas.algorithms.ReferenceObjectLoader`, optional
443 Loader to find objects within a reference catalog.
444 dataId : `dict` or `lsst.daf.butler.DataCoordinate`
445 The dataId of the exposure (and detector) bright stars should be
446 extracted from.
447 skyCorr : `lsst.afw.math.backgroundList.BackgroundList` or ``None``,
448 optional
449 Full focal plane sky correction, obtained by running
450 `lsst.pipe.drivers.skyCorrection.SkyCorrectionTask`.
452 Returns
453 -------
454 result : `lsst.pipe.base.Struct`
455 Result struct with component:
457 - ``brightStarStamps``: ``bSS.BrightStarStamps``
458 """
459 if self.config.doApplySkyCorr:
460 self.log.info("Applying sky correction to exposure %s (exposure will be modified in-place).",
461 dataId)
462 self.applySkyCorr(inputExposure, skyCorr)
463 self.log.info("Extracting bright stars from exposure %s", dataId)
464 # Extract stamps around bright stars
465 extractedStamps = self.extractStamps(inputExposure, refObjLoader=refObjLoader)
466 if not extractedStamps.starIms:
467 self.log.info("No suitable bright star found.")
468 return None
469 # Warp (and shift, and potentially rotate) them
470 self.log.info("Applying warp and/or shift to %i star stamps from exposure %s",
471 len(extractedStamps.starIms), dataId)
472 warpOutputs = self.warpStamps(extractedStamps.starIms, extractedStamps.pixCenters)
473 warpedStars = warpOutputs.warpedStars
474 xy0s = warpOutputs.xy0s
475 brightStarList = [bSS.BrightStarStamp(stamp_im=warp,
476 archive_element=transform,
477 position=xy0s[j],
478 gaiaGMag=extractedStamps.GMags[j],
479 gaiaId=extractedStamps.gaiaIds[j])
480 for j, (warp, transform) in
481 enumerate(zip(warpedStars, warpOutputs.warpTransforms))]
482 # Compute annularFlux and normalize
483 self.log.info("Computing annular flux and normalizing %i bright stars from exposure %s",
484 len(warpedStars), dataId)
485 # annularFlux statistic set-up, excluding mask planes
486 statsControl = afwMath.StatisticsControl()
487 statsControl.setNumSigmaClip(self.config.numSigmaClip)
488 statsControl.setNumIter(self.config.numIter)
489 innerRadius, outerRadius = self.config.annularFluxRadii
490 statsFlag = afwMath.stringToStatisticsProperty(self.config.annularFluxStatistic)
491 brightStarStamps = bSS.BrightStarStamps.initAndNormalize(brightStarList,
492 innerRadius=innerRadius,
493 outerRadius=outerRadius,
494 nb90Rots=warpOutputs.nb90Rots,
495 imCenter=self.modelCenter,
496 use_archive=True,
497 statsControl=statsControl,
498 statsFlag=statsFlag,
499 badMaskPlanes=self.config.badMaskPlanes,
500 discardNanFluxObjects=(
501 self.config.discardNanFluxStars))
502 return pipeBase.Struct(brightStarStamps=brightStarStamps)
504 def runQuantum(self, butlerQC, inputRefs, outputRefs):
505 inputs = butlerQC.get(inputRefs)
506 inputs['dataId'] = str(butlerQC.quantum.dataId)
507 refObjLoader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId
508 for ref in inputRefs.refCat],
509 refCats=inputs.pop("refCat"),
510 name=self.config.connections.refCat,
511 config=self.config.refObjLoader)
512 output = self.run(**inputs, refObjLoader=refObjLoader)
513 if output:
514 butlerQC.put(output, outputRefs)