lsst.pipe.tasks gf10b05e212+a1470f6de0
processBrightStars.py
Go to the documentation of this file.
1# This file is part of pipe_tasks.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
21#
22"""Extract small cutouts around bright stars, normalize and warp them to the
23same arbitrary pixel grid.
24"""
25
26__all__ = ["ProcessBrightStarsTask"]
27
28import numpy as np
29import astropy.units as u
30
31from lsst import geom
32from lsst.afw import math as afwMath
33from lsst.afw import image as afwImage
34from lsst.afw import detection as afwDetect
35from lsst.afw import cameraGeom as cg
36from lsst.afw.geom import transformFactory as tFactory
37import lsst.pex.config as pexConfig
38from lsst.pipe import base as pipeBase
39from lsst.pipe.base import connectionTypes as cT
40from lsst.pex.exceptions import InvalidParameterError
41from lsst.meas.algorithms import LoadReferenceObjectsConfig
42from lsst.meas.algorithms import ReferenceObjectLoader
43from lsst.meas.algorithms import brightStarStamps as bSS
44from lsst.utils.timer import timeMethod
45
46
47class ProcessBrightStarsConnections(pipeBase.PipelineTaskConnections,
48 dimensions=("instrument", "visit", "detector")):
49 inputExposure = cT.Input(
50 doc="Input exposure from which to extract bright star stamps",
51 name="calexp",
52 storageClass="ExposureF",
53 dimensions=("visit", "detector")
54 )
55 skyCorr = cT.Input(
56 doc="Input Sky Correction to be subtracted from the calexp if doApplySkyCorr=True",
57 name="skyCorr",
58 storageClass="Background",
59 dimensions=("instrument", "visit", "detector")
60 )
61 refCat = cT.PrerequisiteInput(
62 doc="Reference catalog that contains bright star positions",
63 name="gaia_dr2_20200414",
64 storageClass="SimpleCatalog",
65 dimensions=("skypix",),
66 multiple=True,
67 deferLoad=True
68 )
69 brightStarStamps = cT.Output(
70 doc="Set of preprocessed postage stamps, each centered on a single bright star.",
71 name="brightStarStamps",
72 storageClass="BrightStarStamps",
73 dimensions=("visit", "detector")
74 )
75
76 def __init__(self, *, config=None):
77 super().__init__(config=config)
78 if not config.doApplySkyCorr:
79 self.inputs.remove("skyCorr")
80
81
82class ProcessBrightStarsConfig(pipeBase.PipelineTaskConfig,
83 pipelineConnections=ProcessBrightStarsConnections):
84 """Configuration parameters for ProcessBrightStarsTask
85 """
86 magLimit = pexConfig.Field(
87 dtype=float,
88 doc="Magnitude limit, in Gaia G; all stars brighter than this value will be processed",
89 default=18
90 )
91 stampSize = pexConfig.ListField(
92 dtype=int,
93 doc="Size of the stamps to be extracted, in pixels",
94 default=(250, 250)
95 )
96 modelStampBuffer = pexConfig.Field(
97 dtype=float,
98 doc="'Buffer' factor to be applied to determine the size of the stamp the processed stars will "
99 "be saved in. This will also be the size of the extended PSF model.",
100 default=1.1
101 )
102 doRemoveDetected = pexConfig.Field(
103 dtype=bool,
104 doc="Whether DETECTION footprints, other than that for the central object, should be changed to "
105 "BAD",
106 default=True
107 )
108 doApplyTransform = pexConfig.Field(
109 dtype=bool,
110 doc="Apply transform to bright star stamps to correct for optical distortions?",
111 default=True
112 )
113 warpingKernelName = pexConfig.ChoiceField(
114 dtype=str,
115 doc="Warping kernel",
116 default="lanczos5",
117 allowed={
118 "bilinear": "bilinear interpolation",
119 "lanczos3": "Lanczos kernel of order 3",
120 "lanczos4": "Lanczos kernel of order 4",
121 "lanczos5": "Lanczos kernel of order 5",
122 }
123 )
124 annularFluxRadii = pexConfig.ListField(
125 dtype=int,
126 doc="Inner and outer radii of the annulus used to compute the AnnularFlux for normalization, "
127 "in pixels.",
128 default=(40, 50)
129 )
130 annularFluxStatistic = pexConfig.ChoiceField(
131 dtype=str,
132 doc="Type of statistic to use to compute annular flux.",
133 default="MEANCLIP",
134 allowed={
135 "MEAN": "mean",
136 "MEDIAN": "median",
137 "MEANCLIP": "clipped mean",
138 }
139 )
140 numSigmaClip = pexConfig.Field(
141 dtype=float,
142 doc="Sigma for outlier rejection; ignored if annularFluxStatistic != 'MEANCLIP'.",
143 default=4
144 )
145 numIter = pexConfig.Field(
146 dtype=int,
147 doc="Number of iterations of outlier rejection; ignored if annularFluxStatistic != 'MEANCLIP'.",
148 default=3
149 )
150 badMaskPlanes = pexConfig.ListField(
151 dtype=str,
152 doc="Mask planes that, if set, lead to associated pixels not being included in the computation of the"
153 " annular flux.",
154 default=('BAD', 'CR', 'CROSSTALK', 'EDGE', 'NO_DATA', 'SAT', 'SUSPECT', 'UNMASKEDNAN')
155 )
156 minPixelsWithinFrame = pexConfig.Field(
157 dtype=int,
158 doc="Minimum number of pixels that must fall within the stamp boundary for the bright star to be"
159 " saved when its center is beyond the exposure boundary.",
160 default=50
161 )
162 doApplySkyCorr = pexConfig.Field(
163 dtype=bool,
164 doc="Apply full focal plane sky correction before extracting stars?",
165 default=True
166 )
167 discardNanFluxStars = pexConfig.Field(
168 dtype=bool,
169 doc="Should stars with NaN annular flux be discarded?",
170 default=False
171 )
172 refObjLoader = pexConfig.ConfigField(
173 dtype=LoadReferenceObjectsConfig,
174 doc="Reference object loader for astrometric calibration.",
175 )
176
177 def setDefaults(self):
178 self.refObjLoader.ref_dataset_name = "gaia_dr2_20200414"
179
180
181class ProcessBrightStarsTask(pipeBase.PipelineTask):
182 """The description of the parameters for this Task are detailed in
183 :lsst-task:`~lsst.pipe.base.PipelineTask`.
184
185 Notes
186 -----
187 `ProcessBrightStarsTask` is used to extract, process, and store small
188 image cut-outs (or "postage stamps") around bright stars. It relies on
189 three methods, called in succession:
190
191 `extractStamps`
192 Find bright stars within the exposure using a reference catalog and
193 extract a stamp centered on each.
194 `warpStamps`
195 Shift and warp each stamp to remove optical distortions and sample all
196 stars on the same pixel grid.
197 `measureAndNormalize`
198 Compute the flux of an object in an annulus and normalize it. This is
199 required to normalize each bright star stamp as their central pixels
200 are likely saturated and/or contain ghosts, and cannot be used.
201 """
202 ConfigClass = ProcessBrightStarsConfig
203 _DefaultName = "processBrightStars"
204
205 def __init__(self, butler=None, initInputs=None, *args, **kwargs):
206 super().__init__(*args, **kwargs)
207 # Compute (model) stamp size depending on provided "buffer" value
208 self.modelStampSize = [int(self.config.stampSize[0]*self.config.modelStampBuffer),
209 int(self.config.stampSize[1]*self.config.modelStampBuffer)]
210 # force it to be odd-sized so we have a central pixel
211 if not self.modelStampSize[0] % 2:
212 self.modelStampSize[0] += 1
213 if not self.modelStampSize[1] % 2:
214 self.modelStampSize[1] += 1
215 # central pixel
216 self.modelCenter = self.modelStampSize[0]//2, self.modelStampSize[1]//2
217 # configure Gaia refcat
218 if butler is not None:
219 self.makeSubtask('refObjLoader', butler=butler)
220
221 def applySkyCorr(self, calexp, skyCorr):
222 """Apply correction to the sky background level.
223
224 Sky corrections can be generated with the 'skyCorrection.py'
225 executable in pipe_drivers. Because the sky model used by that
226 code extends over the entire focal plane, this can produce
227 better sky subtraction.
228 The calexp is updated in-place.
229
230 Parameters
231 ----------
233 Calibrated exposure.
234 skyCorr : `lsst.afw.math.backgroundList.BackgroundList` or None,
235 optional
236 Full focal plane sky correction, obtained by running
237 `lsst.pipe.drivers.skyCorrection.SkyCorrectionTask`.
238 """
239 if isinstance(calexp, afwImage.Exposure):
240 calexp = calexp.getMaskedImage()
241 calexp -= skyCorr.getImage()
242
243 def extractStamps(self, inputExposure, refObjLoader=None):
244 """ Read position of bright stars within `inputExposure` from refCat
245 and extract them.
246
247 Parameters
248 ----------
249 inputExposure : `afwImage.exposure.exposure.ExposureF`
250 The image from which bright star stamps should be extracted.
251 refObjLoader : `lsst.meas.algorithms.ReferenceObjectLoader`, optional
252 Loader to find objects within a reference catalog.
253
254 Returns
255 -------
256 result : `lsst.pipe.base.Struct`
257 Result struct with components:
258
259 - ``starIms``: `list` of stamps
260 - ``pixCenters``: `list` of corresponding coordinates to each
261 star's center, in pixels.
262 - ``GMags``: `list` of corresponding (Gaia) G magnitudes.
263 - ``gaiaIds``: `np.ndarray` of corresponding unique Gaia
264 identifiers.
265 """
266 if refObjLoader is None:
267 refObjLoader = self.refObjLoader
268 starIms = []
269 pixCenters = []
270 GMags = []
271 ids = []
272 wcs = inputExposure.getWcs()
273 # select stars within, or close enough to input exposure from refcat
274 inputIm = inputExposure.maskedImage
275 inputExpBBox = inputExposure.getBBox()
276 dilatationExtent = geom.Extent2I(np.array(self.config.stampSize) - self.config.minPixelsWithinFrame)
277 # TODO (DM-25894): handle catalog with stars missing from Gaia
278 withinCalexp = refObjLoader.loadPixelBox(inputExpBBox.dilatedBy(dilatationExtent), wcs,
279 filterName="phot_g_mean")
280 refCat = withinCalexp.refCat
281 # keep bright objects
282 fluxLimit = ((self.config.magLimit*u.ABmag).to(u.nJy)).to_value()
283 GFluxes = np.array(refCat['phot_g_mean_flux'])
284 bright = GFluxes > fluxLimit
285 # convert to AB magnitudes
286 allGMags = [((gFlux*u.nJy).to(u.ABmag)).to_value() for gFlux in GFluxes[bright]]
287 allIds = refCat.columns.extract("id", where=bright)["id"]
288 selectedColumns = refCat.columns.extract('coord_ra', 'coord_dec', where=bright)
289 for j, (ra, dec) in enumerate(zip(selectedColumns["coord_ra"], selectedColumns["coord_dec"])):
290 sp = geom.SpherePoint(ra, dec, geom.radians)
291 cpix = wcs.skyToPixel(sp)
292 try:
293 starIm = inputExposure.getCutout(sp, geom.Extent2I(self.config.stampSize))
294 except InvalidParameterError:
295 # star is beyond boundary
296 bboxCorner = np.array(cpix) - np.array(self.config.stampSize)/2
297 # compute bbox as it would be otherwise
298 idealBBox = geom.Box2I(geom.Point2I(bboxCorner), geom.Extent2I(self.config.stampSize))
299 clippedStarBBox = geom.Box2I(idealBBox)
300 clippedStarBBox.clip(inputExpBBox)
301 if clippedStarBBox.getArea() > 0:
302 # create full-sized stamp with all pixels
303 # flagged as NO_DATA
304 starIm = afwImage.ExposureF(bbox=idealBBox)
305 starIm.image[:] = np.nan
306 starIm.mask.set(inputExposure.mask.getPlaneBitMask("NO_DATA"))
307 # recover pixels from intersection with the exposure
308 clippedIm = inputIm.Factory(inputIm, clippedStarBBox)
309 starIm.maskedImage[clippedStarBBox] = clippedIm
310 # set detector and wcs, used in warpStars
311 starIm.setDetector(inputExposure.getDetector())
312 starIm.setWcs(inputExposure.getWcs())
313 else:
314 continue
315 if self.config.doRemoveDetected:
316 # give detection footprint of other objects the BAD flag
317 detThreshold = afwDetect.Threshold(starIm.mask.getPlaneBitMask("DETECTED"),
318 afwDetect.Threshold.BITMASK)
319 omask = afwDetect.FootprintSet(starIm.mask, detThreshold)
320 allFootprints = omask.getFootprints()
321 otherFootprints = []
322 for fs in allFootprints:
323 if not fs.contains(geom.Point2I(cpix)):
324 otherFootprints.append(fs)
325 nbMatchingFootprints = len(allFootprints) - len(otherFootprints)
326 if not nbMatchingFootprints == 1:
327 self.log.warning("Failed to uniquely identify central DETECTION footprint for star "
328 "%s; found %d footprints instead.",
329 allIds[j], nbMatchingFootprints)
330 omask.setFootprints(otherFootprints)
331 omask.setMask(starIm.mask, "BAD")
332 starIms.append(starIm)
333 pixCenters.append(cpix)
334 GMags.append(allGMags[j])
335 ids.append(allIds[j])
336 return pipeBase.Struct(starIms=starIms,
337 pixCenters=pixCenters,
338 GMags=GMags,
339 gaiaIds=ids)
340
341 def warpStamps(self, stamps, pixCenters):
342 """Warps and shifts all given stamps so they are sampled on the same
343 pixel grid and centered on the central pixel. This includes rotating
344 the stamp depending on detector orientation.
345
346 Parameters
347 ----------
348 stamps : `collections.abc.Sequence`
349 [`afwImage.exposure.exposure.ExposureF`]
350 Image cutouts centered on a single object.
351 pixCenters : `collections.abc.Sequence` [`geom.Point2D`]
352 Positions of each object's center (as obtained from the refCat),
353 in pixels.
354
355 Returns
356 -------
357 result : `lsst.pipe.base.Struct`
358 Result struct with components:
359
360 - ``warpedStars``:
361 `list` [`afwImage.maskedImage.maskedImage.MaskedImage`] of
362 stamps of warped stars
363 - ``warpTransforms``:
364 `list` [`afwGeom.TransformPoint2ToPoint2`] of
365 the corresponding Transform from the initial star stamp to
366 the common model grid
367 - ``xy0s``:
368 `list` [`geom.Point2I`] of coordinates of the bottom-left
369 pixels of each stamp, before rotation
370 - ``nb90Rots``: `int`, the number of 90 degrees rotations required
371 to compensate for detector orientation
372 """
373 # warping control; only contains shiftingALg provided in config
374 warpCont = afwMath.WarpingControl(self.config.warpingKernelName)
375 # Compare model to star stamp sizes
376 bufferPix = (self.modelStampSize[0] - self.config.stampSize[0],
377 self.modelStampSize[1] - self.config.stampSize[1])
378 # Initialize detector instance (note all stars were extracted from an
379 # exposure from the same detector)
380 det = stamps[0].getDetector()
381 # Define correction for optical distortions
382 if self.config.doApplyTransform:
383 pixToTan = det.getTransform(cg.PIXELS, cg.TAN_PIXELS)
384 else:
385 pixToTan = tFactory.makeIdentityTransform()
386 # Array of all possible rotations for detector orientation:
387 possibleRots = np.array([k*np.pi/2 for k in range(4)])
388 # determine how many, if any, rotations are required
389 yaw = det.getOrientation().getYaw()
390 nb90Rots = np.argmin(np.abs(possibleRots - float(yaw)))
391
392 # apply transformation to each star
393 warpedStars, warpTransforms, xy0s = [], [], []
394 for star, cent in zip(stamps, pixCenters):
395 # (re)create empty destination image
396 destImage = afwImage.MaskedImageF(*self.modelStampSize)
397 bottomLeft = geom.Point2D(star.image.getXY0())
398 newBottomLeft = pixToTan.applyForward(bottomLeft)
399 newBottomLeft.setX(newBottomLeft.getX() - bufferPix[0]/2)
400 newBottomLeft.setY(newBottomLeft.getY() - bufferPix[1]/2)
401 # Convert to int
402 newBottomLeft = geom.Point2I(newBottomLeft)
403 # Set origin and save it
404 destImage.setXY0(newBottomLeft)
405 xy0s.append(newBottomLeft)
406
407 # Define linear shifting to recenter stamps
408 newCenter = pixToTan.applyForward(cent) # center of warped star
409 shift = self.modelCenter[0] + newBottomLeft[0] - newCenter[0],\
410 self.modelCenter[1] + newBottomLeft[1] - newCenter[1]
411 affineShift = geom.AffineTransform(shift)
412 shiftTransform = tFactory.makeTransform(affineShift)
413
414 # Define full transform (warp and shift)
415 starWarper = pixToTan.then(shiftTransform)
416
417 # Apply it
418 goodPix = afwMath.warpImage(destImage, star.getMaskedImage(),
419 starWarper, warpCont)
420 if not goodPix:
421 self.log.debug("Warping of a star failed: no good pixel in output")
422
423 # Arbitrarily set origin of shifted star to 0
424 destImage.setXY0(0, 0)
425
426 # Apply rotation if appropriate
427 if nb90Rots:
428 destImage = afwMath.rotateImageBy90(destImage, nb90Rots)
429 warpedStars.append(destImage.clone())
430 warpTransforms.append(starWarper)
431 return pipeBase.Struct(warpedStars=warpedStars, warpTransforms=warpTransforms, xy0s=xy0s,
432 nb90Rots=nb90Rots)
433
434 @timeMethod
435 def run(self, inputExposure, refObjLoader=None, dataId=None, skyCorr=None):
436 """Identify bright stars within an exposure using a reference catalog,
437 extract stamps around each, then preprocess them. The preprocessing
438 steps are: shifting, warping and potentially rotating them to the same
439 pixel grid; computing their annular flux and normalizing them.
440
441 Parameters
442 ----------
443 inputExposure : `afwImage.exposure.exposure.ExposureF`
444 The image from which bright star stamps should be extracted.
445 refObjLoader : `lsst.meas.algorithms.ReferenceObjectLoader`, optional
446 Loader to find objects within a reference catalog.
447 dataId : `dict` or `lsst.daf.butler.DataCoordinate`
448 The dataId of the exposure (and detector) bright stars should be
449 extracted from.
450 skyCorr : `lsst.afw.math.backgroundList.BackgroundList` or ``None``,
451 optional
452 Full focal plane sky correction, obtained by running
453 `lsst.pipe.drivers.skyCorrection.SkyCorrectionTask`.
454
455 Returns
456 -------
457 result : `lsst.pipe.base.Struct`
458 Result struct with component:
459
460 - ``brightStarStamps``: ``bSS.BrightStarStamps``
461 """
462 if self.config.doApplySkyCorr:
463 self.log.info("Applying sky correction to exposure %s (exposure will be modified in-place).",
464 dataId)
465 self.applySkyCorr(inputExposure, skyCorr)
466 self.log.info("Extracting bright stars from exposure %s", dataId)
467 # Extract stamps around bright stars
468 extractedStamps = self.extractStamps(inputExposure, refObjLoader=refObjLoader)
469 if not extractedStamps.starIms:
470 self.log.info("No suitable bright star found.")
471 return None
472 # Warp (and shift, and potentially rotate) them
473 self.log.info("Applying warp and/or shift to %i star stamps from exposure %s",
474 len(extractedStamps.starIms), dataId)
475 warpOutputs = self.warpStamps(extractedStamps.starIms, extractedStamps.pixCenters)
476 warpedStars = warpOutputs.warpedStars
477 xy0s = warpOutputs.xy0s
478 brightStarList = [bSS.BrightStarStamp(stamp_im=warp,
479 archive_element=transform,
480 position=xy0s[j],
481 gaiaGMag=extractedStamps.GMags[j],
482 gaiaId=extractedStamps.gaiaIds[j])
483 for j, (warp, transform) in
484 enumerate(zip(warpedStars, warpOutputs.warpTransforms))]
485 # Compute annularFlux and normalize
486 self.log.info("Computing annular flux and normalizing %i bright stars from exposure %s",
487 len(warpedStars), dataId)
488 # annularFlux statistic set-up, excluding mask planes
489 statsControl = afwMath.StatisticsControl()
490 statsControl.setNumSigmaClip(self.config.numSigmaClip)
491 statsControl.setNumIter(self.config.numIter)
492 innerRadius, outerRadius = self.config.annularFluxRadii
493 statsFlag = afwMath.stringToStatisticsProperty(self.config.annularFluxStatistic)
494 brightStarStamps = bSS.BrightStarStamps.initAndNormalize(brightStarList,
495 innerRadius=innerRadius,
496 outerRadius=outerRadius,
497 nb90Rots=warpOutputs.nb90Rots,
498 imCenter=self.modelCenter,
499 use_archive=True,
500 statsControl=statsControl,
501 statsFlag=statsFlag,
502 badMaskPlanes=self.config.badMaskPlanes,
503 discardNanFluxObjects=(
504 self.config.discardNanFluxStars))
505 return pipeBase.Struct(brightStarStamps=brightStarStamps)
506
507 def runQuantum(self, butlerQC, inputRefs, outputRefs):
508 inputs = butlerQC.get(inputRefs)
509 inputs['dataId'] = str(butlerQC.quantum.dataId)
510 refObjLoader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId
511 for ref in inputRefs.refCat],
512 refCats=inputs.pop("refCat"),
513 config=self.config.refObjLoader)
514 output = self.run(**inputs, refObjLoader=refObjLoader)
515 if output:
516 butlerQC.put(output, outputRefs)
def runQuantum(self, butlerQC, inputRefs, outputRefs)
def run(self, inputExposure, refObjLoader=None, dataId=None, skyCorr=None)
def extractStamps(self, inputExposure, refObjLoader=None)
def __init__(self, butler=None, initInputs=None, *args, **kwargs)