lsst.pipe.tasks  21.0.0-51-gd3b42663+de9f4996ec
processBrightStars.py
Go to the documentation of this file.
1 # This file is part of pipe_tasks.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (https://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <https://www.gnu.org/licenses/>.
21 #
22 """Extract small cutouts around bright stars, normalize and warp them to the
23 same arbitrary pixel grid.
24 """
25 
26 __all__ = ["ProcessBrightStarsTask"]
27 
28 import numpy as np
29 import astropy.units as u
30 
31 from lsst import geom
32 from lsst.afw import math as afwMath
33 from lsst.afw import image as afwImage
34 from lsst.afw import detection as afwDetect
35 from lsst.afw import cameraGeom as cg
36 from lsst.afw.geom import transformFactory as tFactory
37 import lsst.pex.config as pexConfig
38 from lsst.pipe import base as pipeBase
39 from lsst.pipe.base import connectionTypes as cT
40 from lsst.pex.exceptions import InvalidParameterError
41 from lsst.meas.algorithms.loadIndexedReferenceObjects import LoadIndexedReferenceObjectsTask
42 from lsst.meas.algorithms import ReferenceObjectLoader
43 from lsst.meas.algorithms import brightStarStamps as bSS
44 
45 
46 class ProcessBrightStarsConnections(pipeBase.PipelineTaskConnections, dimensions=("visit", "detector")):
47  inputExposure = cT.Input(
48  doc="Input exposure from which to extract bright star stamps",
49  name="calexp",
50  storageClass="ExposureF",
51  dimensions=("visit", "detector")
52  )
53  skyCorr = cT.Input(
54  doc="Input Sky Correction to be subtracted from the calexp if doApplySkyCorr=True",
55  name="skyCorr",
56  storageClass="Background",
57  dimensions=("instrument", "visit", "detector")
58  )
59  refCat = cT.PrerequisiteInput(
60  doc="Reference catalog that contains bright star positions",
61  name="gaia_dr2_20200414",
62  storageClass="SimpleCatalog",
63  dimensions=("skypix",),
64  multiple=True,
65  deferLoad=True
66  )
67  brightStarStamps = cT.Output(
68  doc="Set of preprocessed postage stamps, each centered on a single bright star.",
69  name="brightStarStamps",
70  storageClass="BrightStarStamps",
71  dimensions=("visit", "detector")
72  )
73 
74  def __init__(self, *, config=None):
75  super().__init__(config=config)
76  if not config.doApplySkyCorr:
77  self.inputs.remove("skyCorr")
78 
79 
80 class ProcessBrightStarsConfig(pipeBase.PipelineTaskConfig,
81  pipelineConnections=ProcessBrightStarsConnections):
82  """Configuration parameters for ProcessBrightStarsTask
83  """
84  magLimit = pexConfig.Field(
85  dtype=float,
86  doc="Magnitude limit, in Gaia G; all stars brighter than this value will be processed",
87  default=18
88  )
89  stampSize = pexConfig.ListField(
90  dtype=int,
91  doc="Size of the stamps to be extracted, in pixels",
92  default=(250, 250)
93  )
94  modelStampBuffer = pexConfig.Field(
95  dtype=float,
96  doc="'Buffer' factor to be applied to determine the size of the stamp the processed stars will "
97  "be saved in. This will also be the size of the extended PSF model.",
98  default=1.1
99  )
100  doRemoveDetected = pexConfig.Field(
101  dtype=bool,
102  doc="Whether DETECTION footprints, other than that for the central object, should be changed to "
103  "BAD",
104  default=True
105  )
106  doApplyTransform = pexConfig.Field(
107  dtype=bool,
108  doc="Apply transform to bright star stamps to correct for optical distortions?",
109  default=True
110  )
111  warpingKernelName = pexConfig.ChoiceField(
112  dtype=str,
113  doc="Warping kernel",
114  default="lanczos5",
115  allowed={
116  "bilinear": "bilinear interpolation",
117  "lanczos3": "Lanczos kernel of order 3",
118  "lanczos4": "Lanczos kernel of order 4",
119  "lanczos5": "Lanczos kernel of order 5",
120  }
121  )
122  annularFluxRadii = pexConfig.ListField(
123  dtype=int,
124  doc="Inner and outer radii of the annulus used to compute the AnnularFlux for normalization, "
125  "in pixels.",
126  default=(40, 50)
127  )
128  annularFluxStatistic = pexConfig.ChoiceField(
129  dtype=str,
130  doc="Type of statistic to use to compute annular flux.",
131  default="MEANCLIP",
132  allowed={
133  "MEAN": "mean",
134  "MEDIAN": "median",
135  "MEANCLIP": "clipped mean",
136  }
137  )
138  numSigmaClip = pexConfig.Field(
139  dtype=float,
140  doc="Sigma for outlier rejection; ignored if annularFluxStatistic != 'MEANCLIP'.",
141  default=4
142  )
143  numIter = pexConfig.Field(
144  dtype=int,
145  doc="Number of iterations of outlier rejection; ignored if annularFluxStatistic != 'MEANCLIP'.",
146  default=3
147  )
148  badMaskPlanes = pexConfig.ListField(
149  dtype=str,
150  doc="Mask planes that, if set, lead to associated pixels not being included in the computation of the"
151  " annular flux.",
152  default=('BAD', 'CR', 'CROSSTALK', 'EDGE', 'NO_DATA', 'SAT', 'SUSPECT', 'UNMASKEDNAN')
153  )
154  minPixelsWithinFrame = pexConfig.Field(
155  dtype=int,
156  doc="Minimum number of pixels that must fall within the stamp boundary for the bright star to be"
157  " saved when its center is beyond the exposure boundary.",
158  default=50
159  )
160  doApplySkyCorr = pexConfig.Field(
161  dtype=bool,
162  doc="Apply full focal plane sky correction before extracting stars?",
163  default=True
164  )
165  refObjLoader = pexConfig.ConfigurableField(
166  target=LoadIndexedReferenceObjectsTask,
167  doc="Reference object loader for astrometric calibration.",
168  )
169 
170  def setDefaults(self):
171  self.refObjLoaderrefObjLoader.ref_dataset_name = "gaia_dr2_20200414"
172 
173 
174 class ProcessBrightStarsTask(pipeBase.PipelineTask, pipeBase.CmdLineTask):
175  """The description of the parameters for this Task are detailed in
176  :lsst-task:`~lsst.pipe.base.PipelineTask`.
177 
178  Notes
179  -----
180  `ProcessBrightStarsTask` is used to extract, process, and store small
181  image cut-outs (or "postage stamps") around bright stars. It relies on
182  three methods, called in succession:
183 
184  `extractStamps`
185  Find bright stars within the exposure using a reference catalog and
186  extract a stamp centered on each.
187  `warpStamps`
188  Shift and warp each stamp to remove optical distortions and sample all
189  stars on the same pixel grid.
190  `measureAndNormalize`
191  Compute the flux of an object in an annulus and normalize it. This is
192  required to normalize each bright star stamp as their central pixels
193  are likely saturated and/or contain ghosts, and cannot be used.
194  """
195  ConfigClass = ProcessBrightStarsConfig
196  _DefaultName = "processBrightStars"
197  RunnerClass = pipeBase.ButlerInitializedTaskRunner
198 
199  def __init__(self, butler=None, initInputs=None, *args, **kwargs):
200  super().__init__(*args, **kwargs)
201  # Compute (model) stamp size depending on provided "buffer" value
202  self.modelStampSizemodelStampSize = [int(self.config.stampSize[0]*self.config.modelStampBuffer),
203  int(self.config.stampSize[1]*self.config.modelStampBuffer)]
204  # force it to be odd-sized so we have a central pixel
205  if not self.modelStampSizemodelStampSize[0] % 2:
206  self.modelStampSizemodelStampSize[0] += 1
207  if not self.modelStampSizemodelStampSize[1] % 2:
208  self.modelStampSizemodelStampSize[1] += 1
209  # central pixel
210  self.modelCentermodelCenter = self.modelStampSizemodelStampSize[0]//2, self.modelStampSizemodelStampSize[1]//2
211  # configure Gaia refcat
212  if butler is not None:
213  self.makeSubtask('refObjLoader', butler=butler)
214 
215  def applySkyCorr(self, calexp, skyCorr):
216  """Apply correction to the sky background level.
217 
218  Sky corrections can be generated with the 'skyCorrection.py'
219  executable in pipe_drivers. Because the sky model used by that
220  code extends over the entire focal plane, this can produce
221  better sky subtraction.
222  The calexp is updated in-place.
223 
224  Parameters
225  ----------
226  calexp : `lsst.afw.image.Exposure` or `lsst.afw.image.MaskedImage`
227  Calibrated exposure.
228  skyCorr : `lsst.afw.math.backgroundList.BackgroundList` or None,
229  optional
230  Full focal plane sky correction, obtained by running
231  `lsst.pipe.drivers.skyCorrection.SkyCorrectionTask`.
232  """
233  if isinstance(calexp, afwImage.Exposure):
234  calexp = calexp.getMaskedImage()
235  calexp -= skyCorr.getImage()
236 
237  def extractStamps(self, inputExposure, refObjLoader=None):
238  """ Read position of bright stars within `inputExposure` from refCat
239  and extract them.
240 
241  Parameters
242  ----------
243  inputExposure : `afwImage.exposure.exposure.ExposureF`
244  The image from which bright star stamps should be extracted.
245  refObjLoader : `LoadIndexedReferenceObjectsTask`, optional
246  Loader to find objects within a reference catalog.
247 
248  Returns
249  -------
250  result : `lsst.pipe.base.Struct`
251  Result struct with components:
252 
253  - ``starIms``: `list` of stamps
254  - ``pixCenters``: `list` of corresponding coordinates to each
255  star's center, in pixels.
256  - ``GMags``: `list` of corresponding (Gaia) G magnitudes.
257  - ``gaiaIds``: `np.ndarray` of corresponding unique Gaia
258  identifiers.
259  """
260  if refObjLoader is None:
261  refObjLoader = self.refObjLoader
262  starIms = []
263  pixCenters = []
264  GMags = []
265  ids = []
266  wcs = inputExposure.getWcs()
267  # select stars within, or close enough to input exposure from refcat
268  inputIm = inputExposure.maskedImage
269  inputExpBBox = inputExposure.getBBox()
270  dilatationExtent = geom.Extent2I(np.array(self.config.stampSize) - self.config.minPixelsWithinFrame)
271  # TODO (DM-25894): handle catalog with stars missing from Gaia
272  withinCalexp = refObjLoader.loadPixelBox(inputExpBBox.dilatedBy(dilatationExtent), wcs,
273  filterName="phot_g_mean")
274  refCat = withinCalexp.refCat
275  # keep bright objects
276  fluxLimit = ((self.config.magLimit*u.ABmag).to(u.nJy)).to_value()
277  GFluxes = np.array(refCat['phot_g_mean_flux'])
278  bright = GFluxes > fluxLimit
279  # convert to AB magnitudes
280  allGMags = [((gFlux*u.nJy).to(u.ABmag)).to_value() for gFlux in GFluxes[bright]]
281  allIds = refCat.columns.extract("id", where=bright)["id"]
282  selectedColumns = refCat.columns.extract('coord_ra', 'coord_dec', where=bright)
283  for j, (ra, dec) in enumerate(zip(selectedColumns["coord_ra"], selectedColumns["coord_dec"])):
284  sp = geom.SpherePoint(ra, dec, geom.radians)
285  cpix = wcs.skyToPixel(sp)
286  try:
287  starIm = inputExposure.getCutout(sp, geom.Extent2I(self.config.stampSize))
288  except InvalidParameterError:
289  # star is beyond boundary
290  bboxCorner = np.array(cpix) - np.array(self.config.stampSize)/2
291  # compute bbox as it would be otherwise
292  idealBBox = geom.Box2I(geom.Point2I(bboxCorner), geom.Extent2I(self.config.stampSize))
293  clippedStarBBox = geom.Box2I(idealBBox)
294  clippedStarBBox.clip(inputExpBBox)
295  if clippedStarBBox.getArea() > 0:
296  # create full-sized stamp with all pixels
297  # flagged as NO_DATA
298  starIm = afwImage.ExposureF(bbox=idealBBox)
299  starIm.image[:] = np.nan
300  starIm.mask.set(inputExposure.mask.getPlaneBitMask("NO_DATA"))
301  # recover pixels from intersection with the exposure
302  clippedIm = inputIm.Factory(inputIm, clippedStarBBox)
303  starIm.maskedImage[clippedStarBBox] = clippedIm
304  # set detector and wcs, used in warpStars
305  starIm.setDetector(inputExposure.getDetector())
306  starIm.setWcs(inputExposure.getWcs())
307  else:
308  continue
309  if self.config.doRemoveDetected:
310  # give detection footprint of other objects the BAD flag
311  detThreshold = afwDetect.Threshold(starIm.mask.getPlaneBitMask("DETECTED"),
312  afwDetect.Threshold.BITMASK)
313  omask = afwDetect.FootprintSet(starIm.mask, detThreshold)
314  allFootprints = omask.getFootprints()
315  otherFootprints = []
316  for fs in allFootprints:
317  if not fs.contains(geom.Point2I(cpix)):
318  otherFootprints.append(fs)
319  nbMatchingFootprints = len(allFootprints) - len(otherFootprints)
320  if not nbMatchingFootprints == 1:
321  self.log.warn("Failed to uniquely identify central DETECTION footprint for star "
322  f"{allIds[j]}; found {nbMatchingFootprints} footprints instead.")
323  omask.setFootprints(otherFootprints)
324  omask.setMask(starIm.mask, "BAD")
325  starIms.append(starIm)
326  pixCenters.append(cpix)
327  GMags.append(allGMags[j])
328  ids.append(allIds[j])
329  return pipeBase.Struct(starIms=starIms,
330  pixCenters=pixCenters,
331  GMags=GMags,
332  gaiaIds=ids)
333 
334  def warpStamps(self, stamps, pixCenters):
335  """Warps and shifts all given stamps so they are sampled on the same
336  pixel grid and centered on the central pixel. This includes rotating
337  the stamp depending on detector orientation.
338 
339  Parameters
340  ----------
341  stamps : `collections.abc.Sequence`
342  [`afwImage.exposure.exposure.ExposureF`]
343  Image cutouts centered on a single object.
344  pixCenters : `collections.abc.Sequence` [`geom.Point2D`]
345  Positions of each object's center (as obtained from the refCat),
346  in pixels.
347 
348  Returns
349  -------
350  warpedStars : `list` [`afwImage.maskedImage.maskedImage.MaskedImage`]
351  """
352  # warping control; only contains shiftingALg provided in config
353  warpCont = afwMath.WarpingControl(self.config.warpingKernelName)
354  # Compare model to star stamp sizes
355  bufferPix = (self.modelStampSizemodelStampSize[0] - self.config.stampSize[0],
356  self.modelStampSizemodelStampSize[1] - self.config.stampSize[1])
357  # Initialize detector instance (note all stars were extracted from an
358  # exposure from the same detector)
359  det = stamps[0].getDetector()
360  # Define correction for optical distortions
361  if self.config.doApplyTransform:
362  pixToTan = det.getTransform(cg.PIXELS, cg.TAN_PIXELS)
363  else:
364  pixToTan = tFactory.makeIdentityTransform()
365  # Array of all possible rotations for detector orientation:
366  possibleRots = np.array([k*np.pi/2 for k in range(4)])
367  # determine how many, if any, rotations are required
368  yaw = det.getOrientation().getYaw()
369  nb90Rots = np.argmin(np.abs(possibleRots - float(yaw)))
370 
371  # apply transformation to each star
372  warpedStars = []
373  for star, cent in zip(stamps, pixCenters):
374  # (re)create empty destination image
375  destImage = afwImage.MaskedImageF(*self.modelStampSizemodelStampSize)
376  bottomLeft = geom.Point2D(star.image.getXY0())
377  newBottomLeft = pixToTan.applyForward(bottomLeft)
378  newBottomLeft.setX(newBottomLeft.getX() - bufferPix[0]/2)
379  newBottomLeft.setY(newBottomLeft.getY() - bufferPix[1]/2)
380  # Convert to int
381  newBottomLeft = geom.Point2I(newBottomLeft)
382  # Set origin
383  destImage.setXY0(newBottomLeft)
384 
385  # Define linear shifting to recenter stamps
386  newCenter = pixToTan.applyForward(cent) # center of warped star
387  shift = self.modelCentermodelCenter[0] + newBottomLeft[0] - newCenter[0],\
388  self.modelCentermodelCenter[1] + newBottomLeft[1] - newCenter[1]
389  affineShift = geom.AffineTransform(shift)
390  shiftTransform = tFactory.makeTransform(affineShift)
391 
392  # Define full transform (warp and shift)
393  starWarper = pixToTan.then(shiftTransform)
394 
395  # Apply it
396  goodPix = afwMath.warpImage(destImage, star.getMaskedImage(),
397  starWarper, warpCont)
398  if not goodPix:
399  self.log.debug("Warping of a star failed: no good pixel in output")
400 
401  # Arbitrarily set origin of shifted star to 0
402  destImage.setXY0(0, 0)
403 
404  # Apply rotation if apropriate
405  if nb90Rots:
406  destImage = afwMath.rotateImageBy90(destImage, nb90Rots)
407  warpedStars.append(destImage.clone())
408  return warpedStars
409 
410  @pipeBase.timeMethod
411  def run(self, inputExposure, refObjLoader=None, dataId=None, skyCorr=None):
412  """Identify bright stars within an exposure using a reference catalog,
413  extract stamps around each, then preprocess them. The preprocessing
414  steps are: shifting, warping and potentially rotating them to the same
415  pixel grid; computing their annular flux and normalizing them.
416 
417  Parameters
418  ----------
419  inputExposure : `afwImage.exposure.exposure.ExposureF`
420  The image from which bright star stamps should be extracted.
421  refObjLoader : `LoadIndexedReferenceObjectsTask`, optional
422  Loader to find objects within a reference catalog.
423  dataId : `dict` or `lsst.daf.butler.DataCoordinate`
424  The dataId of the exposure (and detector) bright stars should be
425  extracted from.
426  skyCorr : `lsst.afw.math.backgroundList.BackgroundList` or ``None``,
427  optional
428  Full focal plane sky correction, obtained by running
429  `lsst.pipe.drivers.skyCorrection.SkyCorrectionTask`.
430 
431  Returns
432  -------
433  result : `lsst.pipe.base.Struct`
434  Result struct with component:
435 
436  - ``brightStarStamps``: ``bSS.BrightStarStamps``
437  """
438  if self.config.doApplySkyCorr:
439  self.log.info("Applying sky correction to exposure %s (exposure will be modified in-place).",
440  dataId)
441  self.applySkyCorrapplySkyCorr(inputExposure, skyCorr)
442  self.log.info("Extracting bright stars from exposure %s", dataId)
443  # Extract stamps around bright stars
444  extractedStamps = self.extractStampsextractStamps(inputExposure, refObjLoader=refObjLoader)
445  # Warp (and shift, and potentially rotate) them
446  self.log.info("Applying warp and/or shift to %i star stamps from exposure %s",
447  len(extractedStamps.starIms), dataId)
448  warpedStars = self.warpStampswarpStamps(extractedStamps.starIms, extractedStamps.pixCenters)
449  brightStarList = [bSS.BrightStarStamp(stamp_im=warp,
450  gaiaGMag=extractedStamps.GMags[j],
451  gaiaId=extractedStamps.gaiaIds[j])
452  for j, warp in enumerate(warpedStars)]
453  # Compute annularFlux and normalize
454  self.log.info("Computing annular flux and normalizing %i bright stars from exposure %s",
455  len(warpedStars), dataId)
456  # annularFlux statistic set-up, excluding mask planes
457  statsControl = afwMath.StatisticsControl()
458  statsControl.setNumSigmaClip(self.config.numSigmaClip)
459  statsControl.setNumIter(self.config.numIter)
460  innerRadius, outerRadius = self.config.annularFluxRadii
461  statsFlag = afwMath.stringToStatisticsProperty(self.config.annularFluxStatistic)
462  brightStarStamps = bSS.BrightStarStamps.initAndNormalize(brightStarList,
463  innerRadius=innerRadius,
464  outerRadius=outerRadius,
465  imCenter=self.modelCentermodelCenter,
466  statsControl=statsControl,
467  statsFlag=statsFlag,
468  badMaskPlanes=self.config.badMaskPlanes)
469  return pipeBase.Struct(brightStarStamps=brightStarStamps)
470 
471  def runDataRef(self, dataRef):
472  """Read in required calexp, extract and process stamps around bright
473  stars and write them to disk.
474 
475  Parameters
476  ----------
477  dataRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef`
478  Data reference to the calexp to extract bright stars from.
479  """
480  calexp = dataRef.get("calexp")
481  skyCorr = dataRef.get("skyCorr") if self.config.doApplySkyCorr else None
482  output = self.runrun(calexp, dataId=dataRef.dataId, skyCorr=skyCorr)
483  # Save processed bright star stamps
484  dataRef.put(output.brightStarStamps, "brightStarStamps")
485  return pipeBase.Struct(brightStarStamps=output.brightStarStamps)
486 
487  def runQuantum(self, butlerQC, inputRefs, outputRefs):
488  inputs = butlerQC.get(inputRefs)
489  inputs['dataId'] = str(butlerQC.quantum.dataId)
490  refObjLoader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId
491  for ref in inputRefs.refCat],
492  refCats=inputs.pop("refCat"),
493  config=self.config.refObjLoader)
494  output = self.runrun(**inputs, refObjLoader=refObjLoader)
495  butlerQC.put(output, outputRefs)
def runQuantum(self, butlerQC, inputRefs, outputRefs)
def run(self, inputExposure, refObjLoader=None, dataId=None, skyCorr=None)
def extractStamps(self, inputExposure, refObjLoader=None)
def __init__(self, butler=None, initInputs=None, *args, **kwargs)