lsst.pipe.tasks  21.0.0-111-gf06e6c5c+89c6c3aec3
processBrightStars.py
Go to the documentation of this file.
1 # This file is part of pipe_tasks.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (https://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <https://www.gnu.org/licenses/>.
21 #
22 """Extract small cutouts around bright stars, normalize and warp them to the
23 same arbitrary pixel grid.
24 """
25 
26 __all__ = ["ProcessBrightStarsTask"]
27 
28 import numpy as np
29 import astropy.units as u
30 
31 from lsst import geom
32 from lsst.afw import math as afwMath
33 from lsst.afw import image as afwImage
34 from lsst.afw import detection as afwDetect
35 from lsst.afw import cameraGeom as cg
36 from lsst.afw.geom import transformFactory as tFactory
37 import lsst.pex.config as pexConfig
38 from lsst.pipe import base as pipeBase
39 from lsst.pipe.base import connectionTypes as cT
40 from lsst.pex.exceptions import InvalidParameterError
41 from lsst.meas.algorithms.loadIndexedReferenceObjects import LoadIndexedReferenceObjectsTask
42 from lsst.meas.algorithms import ReferenceObjectLoader
43 from lsst.meas.algorithms import brightStarStamps as bSS
44 
45 
46 class ProcessBrightStarsConnections(pipeBase.PipelineTaskConnections,
47  dimensions=("instrument", "visit", "detector")):
48  inputExposure = cT.Input(
49  doc="Input exposure from which to extract bright star stamps",
50  name="calexp",
51  storageClass="ExposureF",
52  dimensions=("visit", "detector")
53  )
54  skyCorr = cT.Input(
55  doc="Input Sky Correction to be subtracted from the calexp if doApplySkyCorr=True",
56  name="skyCorr",
57  storageClass="Background",
58  dimensions=("instrument", "visit", "detector")
59  )
60  refCat = cT.PrerequisiteInput(
61  doc="Reference catalog that contains bright star positions",
62  name="gaia_dr2_20200414",
63  storageClass="SimpleCatalog",
64  dimensions=("skypix",),
65  multiple=True,
66  deferLoad=True
67  )
68  brightStarStamps = cT.Output(
69  doc="Set of preprocessed postage stamps, each centered on a single bright star.",
70  name="brightStarStamps",
71  storageClass="BrightStarStamps",
72  dimensions=("visit", "detector")
73  )
74 
75  def __init__(self, *, config=None):
76  super().__init__(config=config)
77  if not config.doApplySkyCorr:
78  self.inputs.remove("skyCorr")
79 
80 
81 class ProcessBrightStarsConfig(pipeBase.PipelineTaskConfig,
82  pipelineConnections=ProcessBrightStarsConnections):
83  """Configuration parameters for ProcessBrightStarsTask
84  """
85  magLimit = pexConfig.Field(
86  dtype=float,
87  doc="Magnitude limit, in Gaia G; all stars brighter than this value will be processed",
88  default=18
89  )
90  stampSize = pexConfig.ListField(
91  dtype=int,
92  doc="Size of the stamps to be extracted, in pixels",
93  default=(250, 250)
94  )
95  modelStampBuffer = pexConfig.Field(
96  dtype=float,
97  doc="'Buffer' factor to be applied to determine the size of the stamp the processed stars will "
98  "be saved in. This will also be the size of the extended PSF model.",
99  default=1.1
100  )
101  doRemoveDetected = pexConfig.Field(
102  dtype=bool,
103  doc="Whether DETECTION footprints, other than that for the central object, should be changed to "
104  "BAD",
105  default=True
106  )
107  doApplyTransform = pexConfig.Field(
108  dtype=bool,
109  doc="Apply transform to bright star stamps to correct for optical distortions?",
110  default=True
111  )
112  warpingKernelName = pexConfig.ChoiceField(
113  dtype=str,
114  doc="Warping kernel",
115  default="lanczos5",
116  allowed={
117  "bilinear": "bilinear interpolation",
118  "lanczos3": "Lanczos kernel of order 3",
119  "lanczos4": "Lanczos kernel of order 4",
120  "lanczos5": "Lanczos kernel of order 5",
121  }
122  )
123  annularFluxRadii = pexConfig.ListField(
124  dtype=int,
125  doc="Inner and outer radii of the annulus used to compute the AnnularFlux for normalization, "
126  "in pixels.",
127  default=(40, 50)
128  )
129  annularFluxStatistic = pexConfig.ChoiceField(
130  dtype=str,
131  doc="Type of statistic to use to compute annular flux.",
132  default="MEANCLIP",
133  allowed={
134  "MEAN": "mean",
135  "MEDIAN": "median",
136  "MEANCLIP": "clipped mean",
137  }
138  )
139  numSigmaClip = pexConfig.Field(
140  dtype=float,
141  doc="Sigma for outlier rejection; ignored if annularFluxStatistic != 'MEANCLIP'.",
142  default=4
143  )
144  numIter = pexConfig.Field(
145  dtype=int,
146  doc="Number of iterations of outlier rejection; ignored if annularFluxStatistic != 'MEANCLIP'.",
147  default=3
148  )
149  badMaskPlanes = pexConfig.ListField(
150  dtype=str,
151  doc="Mask planes that, if set, lead to associated pixels not being included in the computation of the"
152  " annular flux.",
153  default=('BAD', 'CR', 'CROSSTALK', 'EDGE', 'NO_DATA', 'SAT', 'SUSPECT', 'UNMASKEDNAN')
154  )
155  minPixelsWithinFrame = pexConfig.Field(
156  dtype=int,
157  doc="Minimum number of pixels that must fall within the stamp boundary for the bright star to be"
158  " saved when its center is beyond the exposure boundary.",
159  default=50
160  )
161  doApplySkyCorr = pexConfig.Field(
162  dtype=bool,
163  doc="Apply full focal plane sky correction before extracting stars?",
164  default=True
165  )
166  discardNanFluxStars = pexConfig.Field(
167  dtype=bool,
168  doc="Should stars with NaN annular flux be discarded?",
169  default=False
170  )
171  refObjLoader = pexConfig.ConfigurableField(
172  target=LoadIndexedReferenceObjectsTask,
173  doc="Reference object loader for astrometric calibration.",
174  )
175 
176  def setDefaults(self):
177  self.refObjLoaderrefObjLoader.ref_dataset_name = "gaia_dr2_20200414"
178 
179 
180 class ProcessBrightStarsTask(pipeBase.PipelineTask, pipeBase.CmdLineTask):
181  """The description of the parameters for this Task are detailed in
182  :lsst-task:`~lsst.pipe.base.PipelineTask`.
183 
184  Notes
185  -----
186  `ProcessBrightStarsTask` is used to extract, process, and store small
187  image cut-outs (or "postage stamps") around bright stars. It relies on
188  three methods, called in succession:
189 
190  `extractStamps`
191  Find bright stars within the exposure using a reference catalog and
192  extract a stamp centered on each.
193  `warpStamps`
194  Shift and warp each stamp to remove optical distortions and sample all
195  stars on the same pixel grid.
196  `measureAndNormalize`
197  Compute the flux of an object in an annulus and normalize it. This is
198  required to normalize each bright star stamp as their central pixels
199  are likely saturated and/or contain ghosts, and cannot be used.
200  """
201  ConfigClass = ProcessBrightStarsConfig
202  _DefaultName = "processBrightStars"
203  RunnerClass = pipeBase.ButlerInitializedTaskRunner
204 
205  def __init__(self, butler=None, initInputs=None, *args, **kwargs):
206  super().__init__(*args, **kwargs)
207  # Compute (model) stamp size depending on provided "buffer" value
208  self.modelStampSizemodelStampSize = [int(self.config.stampSize[0]*self.config.modelStampBuffer),
209  int(self.config.stampSize[1]*self.config.modelStampBuffer)]
210  # force it to be odd-sized so we have a central pixel
211  if not self.modelStampSizemodelStampSize[0] % 2:
212  self.modelStampSizemodelStampSize[0] += 1
213  if not self.modelStampSizemodelStampSize[1] % 2:
214  self.modelStampSizemodelStampSize[1] += 1
215  # central pixel
216  self.modelCentermodelCenter = self.modelStampSizemodelStampSize[0]//2, self.modelStampSizemodelStampSize[1]//2
217  # configure Gaia refcat
218  if butler is not None:
219  self.makeSubtask('refObjLoader', butler=butler)
220 
221  def applySkyCorr(self, calexp, skyCorr):
222  """Apply correction to the sky background level.
223 
224  Sky corrections can be generated with the 'skyCorrection.py'
225  executable in pipe_drivers. Because the sky model used by that
226  code extends over the entire focal plane, this can produce
227  better sky subtraction.
228  The calexp is updated in-place.
229 
230  Parameters
231  ----------
232  calexp : `lsst.afw.image.Exposure` or `lsst.afw.image.MaskedImage`
233  Calibrated exposure.
234  skyCorr : `lsst.afw.math.backgroundList.BackgroundList` or None,
235  optional
236  Full focal plane sky correction, obtained by running
237  `lsst.pipe.drivers.skyCorrection.SkyCorrectionTask`.
238  """
239  if isinstance(calexp, afwImage.Exposure):
240  calexp = calexp.getMaskedImage()
241  calexp -= skyCorr.getImage()
242 
243  def extractStamps(self, inputExposure, refObjLoader=None):
244  """ Read position of bright stars within `inputExposure` from refCat
245  and extract them.
246 
247  Parameters
248  ----------
249  inputExposure : `afwImage.exposure.exposure.ExposureF`
250  The image from which bright star stamps should be extracted.
251  refObjLoader : `LoadIndexedReferenceObjectsTask`, optional
252  Loader to find objects within a reference catalog.
253 
254  Returns
255  -------
256  result : `lsst.pipe.base.Struct`
257  Result struct with components:
258 
259  - ``starIms``: `list` of stamps
260  - ``pixCenters``: `list` of corresponding coordinates to each
261  star's center, in pixels.
262  - ``GMags``: `list` of corresponding (Gaia) G magnitudes.
263  - ``gaiaIds``: `np.ndarray` of corresponding unique Gaia
264  identifiers.
265  """
266  if refObjLoader is None:
267  refObjLoader = self.refObjLoader
268  starIms = []
269  pixCenters = []
270  GMags = []
271  ids = []
272  wcs = inputExposure.getWcs()
273  # select stars within, or close enough to input exposure from refcat
274  inputIm = inputExposure.maskedImage
275  inputExpBBox = inputExposure.getBBox()
276  dilatationExtent = geom.Extent2I(np.array(self.config.stampSize) - self.config.minPixelsWithinFrame)
277  # TODO (DM-25894): handle catalog with stars missing from Gaia
278  withinCalexp = refObjLoader.loadPixelBox(inputExpBBox.dilatedBy(dilatationExtent), wcs,
279  filterName="phot_g_mean")
280  refCat = withinCalexp.refCat
281  # keep bright objects
282  fluxLimit = ((self.config.magLimit*u.ABmag).to(u.nJy)).to_value()
283  GFluxes = np.array(refCat['phot_g_mean_flux'])
284  bright = GFluxes > fluxLimit
285  # convert to AB magnitudes
286  allGMags = [((gFlux*u.nJy).to(u.ABmag)).to_value() for gFlux in GFluxes[bright]]
287  allIds = refCat.columns.extract("id", where=bright)["id"]
288  selectedColumns = refCat.columns.extract('coord_ra', 'coord_dec', where=bright)
289  for j, (ra, dec) in enumerate(zip(selectedColumns["coord_ra"], selectedColumns["coord_dec"])):
290  sp = geom.SpherePoint(ra, dec, geom.radians)
291  cpix = wcs.skyToPixel(sp)
292  try:
293  starIm = inputExposure.getCutout(sp, geom.Extent2I(self.config.stampSize))
294  except InvalidParameterError:
295  # star is beyond boundary
296  bboxCorner = np.array(cpix) - np.array(self.config.stampSize)/2
297  # compute bbox as it would be otherwise
298  idealBBox = geom.Box2I(geom.Point2I(bboxCorner), geom.Extent2I(self.config.stampSize))
299  clippedStarBBox = geom.Box2I(idealBBox)
300  clippedStarBBox.clip(inputExpBBox)
301  if clippedStarBBox.getArea() > 0:
302  # create full-sized stamp with all pixels
303  # flagged as NO_DATA
304  starIm = afwImage.ExposureF(bbox=idealBBox)
305  starIm.image[:] = np.nan
306  starIm.mask.set(inputExposure.mask.getPlaneBitMask("NO_DATA"))
307  # recover pixels from intersection with the exposure
308  clippedIm = inputIm.Factory(inputIm, clippedStarBBox)
309  starIm.maskedImage[clippedStarBBox] = clippedIm
310  # set detector and wcs, used in warpStars
311  starIm.setDetector(inputExposure.getDetector())
312  starIm.setWcs(inputExposure.getWcs())
313  else:
314  continue
315  if self.config.doRemoveDetected:
316  # give detection footprint of other objects the BAD flag
317  detThreshold = afwDetect.Threshold(starIm.mask.getPlaneBitMask("DETECTED"),
318  afwDetect.Threshold.BITMASK)
319  omask = afwDetect.FootprintSet(starIm.mask, detThreshold)
320  allFootprints = omask.getFootprints()
321  otherFootprints = []
322  for fs in allFootprints:
323  if not fs.contains(geom.Point2I(cpix)):
324  otherFootprints.append(fs)
325  nbMatchingFootprints = len(allFootprints) - len(otherFootprints)
326  if not nbMatchingFootprints == 1:
327  self.log.warn("Failed to uniquely identify central DETECTION footprint for star "
328  f"{allIds[j]}; found {nbMatchingFootprints} footprints instead.")
329  omask.setFootprints(otherFootprints)
330  omask.setMask(starIm.mask, "BAD")
331  starIms.append(starIm)
332  pixCenters.append(cpix)
333  GMags.append(allGMags[j])
334  ids.append(allIds[j])
335  return pipeBase.Struct(starIms=starIms,
336  pixCenters=pixCenters,
337  GMags=GMags,
338  gaiaIds=ids)
339 
340  def warpStamps(self, stamps, pixCenters):
341  """Warps and shifts all given stamps so they are sampled on the same
342  pixel grid and centered on the central pixel. This includes rotating
343  the stamp depending on detector orientation.
344 
345  Parameters
346  ----------
347  stamps : `collections.abc.Sequence`
348  [`afwImage.exposure.exposure.ExposureF`]
349  Image cutouts centered on a single object.
350  pixCenters : `collections.abc.Sequence` [`geom.Point2D`]
351  Positions of each object's center (as obtained from the refCat),
352  in pixels.
353 
354  Returns
355  -------
356  warpedStars : `list` [`afwImage.maskedImage.maskedImage.MaskedImage`]
357  """
358  # warping control; only contains shiftingALg provided in config
359  warpCont = afwMath.WarpingControl(self.config.warpingKernelName)
360  # Compare model to star stamp sizes
361  bufferPix = (self.modelStampSizemodelStampSize[0] - self.config.stampSize[0],
362  self.modelStampSizemodelStampSize[1] - self.config.stampSize[1])
363  # Initialize detector instance (note all stars were extracted from an
364  # exposure from the same detector)
365  det = stamps[0].getDetector()
366  # Define correction for optical distortions
367  if self.config.doApplyTransform:
368  pixToTan = det.getTransform(cg.PIXELS, cg.TAN_PIXELS)
369  else:
370  pixToTan = tFactory.makeIdentityTransform()
371  # Array of all possible rotations for detector orientation:
372  possibleRots = np.array([k*np.pi/2 for k in range(4)])
373  # determine how many, if any, rotations are required
374  yaw = det.getOrientation().getYaw()
375  nb90Rots = np.argmin(np.abs(possibleRots - float(yaw)))
376 
377  # apply transformation to each star
378  warpedStars = []
379  for star, cent in zip(stamps, pixCenters):
380  # (re)create empty destination image
381  destImage = afwImage.MaskedImageF(*self.modelStampSizemodelStampSize)
382  bottomLeft = geom.Point2D(star.image.getXY0())
383  newBottomLeft = pixToTan.applyForward(bottomLeft)
384  newBottomLeft.setX(newBottomLeft.getX() - bufferPix[0]/2)
385  newBottomLeft.setY(newBottomLeft.getY() - bufferPix[1]/2)
386  # Convert to int
387  newBottomLeft = geom.Point2I(newBottomLeft)
388  # Set origin
389  destImage.setXY0(newBottomLeft)
390 
391  # Define linear shifting to recenter stamps
392  newCenter = pixToTan.applyForward(cent) # center of warped star
393  shift = self.modelCentermodelCenter[0] + newBottomLeft[0] - newCenter[0],\
394  self.modelCentermodelCenter[1] + newBottomLeft[1] - newCenter[1]
395  affineShift = geom.AffineTransform(shift)
396  shiftTransform = tFactory.makeTransform(affineShift)
397 
398  # Define full transform (warp and shift)
399  starWarper = pixToTan.then(shiftTransform)
400 
401  # Apply it
402  goodPix = afwMath.warpImage(destImage, star.getMaskedImage(),
403  starWarper, warpCont)
404  if not goodPix:
405  self.log.debug("Warping of a star failed: no good pixel in output")
406 
407  # Arbitrarily set origin of shifted star to 0
408  destImage.setXY0(0, 0)
409 
410  # Apply rotation if apropriate
411  if nb90Rots:
412  destImage = afwMath.rotateImageBy90(destImage, nb90Rots)
413  warpedStars.append(destImage.clone())
414  return warpedStars
415 
416  @pipeBase.timeMethod
417  def run(self, inputExposure, refObjLoader=None, dataId=None, skyCorr=None):
418  """Identify bright stars within an exposure using a reference catalog,
419  extract stamps around each, then preprocess them. The preprocessing
420  steps are: shifting, warping and potentially rotating them to the same
421  pixel grid; computing their annular flux and normalizing them.
422 
423  Parameters
424  ----------
425  inputExposure : `afwImage.exposure.exposure.ExposureF`
426  The image from which bright star stamps should be extracted.
427  refObjLoader : `LoadIndexedReferenceObjectsTask`, optional
428  Loader to find objects within a reference catalog.
429  dataId : `dict` or `lsst.daf.butler.DataCoordinate`
430  The dataId of the exposure (and detector) bright stars should be
431  extracted from.
432  skyCorr : `lsst.afw.math.backgroundList.BackgroundList` or ``None``,
433  optional
434  Full focal plane sky correction, obtained by running
435  `lsst.pipe.drivers.skyCorrection.SkyCorrectionTask`.
436 
437  Returns
438  -------
439  result : `lsst.pipe.base.Struct`
440  Result struct with component:
441 
442  - ``brightStarStamps``: ``bSS.BrightStarStamps``
443  """
444  if self.config.doApplySkyCorr:
445  self.log.info("Applying sky correction to exposure %s (exposure will be modified in-place).",
446  dataId)
447  self.applySkyCorrapplySkyCorr(inputExposure, skyCorr)
448  self.log.info("Extracting bright stars from exposure %s", dataId)
449  # Extract stamps around bright stars
450  extractedStamps = self.extractStampsextractStamps(inputExposure, refObjLoader=refObjLoader)
451  if not extractedStamps.starIms:
452  self.log.info("No suitable bright star found.")
453  return None
454  # Warp (and shift, and potentially rotate) them
455  self.log.info("Applying warp and/or shift to %i star stamps from exposure %s",
456  len(extractedStamps.starIms), dataId)
457  warpedStars = self.warpStampswarpStamps(extractedStamps.starIms, extractedStamps.pixCenters)
458  brightStarList = [bSS.BrightStarStamp(stamp_im=warp,
459  gaiaGMag=extractedStamps.GMags[j],
460  gaiaId=extractedStamps.gaiaIds[j])
461  for j, warp in enumerate(warpedStars)]
462  # Compute annularFlux and normalize
463  self.log.info("Computing annular flux and normalizing %i bright stars from exposure %s",
464  len(warpedStars), dataId)
465  # annularFlux statistic set-up, excluding mask planes
466  statsControl = afwMath.StatisticsControl()
467  statsControl.setNumSigmaClip(self.config.numSigmaClip)
468  statsControl.setNumIter(self.config.numIter)
469  innerRadius, outerRadius = self.config.annularFluxRadii
470  statsFlag = afwMath.stringToStatisticsProperty(self.config.annularFluxStatistic)
471  brightStarStamps = bSS.BrightStarStamps.initAndNormalize(brightStarList,
472  innerRadius=innerRadius,
473  outerRadius=outerRadius,
474  imCenter=self.modelCentermodelCenter,
475  statsControl=statsControl,
476  statsFlag=statsFlag,
477  badMaskPlanes=self.config.badMaskPlanes,
478  discardNanFluxObjects=(
479  self.config.discardNanFluxStars))
480  return pipeBase.Struct(brightStarStamps=brightStarStamps)
481 
482  def runDataRef(self, dataRef):
483  """Read in required calexp, extract and process stamps around bright
484  stars and write them to disk.
485 
486  Parameters
487  ----------
488  dataRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef`
489  Data reference to the calexp to extract bright stars from.
490  """
491  calexp = dataRef.get("calexp")
492  skyCorr = dataRef.get("skyCorr") if self.config.doApplySkyCorr else None
493  output = self.runrun(calexp, dataId=dataRef.dataId, skyCorr=skyCorr)
494  # Save processed bright star stamps
495  dataRef.put(output.brightStarStamps, "brightStarStamps")
496  return pipeBase.Struct(brightStarStamps=output.brightStarStamps)
497 
498  def runQuantum(self, butlerQC, inputRefs, outputRefs):
499  inputs = butlerQC.get(inputRefs)
500  inputs['dataId'] = str(butlerQC.quantum.dataId)
501  refObjLoader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId
502  for ref in inputRefs.refCat],
503  refCats=inputs.pop("refCat"),
504  config=self.config.refObjLoader)
505  output = self.runrun(**inputs, refObjLoader=refObjLoader)
506  if output:
507  butlerQC.put(output, outputRefs)
def runQuantum(self, butlerQC, inputRefs, outputRefs)
def run(self, inputExposure, refObjLoader=None, dataId=None, skyCorr=None)
def extractStamps(self, inputExposure, refObjLoader=None)
def __init__(self, butler=None, initInputs=None, *args, **kwargs)