lsst.pipe.tasks  21.0.0-45-g0dcdce56+90354a0300
processBrightStars.py
Go to the documentation of this file.
1 # This file is part of pipe_tasks.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (https://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <https://www.gnu.org/licenses/>.
21 #
22 """Extract small cutouts around bright stars, normalize and warp them to the
23 same arbitrary pixel grid.
24 """
25 
26 __all__ = ["ProcessBrightStarsTask"]
27 
28 import numpy as np
29 import astropy.units as u
30 
31 from lsst import geom
32 from lsst.afw import math as afwMath
33 from lsst.afw import image as afwImage
34 from lsst.afw import detection as afwDetect
35 from lsst.afw import cameraGeom as cg
36 from lsst.afw.geom import transformFactory as tFactory
37 import lsst.pex.config as pexConfig
38 from lsst.pipe import base as pipeBase
39 from lsst.pipe.base import connectionTypes as cT
40 from lsst.pex.exceptions import InvalidParameterError
41 from lsst.meas.algorithms.loadIndexedReferenceObjects import LoadIndexedReferenceObjectsTask
42 from lsst.meas.algorithms import ReferenceObjectLoader
43 from lsst.meas.algorithms import brightStarStamps as bSS
44 
45 
46 class ProcessBrightStarsConnections(pipeBase.PipelineTaskConnections, dimensions=("visit", "detector")):
47  inputExposure = cT.Input(
48  doc="Input exposure from which to extract bright star stamps",
49  name="calexp",
50  storageClass="ExposureF",
51  dimensions=("visit", "detector")
52  )
53  refCat = cT.PrerequisiteInput(
54  doc="Reference catalog that contains bright star positions",
55  name="gaia_dr2_20200414",
56  storageClass="SimpleCatalog",
57  dimensions=("skypix",),
58  multiple=True,
59  deferLoad=True
60  )
61  brightStarStamps = cT.Output(
62  doc="Set of preprocessed postage stamps, each centered on a single bright star.",
63  name="brightStarStamps",
64  storageClass="BrightStarStamps",
65  dimensions=("visit", "detector")
66  )
67 
68 
69 class ProcessBrightStarsConfig(pipeBase.PipelineTaskConfig,
70  pipelineConnections=ProcessBrightStarsConnections):
71  """Configuration parameters for ProcessBrightStarsTask
72  """
73  magLimit = pexConfig.Field(
74  dtype=float,
75  doc="Magnitude limit, in Gaia G; all stars brighter than this value will be processed",
76  default=18
77  )
78  stampSize = pexConfig.ListField(
79  dtype=int,
80  doc="Size of the stamps to be extracted, in pixels",
81  default=(250, 250)
82  )
83  modelStampBuffer = pexConfig.Field(
84  dtype=float,
85  doc="'Buffer' factor to be applied to determine the size of the stamp the processed stars will "
86  "be saved in. This will also be the size of the extended PSF model.",
87  default=1.1
88  )
89  doRemoveDetected = pexConfig.Field(
90  dtype=bool,
91  doc="Whether DETECTION footprints, other than that for the central object, should be changed to "
92  "BAD",
93  default=True
94  )
95  doApplyTransform = pexConfig.Field(
96  dtype=bool,
97  doc="Apply transform to bright star stamps to correct for optical distortions?",
98  default=True
99  )
100  warpingKernelName = pexConfig.ChoiceField(
101  dtype=str,
102  doc="Warping kernel",
103  default="lanczos5",
104  allowed={
105  "bilinear": "bilinear interpolation",
106  "lanczos3": "Lanczos kernel of order 3",
107  "lanczos4": "Lanczos kernel of order 4",
108  "lanczos5": "Lanczos kernel of order 5",
109  }
110  )
111  annularFluxRadii = pexConfig.ListField(
112  dtype=int,
113  doc="Inner and outer radii of the annulus used to compute the AnnularFlux for normalization, "
114  "in pixels.",
115  default=(40, 50)
116  )
117  annularFluxStatistic = pexConfig.ChoiceField(
118  dtype=str,
119  doc="Type of statistic to use to compute annular flux.",
120  default="MEANCLIP",
121  allowed={
122  "MEAN": "mean",
123  "MEDIAN": "median",
124  "MEANCLIP": "clipped mean",
125  }
126  )
127  numSigmaClip = pexConfig.Field(
128  dtype=float,
129  doc="Sigma for outlier rejection; ignored if annularFluxStatistic != 'MEANCLIP'.",
130  default=4
131  )
132  numIter = pexConfig.Field(
133  dtype=int,
134  doc="Number of iterations of outlier rejection; ignored if annularFluxStatistic != 'MEANCLIP'.",
135  default=3
136  )
137  badMaskPlanes = pexConfig.ListField(
138  dtype=str,
139  doc="Mask planes that, if set, lead to associated pixels not being included in the computation of the"
140  " annular flux.",
141  default=('BAD', 'CR', 'CROSSTALK', 'EDGE', 'NO_DATA', 'SAT', 'SUSPECT', 'UNMASKEDNAN')
142  )
143  minPixelsWithinFrame = pexConfig.Field(
144  dtype=int,
145  doc="Minimum number of pixels that must fall within the stamp boundary for the bright star to be"
146  " saved when its center is beyond the exposure boundary.",
147  default=50
148  )
149  refObjLoader = pexConfig.ConfigurableField(
150  target=LoadIndexedReferenceObjectsTask,
151  doc="Reference object loader for astrometric calibration.",
152  )
153 
154  def setDefaults(self):
155  self.refObjLoaderrefObjLoader.ref_dataset_name = "gaia_dr2_20200414"
156 
157 
158 class ProcessBrightStarsTask(pipeBase.PipelineTask, pipeBase.CmdLineTask):
159  """The description of the parameters for this Task are detailed in
160  :lsst-task:`~lsst.pipe.base.PipelineTask`.
161 
162  Notes
163  -----
164  `ProcessBrightStarsTask` is used to extract, process, and store small
165  image cut-outs (or "postage stamps") around bright stars. It relies on
166  three methods, called in succession:
167 
168  `extractStamps`
169  Find bright stars within the exposure using a reference catalog and
170  extract a stamp centered on each.
171  `warpStamps`
172  Shift and warp each stamp to remove optical distortions and sample all
173  stars on the same pixel grid.
174  `measureAndNormalize`
175  Compute the flux of an object in an annulus and normalize it. This is
176  required to normalize each bright star stamp as their central pixels
177  are likely saturated and/or contain ghosts, and cannot be used.
178  """
179  ConfigClass = ProcessBrightStarsConfig
180  _DefaultName = "processBrightStars"
181  RunnerClass = pipeBase.ButlerInitializedTaskRunner
182 
183  def __init__(self, butler=None, initInputs=None, *args, **kwargs):
184  super().__init__(*args, **kwargs)
185  # Compute (model) stamp size depending on provided "buffer" value
186  self.modelStampSizemodelStampSize = (int(self.config.stampSize[0]*self.config.modelStampBuffer),
187  int(self.config.stampSize[1]*self.config.modelStampBuffer))
188  # force it to be odd-sized so we have a central pixel
189  if not self.modelStampSizemodelStampSize[0] % 2:
190  self.modelStampSizemodelStampSize[0] += 1
191  if not self.modelStampSizemodelStampSize[1] % 2:
192  self.modelStampSizemodelStampSize[1] += 1
193  # central pixel
194  self.modelCentermodelCenter = self.modelStampSizemodelStampSize[0]//2, self.modelStampSizemodelStampSize[1]//2
195  # configure Gaia refcat
196  if butler is not None:
197  self.makeSubtask('refObjLoader', butler=butler)
198 
199  def extractStamps(self, inputExposure, refObjLoader=None):
200  """ Read position of bright stars within `inputExposure` from refCat
201  and extract them.
202 
203  Parameters
204  ----------
205  inputExposure : `afwImage.exposure.exposure.ExposureF`
206  The image from which bright star stamps should be extracted.
207  refObjLoader : `LoadIndexedReferenceObjectsTask`, optional
208  Loader to find objects within a reference catalog.
209 
210  Returns
211  -------
212  result : `lsst.pipe.base.Struct`
213  Result struct with components:
214 
215  - ``starIms``: `list` of stamps
216  - ``pixCenters``: `list` of corresponding coordinates to each
217  star's center, in pixels.
218  - ``GMags``: `list` of corresponding (Gaia) G magnitudes.
219  - ``gaiaIds``: `np.ndarray` of corresponding unique Gaia
220  identifiers.
221  """
222  if refObjLoader is None:
223  refObjLoader = self.refObjLoader
224  starIms = []
225  pixCenters = []
226  GMags = []
227  ids = []
228  wcs = inputExposure.getWcs()
229  # select stars within, or close enough to input exposure from refcat
230  inputIm = inputExposure.maskedImage
231  inputExpBBox = inputExposure.getBBox()
232  dilatationExtent = geom.Extent2I(np.array(self.config.stampSize) - self.config.minPixelsWithinFrame)
233  # TODO (DM-25894): handle catalog with stars missing from Gaia
234  withinCalexp = refObjLoader.loadPixelBox(inputExpBBox.dilatedBy(dilatationExtent), wcs,
235  filterName="phot_g_mean")
236  refCat = withinCalexp.refCat
237  # keep bright objects
238  fluxLimit = ((self.config.magLimit*u.ABmag).to(u.nJy)).to_value()
239  GFluxes = np.array(refCat['phot_g_mean_flux'])
240  bright = GFluxes > fluxLimit
241  # convert to AB magnitudes
242  allGMags = [((gFlux*u.nJy).to(u.ABmag)).to_value() for gFlux in GFluxes[bright]]
243  allIds = refCat.columns.extract("id", where=bright)["id"]
244  selectedColumns = refCat.columns.extract('coord_ra', 'coord_dec', where=bright)
245  for j, (ra, dec) in enumerate(zip(selectedColumns["coord_ra"], selectedColumns["coord_dec"])):
246  sp = geom.SpherePoint(ra, dec, geom.radians)
247  cpix = wcs.skyToPixel(sp)
248  try:
249  starIm = inputExposure.getCutout(sp, geom.Extent2I(self.config.stampSize))
250  except InvalidParameterError:
251  # star is beyond boundary
252  bboxCorner = np.array(cpix) - np.array(self.config.stampSize)/2
253  # compute bbox as it would be otherwise
254  idealBBox = geom.Box2I(geom.Point2I(bboxCorner), geom.Extent2I(self.config.stampSize))
255  clippedStarBBox = geom.Box2I(idealBBox)
256  clippedStarBBox.clip(inputExpBBox)
257  if clippedStarBBox.getArea() > 0:
258  # create full-sized stamp with all pixels
259  # flagged as NO_DATA
260  starIm = afwImage.ExposureF(bbox=idealBBox)
261  starIm.image[:] = np.nan
262  starIm.mask.set(inputExposure.mask.getPlaneBitMask("NO_DATA"))
263  # recover pixels from intersection with the exposure
264  clippedIm = inputIm.Factory(inputIm, clippedStarBBox)
265  starIm.maskedImage[clippedStarBBox] = clippedIm
266  # set detector and wcs, used in warpStars
267  starIm.setDetector(inputExposure.getDetector())
268  starIm.setWcs(inputExposure.getWcs())
269  else:
270  continue
271  if self.config.doRemoveDetected:
272  # give detection footprint of other objects the BAD flag
273  detThreshold = afwDetect.Threshold(starIm.mask.getPlaneBitMask("DETECTED"),
274  afwDetect.Threshold.BITMASK)
275  omask = afwDetect.FootprintSet(starIm.mask, detThreshold)
276  allFootprints = omask.getFootprints()
277  otherFootprints = []
278  for fs in allFootprints:
279  if not fs.contains(geom.Point2I(cpix)):
280  otherFootprints.append(fs)
281  nbMatchingFootprints = len(allFootprints) - len(otherFootprints)
282  if not nbMatchingFootprints == 1:
283  self.log.warn("Failed to uniquely identify central DETECTION footprint for star "
284  f"{allIds[j]}; found {nbMatchingFootprints} footprints instead.")
285  omask.setFootprints(otherFootprints)
286  omask.setMask(starIm.mask, "BAD")
287  starIms.append(starIm)
288  pixCenters.append(cpix)
289  GMags.append(allGMags[j])
290  ids.append(allIds[j])
291  return pipeBase.Struct(starIms=starIms,
292  pixCenters=pixCenters,
293  GMags=GMags,
294  gaiaIds=ids)
295 
296  def warpStamps(self, stamps, pixCenters):
297  """Warps and shifts all given stamps so they are sampled on the same
298  pixel grid and centered on the central pixel. This includes rotating
299  the stamp depending on detector orientation.
300 
301  Parameters
302  ----------
303  stamps : `collections.abc.Sequence`
304  [`afwImage.exposure.exposure.ExposureF`]
305  Image cutouts centered on a single object.
306  pixCenters : `collections.abc.Sequence` [`geom.Point2D`]
307  Positions of each object's center (as obtained from the refCat),
308  in pixels.
309 
310  Returns
311  -------
312  warpedStars : `list` [`afwImage.maskedImage.maskedImage.MaskedImage`]
313  """
314  # warping control; only contains shiftingALg provided in config
315  warpCont = afwMath.WarpingControl(self.config.warpingKernelName)
316  # Compare model to star stamp sizes
317  bufferPix = (self.modelStampSizemodelStampSize[0] - self.config.stampSize[0],
318  self.modelStampSizemodelStampSize[1] - self.config.stampSize[1])
319  # Initialize detector instance (note all stars were extracted from an
320  # exposure from the same detector)
321  det = stamps[0].getDetector()
322  # Define correction for optical distortions
323  if self.config.doApplyTransform:
324  pixToTan = det.getTransform(cg.PIXELS, cg.TAN_PIXELS)
325  else:
326  pixToTan = tFactory.makeIdentityTransform()
327  # Array of all possible rotations for detector orientation:
328  possibleRots = np.array([k*np.pi/2 for k in range(4)])
329  # determine how many, if any, rotations are required
330  yaw = det.getOrientation().getYaw()
331  nb90Rots = np.argmin(np.abs(possibleRots - float(yaw)))
332 
333  # apply transformation to each star
334  warpedStars = []
335  for star, cent in zip(stamps, pixCenters):
336  # (re)create empty destination image
337  destImage = afwImage.MaskedImageF(*self.modelStampSizemodelStampSize)
338  bottomLeft = geom.Point2D(star.image.getXY0())
339  newBottomLeft = pixToTan.applyForward(bottomLeft)
340  newBottomLeft.setX(newBottomLeft.getX() - bufferPix[0]/2)
341  newBottomLeft.setY(newBottomLeft.getY() - bufferPix[1]/2)
342  # Convert to int
343  newBottomLeft = geom.Point2I(newBottomLeft)
344  # Set origin
345  destImage.setXY0(newBottomLeft)
346 
347  # Define linear shifting to recenter stamps
348  newCenter = pixToTan.applyForward(cent) # center of warped star
349  shift = self.modelCentermodelCenter[0] + newBottomLeft[0] - newCenter[0],\
350  self.modelCentermodelCenter[1] + newBottomLeft[1] - newCenter[1]
351  affineShift = geom.AffineTransform(shift)
352  shiftTransform = tFactory.makeTransform(affineShift)
353 
354  # Define full transform (warp and shift)
355  starWarper = pixToTan.then(shiftTransform)
356 
357  # Apply it
358  goodPix = afwMath.warpImage(destImage, star.getMaskedImage(),
359  starWarper, warpCont)
360  if not goodPix:
361  self.log.debug("Warping of a star failed: no good pixel in output")
362 
363  # Arbitrarily set origin of shifted star to 0
364  destImage.setXY0(0, 0)
365 
366  # Apply rotation if apropriate
367  if nb90Rots:
368  destImage = afwMath.rotateImageBy90(destImage, nb90Rots)
369  warpedStars.append(destImage.clone())
370  return warpedStars
371 
372  @pipeBase.timeMethod
373  def run(self, inputExposure, refObjLoader=None, dataId=None):
374  """Identify bright stars within an exposure using a reference catalog,
375  extract stamps around each, then preprocess them. The preprocessing
376  steps are: shifting, warping and potentially rotating them to the same
377  pixel grid; computing their annular flux and normalizing them.
378 
379  Parameters
380  ----------
381  inputExposure : `afwImage.exposure.exposure.ExposureF`
382  The image from which bright star stamps should be extracted.
383  refObjLoader : `LoadIndexedReferenceObjectsTask`, optional
384  Loader to find objects within a reference catalog.
385  dataId : `dict` or `lsst.daf.butler.DataCoordinate`
386  The dataId of the exposure (and detector) bright stars should be
387  extracted from.
388 
389  Returns
390  -------
391  result : `lsst.pipe.base.Struct`
392  Result struct with component:
393 
394  - ``brightStarStamps``: ``bSS.BrightStarStamps``
395  """
396  self.log.info("Extracting bright stars from exposure %s", dataId)
397  # Extract stamps around bright stars
398  extractedStamps = self.extractStampsextractStamps(inputExposure, refObjLoader=refObjLoader)
399  # Warp (and shift, and potentially rotate) them
400  self.log.info("Applying warp and/or shift to %i star stamps from exposure %s",
401  len(extractedStamps.starIms), dataId)
402  warpedStars = self.warpStampswarpStamps(extractedStamps.starIms, extractedStamps.pixCenters)
403  brightStarList = [bSS.BrightStarStamp(stamp_im=warp,
404  gaiaGMag=extractedStamps.GMags[j],
405  gaiaId=extractedStamps.gaiaIds[j])
406  for j, warp in enumerate(warpedStars)]
407  # Compute annularFlux and normalize
408  self.log.info("Computing annular flux and normalizing %i bright stars from exposure %s",
409  len(warpedStars), dataId)
410  # annularFlux statistic set-up, excluding mask planes
411  statsControl = afwMath.StatisticsControl()
412  statsControl.setNumSigmaClip(self.config.numSigmaClip)
413  statsControl.setNumIter(self.config.numIter)
414  innerRadius, outerRadius = self.config.annularFluxRadii
415  statsFlag = afwMath.stringToStatisticsProperty(self.config.annularFluxStatistic)
416  brightStarStamps = bSS.BrightStarStamps.initAndNormalize(brightStarList,
417  innerRadius=innerRadius,
418  outerRadius=outerRadius,
419  imCenter=self.modelCentermodelCenter,
420  statsControl=statsControl,
421  statsFlag=statsFlag,
422  badMaskPlanes=self.config.badMaskPlanes)
423  return pipeBase.Struct(brightStarStamps=brightStarStamps)
424 
425  def runDataRef(self, dataRef):
426  """ Read in required calexp, extract and process stamps around bright
427  stars and write them to disk.
428 
429  Parameters
430  ----------
431  dataRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef`
432  Data reference to the calexp to extract bright stars from.
433  """
434  calexp = dataRef.get("calexp")
435  output = self.runrun(calexp, dataId=dataRef.dataId)
436  # Save processed bright star stamps
437  dataRef.put(output.brightStarStamps, "brightStarStamps")
438  return pipeBase.Struct(brightStarStamps=output.brightStarStamps)
439 
440  def runQuantum(self, butlerQC, inputRefs, outputRefs):
441  inputs = butlerQC.get(inputRefs)
442  inputs['dataId'] = str(butlerQC.quantum.dataId)
443  refObjLoader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId
444  for ref in inputRefs.refCat],
445  refCats=inputs.pop("refCat"),
446  config=self.config.refObjLoader)
447  output = self.runrun(**inputs, refObjLoader=refObjLoader)
448  butlerQC.put(output, outputRefs)
def run(self, inputExposure, refObjLoader=None, dataId=None)
def runQuantum(self, butlerQC, inputRefs, outputRefs)
def extractStamps(self, inputExposure, refObjLoader=None)
def __init__(self, butler=None, initInputs=None, *args, **kwargs)