lsst.pipe.tasks  21.0.0-33-g5ddf1805+888ad90e2c
processBrightStars.py
Go to the documentation of this file.
1 # This file is part of pipe_tasks.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (https://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <https://www.gnu.org/licenses/>.
21 #
22 """Extract small cutouts around bright stars, normalize and warp them to the
23 same arbitrary pixel grid.
24 """
25 
26 __all__ = ["ProcessBrightStarsTask"]
27 
28 import numpy as np
29 import astropy.units as u
30 
31 from lsst import geom
32 from lsst.afw import math as afwMath
33 from lsst.afw import image as afwImage
34 from lsst.afw import detection as afwDetect
35 from lsst.afw import cameraGeom as cg
36 from lsst.afw.geom import transformFactory as tFactory
37 import lsst.pex.config as pexConfig
38 from lsst.pipe import base as pipeBase
39 from lsst.pipe.base import connectionTypes as cT
40 from lsst.meas.algorithms.loadIndexedReferenceObjects import LoadIndexedReferenceObjectsTask
41 from lsst.meas.algorithms import ReferenceObjectLoader
42 from lsst.meas.algorithms import brightStarStamps as bSS
43 
44 
45 class ProcessBrightStarsConnections(pipeBase.PipelineTaskConnections, dimensions=("visit", "detector")):
46  inputExposure = cT.Input(
47  doc="Input exposure from which to extract bright star stamps",
48  name="calexp",
49  storageClass="ExposureF",
50  dimensions=("visit", "detector")
51  )
52  refCat = cT.PrerequisiteInput(
53  doc="Reference catalog that contains bright star positions",
54  name="gaia_dr2_20200414",
55  storageClass="SimpleCatalog",
56  dimensions=("skypix",),
57  multiple=True,
58  deferLoad=True
59  )
60  brightStarStamps = cT.Output(
61  doc="Set of preprocessed postage stamps, each centered on a single bright star.",
62  name="brightStarStamps",
63  storageClass="BrightStarStamps",
64  dimensions=("visit", "detector")
65  )
66 
67 
68 class ProcessBrightStarsConfig(pipeBase.PipelineTaskConfig,
69  pipelineConnections=ProcessBrightStarsConnections):
70  """Configuration parameters for ProcessBrightStarsTask
71  """
72  magLimit = pexConfig.Field(
73  dtype=float,
74  doc="Magnitude limit, in Gaia G; all stars brighter than this value will be processed",
75  default=18
76  )
77  stampSize = pexConfig.ListField(
78  dtype=int,
79  doc="Size of the stamps to be extracted, in pixels",
80  default=(250, 250)
81  )
82  modelStampBuffer = pexConfig.Field(
83  dtype=float,
84  doc="'Buffer' factor to be applied to determine the size of the stamp the processed stars will "
85  "be saved in. This will also be the size of the extended PSF model.",
86  default=1.1
87  )
88  doRemoveDetected = pexConfig.Field(
89  dtype=bool,
90  doc="Whether DETECTION footprints, other than that for the central object, should be changed to "
91  "BAD",
92  default=True
93  )
94  warpingKernelName = pexConfig.ChoiceField(
95  dtype=str,
96  doc="Warping kernel",
97  default="lanczos5",
98  allowed={
99  "bilinear": "bilinear interpolation",
100  "lanczos3": "Lanczos kernel of order 3",
101  "lanczos4": "Lanczos kernel of order 4",
102  "lanczos5": "Lanczos kernel of order 5",
103  }
104  )
105  annularFluxRadii = pexConfig.ListField(
106  dtype=int,
107  doc="Inner and outer radii of the annulus used to compute the AnnularFlux for normalization, "
108  "in pixels.",
109  default=(40, 50)
110  )
111  annularFluxStatistic = pexConfig.ChoiceField(
112  dtype=str,
113  doc="Type of statistic to use to compute annular flux.",
114  default="MEANCLIP",
115  allowed={
116  "MEAN": "mean",
117  "MEDIAN": "median",
118  "MEANCLIP": "clipped mean",
119  }
120  )
121  numSigmaClip = pexConfig.Field(
122  dtype=float,
123  doc="Sigma for outlier rejection; ignored if annularFluxStatistic != 'MEANCLIP'.",
124  default=4
125  )
126  numIter = pexConfig.Field(
127  dtype=int,
128  doc="Number of iterations of outlier rejection; ignored if annularFluxStatistic != 'MEANCLIP'.",
129  default=3
130  )
131  badMaskPlanes = pexConfig.ListField(
132  dtype=str,
133  doc="Mask planes that, if set, lead to associated pixels not being included in the computation of the"
134  " annular flux.",
135  default=('BAD', 'CR', 'CROSSTALK', 'EDGE', 'NO_DATA', 'SAT', 'SUSPECT', 'UNMASKEDNAN')
136  )
137  refObjLoader = pexConfig.ConfigurableField(
138  target=LoadIndexedReferenceObjectsTask,
139  doc="reference object loader for astrometric calibration",
140  )
141 
142  def setDefaults(self):
143  self.refObjLoaderrefObjLoader.ref_dataset_name = "gaia_dr2_20200414"
144 
145 
146 class ProcessBrightStarsTask(pipeBase.PipelineTask, pipeBase.CmdLineTask):
147  """The description of the parameters for this Task are detailed in
148  :lsst-task:`~lsst.pipe.base.PipelineTask`.
149 
150  Notes
151  -----
152  `ProcessBrightStarsTask` is used to extract, process, and store small
153  image cut-outs (or "postage stamps") around bright stars. It relies on
154  three methods, called in succession:
155 
156  `extractStamps`
157  Find bright stars within the exposure using a reference catalog and
158  extract a stamp centered on each.
159  `warpStamps`
160  Shift and warp each stamp to remove optical distortions and sample all
161  stars on the same pixel grid.
162  `measureAndNormalize`
163  Compute the flux of an object in an annulus and normalize it. This is
164  required to normalize each bright star stamp as their central pixels
165  are likely saturated and/or contain ghosts, and cannot be used.
166  """
167  ConfigClass = ProcessBrightStarsConfig
168  _DefaultName = "processBrightStars"
169  RunnerClass = pipeBase.ButlerInitializedTaskRunner
170 
171  def __init__(self, butler=None, initInputs=None, *args, **kwargs):
172  super().__init__(*args, **kwargs)
173  # Compute (model) stamp size depending on provided "buffer" value
174  self.modelStampSizemodelStampSize = (int(self.config.stampSize[0]*self.config.modelStampBuffer),
175  int(self.config.stampSize[1]*self.config.modelStampBuffer))
176  # force it to be odd-sized so we have a central pixel
177  if not self.modelStampSizemodelStampSize[0] % 2:
178  self.modelStampSizemodelStampSize[0] += 1
179  if not self.modelStampSizemodelStampSize[1] % 2:
180  self.modelStampSizemodelStampSize[1] += 1
181  # central pixel
182  self.modelCentermodelCenter = self.modelStampSizemodelStampSize[0]//2, self.modelStampSizemodelStampSize[1]//2
183  # configure Gaia refcat
184  if butler is not None:
185  self.makeSubtask('refObjLoader', butler=butler)
186 
187  def extractStamps(self, inputExposure, refObjLoader=None):
188  """ Read position of bright stars within `inputExposure` from refCat
189  and extract them.
190 
191  Parameters
192  ----------
193  inputExposure : `afwImage.exposure.exposure.ExposureF`
194  The image from which bright star stamps should be extracted.
195  refObjLoader : `LoadIndexedReferenceObjectsTask`, optional
196  Loader to find objects within a reference catalog.
197 
198  Returns
199  -------
200  result : `lsst.pipe.base.Struct`
201  Result struct with components:
202 
203  - ``starIms``: `list` of stamps
204  - ``pixCenters``: `list` of corresponding coordinates to each
205  star's center, in pixels.
206  - ``GMags``: `list` of corresponding (Gaia) G magnitudes.
207  - ``gaiaIds``: `np.ndarray` of corresponding unique Gaia
208  identifiers.
209  """
210  if refObjLoader is None:
211  refObjLoader = self.refObjLoader
212  starIms = []
213  pixCenters = []
214  GMags = []
215  ids = []
216  wcs = inputExposure.getWcs()
217  # select stars within input exposure from refcat
218  withinCalexp = refObjLoader.loadPixelBox(inputExposure.getBBox(), wcs, filterName="phot_g_mean")
219  refCat = withinCalexp.refCat
220  # keep bright objects
221  fluxLimit = ((self.config.magLimit*u.ABmag).to(u.nJy)).to_value()
222  GFluxes = np.array(refCat['phot_g_mean_flux'])
223  bright = GFluxes > fluxLimit
224  # convert to AB magnitudes
225  allGMags = [((gFlux*u.nJy).to(u.ABmag)).to_value() for gFlux in GFluxes[bright]]
226  allIds = refCat.columns.extract("id", where=bright)["id"]
227  selectedColumns = refCat.columns.extract('coord_ra', 'coord_dec', where=bright)
228  for j, (ra, dec) in enumerate(zip(selectedColumns["coord_ra"], selectedColumns["coord_dec"])):
229  sp = geom.SpherePoint(ra, dec, geom.radians)
230  cpix = wcs.skyToPixel(sp)
231  # TODO: DM-25894 keep objects on or slightly beyond CCD edge
232  if (cpix[0] >= self.config.stampSize[0]/2
233  and cpix[0] < inputExposure.getDimensions()[0] - self.config.stampSize[0]/2
234  and cpix[1] >= self.config.stampSize[1]/2
235  and cpix[1] < inputExposure.getDimensions()[1] - self.config.stampSize[1]/2):
236  starIm = inputExposure.getCutout(sp, geom.Extent2I(self.config.stampSize))
237  if self.config.doRemoveDetected:
238  # give detection footprint of other objects the BAD flag
239  detThreshold = afwDetect.Threshold(starIm.mask.getPlaneBitMask("DETECTED"),
240  afwDetect.Threshold.BITMASK)
241  omask = afwDetect.FootprintSet(starIm.mask, detThreshold)
242  allFootprints = omask.getFootprints()
243  otherFootprints = []
244  for fs in allFootprints:
245  if not fs.contains(geom.Point2I(cpix)):
246  otherFootprints.append(fs)
247  nbMatchingFootprints = len(allFootprints) - len(otherFootprints)
248  if not nbMatchingFootprints == 1:
249  self.log.warn("Failed to uniquely identify central DETECTION footprint for star "
250  f"{allIds[j]}; found {nbMatchingFootprints} footprints instead.")
251  omask.setFootprints(otherFootprints)
252  omask.setMask(starIm.mask, "BAD")
253  starIms.append(starIm)
254  pixCenters.append(cpix)
255  GMags.append(allGMags[j])
256  ids.append(allIds[j])
257  return pipeBase.Struct(starIms=starIms,
258  pixCenters=pixCenters,
259  GMags=GMags,
260  gaiaIds=ids)
261 
262  def warpStamps(self, stamps, pixCenters):
263  """Warps and shifts all given stamps so they are sampled on the same
264  pixel grid and centered on the central pixel. This includes rotating
265  the stamp depending on detector orientation.
266 
267  Parameters
268  ----------
269  stamps : `collections.abc.Sequence`
270  [`afwImage.exposure.exposure.ExposureF`]
271  Image cutouts centered on a single object.
272  pixCenters : `collections.abc.Sequence` [`geom.Point2D`]
273  Positions of each object's center (as obtained from the refCat),
274  in pixels.
275 
276  Returns
277  -------
278  warpedStars : `list` [`afwImage.maskedImage.maskedImage.MaskedImage`]
279  """
280  # warping control; only contains shiftingALg provided in config
281  warpCont = afwMath.WarpingControl(self.config.warpingKernelName)
282  # Compare model to star stamp sizes
283  bufferPix = (self.modelStampSizemodelStampSize[0] - self.config.stampSize[0],
284  self.modelStampSizemodelStampSize[1] - self.config.stampSize[1])
285  # Initialize detector instance (note all stars were extracted from an
286  # exposure from the same detector)
287  det = stamps[0].getDetector()
288  # Define correction for optical distortions
289  pixToTan = det.getTransform(cg.PIXELS, cg.TAN_PIXELS)
290  # Array of all possible rotations for detector orientation:
291  possibleRots = np.array([k*np.pi/2 for k in range(4)])
292  # determine how many, if any, rotations are required
293  yaw = det.getOrientation().getYaw()
294  nb90Rots = np.argmin(np.abs(possibleRots - float(yaw)))
295 
296  # apply transformation to each star
297  warpedStars = []
298  for star, cent in zip(stamps, pixCenters):
299  # (re)create empty destination image
300  destImage = afwImage.MaskedImageF(*self.modelStampSizemodelStampSize)
301  bottomLeft = geom.Point2D(star.image.getXY0())
302  newBottomLeft = pixToTan.applyForward(bottomLeft)
303  newBottomLeft.setX(newBottomLeft.getX() - bufferPix[0]/2)
304  newBottomLeft.setY(newBottomLeft.getY() - bufferPix[1]/2)
305  # Convert to int
306  newBottomLeft = geom.Point2I(newBottomLeft)
307  # Set origin
308  destImage.setXY0(newBottomLeft)
309 
310  # Define linear shifting to recenter stamps
311  newCenter = pixToTan.applyForward(cent) # center of warped star
312  shift = self.modelCentermodelCenter[0] + newBottomLeft[0] - newCenter[0],\
313  self.modelCentermodelCenter[1] + newBottomLeft[1] - newCenter[1]
314  affineShift = geom.AffineTransform(shift)
315  shiftTransform = tFactory.makeTransform(affineShift)
316 
317  # Define full transform (warp and shift)
318  starWarper = pixToTan.then(shiftTransform)
319 
320  # Apply it
321  goodPix = afwMath.warpImage(destImage, star.getMaskedImage(),
322  starWarper, warpCont)
323  if not goodPix:
324  self.log.debug("Warping of a star failed: no good pixel in output")
325 
326  # Arbitrarily set origin of shifted star to 0
327  destImage.setXY0(0, 0)
328 
329  # Apply rotation if apropriate
330  if nb90Rots:
331  destImage = afwMath.rotateImageBy90(destImage, nb90Rots)
332  warpedStars.append(destImage.clone())
333  return warpedStars
334 
335  @pipeBase.timeMethod
336  def run(self, inputExposure, refObjLoader=None, dataId=None):
337  """Identify bright stars within an exposure using a reference catalog,
338  extract stamps around each, then preprocess them. The preprocessing
339  steps are: shifting, warping and potentially rotating them to the same
340  pixel grid; computing their annular flux and normalizing them.
341 
342  Parameters
343  ----------
344  inputExposure : `afwImage.exposure.exposure.ExposureF`
345  The image from which bright star stamps should be extracted.
346  refObjLoader : `LoadIndexedReferenceObjectsTask`, optional
347  Loader to find objects within a reference catalog.
348  dataId : `dict` or `lsst.daf.butler.DataCoordinate`
349  The dataId of the exposure (and detector) bright stars should be
350  extracted from.
351 
352  Returns
353  -------
354  result : `lsst.pipe.base.Struct`
355  Result struct with component:
356 
357  - ``brightStarStamps``: ``bSS.BrightStarStamps``
358  """
359  self.log.info("Extracting bright stars from exposure %s", dataId)
360  # Extract stamps around bright stars
361  extractedStamps = self.extractStampsextractStamps(inputExposure, refObjLoader=refObjLoader)
362  # Warp (and shift, and potentially rotate) them
363  self.log.info("Applying warp to %i star stamps from exposure %s",
364  len(extractedStamps.starIms), dataId)
365  warpedStars = self.warpStampswarpStamps(extractedStamps.starIms, extractedStamps.pixCenters)
366  brightStarList = [bSS.BrightStarStamp(stamp_im=warp,
367  gaiaGMag=extractedStamps.GMags[j],
368  gaiaId=extractedStamps.gaiaIds[j])
369  for j, warp in enumerate(warpedStars)]
370  # Compute annularFlux and normalize
371  self.log.info("Computing annular flux and normalizing %i bright stars from exposure %s",
372  len(warpedStars), dataId)
373  # annularFlux statistic set-up, excluding mask planes
374  statsControl = afwMath.StatisticsControl()
375  statsControl.setNumSigmaClip(self.config.numSigmaClip)
376  statsControl.setNumIter(self.config.numIter)
377  innerRadius, outerRadius = self.config.annularFluxRadii
378  statsFlag = afwMath.stringToStatisticsProperty(self.config.annularFluxStatistic)
379  brightStarStamps = bSS.BrightStarStamps.initAndNormalize(brightStarList,
380  innerRadius=innerRadius,
381  outerRadius=outerRadius,
382  imCenter=self.modelCentermodelCenter,
383  statsControl=statsControl,
384  statsFlag=statsFlag,
385  badMaskPlanes=self.config.badMaskPlanes)
386  return pipeBase.Struct(brightStarStamps=brightStarStamps)
387 
388  def runDataRef(self, dataRef):
389  """ Read in required calexp, extract and process stamps around bright
390  stars and write them to disk.
391 
392  Parameters
393  ----------
394  dataRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef`
395  Data reference to the calexp to extract bright stars from.
396  """
397  calexp = dataRef.get("calexp")
398  output = self.runrun(calexp, dataId=dataRef.dataId)
399  # Save processed bright star stamps
400  dataRef.put(output.brightStarStamps, "brightStarStamps")
401  return pipeBase.Struct(brightStarStamps=output.brightStarStamps)
402 
403  def runQuantum(self, butlerQC, inputRefs, outputRefs):
404  inputs = butlerQC.get(inputRefs)
405  inputs['dataId'] = str(butlerQC.quantum.dataId)
406  refObjLoader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId
407  for ref in inputRefs.refCat],
408  refCats=inputs.pop("refCat"),
409  config=self.config.refObjLoader)
410  output = self.runrun(**inputs, refObjLoader=refObjLoader)
411  butlerQC.put(output, outputRefs)
def run(self, inputExposure, refObjLoader=None, dataId=None)
def runQuantum(self, butlerQC, inputRefs, outputRefs)
def extractStamps(self, inputExposure, refObjLoader=None)
def __init__(self, butler=None, initInputs=None, *args, **kwargs)