lsst.pipe.tasks  21.0.0-38-g070523fc+12fa6302a7
processBrightStars.py
Go to the documentation of this file.
1 # This file is part of pipe_tasks.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (https://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <https://www.gnu.org/licenses/>.
21 #
22 """Extract small cutouts around bright stars, normalize and warp them to the
23 same arbitrary pixel grid.
24 """
25 
26 __all__ = ["ProcessBrightStarsTask"]
27 
28 import numpy as np
29 import astropy.units as u
30 
31 from lsst import geom
32 from lsst.afw import math as afwMath
33 from lsst.afw import image as afwImage
34 from lsst.afw import detection as afwDetect
35 from lsst.afw import cameraGeom as cg
36 from lsst.afw.geom import transformFactory as tFactory
37 import lsst.pex.config as pexConfig
38 from lsst.pipe import base as pipeBase
39 from lsst.pipe.base import connectionTypes as cT
40 from lsst.meas.algorithms.loadIndexedReferenceObjects import LoadIndexedReferenceObjectsTask
41 from lsst.meas.algorithms import ReferenceObjectLoader
42 from lsst.meas.algorithms import brightStarStamps as bSS
43 
44 
45 class ProcessBrightStarsConnections(pipeBase.PipelineTaskConnections, dimensions=("visit", "detector")):
46  inputExposure = cT.Input(
47  doc="Input exposure from which to extract bright star stamps",
48  name="calexp",
49  storageClass="ExposureF",
50  dimensions=("visit", "detector")
51  )
52  refCat = cT.PrerequisiteInput(
53  doc="Reference catalog that contains bright star positions",
54  name="gaia_dr2_20200414",
55  storageClass="SimpleCatalog",
56  dimensions=("skypix",),
57  multiple=True,
58  deferLoad=True
59  )
60  brightStarStamps = cT.Output(
61  doc="Set of preprocessed postage stamps, each centered on a single bright star.",
62  name="brightStarStamps",
63  storageClass="BrightStarStamps",
64  dimensions=("visit", "detector")
65  )
66 
67 
68 class ProcessBrightStarsConfig(pipeBase.PipelineTaskConfig,
69  pipelineConnections=ProcessBrightStarsConnections):
70  """Configuration parameters for ProcessBrightStarsTask
71  """
72  magLimit = pexConfig.Field(
73  dtype=float,
74  doc="Magnitude limit, in Gaia G; all stars brighter than this value will be processed",
75  default=18
76  )
77  stampSize = pexConfig.ListField(
78  dtype=int,
79  doc="Size of the stamps to be extracted, in pixels",
80  default=(250, 250)
81  )
82  modelStampBuffer = pexConfig.Field(
83  dtype=float,
84  doc="'Buffer' factor to be applied to determine the size of the stamp the processed stars will "
85  "be saved in. This will also be the size of the extended PSF model.",
86  default=1.1
87  )
88  doRemoveDetected = pexConfig.Field(
89  dtype=bool,
90  doc="Whether DETECTION footprints, other than that for the central object, should be changed to "
91  "BAD",
92  default=True
93  )
94  doApplyTransform = pexConfig.Field(
95  dtype=bool,
96  doc="Apply transform to bright star stamps to correct for optical distortions?",
97  default=True
98  )
99  warpingKernelName = pexConfig.ChoiceField(
100  dtype=str,
101  doc="Warping kernel",
102  default="lanczos5",
103  allowed={
104  "bilinear": "bilinear interpolation",
105  "lanczos3": "Lanczos kernel of order 3",
106  "lanczos4": "Lanczos kernel of order 4",
107  "lanczos5": "Lanczos kernel of order 5",
108  }
109  )
110  annularFluxRadii = pexConfig.ListField(
111  dtype=int,
112  doc="Inner and outer radii of the annulus used to compute the AnnularFlux for normalization, "
113  "in pixels.",
114  default=(40, 50)
115  )
116  annularFluxStatistic = pexConfig.ChoiceField(
117  dtype=str,
118  doc="Type of statistic to use to compute annular flux.",
119  default="MEANCLIP",
120  allowed={
121  "MEAN": "mean",
122  "MEDIAN": "median",
123  "MEANCLIP": "clipped mean",
124  }
125  )
126  numSigmaClip = pexConfig.Field(
127  dtype=float,
128  doc="Sigma for outlier rejection; ignored if annularFluxStatistic != 'MEANCLIP'.",
129  default=4
130  )
131  numIter = pexConfig.Field(
132  dtype=int,
133  doc="Number of iterations of outlier rejection; ignored if annularFluxStatistic != 'MEANCLIP'.",
134  default=3
135  )
136  badMaskPlanes = pexConfig.ListField(
137  dtype=str,
138  doc="Mask planes that, if set, lead to associated pixels not being included in the computation of the"
139  " annular flux.",
140  default=('BAD', 'CR', 'CROSSTALK', 'EDGE', 'NO_DATA', 'SAT', 'SUSPECT', 'UNMASKEDNAN')
141  )
142  refObjLoader = pexConfig.ConfigurableField(
143  target=LoadIndexedReferenceObjectsTask,
144  doc="reference object loader for astrometric calibration",
145  )
146 
147  def setDefaults(self):
148  self.refObjLoaderrefObjLoader.ref_dataset_name = "gaia_dr2_20200414"
149 
150 
151 class ProcessBrightStarsTask(pipeBase.PipelineTask, pipeBase.CmdLineTask):
152  """The description of the parameters for this Task are detailed in
153  :lsst-task:`~lsst.pipe.base.PipelineTask`.
154 
155  Notes
156  -----
157  `ProcessBrightStarsTask` is used to extract, process, and store small
158  image cut-outs (or "postage stamps") around bright stars. It relies on
159  three methods, called in succession:
160 
161  `extractStamps`
162  Find bright stars within the exposure using a reference catalog and
163  extract a stamp centered on each.
164  `warpStamps`
165  Shift and warp each stamp to remove optical distortions and sample all
166  stars on the same pixel grid.
167  `measureAndNormalize`
168  Compute the flux of an object in an annulus and normalize it. This is
169  required to normalize each bright star stamp as their central pixels
170  are likely saturated and/or contain ghosts, and cannot be used.
171  """
172  ConfigClass = ProcessBrightStarsConfig
173  _DefaultName = "processBrightStars"
174  RunnerClass = pipeBase.ButlerInitializedTaskRunner
175 
176  def __init__(self, butler=None, initInputs=None, *args, **kwargs):
177  super().__init__(*args, **kwargs)
178  # Compute (model) stamp size depending on provided "buffer" value
179  self.modelStampSizemodelStampSize = (int(self.config.stampSize[0]*self.config.modelStampBuffer),
180  int(self.config.stampSize[1]*self.config.modelStampBuffer))
181  # force it to be odd-sized so we have a central pixel
182  if not self.modelStampSizemodelStampSize[0] % 2:
183  self.modelStampSizemodelStampSize[0] += 1
184  if not self.modelStampSizemodelStampSize[1] % 2:
185  self.modelStampSizemodelStampSize[1] += 1
186  # central pixel
187  self.modelCentermodelCenter = self.modelStampSizemodelStampSize[0]//2, self.modelStampSizemodelStampSize[1]//2
188  # configure Gaia refcat
189  if butler is not None:
190  self.makeSubtask('refObjLoader', butler=butler)
191 
192  def extractStamps(self, inputExposure, refObjLoader=None):
193  """ Read position of bright stars within `inputExposure` from refCat
194  and extract them.
195 
196  Parameters
197  ----------
198  inputExposure : `afwImage.exposure.exposure.ExposureF`
199  The image from which bright star stamps should be extracted.
200  refObjLoader : `LoadIndexedReferenceObjectsTask`, optional
201  Loader to find objects within a reference catalog.
202 
203  Returns
204  -------
205  result : `lsst.pipe.base.Struct`
206  Result struct with components:
207 
208  - ``starIms``: `list` of stamps
209  - ``pixCenters``: `list` of corresponding coordinates to each
210  star's center, in pixels.
211  - ``GMags``: `list` of corresponding (Gaia) G magnitudes.
212  - ``gaiaIds``: `np.ndarray` of corresponding unique Gaia
213  identifiers.
214  """
215  if refObjLoader is None:
216  refObjLoader = self.refObjLoader
217  starIms = []
218  pixCenters = []
219  GMags = []
220  ids = []
221  wcs = inputExposure.getWcs()
222  # select stars within input exposure from refcat
223  withinCalexp = refObjLoader.loadPixelBox(inputExposure.getBBox(), wcs, filterName="phot_g_mean")
224  refCat = withinCalexp.refCat
225  # keep bright objects
226  fluxLimit = ((self.config.magLimit*u.ABmag).to(u.nJy)).to_value()
227  GFluxes = np.array(refCat['phot_g_mean_flux'])
228  bright = GFluxes > fluxLimit
229  # convert to AB magnitudes
230  allGMags = [((gFlux*u.nJy).to(u.ABmag)).to_value() for gFlux in GFluxes[bright]]
231  allIds = refCat.columns.extract("id", where=bright)["id"]
232  selectedColumns = refCat.columns.extract('coord_ra', 'coord_dec', where=bright)
233  for j, (ra, dec) in enumerate(zip(selectedColumns["coord_ra"], selectedColumns["coord_dec"])):
234  sp = geom.SpherePoint(ra, dec, geom.radians)
235  cpix = wcs.skyToPixel(sp)
236  # TODO: DM-25894 keep objects on or slightly beyond CCD edge
237  if (cpix[0] >= self.config.stampSize[0]/2
238  and cpix[0] < inputExposure.getDimensions()[0] - self.config.stampSize[0]/2
239  and cpix[1] >= self.config.stampSize[1]/2
240  and cpix[1] < inputExposure.getDimensions()[1] - self.config.stampSize[1]/2):
241  starIm = inputExposure.getCutout(sp, geom.Extent2I(self.config.stampSize))
242  if self.config.doRemoveDetected:
243  # give detection footprint of other objects the BAD flag
244  detThreshold = afwDetect.Threshold(starIm.mask.getPlaneBitMask("DETECTED"),
245  afwDetect.Threshold.BITMASK)
246  omask = afwDetect.FootprintSet(starIm.mask, detThreshold)
247  allFootprints = omask.getFootprints()
248  otherFootprints = []
249  for fs in allFootprints:
250  if not fs.contains(geom.Point2I(cpix)):
251  otherFootprints.append(fs)
252  nbMatchingFootprints = len(allFootprints) - len(otherFootprints)
253  if not nbMatchingFootprints == 1:
254  self.log.warn("Failed to uniquely identify central DETECTION footprint for star "
255  f"{allIds[j]}; found {nbMatchingFootprints} footprints instead.")
256  omask.setFootprints(otherFootprints)
257  omask.setMask(starIm.mask, "BAD")
258  starIms.append(starIm)
259  pixCenters.append(cpix)
260  GMags.append(allGMags[j])
261  ids.append(allIds[j])
262  return pipeBase.Struct(starIms=starIms,
263  pixCenters=pixCenters,
264  GMags=GMags,
265  gaiaIds=ids)
266 
267  def warpStamps(self, stamps, pixCenters):
268  """Warps and shifts all given stamps so they are sampled on the same
269  pixel grid and centered on the central pixel. This includes rotating
270  the stamp depending on detector orientation.
271 
272  Parameters
273  ----------
274  stamps : `collections.abc.Sequence`
275  [`afwImage.exposure.exposure.ExposureF`]
276  Image cutouts centered on a single object.
277  pixCenters : `collections.abc.Sequence` [`geom.Point2D`]
278  Positions of each object's center (as obtained from the refCat),
279  in pixels.
280 
281  Returns
282  -------
283  warpedStars : `list` [`afwImage.maskedImage.maskedImage.MaskedImage`]
284  """
285  # warping control; only contains shiftingALg provided in config
286  warpCont = afwMath.WarpingControl(self.config.warpingKernelName)
287  # Compare model to star stamp sizes
288  bufferPix = (self.modelStampSizemodelStampSize[0] - self.config.stampSize[0],
289  self.modelStampSizemodelStampSize[1] - self.config.stampSize[1])
290  # Initialize detector instance (note all stars were extracted from an
291  # exposure from the same detector)
292  det = stamps[0].getDetector()
293  # Define correction for optical distortions
294  if self.config.doApplyTransform:
295  pixToTan = det.getTransform(cg.PIXELS, cg.TAN_PIXELS)
296  else:
297  pixToTan = tFactory.makeIdentityTransform()
298  # Array of all possible rotations for detector orientation:
299  possibleRots = np.array([k*np.pi/2 for k in range(4)])
300  # determine how many, if any, rotations are required
301  yaw = det.getOrientation().getYaw()
302  nb90Rots = np.argmin(np.abs(possibleRots - float(yaw)))
303 
304  # apply transformation to each star
305  warpedStars = []
306  for star, cent in zip(stamps, pixCenters):
307  # (re)create empty destination image
308  destImage = afwImage.MaskedImageF(*self.modelStampSizemodelStampSize)
309  bottomLeft = geom.Point2D(star.image.getXY0())
310  newBottomLeft = pixToTan.applyForward(bottomLeft)
311  newBottomLeft.setX(newBottomLeft.getX() - bufferPix[0]/2)
312  newBottomLeft.setY(newBottomLeft.getY() - bufferPix[1]/2)
313  # Convert to int
314  newBottomLeft = geom.Point2I(newBottomLeft)
315  # Set origin
316  destImage.setXY0(newBottomLeft)
317 
318  # Define linear shifting to recenter stamps
319  newCenter = pixToTan.applyForward(cent) # center of warped star
320  shift = self.modelCentermodelCenter[0] + newBottomLeft[0] - newCenter[0],\
321  self.modelCentermodelCenter[1] + newBottomLeft[1] - newCenter[1]
322  affineShift = geom.AffineTransform(shift)
323  shiftTransform = tFactory.makeTransform(affineShift)
324 
325  # Define full transform (warp and shift)
326  starWarper = pixToTan.then(shiftTransform)
327 
328  # Apply it
329  goodPix = afwMath.warpImage(destImage, star.getMaskedImage(),
330  starWarper, warpCont)
331  if not goodPix:
332  self.log.debug("Warping of a star failed: no good pixel in output")
333 
334  # Arbitrarily set origin of shifted star to 0
335  destImage.setXY0(0, 0)
336 
337  # Apply rotation if apropriate
338  if nb90Rots:
339  destImage = afwMath.rotateImageBy90(destImage, nb90Rots)
340  warpedStars.append(destImage.clone())
341  return warpedStars
342 
343  @pipeBase.timeMethod
344  def run(self, inputExposure, refObjLoader=None, dataId=None):
345  """Identify bright stars within an exposure using a reference catalog,
346  extract stamps around each, then preprocess them. The preprocessing
347  steps are: shifting, warping and potentially rotating them to the same
348  pixel grid; computing their annular flux and normalizing them.
349 
350  Parameters
351  ----------
352  inputExposure : `afwImage.exposure.exposure.ExposureF`
353  The image from which bright star stamps should be extracted.
354  refObjLoader : `LoadIndexedReferenceObjectsTask`, optional
355  Loader to find objects within a reference catalog.
356  dataId : `dict` or `lsst.daf.butler.DataCoordinate`
357  The dataId of the exposure (and detector) bright stars should be
358  extracted from.
359 
360  Returns
361  -------
362  result : `lsst.pipe.base.Struct`
363  Result struct with component:
364 
365  - ``brightStarStamps``: ``bSS.BrightStarStamps``
366  """
367  self.log.info("Extracting bright stars from exposure %s", dataId)
368  # Extract stamps around bright stars
369  extractedStamps = self.extractStampsextractStamps(inputExposure, refObjLoader=refObjLoader)
370  # Warp (and shift, and potentially rotate) them
371  self.log.info("Applying warp and/or shift to %i star stamps from exposure %s",
372  len(extractedStamps.starIms), dataId)
373  warpedStars = self.warpStampswarpStamps(extractedStamps.starIms, extractedStamps.pixCenters)
374  brightStarList = [bSS.BrightStarStamp(stamp_im=warp,
375  gaiaGMag=extractedStamps.GMags[j],
376  gaiaId=extractedStamps.gaiaIds[j])
377  for j, warp in enumerate(warpedStars)]
378  # Compute annularFlux and normalize
379  self.log.info("Computing annular flux and normalizing %i bright stars from exposure %s",
380  len(warpedStars), dataId)
381  # annularFlux statistic set-up, excluding mask planes
382  statsControl = afwMath.StatisticsControl()
383  statsControl.setNumSigmaClip(self.config.numSigmaClip)
384  statsControl.setNumIter(self.config.numIter)
385  innerRadius, outerRadius = self.config.annularFluxRadii
386  statsFlag = afwMath.stringToStatisticsProperty(self.config.annularFluxStatistic)
387  brightStarStamps = bSS.BrightStarStamps.initAndNormalize(brightStarList,
388  innerRadius=innerRadius,
389  outerRadius=outerRadius,
390  imCenter=self.modelCentermodelCenter,
391  statsControl=statsControl,
392  statsFlag=statsFlag,
393  badMaskPlanes=self.config.badMaskPlanes)
394  return pipeBase.Struct(brightStarStamps=brightStarStamps)
395 
396  def runDataRef(self, dataRef):
397  """ Read in required calexp, extract and process stamps around bright
398  stars and write them to disk.
399 
400  Parameters
401  ----------
402  dataRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef`
403  Data reference to the calexp to extract bright stars from.
404  """
405  calexp = dataRef.get("calexp")
406  output = self.runrun(calexp, dataId=dataRef.dataId)
407  # Save processed bright star stamps
408  dataRef.put(output.brightStarStamps, "brightStarStamps")
409  return pipeBase.Struct(brightStarStamps=output.brightStarStamps)
410 
411  def runQuantum(self, butlerQC, inputRefs, outputRefs):
412  inputs = butlerQC.get(inputRefs)
413  inputs['dataId'] = str(butlerQC.quantum.dataId)
414  refObjLoader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId
415  for ref in inputRefs.refCat],
416  refCats=inputs.pop("refCat"),
417  config=self.config.refObjLoader)
418  output = self.runrun(**inputs, refObjLoader=refObjLoader)
419  butlerQC.put(output, outputRefs)
def run(self, inputExposure, refObjLoader=None, dataId=None)
def runQuantum(self, butlerQC, inputRefs, outputRefs)
def extractStamps(self, inputExposure, refObjLoader=None)
def __init__(self, butler=None, initInputs=None, *args, **kwargs)