lsst.pipe.tasks  21.0.0-25-g85b8e57b+773e41f820
processBrightStars.py
Go to the documentation of this file.
1 # This file is part of pipe_tasks.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (https://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <https://www.gnu.org/licenses/>.
21 #
22 """Extract small cutouts around bright stars, normalize and warp them to the
23 same arbitrary pixel grid.
24 """
25 
26 __all__ = ["ProcessBrightStarsTask"]
27 
28 import numpy as np
29 import astropy.units as u
30 
31 from lsst import geom
32 from lsst.afw import math as afwMath
33 from lsst.afw import image as afwImage
34 from lsst.afw import cameraGeom as cg
35 from lsst.afw.geom import transformFactory as tFactory
36 import lsst.pex.config as pexConfig
37 from lsst.pipe import base as pipeBase
38 from lsst.pipe.base import connectionTypes as cT
39 from lsst.meas.algorithms.loadIndexedReferenceObjects import LoadIndexedReferenceObjectsTask
40 from lsst.meas.algorithms import ReferenceObjectLoader
41 from lsst.meas.algorithms import brightStarStamps as bSS
42 
43 
44 class ProcessBrightStarsConnections(pipeBase.PipelineTaskConnections, dimensions=("visit", "detector")):
45  inputExposure = cT.Input(
46  doc="Input exposure from which to extract bright star stamps",
47  name="calexp",
48  storageClass="ExposureF",
49  dimensions=("visit", "detector")
50  )
51  refCat = cT.PrerequisiteInput(
52  doc="Reference catalog that contains bright star positions",
53  name="gaia_dr2_20200414",
54  storageClass="SimpleCatalog",
55  dimensions=("skypix",),
56  multiple=True,
57  deferLoad=True
58  )
59  brightStarStamps = cT.Output(
60  doc="Set of preprocessed postage stamps, each centered on a single bright star.",
61  name="brightStarStamps",
62  storageClass="BrightStarStamps",
63  dimensions=("visit", "detector")
64  )
65 
66 
67 class ProcessBrightStarsConfig(pipeBase.PipelineTaskConfig,
68  pipelineConnections=ProcessBrightStarsConnections):
69  """Configuration parameters for ProcessBrightStarsTask
70  """
71  magLimit = pexConfig.Field(
72  dtype=float,
73  doc="Magnitude limit, in Gaia G; all stars brighter than this value will be processed",
74  default=18
75  )
76  stampSize = pexConfig.ListField(
77  dtype=int,
78  doc="Size of the stamps to be extracted, in pixels",
79  default=(250, 250)
80  )
81  modelStampBuffer = pexConfig.Field(
82  dtype=float,
83  doc="'Buffer' factor to be applied to determine the size of the stamp the processed stars will "
84  "be saved in. This will also be the size of the extended PSF model.",
85  default=1.1
86  )
87  warpingKernelName = pexConfig.ChoiceField(
88  dtype=str,
89  doc="Warping kernel",
90  default="lanczos5",
91  allowed={
92  "bilinear": "bilinear interpolation",
93  "lanczos3": "Lanczos kernel of order 3",
94  "lanczos4": "Lanczos kernel of order 4",
95  "lanczos5": "Lanczos kernel of order 5",
96  }
97  )
98  annularFluxRadii = pexConfig.ListField(
99  dtype=int,
100  doc="Inner and outer radii of the annulus used to compute the AnnularFlux for normalization, "
101  "in pixels.",
102  default=(40, 50)
103  )
104  annularFluxStatistic = pexConfig.ChoiceField(
105  dtype=str,
106  doc="Type of statistic to use to compute annular flux.",
107  default="MEANCLIP",
108  allowed={
109  "MEAN": "mean",
110  "MEDIAN": "median",
111  "MEANCLIP": "clipped mean",
112  }
113  )
114  numSigmaClip = pexConfig.Field(
115  dtype=float,
116  doc="Sigma for outlier rejection; ignored if annularFluxStatistic != 'MEANCLIP'.",
117  default=4
118  )
119  numIter = pexConfig.Field(
120  dtype=int,
121  doc="Number of iterations of outlier rejection; ignored if annularFluxStatistic != 'MEANCLIP'.",
122  default=3
123  )
124  badMaskPlanes = pexConfig.ListField(
125  dtype=str,
126  doc="Mask planes that, if set, lead to associated pixels not being included in the computation of the"
127  " annular flux.",
128  default=('BAD', 'CR', 'CROSSTALK', 'EDGE', 'NO_DATA', 'SAT', 'SUSPECT', 'UNMASKEDNAN')
129  )
130  refObjLoader = pexConfig.ConfigurableField(
131  target=LoadIndexedReferenceObjectsTask,
132  doc="reference object loader for astrometric calibration",
133  )
134 
135  def setDefaults(self):
136  self.refObjLoaderrefObjLoader.ref_dataset_name = "gaia_dr2_20200414"
137 
138 
139 class ProcessBrightStarsTask(pipeBase.PipelineTask, pipeBase.CmdLineTask):
140  """The description of the parameters for this Task are detailed in
141  :lsst-task:`~lsst.pipe.base.PipelineTask`.
142 
143  Notes
144  -----
145  `ProcessBrightStarsTask` is used to extract, process, and store small
146  image cut-outs (or "postage stamps") around bright stars. It relies on
147  three methods, called in succession:
148 
149  `extractStamps`
150  Find bright stars within the exposure using a reference catalog and
151  extract a stamp centered on each.
152  `warpStamps`
153  Shift and warp each stamp to remove optical distortions and sample all
154  stars on the same pixel grid.
155  `measureAndNormalize`
156  Compute the flux of an object in an annulus and normalize it. This is
157  required to normalize each bright star stamp as their central pixels
158  are likely saturated and/or contain ghosts, and cannot be used.
159  """
160  ConfigClass = ProcessBrightStarsConfig
161  _DefaultName = "processBrightStars"
162  RunnerClass = pipeBase.ButlerInitializedTaskRunner
163 
164  def __init__(self, butler=None, initInputs=None, *args, **kwargs):
165  super().__init__(*args, **kwargs)
166  # Compute (model) stamp size depending on provided "buffer" value
167  self.modelStampSizemodelStampSize = (int(self.config.stampSize[0]*self.config.modelStampBuffer),
168  int(self.config.stampSize[1]*self.config.modelStampBuffer))
169  # force it to be odd-sized so we have a central pixel
170  if not self.modelStampSizemodelStampSize[0] % 2:
171  self.modelStampSizemodelStampSize[0] += 1
172  if not self.modelStampSizemodelStampSize[1] % 2:
173  self.modelStampSizemodelStampSize[1] += 1
174  # central pixel
175  self.modelCentermodelCenter = self.modelStampSizemodelStampSize[0]//2, self.modelStampSizemodelStampSize[1]//2
176  # configure Gaia refcat
177  if butler is not None:
178  self.makeSubtask('refObjLoader', butler=butler)
179 
180  def extractStamps(self, inputExposure, refObjLoader=None):
181  """ Read position of bright stars within `inputExposure` from refCat
182  and extract them.
183 
184  Parameters
185  ----------
186  inputExposure : `afwImage.exposure.exposure.ExposureF`
187  The image from which bright star stamps should be extracted.
188  refObjLoader : `LoadIndexedReferenceObjectsTask`, optional
189  Loader to find objects within a reference catalog.
190 
191  Returns
192  -------
193  result : `lsst.pipe.base.Struct`
194  Result struct with components:
195 
196  - ``starIms``: `list` of stamps
197  - ``pixCenters``: `list` of corresponding coordinates to each
198  star's center, in pixels.
199  - ``GMags``: `list` of corresponding (Gaia) G magnitudes.
200  - ``gaiaIds``: `np.ndarray` of corresponding unique Gaia
201  identifiers.
202  """
203  if refObjLoader is None:
204  refObjLoader = self.refObjLoader
205  starIms = []
206  pixCenters = []
207  GMags = []
208  ids = []
209  wcs = inputExposure.getWcs()
210  # select stars within input exposure from refcat
211  withinCalexp = refObjLoader.loadPixelBox(inputExposure.getBBox(), wcs, filterName="phot_g_mean")
212  refCat = withinCalexp.refCat
213  # keep bright objects
214  fluxLimit = ((self.config.magLimit*u.ABmag).to(u.nJy)).to_value()
215  GFluxes = np.array(refCat['phot_g_mean_flux'])
216  bright = GFluxes > fluxLimit
217  # convert to AB magnitudes
218  allGMags = [((gFlux*u.nJy).to(u.ABmag)).to_value() for gFlux in GFluxes[bright]]
219  allIds = refCat.columns.extract("id", where=bright)["id"]
220  selectedColumns = refCat.columns.extract('coord_ra', 'coord_dec', where=bright)
221  for j, (ra, dec) in enumerate(zip(selectedColumns["coord_ra"], selectedColumns["coord_dec"])):
222  sp = geom.SpherePoint(ra, dec, geom.radians)
223  cpix = wcs.skyToPixel(sp)
224  # TODO: DM-25894 keep objects on or slightly beyond CCD edge
225  if (cpix[0] >= self.config.stampSize[0]/2
226  and cpix[0] < inputExposure.getDimensions()[0] - self.config.stampSize[0]/2
227  and cpix[1] >= self.config.stampSize[1]/2
228  and cpix[1] < inputExposure.getDimensions()[1] - self.config.stampSize[1]/2):
229  starIms.append(inputExposure.getCutout(sp, geom.Extent2I(self.config.stampSize)))
230  pixCenters.append(cpix)
231  GMags.append(allGMags[j])
232  ids.append(allIds[j])
233  return pipeBase.Struct(starIms=starIms,
234  pixCenters=pixCenters,
235  GMags=GMags,
236  gaiaIds=ids)
237 
238  def warpStamps(self, stamps, pixCenters):
239  """Warps and shifts all given stamps so they are sampled on the same
240  pixel grid and centered on the central pixel. This includes rotating
241  the stamp depending on detector orientation.
242 
243  Parameters
244  ----------
245  stamps : `collections.abc.Sequence`
246  [`afwImage.exposure.exposure.ExposureF`]
247  Image cutouts centered on a single object.
248  pixCenters : `collections.abc.Sequence` [`geom.Point2D`]
249  Positions of each object's center (as obtained from the refCat),
250  in pixels.
251 
252  Returns
253  -------
254  warpedStars : `list` [`afwImage.maskedImage.maskedImage.MaskedImage`]
255  """
256  # warping control; only contains shiftingALg provided in config
257  warpCont = afwMath.WarpingControl(self.config.warpingKernelName)
258  # Compare model to star stamp sizes
259  bufferPix = (self.modelStampSizemodelStampSize[0] - self.config.stampSize[0],
260  self.modelStampSizemodelStampSize[1] - self.config.stampSize[1])
261  # Initialize detector instance (note all stars were extracted from an
262  # exposure from the same detector)
263  det = stamps[0].getDetector()
264  # Define correction for optical distortions
265  pixToTan = det.getTransform(cg.PIXELS, cg.TAN_PIXELS)
266  # Array of all possible rotations for detector orientation:
267  possibleRots = np.array([k*np.pi/2 for k in range(4)])
268  # determine how many, if any, rotations are required
269  yaw = det.getOrientation().getYaw()
270  nb90Rots = np.argmin(np.abs(possibleRots - float(yaw)))
271 
272  # apply transformation to each star
273  warpedStars = []
274  for star, cent in zip(stamps, pixCenters):
275  # (re)create empty destination image
276  destImage = afwImage.MaskedImageF(*self.modelStampSizemodelStampSize)
277  bottomLeft = geom.Point2D(star.getImage().getXY0())
278  newBottomLeft = pixToTan.applyForward(bottomLeft)
279  newBottomLeft.setX(newBottomLeft.getX() - bufferPix[0]/2)
280  newBottomLeft.setY(newBottomLeft.getY() - bufferPix[1]/2)
281  # Convert to int
282  newBottomLeft = geom.Point2I(newBottomLeft)
283  # Set origin
284  destImage.setXY0(newBottomLeft)
285 
286  # Define linear shifting to recenter stamps
287  newCenter = pixToTan.applyForward(cent) # center of warped star
288  shift = self.modelCentermodelCenter[0] + newBottomLeft[0] - newCenter[0],\
289  self.modelCentermodelCenter[1] + newBottomLeft[1] - newCenter[1]
290  affineShift = geom.AffineTransform(shift)
291  shiftTransform = tFactory.makeTransform(affineShift)
292 
293  # Define full transform (warp and shift)
294  starWarper = pixToTan.then(shiftTransform)
295 
296  # Apply it
297  goodPix = afwMath.warpImage(destImage, star.getMaskedImage(),
298  starWarper, warpCont)
299  if not goodPix:
300  self.log.debug("Warping of a star failed: no good pixel in output")
301 
302  # Arbitrarily set origin of shifted star to 0
303  destImage.setXY0(0, 0)
304 
305  # Apply rotation if apropriate
306  if nb90Rots:
307  destImage = afwMath.rotateImageBy90(destImage, nb90Rots)
308  warpedStars.append(destImage.clone())
309  return warpedStars
310 
311  @pipeBase.timeMethod
312  def run(self, inputExposure, refObjLoader=None, dataId=None):
313  """Identify bright stars within an exposure using a reference catalog,
314  extract stamps around each, then preprocess them. The preprocessing
315  steps are: shifting, warping and potentially rotating them to the same
316  pixel grid; computing their annular flux and normalizing them.
317 
318  Parameters
319  ----------
320  inputExposure : `afwImage.exposure.exposure.ExposureF`
321  The image from which bright star stamps should be extracted.
322  refObjLoader : `LoadIndexedReferenceObjectsTask`, optional
323  Loader to find objects within a reference catalog.
324  dataId : `dict` or `lsst.daf.butler.DataCoordinate`
325  The dataId of the exposure (and detector) bright stars should be
326  extracted from.
327 
328  Returns
329  -------
330  result : `lsst.pipe.base.Struct`
331  Result struct with component:
332 
333  - ``brightStarStamps``: ``bSS.BrightStarStamps``
334  """
335  self.log.info("Extracting bright stars from exposure %s", dataId)
336  # Extract stamps around bright stars
337  extractedStamps = self.extractStampsextractStamps(inputExposure, refObjLoader=refObjLoader)
338  # Warp (and shift, and potentially rotate) them
339  self.log.info("Applying warp to %i star stamps from exposure %s",
340  len(extractedStamps.starIms), dataId)
341  warpedStars = self.warpStampswarpStamps(extractedStamps.starIms, extractedStamps.pixCenters)
342  brightStarList = [bSS.BrightStarStamp(stamp_im=warp,
343  gaiaGMag=extractedStamps.GMags[j],
344  gaiaId=extractedStamps.gaiaIds[j])
345  for j, warp in enumerate(warpedStars)]
346  # Compute annularFlux and normalize
347  self.log.info("Computing annular flux and normalizing %i bright stars from exposure %s",
348  len(warpedStars), dataId)
349  # annularFlux statistic set-up, excluding mask planes
350  statsControl = afwMath.StatisticsControl()
351  statsControl.setNumSigmaClip(self.config.numSigmaClip)
352  statsControl.setNumIter(self.config.numIter)
353  innerRadius, outerRadius = self.config.annularFluxRadii
354  statsFlag = afwMath.stringToStatisticsProperty(self.config.annularFluxStatistic)
355  brightStarStamps = bSS.BrightStarStamps.initAndNormalize(brightStarList,
356  innerRadius=innerRadius,
357  outerRadius=outerRadius,
358  imCenter=self.modelCentermodelCenter,
359  statsControl=statsControl,
360  statsFlag=statsFlag,
361  badMaskPlanes=self.config.badMaskPlanes)
362  return pipeBase.Struct(brightStarStamps=brightStarStamps)
363 
364  def runDataRef(self, dataRef):
365  """ Read in required calexp, extract and process stamps around bright
366  stars and write them to disk.
367 
368  Parameters
369  ----------
370  dataRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef`
371  Data reference to the calexp to extract bright stars from.
372  """
373  calexp = dataRef.get("calexp")
374  output = self.runrun(calexp, dataId=dataRef.dataId)
375  # Save processed bright star stamps
376  dataRef.put(output.brightStarStamps, "brightStarStamps")
377  return pipeBase.Struct(brightStarStamps=output.brightStarStamps)
378 
379  def runQuantum(self, butlerQC, inputRefs, outputRefs):
380  inputs = butlerQC.get(inputRefs)
381  inputs['dataId'] = str(butlerQC.quantum.dataId)
382  refObjLoader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId
383  for ref in inputRefs.refCat],
384  refCats=inputs.pop("refCat"),
385  config=self.config.refObjLoader)
386  output = self.runrun(**inputs, refObjLoader=refObjLoader)
387  butlerQC.put(output, outputRefs)
def run(self, inputExposure, refObjLoader=None, dataId=None)
def runQuantum(self, butlerQC, inputRefs, outputRefs)
def extractStamps(self, inputExposure, refObjLoader=None)
def __init__(self, butler=None, initInputs=None, *args, **kwargs)