lsst.pipe.tasks  21.0.0-165-g4927fbd8+b0a7cc4645
selectImages.py
Go to the documentation of this file.
1 #
2 # LSST Data Management System
3 # Copyright 2008, 2009, 2010 LSST Corporation.
4 #
5 # This product includes software developed by the
6 # LSST Project (http://www.lsst.org/).
7 #
8 # This program is free software: you can redistribute it and/or modify
9 # it under the terms of the GNU General Public License as published by
10 # the Free Software Foundation, either version 3 of the License, or
11 # (at your option) any later version.
12 #
13 # This program is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU General Public License for more details.
17 #
18 # You should have received a copy of the LSST License Statement and
19 # the GNU General Public License along with this program. If not,
20 # see <http://www.lsstcorp.org/LegalNotices/>.
21 #
22 import numpy as np
23 import lsst.sphgeom
24 import lsst.utils as utils
25 import lsst.pex.config as pexConfig
26 import lsst.pex.exceptions as pexExceptions
27 import lsst.geom as geom
28 import lsst.pipe.base as pipeBase
29 from lsst.skymap import BaseSkyMap
30 from lsst.daf.base import DateTime
31 from lsst.utils.timer import timeMethod
32 
33 __all__ = ["BaseSelectImagesTask", "BaseExposureInfo", "WcsSelectImagesTask", "PsfWcsSelectImagesTask",
34  "DatabaseSelectImagesConfig", "BestSeeingWcsSelectImagesTask", "BestSeeingSelectVisitsTask",
35  "BestSeeingQuantileSelectVisitsTask"]
36 
37 
38 class DatabaseSelectImagesConfig(pexConfig.Config):
39  """Base configuration for subclasses of BaseSelectImagesTask that use a database"""
40  host = pexConfig.Field(
41  doc="Database server host name",
42  dtype=str,
43  )
44  port = pexConfig.Field(
45  doc="Database server port",
46  dtype=int,
47  )
48  database = pexConfig.Field(
49  doc="Name of database",
50  dtype=str,
51  )
52  maxExposures = pexConfig.Field(
53  doc="maximum exposures to select; intended for debugging; ignored if None",
54  dtype=int,
55  optional=True,
56  )
57 
58 
59 class BaseExposureInfo(pipeBase.Struct):
60  """Data about a selected exposure
61  """
62 
63  def __init__(self, dataId, coordList):
64  """Create exposure information that can be used to generate data references
65 
66  The object has the following fields:
67  - dataId: data ID of exposure (a dict)
68  - coordList: ICRS coordinates of the corners of the exposure (list of lsst.geom.SpherePoint)
69  plus any others items that are desired
70  """
71  super(BaseExposureInfo, self).__init__(dataId=dataId, coordList=coordList)
72 
73 
74 class BaseSelectImagesTask(pipeBase.Task):
75  """Base task for selecting images suitable for coaddition
76  """
77  ConfigClass = pexConfig.Config
78  _DefaultName = "selectImages"
79 
80  @timeMethod
81  def run(self, coordList):
82  """Select images suitable for coaddition in a particular region
83 
84  @param[in] coordList: list of coordinates defining region of interest; if None then select all images
85  subclasses may add additional keyword arguments, as required
86 
87  @return a pipeBase Struct containing:
88  - exposureInfoList: a list of exposure information objects (subclasses of BaseExposureInfo),
89  which have at least the following fields:
90  - dataId: data ID dictionary
91  - coordList: ICRS coordinates of the corners of the exposure (list of lsst.geom.SpherePoint)
92  """
93  raise NotImplementedError()
94 
95  def _runArgDictFromDataId(self, dataId):
96  """Extract keyword arguments for run (other than coordList) from a data ID
97 
98  @return keyword arguments for run (other than coordList), as a dict
99  """
100  raise NotImplementedError()
101 
102  def runDataRef(self, dataRef, coordList, makeDataRefList=True, selectDataList=[]):
103  """Run based on a data reference
104 
105  This delegates to run() and _runArgDictFromDataId() to do the actual
106  selection. In the event that the selectDataList is non-empty, this will
107  be used to further restrict the selection, providing the user with
108  additional control over the selection.
109 
110  @param[in] dataRef: data reference; must contain any extra keys needed by the subclass
111  @param[in] coordList: list of coordinates defining region of interest; if None, search the whole sky
112  @param[in] makeDataRefList: if True, return dataRefList
113  @param[in] selectDataList: List of SelectStruct with dataRefs to consider for selection
114  @return a pipeBase Struct containing:
115  - exposureInfoList: a list of objects derived from ExposureInfo
116  - dataRefList: a list of data references (None if makeDataRefList False)
117  """
118  runArgDict = self._runArgDictFromDataId_runArgDictFromDataId(dataRef.dataId)
119  exposureInfoList = self.runrun(coordList, **runArgDict).exposureInfoList
120 
121  if len(selectDataList) > 0 and len(exposureInfoList) > 0:
122  # Restrict the exposure selection further
123  ccdKeys, ccdValues = _extractKeyValue(exposureInfoList)
124  inKeys, inValues = _extractKeyValue([s.dataRef for s in selectDataList], keys=ccdKeys)
125  inValues = set(inValues)
126  newExposureInfoList = []
127  for info, ccdVal in zip(exposureInfoList, ccdValues):
128  if ccdVal in inValues:
129  newExposureInfoList.append(info)
130  else:
131  self.log.info("De-selecting exposure %s: not in selectDataList", info.dataId)
132  exposureInfoList = newExposureInfoList
133 
134  if makeDataRefList:
135  butler = dataRef.butlerSubset.butler
136  dataRefList = [butler.dataRef(datasetType="calexp",
137  dataId=expInfo.dataId,
138  ) for expInfo in exposureInfoList]
139  else:
140  dataRefList = None
141 
142  return pipeBase.Struct(
143  dataRefList=dataRefList,
144  exposureInfoList=exposureInfoList,
145  )
146 
147 
148 def _extractKeyValue(dataList, keys=None):
149  """Extract the keys and values from a list of dataIds
150 
151  The input dataList is a list of objects that have 'dataId' members.
152  This allows it to be used for both a list of data references and a
153  list of ExposureInfo
154  """
155  assert len(dataList) > 0
156  if keys is None:
157  keys = sorted(dataList[0].dataId.keys())
158  keySet = set(keys)
159  values = list()
160  for data in dataList:
161  thisKeys = set(data.dataId.keys())
162  if thisKeys != keySet:
163  raise RuntimeError("DataId keys inconsistent: %s vs %s" % (keySet, thisKeys))
164  values.append(tuple(data.dataId[k] for k in keys))
165  return keys, values
166 
167 
168 class SelectStruct(pipeBase.Struct):
169  """A container for data to be passed to the WcsSelectImagesTask"""
170 
171  def __init__(self, dataRef, wcs, bbox):
172  super(SelectStruct, self).__init__(dataRef=dataRef, wcs=wcs, bbox=bbox)
173 
174 
176  """Select images using their Wcs
177 
178  We use the "convexHull" method of lsst.sphgeom.ConvexPolygon to define
179  polygons on the celestial sphere, and test the polygon of the
180  patch for overlap with the polygon of the image.
181 
182  We use "convexHull" instead of generating a ConvexPolygon
183  directly because the standard for the inputs to ConvexPolygon
184  are pretty high and we don't want to be responsible for reaching them.
185  """
186 
187  def runDataRef(self, dataRef, coordList, makeDataRefList=True, selectDataList=[]):
188  """Select images in the selectDataList that overlap the patch
189 
190  This method is the old entry point for the Gen2 commandline tasks and drivers
191  Will be deprecated in v22.
192 
193  @param dataRef: Data reference for coadd/tempExp (with tract, patch)
194  @param coordList: List of ICRS coordinates (lsst.geom.SpherePoint) specifying boundary of patch
195  @param makeDataRefList: Construct a list of data references?
196  @param selectDataList: List of SelectStruct, to consider for selection
197  """
198  dataRefList = []
199  exposureInfoList = []
200 
201  patchVertices = [coord.getVector() for coord in coordList]
202  patchPoly = lsst.sphgeom.ConvexPolygon.convexHull(patchVertices)
203 
204  for data in selectDataList:
205  dataRef = data.dataRef
206  imageWcs = data.wcs
207  imageBox = data.bbox
208 
209  imageCorners = self.getValidImageCornersgetValidImageCorners(imageWcs, imageBox, patchPoly, dataId=None)
210  if imageCorners:
211  dataRefList.append(dataRef)
212  exposureInfoList.append(BaseExposureInfo(dataRef.dataId, imageCorners))
213 
214  return pipeBase.Struct(
215  dataRefList=dataRefList if makeDataRefList else None,
216  exposureInfoList=exposureInfoList,
217  )
218 
219  def run(self, wcsList, bboxList, coordList, dataIds=None, **kwargs):
220  """Return indices of provided lists that meet the selection criteria
221 
222  Parameters:
223  -----------
224  wcsList : `list` of `lsst.afw.geom.SkyWcs`
225  specifying the WCS's of the input ccds to be selected
226  bboxList : `list` of `lsst.geom.Box2I`
227  specifying the bounding boxes of the input ccds to be selected
228  coordList : `list` of `lsst.geom.SpherePoint`
229  ICRS coordinates specifying boundary of the patch.
230 
231  Returns:
232  --------
233  result: `list` of `int`
234  of indices of selected ccds
235  """
236  if dataIds is None:
237  dataIds = [None] * len(wcsList)
238  patchVertices = [coord.getVector() for coord in coordList]
239  patchPoly = lsst.sphgeom.ConvexPolygon.convexHull(patchVertices)
240  result = []
241  for i, (imageWcs, imageBox, dataId) in enumerate(zip(wcsList, bboxList, dataIds)):
242  imageCorners = self.getValidImageCornersgetValidImageCorners(imageWcs, imageBox, patchPoly, dataId)
243  if imageCorners:
244  result.append(i)
245  return result
246 
247  def getValidImageCorners(self, imageWcs, imageBox, patchPoly, dataId=None):
248  "Return corners or None if bad"
249  try:
250  imageCorners = [imageWcs.pixelToSky(pix) for pix in geom.Box2D(imageBox).getCorners()]
251  except (pexExceptions.DomainError, pexExceptions.RuntimeError) as e:
252  # Protecting ourselves from awful Wcs solutions in input images
253  self.log.debug("WCS error in testing calexp %s (%s): deselecting", dataId, e)
254  return
255 
256  imagePoly = lsst.sphgeom.ConvexPolygon.convexHull([coord.getVector() for coord in imageCorners])
257  if imagePoly is None:
258  self.log.debug("Unable to create polygon from image %s: deselecting", dataId)
259  return
260 
261  if patchPoly.intersects(imagePoly):
262  # "intersects" also covers "contains" or "is contained by"
263  self.log.info("Selecting calexp %s", dataId)
264  return imageCorners
265 
266 
267 def sigmaMad(array):
268  "Return median absolute deviation scaled to normally distributed data"
269  return 1.4826*np.median(np.abs(array - np.median(array)))
270 
271 
272 class PsfWcsSelectImagesConnections(pipeBase.PipelineTaskConnections,
273  dimensions=("tract", "patch", "skymap", "instrument", "visit"),
274  defaultTemplates={"coaddName": "deep"}):
275  pass
276 
277 
278 class PsfWcsSelectImagesConfig(pipeBase.PipelineTaskConfig,
279  pipelineConnections=PsfWcsSelectImagesConnections):
280  maxEllipResidual = pexConfig.Field(
281  doc="Maximum median ellipticity residual",
282  dtype=float,
283  default=0.007,
284  optional=True,
285  )
286  maxSizeScatter = pexConfig.Field(
287  doc="Maximum scatter in the size residuals",
288  dtype=float,
289  optional=True,
290  )
291  maxScaledSizeScatter = pexConfig.Field(
292  doc="Maximum scatter in the size residuals, scaled by the median size",
293  dtype=float,
294  default=0.009,
295  optional=True,
296  )
297  starSelection = pexConfig.Field(
298  doc="select star with this field",
299  dtype=str,
300  default='calib_psf_used'
301  )
302  starShape = pexConfig.Field(
303  doc="name of star shape",
304  dtype=str,
305  default='base_SdssShape'
306  )
307  psfShape = pexConfig.Field(
308  doc="name of psf shape",
309  dtype=str,
310  default='base_SdssShape_psf'
311  )
312 
313 
314 class PsfWcsSelectImagesTask(WcsSelectImagesTask):
315  """Select images using their Wcs and cuts on the PSF properties
316 
317  The PSF quality criteria are based on the size and ellipticity residuals from the
318  adaptive second moments of the star and the PSF.
319 
320  The criteria are:
321  - the median of the ellipticty residuals
322  - the robust scatter of the size residuals (using the median absolute deviation)
323  - the robust scatter of the size residuals scaled by the square of
324  the median size
325  """
326 
327  ConfigClass = PsfWcsSelectImagesConfig
328  _DefaultName = "PsfWcsSelectImages"
329 
330  def runDataRef(self, dataRef, coordList, makeDataRefList=True, selectDataList=[]):
331  """Select images in the selectDataList that overlap the patch and satisfy PSF quality critera.
332 
333  This method is the old entry point for the Gen2 commandline tasks and drivers
334  Will be deprecated in v22.
335 
336  @param dataRef: Data reference for coadd/tempExp (with tract, patch)
337  @param coordList: List of ICRS coordinates (lsst.geom.SpherePoint) specifying boundary of patch
338  @param makeDataRefList: Construct a list of data references?
339  @param selectDataList: List of SelectStruct, to consider for selection
340  """
341  result = super(PsfWcsSelectImagesTask, self).runDataRef(dataRef, coordList, makeDataRefList,
342  selectDataList)
343 
344  dataRefList = []
345  exposureInfoList = []
346  for dataRef, exposureInfo in zip(result.dataRefList, result.exposureInfoList):
347  butler = dataRef.butlerSubset.butler
348  srcCatalog = butler.get('src', dataRef.dataId)
349  valid = self.isValid(srcCatalog, dataRef.dataId)
350  if valid is False:
351  continue
352 
353  dataRefList.append(dataRef)
354  exposureInfoList.append(exposureInfo)
355 
356  return pipeBase.Struct(
357  dataRefList=dataRefList,
358  exposureInfoList=exposureInfoList,
359  )
360 
361  def run(self, wcsList, bboxList, coordList, srcList, dataIds=None, **kwargs):
362  """Return indices of provided lists that meet the selection criteria
363 
364  Parameters:
365  -----------
366  wcsList : `list` of `lsst.afw.geom.SkyWcs`
367  specifying the WCS's of the input ccds to be selected
368  bboxList : `list` of `lsst.geom.Box2I`
369  specifying the bounding boxes of the input ccds to be selected
370  coordList : `list` of `lsst.geom.SpherePoint`
371  ICRS coordinates specifying boundary of the patch.
372  srcList : `list` of `lsst.afw.table.SourceCatalog`
373  containing the PSF shape information for the input ccds to be selected
374 
375  Returns:
376  --------
377  goodPsf: `list` of `int`
378  of indices of selected ccds
379  """
380  goodWcs = super(PsfWcsSelectImagesTask, self).run(wcsList=wcsList, bboxList=bboxList,
381  coordList=coordList, dataIds=dataIds)
382 
383  goodPsf = []
384  if dataIds is None:
385  dataIds = [None] * len(srcList)
386  for i, (srcCatalog, dataId) in enumerate(zip(srcList, dataIds)):
387  if i not in goodWcs:
388  continue
389  if self.isValid(srcCatalog, dataId):
390  goodPsf.append(i)
391 
392  return goodPsf
393 
394  def isValid(self, srcCatalog, dataId=None):
395  """Should this ccd be selected based on its PSF shape information
396 
397  Parameters
398  ----------
399  srcCatalog : `lsst.afw.table.SourceCatalog`
400  dataId : `dict` of dataId keys, optional.
401  Used only for logging. Defaults to None.
402 
403  Returns
404  -------
405  valid : `bool`
406  True if selected.
407  """
408  mask = srcCatalog[self.config.starSelection]
409 
410  starXX = srcCatalog[self.config.starShape+'_xx'][mask]
411  starYY = srcCatalog[self.config.starShape+'_yy'][mask]
412  starXY = srcCatalog[self.config.starShape+'_xy'][mask]
413  psfXX = srcCatalog[self.config.psfShape+'_xx'][mask]
414  psfYY = srcCatalog[self.config.psfShape+'_yy'][mask]
415  psfXY = srcCatalog[self.config.psfShape+'_xy'][mask]
416 
417  starSize = np.power(starXX*starYY - starXY**2, 0.25)
418  starE1 = (starXX - starYY)/(starXX + starYY)
419  starE2 = 2*starXY/(starXX + starYY)
420  medianSize = np.median(starSize)
421 
422  psfSize = np.power(psfXX*psfYY - psfXY**2, 0.25)
423  psfE1 = (psfXX - psfYY)/(psfXX + psfYY)
424  psfE2 = 2*psfXY/(psfXX + psfYY)
425 
426  medianE1 = np.abs(np.median(starE1 - psfE1))
427  medianE2 = np.abs(np.median(starE2 - psfE2))
428  medianE = np.sqrt(medianE1**2 + medianE2**2)
429 
430  scatterSize = sigmaMad(starSize - psfSize)
431  scaledScatterSize = scatterSize/medianSize**2
432 
433  valid = True
434  if self.config.maxEllipResidual and medianE > self.config.maxEllipResidual:
435  self.log.info("Removing visit %s because median e residual too large: %f vs %f",
436  dataId, medianE, self.config.maxEllipResidual)
437  valid = False
438  elif self.config.maxSizeScatter and scatterSize > self.config.maxSizeScatter:
439  self.log.info("Removing visit %s because size scatter is too large: %f vs %f",
440  dataId, scatterSize, self.config.maxSizeScatter)
441  valid = False
442  elif self.config.maxScaledSizeScatter and scaledScatterSize > self.config.maxScaledSizeScatter:
443  self.log.info("Removing visit %s because scaled size scatter is too large: %f vs %f",
444  dataId, scaledScatterSize, self.config.maxScaledSizeScatter)
445  valid = False
446 
447  return valid
448 
449 
450 class BestSeeingWcsSelectImageConfig(WcsSelectImagesTask.ConfigClass):
451  """Base configuration for BestSeeingSelectImagesTask.
452  """
453  nImagesMax = pexConfig.RangeField(
454  dtype=int,
455  doc="Maximum number of images to select",
456  default=5,
457  min=0)
458  maxPsfFwhm = pexConfig.Field(
459  dtype=float,
460  doc="Maximum PSF FWHM (in arcseconds) to select",
461  default=1.5,
462  optional=True)
463  minPsfFwhm = pexConfig.Field(
464  dtype=float,
465  doc="Minimum PSF FWHM (in arcseconds) to select",
466  default=0.,
467  optional=True)
468 
469 
470 class BestSeeingWcsSelectImagesTask(WcsSelectImagesTask):
471  """Select up to a maximum number of the best-seeing images using their Wcs.
472  """
473  ConfigClass = BestSeeingWcsSelectImageConfig
474 
475  def runDataRef(self, dataRef, coordList, makeDataRefList=True,
476  selectDataList=None):
477  """Select the best-seeing images in the selectDataList that overlap the patch.
478 
479  This method is the old entry point for the Gen2 commandline tasks and drivers
480  Will be deprecated in v22.
481 
482  Parameters
483  ----------
484  dataRef : `lsst.daf.persistence.ButlerDataRef`
485  Data reference for coadd/tempExp (with tract, patch)
486  coordList : `list` of `lsst.geom.SpherePoint`
487  List of ICRS sky coordinates specifying boundary of patch
488  makeDataRefList : `boolean`, optional
489  Construct a list of data references?
490  selectDataList : `list` of `SelectStruct`
491  List of SelectStruct, to consider for selection
492 
493  Returns
494  -------
495  result : `lsst.pipe.base.Struct`
496  Result struct with components:
497  - ``exposureList``: the selected exposures
498  (`list` of `lsst.pipe.tasks.selectImages.BaseExposureInfo`).
499  - ``dataRefList``: the optional data references corresponding to
500  each element of ``exposureList``
501  (`list` of `lsst.daf.persistence.ButlerDataRef`, or `None`).
502  """
503  psfSizes = []
504  dataRefList = []
505  exposureInfoList = []
506 
507  if selectDataList is None:
508  selectDataList = []
509 
510  result = super().runDataRef(dataRef, coordList, makeDataRefList=True, selectDataList=selectDataList)
511 
512  for dataRef, exposureInfo in zip(result.dataRefList, result.exposureInfoList):
513  cal = dataRef.get("calexp", immediate=True)
514 
515  # if min/max PSF values are defined, remove images out of bounds
516  pixToArcseconds = cal.getWcs().getPixelScale().asArcseconds()
517  psfSize = cal.getPsf().computeShape().getDeterminantRadius()*pixToArcseconds
518  sizeFwhm = psfSize * np.sqrt(8.*np.log(2.))
519  if self.config.maxPsfFwhm and sizeFwhm > self.config.maxPsfFwhm:
520  continue
521  if self.config.minPsfFwhm and sizeFwhm < self.config.minPsfFwhm:
522  continue
523  psfSizes.append(sizeFwhm)
524  dataRefList.append(dataRef)
525  exposureInfoList.append(exposureInfo)
526 
527  if len(psfSizes) > self.config.nImagesMax:
528  sortedIndices = np.argsort(psfSizes)[:self.config.nImagesMax]
529  filteredDataRefList = [dataRefList[i] for i in sortedIndices]
530  filteredExposureInfoList = [exposureInfoList[i] for i in sortedIndices]
531  self.log.info("%d images selected with FWHM range of %f--%f arcseconds",
532  len(sortedIndices), psfSizes[sortedIndices[0]], psfSizes[sortedIndices[-1]])
533 
534  else:
535  if len(psfSizes) == 0:
536  self.log.warning("0 images selected.")
537  else:
538  self.log.debug("%d images selected with FWHM range of %d--%d arcseconds",
539  len(psfSizes), psfSizes[0], psfSizes[-1])
540  filteredDataRefList = dataRefList
541  filteredExposureInfoList = exposureInfoList
542 
543  return pipeBase.Struct(
544  dataRefList=filteredDataRefList if makeDataRefList else None,
545  exposureInfoList=filteredExposureInfoList,
546  )
547 
548 
549 class BestSeeingSelectVisitsConnections(pipeBase.PipelineTaskConnections,
550  dimensions=("tract", "patch", "skymap", "band", "instrument"),
551  defaultTemplates={"coaddName": "goodSeeing"}):
552  skyMap = pipeBase.connectionTypes.Input(
553  doc="Input definition of geometry/bbox and projection/wcs for coadded exposures",
554  name=BaseSkyMap.SKYMAP_DATASET_TYPE_NAME,
555  storageClass="SkyMap",
556  dimensions=("skymap",),
557  )
558  visitSummaries = pipeBase.connectionTypes.Input(
559  doc="Per-visit consolidated exposure metadata from ConsolidateVisitSummaryTask",
560  name="visitSummary",
561  storageClass="ExposureCatalog",
562  dimensions=("instrument", "visit",),
563  multiple=True,
564  deferLoad=True
565  )
566  goodVisits = pipeBase.connectionTypes.Output(
567  doc="Selected visits to be coadded.",
568  name="{coaddName}Visits",
569  storageClass="StructuredDataDict",
570  dimensions=("instrument", "tract", "patch", "skymap", "band"),
571  )
572 
573 
574 class BestSeeingSelectVisitsConfig(pipeBase.PipelineTaskConfig,
575  pipelineConnections=BestSeeingSelectVisitsConnections):
576  nVisitsMax = pexConfig.RangeField(
577  dtype=int,
578  doc="Maximum number of visits to select",
579  default=12,
580  min=0
581  )
582  maxPsfFwhm = pexConfig.Field(
583  dtype=float,
584  doc="Maximum PSF FWHM (in arcseconds) to select",
585  default=1.5,
586  optional=True
587  )
588  minPsfFwhm = pexConfig.Field(
589  dtype=float,
590  doc="Minimum PSF FWHM (in arcseconds) to select",
591  default=0.,
592  optional=True
593  )
594  doConfirmOverlap = pexConfig.Field(
595  dtype=bool,
596  doc="Do remove visits that do not actually overlap the patch?",
597  default=True,
598  )
599  minMJD = pexConfig.Field(
600  dtype=float,
601  doc="Minimum visit MJD to select",
602  default=None,
603  optional=True
604  )
605  maxMJD = pexConfig.Field(
606  dtype=float,
607  doc="Maximum visit MJD to select",
608  default=None,
609  optional=True
610  )
611 
612 
613 class BestSeeingSelectVisitsTask(pipeBase.PipelineTask):
614  """Select up to a maximum number of the best-seeing visits
615 
616  Don't exceed the FWHM range specified by configs min(max)PsfFwhm.
617  This Task is a port of the Gen2 image-selector used in the AP pipeline:
618  BestSeeingSelectImagesTask. This Task selects full visits based on the
619  average PSF of the entire visit.
620  """
621  ConfigClass = BestSeeingSelectVisitsConfig
622  _DefaultName = 'bestSeeingSelectVisits'
623 
624  def runQuantum(self, butlerQC, inputRefs, outputRefs):
625  inputs = butlerQC.get(inputRefs)
626  quantumDataId = butlerQC.quantum.dataId
627  outputs = self.run(**inputs, dataId=quantumDataId)
628  butlerQC.put(outputs, outputRefs)
629 
630  def run(self, visitSummaries, skyMap, dataId):
631  """Run task
632 
633  Parameters:
634  -----------
635  visitSummary : `list`
636  List of `lsst.pipe.base.connections.DeferredDatasetRef` of
637  visitSummary tables of type `lsst.afw.table.ExposureCatalog`
638  skyMap : `lsst.skyMap.SkyMap`
639  SkyMap for checking visits overlap patch
640  dataId : `dict` of dataId keys
641  For retrieving patch info for checking visits overlap patch
642 
643  Returns
644  -------
645  result : `lsst.pipe.base.Struct`
646  Result struct with components:
647 
648  - `goodVisits`: `dict` with selected visit ids as keys,
649  so that it can be be saved as a StructuredDataDict.
650  StructuredDataList's are currently limited.
651  """
652 
653  if self.config.doConfirmOverlap:
654  patchPolygon = self.makePatchPolygon(skyMap, dataId)
655 
656  inputVisits = [visitSummary.ref.dataId['visit'] for visitSummary in visitSummaries]
657  fwhmSizes = []
658  visits = []
659  for visit, visitSummary in zip(inputVisits, visitSummaries):
660  # read in one-by-one and only once. There may be hundreds
661  visitSummary = visitSummary.get()
662 
663  # mjd is guaranteed to be the same for every detector in the visitSummary.
664  mjd = visitSummary[0].getVisitInfo().getDate().get(system=DateTime.MJD)
665 
666  pixToArcseconds = [vs.getWcs().getPixelScale(vs.getBBox().getCenter()).asArcseconds()
667  for vs in visitSummary]
668  # psfSigma is PSF model determinant radius at chip center in pixels
669  psfSigmas = np.array([vs['psfSigma'] for vs in visitSummary])
670  fwhm = np.nanmean(psfSigmas * pixToArcseconds) * np.sqrt(8.*np.log(2.))
671 
672  if self.config.maxPsfFwhm and fwhm > self.config.maxPsfFwhm:
673  continue
674  if self.config.minPsfFwhm and fwhm < self.config.minPsfFwhm:
675  continue
676  if self.config.minMJD and mjd < self.config.minMJD:
677  self.log.debug('MJD %f earlier than %.2f; rejecting', mjd, self.config.minMJD)
678  continue
679  if self.config.maxMJD and mjd > self.config.maxMJD:
680  self.log.debug('MJD %f later than %.2f; rejecting', mjd, self.config.maxMJD)
681  continue
682  if self.config.doConfirmOverlap and not self.doesIntersectPolygon(visitSummary, patchPolygon):
683  continue
684 
685  fwhmSizes.append(fwhm)
686  visits.append(visit)
687 
688  sortedVisits = [ind for (_, ind) in sorted(zip(fwhmSizes, visits))]
689  output = sortedVisits[:self.config.nVisitsMax]
690  self.log.info("%d images selected with FWHM range of %d--%d arcseconds",
691  len(output), fwhmSizes[visits.index(output[0])], fwhmSizes[visits.index(output[-1])])
692 
693  # In order to store as a StructuredDataDict, convert list to dict
694  goodVisits = {key: True for key in output}
695  return pipeBase.Struct(goodVisits=goodVisits)
696 
697  def makePatchPolygon(self, skyMap, dataId):
698  """Return True if sky polygon overlaps visit
699 
700  Parameters:
701  -----------
702  skyMap : `lsst.afw.table.ExposureCatalog`
703  Exposure catalog with per-detector geometry
704  dataId : `dict` of dataId keys
705  For retrieving patch info
706 
707  Returns:
708  --------
709  result :` lsst.sphgeom.ConvexPolygon.convexHull`
710  Polygon of patch's outer bbox
711  """
712  wcs = skyMap[dataId['tract']].getWcs()
713  bbox = skyMap[dataId['tract']][dataId['patch']].getOuterBBox()
714  sphCorners = wcs.pixelToSky(lsst.geom.Box2D(bbox).getCorners())
715  result = lsst.sphgeom.ConvexPolygon.convexHull([coord.getVector() for coord in sphCorners])
716  return result
717 
718  def doesIntersectPolygon(self, visitSummary, polygon):
719  """Return True if sky polygon overlaps visit
720 
721  Parameters:
722  -----------
723  visitSummary : `lsst.afw.table.ExposureCatalog`
724  Exposure catalog with per-detector geometry
725  polygon :` lsst.sphgeom.ConvexPolygon.convexHull`
726  Polygon to check overlap
727 
728  Returns:
729  --------
730  doesIntersect: `bool`
731  Does the visit overlap the polygon
732  """
733  doesIntersect = False
734  for detectorSummary in visitSummary:
735  corners = [lsst.geom.SpherePoint(ra, decl, units=lsst.geom.degrees).getVector() for (ra, decl) in
736  zip(detectorSummary['raCorners'], detectorSummary['decCorners'])]
737  detectorPolygon = lsst.sphgeom.ConvexPolygon.convexHull(corners)
738  if detectorPolygon.intersects(polygon):
739  doesIntersect = True
740  break
741  return doesIntersect
742 
743 
744 class BestSeeingQuantileSelectVisitsConfig(pipeBase.PipelineTaskConfig,
745  pipelineConnections=BestSeeingSelectVisitsConnections):
746  qMin = pexConfig.RangeField(
747  doc="Lower bound of quantile range to select. Sorts visits by seeing from narrow to wide, "
748  "and select those in the interquantile range (qMin, qMax). Set qMin to 0 for Best Seeing. "
749  "This config should be changed from zero only for exploratory diffIm testing.",
750  dtype=float,
751  default=0,
752  min=0,
753  max=1,
754  )
755  qMax = pexConfig.RangeField(
756  doc="Upper bound of quantile range to select. Sorts visits by seeing from narrow to wide, "
757  "and select those in the interquantile range (qMin, qMax). Set qMax to 1 for Worst Seeing.",
758  dtype=float,
759  default=0.33,
760  min=0,
761  max=1,
762  )
763  nVisitsMin = pexConfig.Field(
764  doc="At least this number of visits selected and supercedes quantile. For example, if 10 visits "
765  "cover this patch, qMin=0.33, and nVisitsMin=5, the best 5 visits will be selected.",
766  dtype=int,
767  default=6,
768  )
769  doConfirmOverlap = pexConfig.Field(
770  dtype=bool,
771  doc="Do remove visits that do not actually overlap the patch?",
772  default=True,
773  )
774  minMJD = pexConfig.Field(
775  dtype=float,
776  doc="Minimum visit MJD to select",
777  default=None,
778  optional=True
779  )
780  maxMJD = pexConfig.Field(
781  dtype=float,
782  doc="Maximum visit MJD to select",
783  default=None,
784  optional=True
785  )
786 
787 
788 class BestSeeingQuantileSelectVisitsTask(BestSeeingSelectVisitsTask):
789  """Select a quantile of the best-seeing visits
790 
791  Selects the best (for example, third) full visits based on the average
792  PSF width in the entire visit. It can also be used for difference imaging
793  experiments that require templates with the worst seeing visits.
794  For example, selecting the worst third can be acheived by
795  changing the config parameters qMin to 0.66 and qMax to 1.
796  """
797  ConfigClass = BestSeeingQuantileSelectVisitsConfig
798  _DefaultName = 'bestSeeingQuantileSelectVisits'
799 
800  @utils.inheritDoc(BestSeeingSelectVisitsTask)
801  def run(self, visitSummaries, skyMap, dataId):
802  if self.config.doConfirmOverlap:
803  patchPolygon = self.makePatchPolygon(skyMap, dataId)
804  visits = np.array([visitSummary.ref.dataId['visit'] for visitSummary in visitSummaries])
805  radius = np.empty(len(visits))
806  intersects = np.full(len(visits), True)
807  for i, visitSummary in enumerate(visitSummaries):
808  # read in one-by-one and only once. There may be hundreds
809  visitSummary = visitSummary.get()
810  # psfSigma is PSF model determinant radius at chip center in pixels
811  psfSigma = np.nanmedian([vs['psfSigma'] for vs in visitSummary])
812  radius[i] = psfSigma
813  if self.config.doConfirmOverlap:
814  intersects[i] = self.doesIntersectPolygon(visitSummary, patchPolygon)
815  if self.config.minMJD or self.config.maxMJD:
816  # mjd is guaranteed to be the same for every detector in the visitSummary.
817  mjd = visitSummary[0].getVisitInfo().getDate().get(system=DateTime.MJD)
818  aboveMin = mjd > self.config.minMJD if self.config.minMJD else True
819  belowMax = mjd < self.config.maxMJD if self.config.maxMJD else True
820  intersects[i] = intersects[i] and aboveMin and belowMax
821 
822  sortedVisits = [v for rad, v in sorted(zip(radius[intersects], visits[intersects]))]
823  lowerBound = min(int(np.round(self.config.qMin*len(visits[intersects]))),
824  max(0, len(visits[intersects]) - self.config.nVisitsMin))
825  upperBound = max(int(np.round(self.config.qMax*len(visits[intersects]))), self.config.nVisitsMin)
826 
827  # In order to store as a StructuredDataDict, convert list to dict
828  goodVisits = {int(visit): True for visit in sortedVisits[lowerBound:upperBound]}
829  return pipeBase.Struct(goodVisits=goodVisits)
def __init__(self, dataId, coordList)
Definition: selectImages.py:63
def runDataRef(self, dataRef, coordList, makeDataRefList=True, selectDataList=[])
def __init__(self, dataRef, wcs, bbox)
def getValidImageCorners(self, imageWcs, imageBox, patchPoly, dataId=None)
def runDataRef(self, dataRef, coordList, makeDataRefList=True, selectDataList=[])
def run(self, wcsList, bboxList, coordList, dataIds=None, **kwargs)
static ConvexPolygon convexHull(std::vector< UnitVector3d > const &points)