lsst.pipe.tasks  v23.0.x-g0162f352a6+f2fbba1123
selectImages.py
Go to the documentation of this file.
1 #
2 # LSST Data Management System
3 # Copyright 2008, 2009, 2010 LSST Corporation.
4 #
5 # This product includes software developed by the
6 # LSST Project (http://www.lsst.org/).
7 #
8 # This program is free software: you can redistribute it and/or modify
9 # it under the terms of the GNU General Public License as published by
10 # the Free Software Foundation, either version 3 of the License, or
11 # (at your option) any later version.
12 #
13 # This program is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU General Public License for more details.
17 #
18 # You should have received a copy of the LSST License Statement and
19 # the GNU General Public License along with this program. If not,
20 # see <http://www.lsstcorp.org/LegalNotices/>.
21 #
22 import numpy as np
23 import lsst.sphgeom
24 import lsst.utils as utils
25 import lsst.pex.config as pexConfig
26 import lsst.pex.exceptions as pexExceptions
27 import lsst.geom as geom
28 import lsst.pipe.base as pipeBase
29 from lsst.skymap import BaseSkyMap
30 from lsst.daf.base import DateTime
31 
32 __all__ = ["BaseSelectImagesTask", "BaseExposureInfo", "WcsSelectImagesTask", "PsfWcsSelectImagesTask",
33  "DatabaseSelectImagesConfig", "BestSeeingWcsSelectImagesTask", "BestSeeingSelectVisitsTask",
34  "BestSeeingQuantileSelectVisitsTask"]
35 
36 
37 class DatabaseSelectImagesConfig(pexConfig.Config):
38  """Base configuration for subclasses of BaseSelectImagesTask that use a database"""
39  host = pexConfig.Field(
40  doc="Database server host name",
41  dtype=str,
42  )
43  port = pexConfig.Field(
44  doc="Database server port",
45  dtype=int,
46  )
47  database = pexConfig.Field(
48  doc="Name of database",
49  dtype=str,
50  )
51  maxExposures = pexConfig.Field(
52  doc="maximum exposures to select; intended for debugging; ignored if None",
53  dtype=int,
54  optional=True,
55  )
56 
57 
58 class BaseExposureInfo(pipeBase.Struct):
59  """Data about a selected exposure
60  """
61 
62  def __init__(self, dataId, coordList):
63  """Create exposure information that can be used to generate data references
64 
65  The object has the following fields:
66  - dataId: data ID of exposure (a dict)
67  - coordList: ICRS coordinates of the corners of the exposure (list of lsst.geom.SpherePoint)
68  plus any others items that are desired
69  """
70  super(BaseExposureInfo, self).__init__(dataId=dataId, coordList=coordList)
71 
72 
73 class BaseSelectImagesTask(pipeBase.Task):
74  """Base task for selecting images suitable for coaddition
75  """
76  ConfigClass = pexConfig.Config
77  _DefaultName = "selectImages"
78 
79  @pipeBase.timeMethod
80  def run(self, coordList):
81  """Select images suitable for coaddition in a particular region
82 
83  @param[in] coordList: list of coordinates defining region of interest; if None then select all images
84  subclasses may add additional keyword arguments, as required
85 
86  @return a pipeBase Struct containing:
87  - exposureInfoList: a list of exposure information objects (subclasses of BaseExposureInfo),
88  which have at least the following fields:
89  - dataId: data ID dictionary
90  - coordList: ICRS coordinates of the corners of the exposure (list of lsst.geom.SpherePoint)
91  """
92  raise NotImplementedError()
93 
94  def _runArgDictFromDataId(self, dataId):
95  """Extract keyword arguments for run (other than coordList) from a data ID
96 
97  @return keyword arguments for run (other than coordList), as a dict
98  """
99  raise NotImplementedError()
100 
101  def runDataRef(self, dataRef, coordList, makeDataRefList=True, selectDataList=[]):
102  """Run based on a data reference
103 
104  This delegates to run() and _runArgDictFromDataId() to do the actual
105  selection. In the event that the selectDataList is non-empty, this will
106  be used to further restrict the selection, providing the user with
107  additional control over the selection.
108 
109  @param[in] dataRef: data reference; must contain any extra keys needed by the subclass
110  @param[in] coordList: list of coordinates defining region of interest; if None, search the whole sky
111  @param[in] makeDataRefList: if True, return dataRefList
112  @param[in] selectDataList: List of SelectStruct with dataRefs to consider for selection
113  @return a pipeBase Struct containing:
114  - exposureInfoList: a list of objects derived from ExposureInfo
115  - dataRefList: a list of data references (None if makeDataRefList False)
116  """
117  runArgDict = self._runArgDictFromDataId_runArgDictFromDataId(dataRef.dataId)
118  exposureInfoList = self.runrun(coordList, **runArgDict).exposureInfoList
119 
120  if len(selectDataList) > 0 and len(exposureInfoList) > 0:
121  # Restrict the exposure selection further
122  ccdKeys, ccdValues = _extractKeyValue(exposureInfoList)
123  inKeys, inValues = _extractKeyValue([s.dataRef for s in selectDataList], keys=ccdKeys)
124  inValues = set(inValues)
125  newExposureInfoList = []
126  for info, ccdVal in zip(exposureInfoList, ccdValues):
127  if ccdVal in inValues:
128  newExposureInfoList.append(info)
129  else:
130  self.log.info("De-selecting exposure %s: not in selectDataList", info.dataId)
131  exposureInfoList = newExposureInfoList
132 
133  if makeDataRefList:
134  butler = dataRef.butlerSubset.butler
135  dataRefList = [butler.dataRef(datasetType="calexp",
136  dataId=expInfo.dataId,
137  ) for expInfo in exposureInfoList]
138  else:
139  dataRefList = None
140 
141  return pipeBase.Struct(
142  dataRefList=dataRefList,
143  exposureInfoList=exposureInfoList,
144  )
145 
146 
147 def _extractKeyValue(dataList, keys=None):
148  """Extract the keys and values from a list of dataIds
149 
150  The input dataList is a list of objects that have 'dataId' members.
151  This allows it to be used for both a list of data references and a
152  list of ExposureInfo
153  """
154  assert len(dataList) > 0
155  if keys is None:
156  keys = sorted(dataList[0].dataId.keys())
157  keySet = set(keys)
158  values = list()
159  for data in dataList:
160  thisKeys = set(data.dataId.keys())
161  if thisKeys != keySet:
162  raise RuntimeError("DataId keys inconsistent: %s vs %s" % (keySet, thisKeys))
163  values.append(tuple(data.dataId[k] for k in keys))
164  return keys, values
165 
166 
167 class SelectStruct(pipeBase.Struct):
168  """A container for data to be passed to the WcsSelectImagesTask"""
169 
170  def __init__(self, dataRef, wcs, bbox):
171  super(SelectStruct, self).__init__(dataRef=dataRef, wcs=wcs, bbox=bbox)
172 
173 
175  """Select images using their Wcs
176 
177  We use the "convexHull" method of lsst.sphgeom.ConvexPolygon to define
178  polygons on the celestial sphere, and test the polygon of the
179  patch for overlap with the polygon of the image.
180 
181  We use "convexHull" instead of generating a ConvexPolygon
182  directly because the standard for the inputs to ConvexPolygon
183  are pretty high and we don't want to be responsible for reaching them.
184  """
185 
186  def runDataRef(self, dataRef, coordList, makeDataRefList=True, selectDataList=[]):
187  """Select images in the selectDataList that overlap the patch
188 
189  This method is the old entry point for the Gen2 commandline tasks and drivers
190  Will be deprecated in v22.
191 
192  @param dataRef: Data reference for coadd/tempExp (with tract, patch)
193  @param coordList: List of ICRS coordinates (lsst.geom.SpherePoint) specifying boundary of patch
194  @param makeDataRefList: Construct a list of data references?
195  @param selectDataList: List of SelectStruct, to consider for selection
196  """
197  dataRefList = []
198  exposureInfoList = []
199 
200  patchVertices = [coord.getVector() for coord in coordList]
201  patchPoly = lsst.sphgeom.ConvexPolygon.convexHull(patchVertices)
202 
203  for data in selectDataList:
204  dataRef = data.dataRef
205  imageWcs = data.wcs
206  imageBox = data.bbox
207 
208  imageCorners = self.getValidImageCornersgetValidImageCorners(imageWcs, imageBox, patchPoly, dataId=None)
209  if imageCorners:
210  dataRefList.append(dataRef)
211  exposureInfoList.append(BaseExposureInfo(dataRef.dataId, imageCorners))
212 
213  return pipeBase.Struct(
214  dataRefList=dataRefList if makeDataRefList else None,
215  exposureInfoList=exposureInfoList,
216  )
217 
218  def run(self, wcsList, bboxList, coordList, dataIds=None, **kwargs):
219  """Return indices of provided lists that meet the selection criteria
220 
221  Parameters:
222  -----------
223  wcsList : `list` of `lsst.afw.geom.SkyWcs`
224  specifying the WCS's of the input ccds to be selected
225  bboxList : `list` of `lsst.geom.Box2I`
226  specifying the bounding boxes of the input ccds to be selected
227  coordList : `list` of `lsst.geom.SpherePoint`
228  ICRS coordinates specifying boundary of the patch.
229 
230  Returns:
231  --------
232  result: `list` of `int`
233  of indices of selected ccds
234  """
235  if dataIds is None:
236  dataIds = [None] * len(wcsList)
237  patchVertices = [coord.getVector() for coord in coordList]
238  patchPoly = lsst.sphgeom.ConvexPolygon.convexHull(patchVertices)
239  result = []
240  for i, (imageWcs, imageBox, dataId) in enumerate(zip(wcsList, bboxList, dataIds)):
241  imageCorners = self.getValidImageCornersgetValidImageCorners(imageWcs, imageBox, patchPoly, dataId)
242  if imageCorners:
243  result.append(i)
244  return result
245 
246  def getValidImageCorners(self, imageWcs, imageBox, patchPoly, dataId=None):
247  "Return corners or None if bad"
248  try:
249  imageCorners = [imageWcs.pixelToSky(pix) for pix in geom.Box2D(imageBox).getCorners()]
250  except (pexExceptions.DomainError, pexExceptions.RuntimeError) as e:
251  # Protecting ourselves from awful Wcs solutions in input images
252  self.log.debug("WCS error in testing calexp %s (%s): deselecting", dataId, e)
253  return
254 
255  imagePoly = lsst.sphgeom.ConvexPolygon.convexHull([coord.getVector() for coord in imageCorners])
256  if imagePoly is None:
257  self.log.debug("Unable to create polygon from image %s: deselecting", dataId)
258  return
259 
260  if patchPoly.intersects(imagePoly):
261  # "intersects" also covers "contains" or "is contained by"
262  self.log.info("Selecting calexp %s", dataId)
263  return imageCorners
264 
265 
266 def sigmaMad(array):
267  "Return median absolute deviation scaled to normally distributed data"
268  return 1.4826*np.median(np.abs(array - np.median(array)))
269 
270 
271 class PsfWcsSelectImagesConnections(pipeBase.PipelineTaskConnections,
272  dimensions=("tract", "patch", "skymap", "instrument", "visit"),
273  defaultTemplates={"coaddName": "deep"}):
274  pass
275 
276 
277 class PsfWcsSelectImagesConfig(pipeBase.PipelineTaskConfig,
278  pipelineConnections=PsfWcsSelectImagesConnections):
279  maxEllipResidual = pexConfig.Field(
280  doc="Maximum median ellipticity residual",
281  dtype=float,
282  default=0.007,
283  optional=True,
284  )
285  maxSizeScatter = pexConfig.Field(
286  doc="Maximum scatter in the size residuals",
287  dtype=float,
288  optional=True,
289  )
290  maxScaledSizeScatter = pexConfig.Field(
291  doc="Maximum scatter in the size residuals, scaled by the median size",
292  dtype=float,
293  default=0.009,
294  optional=True,
295  )
296  starSelection = pexConfig.Field(
297  doc="select star with this field",
298  dtype=str,
299  default='calib_psf_used',
300  deprecated=('This field has been moved to ComputeExposureSummaryStatsTask and '
301  'will be removed after v24.')
302  )
303  starShape = pexConfig.Field(
304  doc="name of star shape",
305  dtype=str,
306  default='base_SdssShape',
307  deprecated=('This field has been moved to ComputeExposureSummaryStatsTask and '
308  'will be removed after v24.')
309  )
310  psfShape = pexConfig.Field(
311  doc="name of psf shape",
312  dtype=str,
313  default='base_SdssShape_psf',
314  deprecated=('This field has been moved to ComputeExposureSummaryStatsTask and '
315  'will be removed after v24.')
316  )
317 
318 
319 class PsfWcsSelectImagesTask(WcsSelectImagesTask):
320  """Select images using their Wcs and cuts on the PSF properties
321 
322  The PSF quality criteria are based on the size and ellipticity residuals from the
323  adaptive second moments of the star and the PSF.
324 
325  The criteria are:
326  - the median of the ellipticty residuals
327  - the robust scatter of the size residuals (using the median absolute deviation)
328  - the robust scatter of the size residuals scaled by the square of
329  the median size
330  """
331 
332  ConfigClass = PsfWcsSelectImagesConfig
333  _DefaultName = "PsfWcsSelectImages"
334 
335  def runDataRef(self, dataRef, coordList, makeDataRefList=True, selectDataList=[]):
336  """Select images in the selectDataList that overlap the patch and satisfy PSF quality critera.
337 
338  This method is the old entry point for the Gen2 commandline tasks and drivers
339  Will be deprecated in v22.
340 
341  @param dataRef: Data reference for coadd/tempExp (with tract, patch)
342  @param coordList: List of ICRS coordinates (lsst.geom.SpherePoint) specifying boundary of patch
343  @param makeDataRefList: Construct a list of data references?
344  @param selectDataList: List of SelectStruct, to consider for selection
345  """
346  result = super(PsfWcsSelectImagesTask, self).runDataRef(dataRef, coordList, makeDataRefList,
347  selectDataList)
348 
349  dataRefList = []
350  exposureInfoList = []
351  for dataRef, exposureInfo in zip(result.dataRefList, result.exposureInfoList):
352  butler = dataRef.butlerSubset.butler
353  srcCatalog = butler.get('src', dataRef.dataId)
354  valid = self.isValidGen2(srcCatalog, dataRef.dataId)
355  if valid is False:
356  continue
357 
358  dataRefList.append(dataRef)
359  exposureInfoList.append(exposureInfo)
360 
361  return pipeBase.Struct(
362  dataRefList=dataRefList,
363  exposureInfoList=exposureInfoList,
364  )
365 
366  def run(self, wcsList, bboxList, coordList, visitSummary, dataIds=None, **kwargs):
367  """Return indices of provided lists that meet the selection criteria
368 
369  Parameters:
370  -----------
371  wcsList : `list` of `lsst.afw.geom.SkyWcs`
372  specifying the WCS's of the input ccds to be selected
373  bboxList : `list` of `lsst.geom.Box2I`
374  specifying the bounding boxes of the input ccds to be selected
375  coordList : `list` of `lsst.geom.SpherePoint`
376  ICRS coordinates specifying boundary of the patch.
377  visitSummary : `list` of `lsst.afw.table.ExposureCatalog`
378  containing the PSF shape information for the input ccds to be selected
379 
380  Returns:
381  --------
382  goodPsf: `list` of `int`
383  of indices of selected ccds
384  """
385  goodWcs = super(PsfWcsSelectImagesTask, self).run(wcsList=wcsList, bboxList=bboxList,
386  coordList=coordList, dataIds=dataIds)
387 
388  goodPsf = []
389 
390  for i, dataId in enumerate(dataIds):
391  if i not in goodWcs:
392  continue
393  if self.isValid(visitSummary, dataId['detector']):
394  goodPsf.append(i)
395 
396  return goodPsf
397 
398  def isValid(self, visitSummary, detectorId):
399  """Should this ccd be selected based on its PSF shape information.
400 
401  Parameters
402  ----------
403  visitSummary : `lsst.afw.table.ExposureCatalog`
404  detectorId : `int`
405  Detector identifier.
406 
407  Returns
408  -------
409  valid : `bool`
410  True if selected.
411  """
412  row = visitSummary.find(detectorId)
413  if row is None:
414  # This is not listed, so it must be bad.
415  self.log.warning("Removing visit %d detector %d because summary stats not available.",
416  row["visit"], detectorId)
417  return False
418 
419  medianE = np.sqrt(row["psfStarDeltaE1Median"]**2. + row["psfStarDeltaE2Median"]**2.)
420  scatterSize = row["psfStarDeltaSizeScatter"]
421  scaledScatterSize = row["psfStarScaledDeltaSizeScatter"]
422 
423  valid = True
424  if self.config.maxEllipResidual and medianE > self.config.maxEllipResidual:
425  self.log.info("Removing visit %d detector %d because median e residual too large: %f vs %f",
426  row["visit"], detectorId, medianE, self.config.maxEllipResidual)
427  valid = False
428  elif self.config.maxSizeScatter and scatterSize > self.config.maxSizeScatter:
429  self.log.info("Removing visit %d detector %d because size scatter too large: %f vs %f",
430  row["visit"], detectorId, scatterSize, self.config.maxSizeScatter)
431  valid = False
432  elif self.config.maxScaledSizeScatter and scaledScatterSize > self.config.maxScaledSizeScatter:
433  self.log.info("Removing visit %d detector %d because scaled size scatter too large: %f vs %f",
434  row["visit"], detectorId, scaledScatterSize, self.config.maxScaledSizeScatter)
435  valid = False
436 
437  return valid
438 
439  def isValidGen2(self, srcCatalog, dataId=None):
440  """Should this ccd be selected based on its PSF shape information.
441 
442  This routine is only used in Gen2 processing, and can be
443  removed when Gen2 is retired.
444 
445  Parameters
446  ----------
447  srcCatalog : `lsst.afw.table.SourceCatalog`
448  dataId : `dict` of dataId keys, optional.
449  Used only for logging. Defaults to None.
450 
451  Returns
452  -------
453  valid : `bool`
454  True if selected.
455  """
456  mask = srcCatalog[self.config.starSelection]
457 
458  starXX = srcCatalog[self.config.starShape+'_xx'][mask]
459  starYY = srcCatalog[self.config.starShape+'_yy'][mask]
460  starXY = srcCatalog[self.config.starShape+'_xy'][mask]
461  psfXX = srcCatalog[self.config.psfShape+'_xx'][mask]
462  psfYY = srcCatalog[self.config.psfShape+'_yy'][mask]
463  psfXY = srcCatalog[self.config.psfShape+'_xy'][mask]
464 
465  starSize = np.power(starXX*starYY - starXY**2, 0.25)
466  starE1 = (starXX - starYY)/(starXX + starYY)
467  starE2 = 2*starXY/(starXX + starYY)
468  medianSize = np.median(starSize)
469 
470  psfSize = np.power(psfXX*psfYY - psfXY**2, 0.25)
471  psfE1 = (psfXX - psfYY)/(psfXX + psfYY)
472  psfE2 = 2*psfXY/(psfXX + psfYY)
473 
474  medianE1 = np.abs(np.median(starE1 - psfE1))
475  medianE2 = np.abs(np.median(starE2 - psfE2))
476  medianE = np.sqrt(medianE1**2 + medianE2**2)
477 
478  scatterSize = sigmaMad(starSize - psfSize)
479  scaledScatterSize = scatterSize/medianSize**2
480 
481  valid = True
482  if self.config.maxEllipResidual and medianE > self.config.maxEllipResidual:
483  self.log.info("Removing visit %s because median e residual too large: %f vs %f",
484  dataId, medianE, self.config.maxEllipResidual)
485  valid = False
486  elif self.config.maxSizeScatter and scatterSize > self.config.maxSizeScatter:
487  self.log.info("Removing visit %s because size scatter is too large: %f vs %f",
488  dataId, scatterSize, self.config.maxSizeScatter)
489  valid = False
490  elif self.config.maxScaledSizeScatter and scaledScatterSize > self.config.maxScaledSizeScatter:
491  self.log.info("Removing visit %s because scaled size scatter is too large: %f vs %f",
492  dataId, scaledScatterSize, self.config.maxScaledSizeScatter)
493  valid = False
494 
495  return valid
496 
497 
498 class BestSeeingWcsSelectImageConfig(WcsSelectImagesTask.ConfigClass):
499  """Base configuration for BestSeeingSelectImagesTask.
500  """
501  nImagesMax = pexConfig.RangeField(
502  dtype=int,
503  doc="Maximum number of images to select",
504  default=5,
505  min=0)
506  maxPsfFwhm = pexConfig.Field(
507  dtype=float,
508  doc="Maximum PSF FWHM (in arcseconds) to select",
509  default=1.5,
510  optional=True)
511  minPsfFwhm = pexConfig.Field(
512  dtype=float,
513  doc="Minimum PSF FWHM (in arcseconds) to select",
514  default=0.,
515  optional=True)
516 
517 
518 class BestSeeingWcsSelectImagesTask(WcsSelectImagesTask):
519  """Select up to a maximum number of the best-seeing images using their Wcs.
520  """
521  ConfigClass = BestSeeingWcsSelectImageConfig
522 
523  def runDataRef(self, dataRef, coordList, makeDataRefList=True,
524  selectDataList=None):
525  """Select the best-seeing images in the selectDataList that overlap the patch.
526 
527  This method is the old entry point for the Gen2 commandline tasks and drivers
528  Will be deprecated in v22.
529 
530  Parameters
531  ----------
532  dataRef : `lsst.daf.persistence.ButlerDataRef`
533  Data reference for coadd/tempExp (with tract, patch)
534  coordList : `list` of `lsst.geom.SpherePoint`
535  List of ICRS sky coordinates specifying boundary of patch
536  makeDataRefList : `boolean`, optional
537  Construct a list of data references?
538  selectDataList : `list` of `SelectStruct`
539  List of SelectStruct, to consider for selection
540 
541  Returns
542  -------
543  result : `lsst.pipe.base.Struct`
544  Result struct with components:
545  - ``exposureList``: the selected exposures
546  (`list` of `lsst.pipe.tasks.selectImages.BaseExposureInfo`).
547  - ``dataRefList``: the optional data references corresponding to
548  each element of ``exposureList``
549  (`list` of `lsst.daf.persistence.ButlerDataRef`, or `None`).
550  """
551  psfSizes = []
552  dataRefList = []
553  exposureInfoList = []
554 
555  if selectDataList is None:
556  selectDataList = []
557 
558  result = super().runDataRef(dataRef, coordList, makeDataRefList=True, selectDataList=selectDataList)
559 
560  for dataRef, exposureInfo in zip(result.dataRefList, result.exposureInfoList):
561  cal = dataRef.get("calexp", immediate=True)
562 
563  # if min/max PSF values are defined, remove images out of bounds
564  pixToArcseconds = cal.getWcs().getPixelScale().asArcseconds()
565  psfSize = cal.getPsf().computeShape().getDeterminantRadius()*pixToArcseconds
566  sizeFwhm = psfSize * np.sqrt(8.*np.log(2.))
567  if self.config.maxPsfFwhm and sizeFwhm > self.config.maxPsfFwhm:
568  continue
569  if self.config.minPsfFwhm and sizeFwhm < self.config.minPsfFwhm:
570  continue
571  psfSizes.append(sizeFwhm)
572  dataRefList.append(dataRef)
573  exposureInfoList.append(exposureInfo)
574 
575  if len(psfSizes) > self.config.nImagesMax:
576  sortedIndices = np.argsort(psfSizes)[:self.config.nImagesMax]
577  filteredDataRefList = [dataRefList[i] for i in sortedIndices]
578  filteredExposureInfoList = [exposureInfoList[i] for i in sortedIndices]
579  self.log.info("%d images selected with FWHM range of %f--%f arcseconds",
580  len(sortedIndices), psfSizes[sortedIndices[0]], psfSizes[sortedIndices[-1]])
581 
582  else:
583  if len(psfSizes) == 0:
584  self.log.warning("0 images selected.")
585  else:
586  self.log.debug("%d images selected with FWHM range of %d--%d arcseconds",
587  len(psfSizes), psfSizes[0], psfSizes[-1])
588  filteredDataRefList = dataRefList
589  filteredExposureInfoList = exposureInfoList
590 
591  return pipeBase.Struct(
592  dataRefList=filteredDataRefList if makeDataRefList else None,
593  exposureInfoList=filteredExposureInfoList,
594  )
595 
596 
597 class BestSeeingSelectVisitsConnections(pipeBase.PipelineTaskConnections,
598  dimensions=("tract", "patch", "skymap", "band", "instrument"),
599  defaultTemplates={"coaddName": "goodSeeing"}):
600  skyMap = pipeBase.connectionTypes.Input(
601  doc="Input definition of geometry/bbox and projection/wcs for coadded exposures",
602  name=BaseSkyMap.SKYMAP_DATASET_TYPE_NAME,
603  storageClass="SkyMap",
604  dimensions=("skymap",),
605  )
606  visitSummaries = pipeBase.connectionTypes.Input(
607  doc="Per-visit consolidated exposure metadata from ConsolidateVisitSummaryTask",
608  name="visitSummary",
609  storageClass="ExposureCatalog",
610  dimensions=("instrument", "visit",),
611  multiple=True,
612  deferLoad=True
613  )
614  goodVisits = pipeBase.connectionTypes.Output(
615  doc="Selected visits to be coadded.",
616  name="{coaddName}Visits",
617  storageClass="StructuredDataDict",
618  dimensions=("instrument", "tract", "patch", "skymap", "band"),
619  )
620 
621 
622 class BestSeeingSelectVisitsConfig(pipeBase.PipelineTaskConfig,
623  pipelineConnections=BestSeeingSelectVisitsConnections):
624  nVisitsMax = pexConfig.RangeField(
625  dtype=int,
626  doc="Maximum number of visits to select",
627  default=12,
628  min=0
629  )
630  maxPsfFwhm = pexConfig.Field(
631  dtype=float,
632  doc="Maximum PSF FWHM (in arcseconds) to select",
633  default=1.5,
634  optional=True
635  )
636  minPsfFwhm = pexConfig.Field(
637  dtype=float,
638  doc="Minimum PSF FWHM (in arcseconds) to select",
639  default=0.,
640  optional=True
641  )
642  doConfirmOverlap = pexConfig.Field(
643  dtype=bool,
644  doc="Do remove visits that do not actually overlap the patch?",
645  default=True,
646  )
647  minMJD = pexConfig.Field(
648  dtype=float,
649  doc="Minimum visit MJD to select",
650  default=None,
651  optional=True
652  )
653  maxMJD = pexConfig.Field(
654  dtype=float,
655  doc="Maximum visit MJD to select",
656  default=None,
657  optional=True
658  )
659 
660 
661 class BestSeeingSelectVisitsTask(pipeBase.PipelineTask):
662  """Select up to a maximum number of the best-seeing visits
663 
664  Don't exceed the FWHM range specified by configs min(max)PsfFwhm.
665  This Task is a port of the Gen2 image-selector used in the AP pipeline:
666  BestSeeingSelectImagesTask. This Task selects full visits based on the
667  average PSF of the entire visit.
668  """
669  ConfigClass = BestSeeingSelectVisitsConfig
670  _DefaultName = 'bestSeeingSelectVisits'
671 
672  def runQuantum(self, butlerQC, inputRefs, outputRefs):
673  inputs = butlerQC.get(inputRefs)
674  quantumDataId = butlerQC.quantum.dataId
675  outputs = self.run(**inputs, dataId=quantumDataId)
676  butlerQC.put(outputs, outputRefs)
677 
678  def run(self, visitSummaries, skyMap, dataId):
679  """Run task
680 
681  Parameters:
682  -----------
683  visitSummary : `list`
684  List of `lsst.pipe.base.connections.DeferredDatasetRef` of
685  visitSummary tables of type `lsst.afw.table.ExposureCatalog`
686  skyMap : `lsst.skyMap.SkyMap`
687  SkyMap for checking visits overlap patch
688  dataId : `dict` of dataId keys
689  For retrieving patch info for checking visits overlap patch
690 
691  Returns
692  -------
693  result : `lsst.pipe.base.Struct`
694  Result struct with components:
695 
696  - `goodVisits`: `dict` with selected visit ids as keys,
697  so that it can be be saved as a StructuredDataDict.
698  StructuredDataList's are currently limited.
699  """
700 
701  if self.config.doConfirmOverlap:
702  patchPolygon = self.makePatchPolygon(skyMap, dataId)
703 
704  inputVisits = [visitSummary.ref.dataId['visit'] for visitSummary in visitSummaries]
705  fwhmSizes = []
706  visits = []
707  for visit, visitSummary in zip(inputVisits, visitSummaries):
708  # read in one-by-one and only once. There may be hundreds
709  visitSummary = visitSummary.get()
710 
711  # mjd is guaranteed to be the same for every detector in the visitSummary.
712  mjd = visitSummary[0].getVisitInfo().getDate().get(system=DateTime.MJD)
713 
714  pixToArcseconds = [vs.getWcs().getPixelScale(vs.getBBox().getCenter()).asArcseconds()
715  for vs in visitSummary]
716  # psfSigma is PSF model determinant radius at chip center in pixels
717  psfSigmas = np.array([vs['psfSigma'] for vs in visitSummary])
718  fwhm = np.nanmean(psfSigmas * pixToArcseconds) * np.sqrt(8.*np.log(2.))
719 
720  if self.config.maxPsfFwhm and fwhm > self.config.maxPsfFwhm:
721  continue
722  if self.config.minPsfFwhm and fwhm < self.config.minPsfFwhm:
723  continue
724  if self.config.minMJD and mjd < self.config.minMJD:
725  self.log.debug('MJD %f earlier than %.2f; rejecting', mjd, self.config.minMJD)
726  continue
727  if self.config.maxMJD and mjd > self.config.maxMJD:
728  self.log.debug('MJD %f later than %.2f; rejecting', mjd, self.config.maxMJD)
729  continue
730  if self.config.doConfirmOverlap and not self.doesIntersectPolygon(visitSummary, patchPolygon):
731  continue
732 
733  fwhmSizes.append(fwhm)
734  visits.append(visit)
735 
736  sortedVisits = [ind for (_, ind) in sorted(zip(fwhmSizes, visits))]
737  output = sortedVisits[:self.config.nVisitsMax]
738  self.log.info("%d images selected with FWHM range of %d--%d arcseconds",
739  len(output), fwhmSizes[visits.index(output[0])], fwhmSizes[visits.index(output[-1])])
740 
741  # In order to store as a StructuredDataDict, convert list to dict
742  goodVisits = {key: True for key in output}
743  return pipeBase.Struct(goodVisits=goodVisits)
744 
745  def makePatchPolygon(self, skyMap, dataId):
746  """Return True if sky polygon overlaps visit
747 
748  Parameters:
749  -----------
750  skyMap : `lsst.afw.table.ExposureCatalog`
751  Exposure catalog with per-detector geometry
752  dataId : `dict` of dataId keys
753  For retrieving patch info
754 
755  Returns:
756  --------
757  result :` lsst.sphgeom.ConvexPolygon.convexHull`
758  Polygon of patch's outer bbox
759  """
760  wcs = skyMap[dataId['tract']].getWcs()
761  bbox = skyMap[dataId['tract']][dataId['patch']].getOuterBBox()
762  sphCorners = wcs.pixelToSky(lsst.geom.Box2D(bbox).getCorners())
763  result = lsst.sphgeom.ConvexPolygon.convexHull([coord.getVector() for coord in sphCorners])
764  return result
765 
766  def doesIntersectPolygon(self, visitSummary, polygon):
767  """Return True if sky polygon overlaps visit
768 
769  Parameters:
770  -----------
771  visitSummary : `lsst.afw.table.ExposureCatalog`
772  Exposure catalog with per-detector geometry
773  polygon :` lsst.sphgeom.ConvexPolygon.convexHull`
774  Polygon to check overlap
775 
776  Returns:
777  --------
778  doesIntersect: `bool`
779  Does the visit overlap the polygon
780  """
781  doesIntersect = False
782  for detectorSummary in visitSummary:
783  corners = [lsst.geom.SpherePoint(ra, decl, units=lsst.geom.degrees).getVector() for (ra, decl) in
784  zip(detectorSummary['raCorners'], detectorSummary['decCorners'])]
785  detectorPolygon = lsst.sphgeom.ConvexPolygon.convexHull(corners)
786  if detectorPolygon.intersects(polygon):
787  doesIntersect = True
788  break
789  return doesIntersect
790 
791 
792 class BestSeeingQuantileSelectVisitsConfig(pipeBase.PipelineTaskConfig,
793  pipelineConnections=BestSeeingSelectVisitsConnections):
794  qMin = pexConfig.RangeField(
795  doc="Lower bound of quantile range to select. Sorts visits by seeing from narrow to wide, "
796  "and select those in the interquantile range (qMin, qMax). Set qMin to 0 for Best Seeing. "
797  "This config should be changed from zero only for exploratory diffIm testing.",
798  dtype=float,
799  default=0,
800  min=0,
801  max=1,
802  )
803  qMax = pexConfig.RangeField(
804  doc="Upper bound of quantile range to select. Sorts visits by seeing from narrow to wide, "
805  "and select those in the interquantile range (qMin, qMax). Set qMax to 1 for Worst Seeing.",
806  dtype=float,
807  default=0.33,
808  min=0,
809  max=1,
810  )
811  nVisitsMin = pexConfig.Field(
812  doc="At least this number of visits selected and supercedes quantile. For example, if 10 visits "
813  "cover this patch, qMin=0.33, and nVisitsMin=5, the best 5 visits will be selected.",
814  dtype=int,
815  default=6,
816  )
817  doConfirmOverlap = pexConfig.Field(
818  dtype=bool,
819  doc="Do remove visits that do not actually overlap the patch?",
820  default=True,
821  )
822  minMJD = pexConfig.Field(
823  dtype=float,
824  doc="Minimum visit MJD to select",
825  default=None,
826  optional=True
827  )
828  maxMJD = pexConfig.Field(
829  dtype=float,
830  doc="Maximum visit MJD to select",
831  default=None,
832  optional=True
833  )
834 
835 
836 class BestSeeingQuantileSelectVisitsTask(BestSeeingSelectVisitsTask):
837  """Select a quantile of the best-seeing visits
838 
839  Selects the best (for example, third) full visits based on the average
840  PSF width in the entire visit. It can also be used for difference imaging
841  experiments that require templates with the worst seeing visits.
842  For example, selecting the worst third can be acheived by
843  changing the config parameters qMin to 0.66 and qMax to 1.
844  """
845  ConfigClass = BestSeeingQuantileSelectVisitsConfig
846  _DefaultName = 'bestSeeingQuantileSelectVisits'
847 
848  @utils.inheritDoc(BestSeeingSelectVisitsTask)
849  def run(self, visitSummaries, skyMap, dataId):
850  if self.config.doConfirmOverlap:
851  patchPolygon = self.makePatchPolygon(skyMap, dataId)
852  visits = np.array([visitSummary.ref.dataId['visit'] for visitSummary in visitSummaries])
853  radius = np.empty(len(visits))
854  intersects = np.full(len(visits), True)
855  for i, visitSummary in enumerate(visitSummaries):
856  # read in one-by-one and only once. There may be hundreds
857  visitSummary = visitSummary.get()
858  # psfSigma is PSF model determinant radius at chip center in pixels
859  psfSigma = np.nanmedian([vs['psfSigma'] for vs in visitSummary])
860  radius[i] = psfSigma
861  if self.config.doConfirmOverlap:
862  intersects[i] = self.doesIntersectPolygon(visitSummary, patchPolygon)
863  if self.config.minMJD or self.config.maxMJD:
864  # mjd is guaranteed to be the same for every detector in the visitSummary.
865  mjd = visitSummary[0].getVisitInfo().getDate().get(system=DateTime.MJD)
866  aboveMin = mjd > self.config.minMJD if self.config.minMJD else True
867  belowMax = mjd < self.config.maxMJD if self.config.maxMJD else True
868  intersects[i] = intersects[i] and aboveMin and belowMax
869 
870  sortedVisits = [v for rad, v in sorted(zip(radius[intersects], visits[intersects]))]
871  lowerBound = min(int(np.round(self.config.qMin*len(visits[intersects]))),
872  max(0, len(visits[intersects]) - self.config.nVisitsMin))
873  upperBound = max(int(np.round(self.config.qMax*len(visits[intersects]))), self.config.nVisitsMin)
874 
875  # In order to store as a StructuredDataDict, convert list to dict
876  goodVisits = {int(visit): True for visit in sortedVisits[lowerBound:upperBound]}
877  return pipeBase.Struct(goodVisits=goodVisits)
def __init__(self, dataId, coordList)
Definition: selectImages.py:62
def runDataRef(self, dataRef, coordList, makeDataRefList=True, selectDataList=[])
def __init__(self, dataRef, wcs, bbox)
def getValidImageCorners(self, imageWcs, imageBox, patchPoly, dataId=None)
def runDataRef(self, dataRef, coordList, makeDataRefList=True, selectDataList=[])
def run(self, wcsList, bboxList, coordList, dataIds=None, **kwargs)
static ConvexPolygon convexHull(std::vector< UnitVector3d > const &points)