24import lsst.utils
as utils
31from lsst.utils.timer
import timeMethod
33__all__ = [
"BaseSelectImagesTask",
"BaseExposureInfo",
"WcsSelectImagesTask",
"PsfWcsSelectImagesTask",
34 "DatabaseSelectImagesConfig",
"BestSeeingSelectVisitsTask",
35 "BestSeeingQuantileSelectVisitsTask"]
39 """Base configuration for subclasses of BaseSelectImagesTask that use a database"""
40 host = pexConfig.Field(
41 doc=
"Database server host name",
44 port = pexConfig.Field(
45 doc=
"Database server port",
48 database = pexConfig.Field(
49 doc=
"Name of database",
52 maxExposures = pexConfig.Field(
53 doc=
"maximum exposures to select; intended for debugging; ignored if None",
60 """Data about a selected exposure
64 """Create exposure information that can be used to generate data references
66 The object has the following fields:
67 - dataId: data ID of exposure (a dict)
69 plus any others items that are desired
71 super(BaseExposureInfo, self).__init__(dataId=dataId, coordList=coordList)
75 """Base task for selecting images suitable for coaddition
77 ConfigClass = pexConfig.Config
78 _DefaultName = "selectImages"
81 def run(self, coordList):
82 """Select images suitable for coaddition in a particular region
84 @param[
in] coordList: list of coordinates defining region of interest;
if None then select all images
85 subclasses may add additional keyword arguments,
as required
87 @return a pipeBase Struct containing:
88 - exposureInfoList: a list of exposure information objects (subclasses of BaseExposureInfo),
89 which have at least the following fields:
90 - dataId: data ID dictionary
93 raise NotImplementedError()
96def _extractKeyValue(dataList, keys=None):
97 """Extract the keys and values from a list of dataIds
99 The input dataList is a list of objects that have
'dataId' members.
100 This allows it to be used
for both a list of data references
and a
103 assert len(dataList) > 0
105 keys = sorted(dataList[0].dataId.keys())
108 for data
in dataList:
109 thisKeys = set(data.dataId.keys())
110 if thisKeys != keySet:
111 raise RuntimeError(
"DataId keys inconsistent: %s vs %s" % (keySet, thisKeys))
112 values.append(tuple(data.dataId[k]
for k
in keys))
117 """A container for data to be passed to the WcsSelectImagesTask"""
120 super(SelectStruct, self).
__init__(dataRef=dataRef, wcs=wcs, bbox=bbox)
124 """Select images using their Wcs
127 polygons on the celestial sphere,
and test the polygon of the
128 patch
for overlap
with the polygon of the image.
130 We use
"convexHull" instead of generating a ConvexPolygon
131 directly because the standard
for the inputs to ConvexPolygon
132 are pretty high
and we don
't want to be responsible for reaching them.
135 def run(self, wcsList, bboxList, coordList, dataIds=None, **kwargs):
136 """Return indices of provided lists that meet the selection criteria
141 specifying the WCS's of the input ccds to be selected
143 specifying the bounding boxes of the input ccds to be selected
145 ICRS coordinates specifying boundary of the patch.
149 result: `list` of `int`
150 of indices of selected ccds
153 dataIds = [
None] * len(wcsList)
154 patchVertices = [coord.getVector()
for coord
in coordList]
157 for i, (imageWcs, imageBox, dataId)
in enumerate(zip(wcsList, bboxList, dataIds)):
164 "Return corners or None if bad"
166 imageCorners = [imageWcs.pixelToSky(pix)
for pix
in geom.Box2D(imageBox).getCorners()]
167 except (pexExceptions.DomainError, pexExceptions.RuntimeError)
as e:
169 self.log.debug(
"WCS error in testing calexp %s (%s): deselecting", dataId, e)
173 if imagePoly
is None:
174 self.log.debug(
"Unable to create polygon from image %s: deselecting", dataId)
177 if patchPoly.intersects(imagePoly):
179 self.log.info(
"Selecting calexp %s", dataId)
184 dimensions=(
"tract",
"patch",
"skymap",
"instrument",
"visit"),
185 defaultTemplates={
"coaddName":
"deep"}):
189class PsfWcsSelectImagesConfig(pipeBase.PipelineTaskConfig,
190 pipelineConnections=PsfWcsSelectImagesConnections):
191 maxEllipResidual = pexConfig.Field(
192 doc=
"Maximum median ellipticity residual",
197 maxSizeScatter = pexConfig.Field(
198 doc=
"Maximum scatter in the size residuals",
202 maxScaledSizeScatter = pexConfig.Field(
203 doc=
"Maximum scatter in the size residuals, scaled by the median size",
211 """Select images using their Wcs and cuts on the PSF properties
213 The PSF quality criteria are based on the size and ellipticity residuals
from the
214 adaptive second moments of the star
and the PSF.
217 - the median of the ellipticty residuals
218 - the robust scatter of the size residuals (using the median absolute deviation)
219 - the robust scatter of the size residuals scaled by the square of
223 ConfigClass = PsfWcsSelectImagesConfig
224 _DefaultName = "PsfWcsSelectImages"
226 def run(self, wcsList, bboxList, coordList, visitSummary, dataIds=None, **kwargs):
227 """Return indices of provided lists that meet the selection criteria
232 specifying the WCS's of the input ccds to be selected
234 specifying the bounding boxes of the input ccds to be selected
236 ICRS coordinates specifying boundary of the patch.
238 containing the PSF shape information for the input ccds to be selected.
242 goodPsf: `list` of `int`
243 of indices of selected ccds
245 goodWcs = super(PsfWcsSelectImagesTask, self).run(wcsList=wcsList, bboxList=bboxList,
246 coordList=coordList, dataIds=dataIds)
250 for i, dataId
in enumerate(dataIds):
253 if self.isValid(visitSummary, dataId[
"detector"]):
258 def isValid(self, visitSummary, detectorId):
259 """Should this ccd be selected based on its PSF shape information.
272 row = visitSummary.find(detectorId)
275 self.log.warning(
"Removing detector %d because summary stats not available.", detectorId)
278 medianE = np.sqrt(row[
"psfStarDeltaE1Median"]**2. + row[
"psfStarDeltaE2Median"]**2.)
279 scatterSize = row[
"psfStarDeltaSizeScatter"]
280 scaledScatterSize = row[
"psfStarScaledDeltaSizeScatter"]
283 if self.config.maxEllipResidual
and medianE > self.config.maxEllipResidual:
284 self.log.info(
"Removing visit %d detector %d because median e residual too large: %f vs %f",
285 row[
"visit"], detectorId, medianE, self.config.maxEllipResidual)
287 elif self.config.maxSizeScatter
and scatterSize > self.config.maxSizeScatter:
288 self.log.info(
"Removing visit %d detector %d because size scatter too large: %f vs %f",
289 row[
"visit"], detectorId, scatterSize, self.config.maxSizeScatter)
291 elif self.config.maxScaledSizeScatter
and scaledScatterSize > self.config.maxScaledSizeScatter:
292 self.log.info(
"Removing visit %d detector %d because scaled size scatter too large: %f vs %f",
293 row[
"visit"], detectorId, scaledScatterSize, self.config.maxScaledSizeScatter)
299class BestSeeingSelectVisitsConnections(pipeBase.PipelineTaskConnections,
300 dimensions=(
"tract",
"patch",
"skymap",
"band",
"instrument"),
301 defaultTemplates={
"coaddName":
"goodSeeing"}):
302 skyMap = pipeBase.connectionTypes.Input(
303 doc=
"Input definition of geometry/bbox and projection/wcs for coadded exposures",
304 name=BaseSkyMap.SKYMAP_DATASET_TYPE_NAME,
305 storageClass=
"SkyMap",
306 dimensions=(
"skymap",),
308 visitSummaries = pipeBase.connectionTypes.Input(
309 doc=
"Per-visit consolidated exposure metadata from ConsolidateVisitSummaryTask",
311 storageClass=
"ExposureCatalog",
312 dimensions=(
"instrument",
"visit",),
316 goodVisits = pipeBase.connectionTypes.Output(
317 doc=
"Selected visits to be coadded.",
318 name=
"{coaddName}Visits",
319 storageClass=
"StructuredDataDict",
320 dimensions=(
"instrument",
"tract",
"patch",
"skymap",
"band"),
324class BestSeeingSelectVisitsConfig(pipeBase.PipelineTaskConfig,
325 pipelineConnections=BestSeeingSelectVisitsConnections):
326 nVisitsMax = pexConfig.RangeField(
328 doc=
"Maximum number of visits to select",
332 maxPsfFwhm = pexConfig.Field(
334 doc=
"Maximum PSF FWHM (in arcseconds) to select",
338 minPsfFwhm = pexConfig.Field(
340 doc=
"Minimum PSF FWHM (in arcseconds) to select",
344 doConfirmOverlap = pexConfig.Field(
346 doc=
"Do remove visits that do not actually overlap the patch?",
349 minMJD = pexConfig.Field(
351 doc=
"Minimum visit MJD to select",
355 maxMJD = pexConfig.Field(
357 doc=
"Maximum visit MJD to select",
363class BestSeeingSelectVisitsTask(pipeBase.PipelineTask):
364 """Select up to a maximum number of the best-seeing visits
366 Don't exceed the FWHM range specified by configs min(max)PsfFwhm.
367 This Task is a port of the Gen2 image-selector used
in the AP pipeline:
368 BestSeeingSelectImagesTask. This Task selects full visits based on the
369 average PSF of the entire visit.
371 ConfigClass = BestSeeingSelectVisitsConfig
372 _DefaultName = 'bestSeeingSelectVisits'
374 def runQuantum(self, butlerQC, inputRefs, outputRefs):
375 inputs = butlerQC.get(inputRefs)
376 quantumDataId = butlerQC.quantum.dataId
377 outputs = self.run(**inputs, dataId=quantumDataId)
378 butlerQC.put(outputs, outputRefs)
380 def run(self, visitSummaries, skyMap, dataId):
385 visitSummary : `list`
386 List of `lsst.pipe.base.connections.DeferredDatasetRef` of
388 skyMap : `lsst.skyMap.SkyMap`
389 SkyMap for checking visits overlap patch
390 dataId : `dict` of dataId keys
391 For retrieving patch info
for checking visits overlap patch
395 result : `lsst.pipe.base.Struct`
396 Result struct
with components:
398 - `goodVisits`: `dict`
with selected visit ids
as keys,
399 so that it can be be saved
as a StructuredDataDict.
400 StructuredDataList
's are currently limited.
403 if self.config.doConfirmOverlap:
404 patchPolygon = self.makePatchPolygon(skyMap, dataId)
406 inputVisits = [visitSummary.ref.dataId[
'visit']
for visitSummary
in visitSummaries]
409 for visit, visitSummary
in zip(inputVisits, visitSummaries):
411 visitSummary = visitSummary.get()
414 mjd = visitSummary[0].getVisitInfo().getDate().get(system=DateTime.MJD)
416 pixToArcseconds = [vs.getWcs().getPixelScale(vs.getBBox().getCenter()).asArcseconds()
417 for vs
in visitSummary]
419 psfSigmas = np.array([vs[
'psfSigma']
for vs
in visitSummary])
420 fwhm = np.nanmean(psfSigmas * pixToArcseconds) * np.sqrt(8.*np.log(2.))
422 if self.config.maxPsfFwhm
and fwhm > self.config.maxPsfFwhm:
424 if self.config.minPsfFwhm
and fwhm < self.config.minPsfFwhm:
426 if self.config.minMJD
and mjd < self.config.minMJD:
427 self.log.debug(
'MJD %f earlier than %.2f; rejecting', mjd, self.config.minMJD)
429 if self.config.maxMJD
and mjd > self.config.maxMJD:
430 self.log.debug(
'MJD %f later than %.2f; rejecting', mjd, self.config.maxMJD)
432 if self.config.doConfirmOverlap
and not self.doesIntersectPolygon(visitSummary, patchPolygon):
435 fwhmSizes.append(fwhm)
438 sortedVisits = [ind
for (_, ind)
in sorted(zip(fwhmSizes, visits))]
439 output = sortedVisits[:self.config.nVisitsMax]
440 self.log.info(
"%d images selected with FWHM range of %d--%d arcseconds",
441 len(output), fwhmSizes[visits.index(output[0])], fwhmSizes[visits.index(output[-1])])
444 goodVisits = {key:
True for key
in output}
445 return pipeBase.Struct(goodVisits=goodVisits)
447 def makePatchPolygon(self, skyMap, dataId):
448 """Return True if sky polygon overlaps visit
453 Exposure catalog with per-detector geometry
454 dataId : `dict` of dataId keys
455 For retrieving patch info
459 result :` lsst.sphgeom.ConvexPolygon.convexHull`
460 Polygon of patch
's outer bbox
462 wcs = skyMap[dataId['tract']].getWcs()
463 bbox = skyMap[dataId[
'tract']][dataId[
'patch']].getOuterBBox()
468 def doesIntersectPolygon(self, visitSummary, polygon):
469 """Return True if sky polygon overlaps visit
474 Exposure catalog with per-detector geometry
475 polygon :` lsst.sphgeom.ConvexPolygon.convexHull`
476 Polygon to check overlap
480 doesIntersect: `bool`
481 Does the visit overlap the polygon
483 doesIntersect = False
484 for detectorSummary
in visitSummary:
486 zip(detectorSummary[
'raCorners'], detectorSummary[
'decCorners'])]
488 if detectorPolygon.intersects(polygon):
494class BestSeeingQuantileSelectVisitsConfig(pipeBase.PipelineTaskConfig,
495 pipelineConnections=BestSeeingSelectVisitsConnections):
496 qMin = pexConfig.RangeField(
497 doc=
"Lower bound of quantile range to select. Sorts visits by seeing from narrow to wide, "
498 "and select those in the interquantile range (qMin, qMax). Set qMin to 0 for Best Seeing. "
499 "This config should be changed from zero only for exploratory diffIm testing.",
505 qMax = pexConfig.RangeField(
506 doc=
"Upper bound of quantile range to select. Sorts visits by seeing from narrow to wide, "
507 "and select those in the interquantile range (qMin, qMax). Set qMax to 1 for Worst Seeing.",
513 nVisitsMin = pexConfig.Field(
514 doc=
"At least this number of visits selected and supercedes quantile. For example, if 10 visits "
515 "cover this patch, qMin=0.33, and nVisitsMin=5, the best 5 visits will be selected.",
519 doConfirmOverlap = pexConfig.Field(
521 doc=
"Do remove visits that do not actually overlap the patch?",
524 minMJD = pexConfig.Field(
526 doc=
"Minimum visit MJD to select",
530 maxMJD = pexConfig.Field(
532 doc=
"Maximum visit MJD to select",
538class BestSeeingQuantileSelectVisitsTask(BestSeeingSelectVisitsTask):
539 """Select a quantile of the best-seeing visits
541 Selects the best (for example, third) full visits based on the average
542 PSF width
in the entire visit. It can also be used
for difference imaging
543 experiments that require templates
with the worst seeing visits.
544 For example, selecting the worst third can be acheived by
545 changing the config parameters qMin to 0.66
and qMax to 1.
547 ConfigClass = BestSeeingQuantileSelectVisitsConfig
548 _DefaultName = 'bestSeeingQuantileSelectVisits'
550 @utils.inheritDoc(BestSeeingSelectVisitsTask)
551 def run(self, visitSummaries, skyMap, dataId):
552 if self.config.doConfirmOverlap:
553 patchPolygon = self.makePatchPolygon(skyMap, dataId)
554 visits = np.array([visitSummary.ref.dataId[
'visit']
for visitSummary
in visitSummaries])
555 radius = np.empty(len(visits))
556 intersects = np.full(len(visits),
True)
557 for i, visitSummary
in enumerate(visitSummaries):
559 visitSummary = visitSummary.get()
561 psfSigma = np.nanmedian([vs[
'psfSigma']
for vs
in visitSummary])
563 if self.config.doConfirmOverlap:
564 intersects[i] = self.doesIntersectPolygon(visitSummary, patchPolygon)
565 if self.config.minMJD
or self.config.maxMJD:
567 mjd = visitSummary[0].getVisitInfo().getDate().get(system=DateTime.MJD)
568 aboveMin = mjd > self.config.minMJD
if self.config.minMJD
else True
569 belowMax = mjd < self.config.maxMJD
if self.config.maxMJD
else True
570 intersects[i] = intersects[i]
and aboveMin
and belowMax
572 sortedVisits = [v
for rad, v
in sorted(zip(radius[intersects], visits[intersects]))]
573 lowerBound = min(int(np.round(self.config.qMin*len(visits[intersects]))),
574 max(0, len(visits[intersects]) - self.config.nVisitsMin))
575 upperBound = max(int(np.round(self.config.qMax*len(visits[intersects]))), self.config.nVisitsMin)
578 goodVisits = {int(visit):
True for visit
in sortedVisits[lowerBound:upperBound]}
579 return pipeBase.Struct(goodVisits=goodVisits)
def __init__(self, dataId, coordList)
def __init__(self, dataRef, wcs, bbox)
def getValidImageCorners(self, imageWcs, imageBox, patchPoly, dataId=None)
def run(self, wcsList, bboxList, coordList, dataIds=None, **kwargs)
static ConvexPolygon convexHull(std::vector< UnitVector3d > const &points)