32 from astro_metadata_translator
import merge_headers, ObservationGroup
33 from astro_metadata_translator.serialize
import dates_to_fits
38 """Parameters controlling the measurement of background statistics.
40 stat = pexConfig.Field(
43 doc=
"Statistic name to use to estimate background (from lsst.afw.math)",
45 clip = pexConfig.Field(
48 doc=
"Clipping threshold for background",
50 nIter = pexConfig.Field(
53 doc=
"Clipping iterations for background",
55 mask = pexConfig.ListField(
57 default=[
"DETECTED",
"BAD",
"NO_DATA"],
58 doc=
"Mask planes to reject",
63 """Measure statistics on the background
65 This can be useful for scaling the background, e.g., for flats and fringe frames.
67 ConfigClass = CalibStatsConfig
69 def run(self, exposureOrImage):
70 """Measure a particular statistic on an image (of some sort).
74 exposureOrImage : `lsst.afw.image.Exposure`, `lsst.afw.image.MaskedImage`, or `lsst.afw.image.Image`
75 Exposure or image to calculate statistics on.
80 Resulting statistic value.
82 stats = afwMath.StatisticsControl(self.config.clip, self.config.nIter,
83 afwImage.Mask.getPlaneBitMask(self.config.mask))
85 image = exposureOrImage.getMaskedImage()
88 image = exposureOrImage.getImage()
90 image = exposureOrImage
91 statType = afwMath.stringToStatisticsProperty(self.config.stat)
92 return afwMath.makeStatistics(image, statType, stats).getValue()
96 dimensions=(
"instrument",
"detector")):
99 doc=
"Input pre-processed exposures to combine.",
100 storageClass=
"Exposure",
101 dimensions=(
"instrument",
"detector",
"exposure"),
104 inputScales = cT.Input(
106 doc=
"Input scale factors to use.",
107 storageClass=
"StructuredDataDict",
108 dimensions=(
"instrument", ),
112 outputData = cT.Output(
114 doc=
"Output combined proposed calibration to be validated and certified..",
115 storageClass=
"ExposureF",
116 dimensions=(
"instrument",
"detector"),
123 if config
and config.exposureScaling !=
'InputList':
124 self.inputs.discard(
"inputScales")
129 pipelineConnections=CalibCombineConnections):
130 """Configuration for combining calib exposures.
132 calibrationType = pexConfig.Field(
134 default=
"calibration",
135 doc=
"Name of calibration to be generated.",
138 exposureScaling = pexConfig.ChoiceField(
141 "Unity":
"Do not scale inputs. Scale factor is 1.0.",
142 "ExposureTime":
"Scale inputs by their exposure time.",
143 "DarkTime":
"Scale inputs by their dark time.",
144 "MeanStats":
"Scale inputs based on their mean values.",
145 "InputList":
"Scale inputs based on a list of values.",
148 doc=
"Scaling to be applied to each input exposure.",
150 scalingLevel = pexConfig.ChoiceField(
153 "DETECTOR":
"Scale by detector.",
154 "AMP":
"Scale by amplifier.",
157 doc=
"Region to scale.",
159 maxVisitsToCalcErrorFromInputVariance = pexConfig.Field(
162 doc=
"Maximum number of visits to estimate variance from input variance, not per-pixel spread",
165 doVignette = pexConfig.Field(
168 doc=
"Copy vignette polygon to output and censor vignetted pixels?"
171 mask = pexConfig.ListField(
173 default=[
"SAT",
"DETECTED",
"INTRP"],
174 doc=
"Mask planes to respect",
176 combine = pexConfig.Field(
179 doc=
"Statistic name to use for combination (from lsst.afw.math)",
181 clip = pexConfig.Field(
184 doc=
"Clipping threshold for combination",
186 nIter = pexConfig.Field(
189 doc=
"Clipping iterations for combination",
191 stats = pexConfig.ConfigurableField(
192 target=CalibStatsTask,
193 doc=
"Background statistics configuration",
198 pipeBase.CmdLineTask):
199 """Task to combine calib exposures."""
200 ConfigClass = CalibCombineConfig
201 _DefaultName =
'cpCombine'
205 self.makeSubtask(
"stats")
208 inputs = butlerQC.get(inputRefs)
210 dimensions = [exp.dataId.byName()
for exp
in inputRefs.inputExps]
211 inputs[
'inputDims'] = dimensions
213 outputs = self.
runrun(**inputs)
214 butlerQC.put(outputs, outputRefs)
216 def run(self, inputExps, inputScales=None, inputDims=None):
217 """Combine calib exposures for a single detector.
221 inputExps : `list` [`lsst.afw.image.Exposure`]
222 Input list of exposures to combine.
223 inputScales : `dict` [`dict` [`dict` [`float`]]], optional
224 Dictionary of scales, indexed by detector (`int`),
225 amplifier (`int`), and exposure (`int`). Used for
227 inputDims : `list` [`dict`]
228 List of dictionaries of input data dimensions/values.
229 Each list entry should contain:
232 exposure id value (`int`)
234 detector id value (`int`)
238 combinedExp : `lsst.afw.image.Exposure`
239 Final combined exposure generated from the inputs.
244 Raised if no input data is found. Also raised if
245 config.exposureScaling == InputList, and a necessary scale
249 stats = afwMath.StatisticsControl(self.config.clip, self.config.nIter,
250 afwImage.Mask.getPlaneBitMask(self.config.mask))
251 numExps = len(inputExps)
253 raise RuntimeError(
"No valid input data")
254 if numExps < self.config.maxVisitsToCalcErrorFromInputVariance:
255 stats.setCalcErrorFromInputVariance(
True)
258 combined = afwImage.MaskedImageF(width, height)
259 combinedExp = afwImage.makeExposure(combined)
263 if inputDims
is None:
264 inputDims = [dict()
for i
in inputExps]
266 for index, (exp, dims)
in enumerate(zip(inputExps, inputDims)):
269 self.log.warn(
"Input %d is None (%s); unable to scale exp.", index, dims)
272 if self.config.exposureScaling ==
"ExposureTime":
273 scale = exp.getInfo().getVisitInfo().getExposureTime()
274 elif self.config.exposureScaling ==
"DarkTime":
275 scale = exp.getInfo().getVisitInfo().getDarkTime()
276 elif self.config.exposureScaling ==
"MeanStats":
277 scale = self.stats.
run(exp)
278 elif self.config.exposureScaling ==
"InputList":
279 visitId = dims.get(
'exposure',
None)
280 detectorId = dims.get(
'detector',
None)
281 if visitId
is None or detectorId
is None:
282 raise RuntimeError(f
"Could not identify scaling for input {index} ({dims})")
283 if detectorId
not in inputScales[
'expScale']:
284 raise RuntimeError(f
"Could not identify a scaling for input {index}"
285 f
" detector {detectorId}")
287 if self.config.scalingLevel ==
"DETECTOR":
288 if visitId
not in inputScales[
'expScale'][detectorId]:
289 raise RuntimeError(f
"Could not identify a scaling for input {index}"
290 f
"detector {detectorId} visit {visitId}")
291 scale = inputScales[
'expScale'][detectorId][visitId]
292 elif self.config.scalingLevel ==
'AMP':
293 scale = [inputScales[
'expScale'][detectorId][amp.getName()][visitId]
294 for amp
in exp.getDetector()]
296 raise RuntimeError(f
"Unknown scaling level: {self.config.scalingLevel}")
297 elif self.config.exposureScaling ==
'Unity':
300 raise RuntimeError(f
"Unknown scaling type: {self.config.exposureScaling}.")
302 expScales.append(scale)
303 self.log.info(
"Scaling input %d by %s", index, scale)
306 self.
combinecombine(combined, inputExps, stats)
310 if self.config.doVignette:
311 polygon = inputExps[0].getInfo().getValidPolygon()
313 doSetValue=
True, vignetteValue=0.0)
317 calibType=self.config.calibrationType, scales=expScales)
320 return pipeBase.Struct(
321 outputData=combinedExp,
325 """Get dimensions of the inputs.
329 expList : `list` [`lsst.afw.image.Exposure`]
330 Exps to check the sizes of.
334 width, height : `int`
335 Unique set of input dimensions.
337 dimList = [exp.getDimensions()
for exp
in expList
if exp
is not None]
338 return self.
getSizegetSize(dimList)
341 """Determine a consistent size, given a list of image sizes.
345 dimList : iterable of `tuple` (`int`, `int`)
351 If input dimensions are inconsistent.
355 width, height : `int`
358 dim = set((w, h)
for w, h
in dimList)
360 raise RuntimeError(
"Inconsistent dimensions: %s" % dim)
364 """Apply scale to input exposure.
366 This implementation applies a flux scaling: the input exposure is
367 divided by the provided scale.
371 exposure : `lsst.afw.image.Exposure`
373 scale : `float` or `list` [`float`], optional
374 Constant scale to divide the exposure by.
376 if scale
is not None:
377 mi = exposure.getMaskedImage()
378 if isinstance(scale, list):
379 for amp, ampScale
in zip(exposure.getDetector(), scale):
380 ampIm = mi[amp.getBBox()]
386 """Combine multiple images.
390 target : `lsst.afw.image.Exposure`
391 Output exposure to construct.
392 expList : `list` [`lsst.afw.image.Exposure`]
393 Input exposures to combine.
394 stats : `lsst.afw.math.StatisticsControl`
395 Control explaining how to combine the input images.
397 images = [img.getMaskedImage()
for img
in expList
if img
is not None]
398 combineType = afwMath.stringToStatisticsProperty(self.config.combine)
399 afwMath.statisticsStack(target, images, combineType, stats)
402 """Combine input headers to determine the set of common headers,
403 supplemented by calibration inputs.
407 expList : `list` of `lsst.afw.image.Exposure`
408 Input list of exposures to combine.
409 calib : `lsst.afw.image.Exposure`
410 Output calibration to construct headers for.
411 calibType: `str`, optional
412 OBSTYPE the output should claim.
413 scales: `list` of `float`, optional
414 Scale values applied to each input to record.
418 header : `lsst.daf.base.PropertyList`
422 header = calib.getMetadata()
423 header.set(
"OBSTYPE", calibType)
426 comments = {
"TIMESYS":
"Time scale for all dates",
427 "DATE-OBS":
"Start date of earliest input observation",
428 "MJD-OBS":
"[d] Start MJD of earliest input observation",
429 "DATE-END":
"End date of oldest input observation",
430 "MJD-END":
"[d] End MJD of oldest input observation",
431 "MJD-AVG":
"[d] MJD midpoint of all input observations",
432 "DATE-AVG":
"Midpoint date of all input observations"}
435 now = time.localtime()
436 calibDate = time.strftime(
"%Y-%m-%d", now)
437 calibTime = time.strftime(
"%X %Z", now)
438 header.set(
"CALIB_CREATE_DATE", calibDate)
439 header.set(
"CALIB_CREATE_TIME", calibTime)
442 inputHeaders = [exp.getMetadata()
for exp
in expList
if exp
is not None]
443 merged = merge_headers(inputHeaders, mode=
'drop')
444 for k, v
in merged.items():
446 md = expList[0].getMetadata()
447 comment = md.getComment(k)
if k
in md
else None
448 header.set(k, v, comment=comment)
451 visitInfoList = [exp.getInfo().getVisitInfo()
for exp
in expList
if exp
is not None]
452 for i, visit
in enumerate(visitInfoList):
455 header.set(
"CPP_INPUT_%d" % (i,), visit.getExposureId())
456 header.set(
"CPP_INPUT_DATE_%d" % (i,), str(visit.getDate()))
457 header.set(
"CPP_INPUT_EXPT_%d" % (i,), visit.getExposureTime())
458 if scales
is not None:
459 header.set(
"CPP_INPUT_SCALE_%d" % (i,), scales[i])
466 group = ObservationGroup(visitInfoList, pedantic=
False)
468 self.log.warn(
"Exception making an obs group for headers. Continuing.")
470 dateCards = {
"DATE-OBS":
"{}T00:00:00.00".format(calibDate)}
471 comments[
"DATE-OBS"] =
"Date of start of day of calibration midpoint"
473 oldest, newest = group.extremes()
474 dateCards = dates_to_fits(oldest.datetime_begin, newest.datetime_end)
476 for k, v
in dateCards.items():
477 header.set(k, v, comment=comments.get(k,
None))
482 """Interpolate over NANs in the combined image.
484 NANs can result from masked areas on the CCD. We don't want them getting
485 into our science images, so we replace them with the median of the image.
489 exp : `lsst.afw.image.Exposure`
490 Exp to check for NaNs.
492 array = exp.getImage().getArray()
493 bad = np.isnan(array)
495 median = np.median(array[np.logical_not(bad)])
496 count = np.sum(np.logical_not(bad))
499 self.log.warn(
"Found %s NAN pixels", count)
504 dimensions=(
"instrument",
"detector",
"physical_filter")):
505 inputScales = cT.Input(
506 name=
"cpFilterScales",
507 doc=
"Input scale factors to use.",
508 storageClass=
"StructuredDataDict",
509 dimensions=(
"instrument",
"physical_filter"),
513 outputData = cT.Output(
514 name=
"cpFilterProposal",
515 doc=
"Output combined proposed calibration to be validated and certified.",
516 storageClass=
"ExposureF",
517 dimensions=(
"instrument",
"detector",
"physical_filter"),
524 if config
and config.exposureScaling !=
'InputList':
525 self.inputs.discard(
"inputScales")
529 pipelineConnections=CalibCombineByFilterConnections):
534 """Task to combine calib exposures."""
535 ConfigClass = CalibCombineByFilterConfig
536 _DefaultName =
'cpFilterCombine'
541 doUpdateMask=True, maskPlane='BAD',
542 doSetValue=False, vignetteValue=0.0,
544 """Apply vignetted polygon to image pixels.
548 exposure : `lsst.afw.image.Exposure`
550 doUpdateMask : `bool`, optional
551 Update the exposure mask for vignetted area?
552 maskPlane : `str`, optional,
553 Mask plane to assign.
554 doSetValue : `bool`, optional
555 Set image value for vignetted area?
556 vignetteValue : `float`, optional
558 log : `lsst.log.Log`, optional
564 Raised if no valid polygon exists.
566 polygon = polygon
if polygon
else exposure.getInfo().getValidPolygon()
568 raise RuntimeError(
"Could not find valid polygon!")
569 log = log
if log
else Log.getLogger(__name__.partition(
".")[2])
571 fullyIlluminated =
True
572 for corner
in exposure.getBBox().getCorners():
573 if not polygon.contains(
Point2D(corner)):
574 fullyIlluminated =
False
576 log.info(
"Exposure is fully illuminated? %s", fullyIlluminated)
578 if not fullyIlluminated:
580 mask = exposure.getMask()
581 numPixels = mask.getBBox().getArea()
583 xx, yy = np.meshgrid(np.arange(0, mask.getWidth(), dtype=int),
584 np.arange(0, mask.getHeight(), dtype=int))
586 vignMask = np.array([
not polygon.contains(
Point2D(x, y))
for x, y
in
587 zip(xx.reshape(numPixels), yy.reshape(numPixels))])
588 vignMask = vignMask.reshape(mask.getHeight(), mask.getWidth())
591 bitMask = mask.getPlaneBitMask(maskPlane)
592 maskArray = mask.getArray()
593 maskArray[vignMask] |= bitMask
595 imageArray = exposure.getImage().getArray()
596 imageArray[vignMask] = vignetteValue
597 log.info(
"Exposure contains %d vignetted pixels.",
598 np.count_nonzero(vignMask))
def __init__(self, *config=None)
def __init__(self, *config=None)
def applyScale(self, exposure, scale=None)
def runQuantum(self, butlerQC, inputRefs, outputRefs)
def __init__(self, **kwargs)
def run(self, inputExps, inputScales=None, inputDims=None)
def combine(self, target, expList, stats)
def getDimensions(self, expList)
def getSize(self, dimList)
def interpolateNans(self, exp)
def combineHeaders(self, expList, calib, calibType="CALIB", scales=None)
def run(self, exposureOrImage)
def VignetteExposure(exposure, polygon=None, doUpdateMask=True, maskPlane='BAD', doSetValue=False, vignetteValue=0.0, log=None)