lsst.pipe.drivers  14.0-13-g1010e0d+4
constructCalibs.py
Go to the documentation of this file.
1 from __future__ import absolute_import, division, print_function
2 
3 import sys
4 import math
5 import time
6 import argparse
7 import traceback
8 
9 import numpy as np
10 from builtins import zip
11 from builtins import range
12 
13 from lsst.pex.config import Config, ConfigurableField, Field, ListField, ConfigField
14 from lsst.pipe.base import Task, Struct, TaskRunner, ArgumentParser
15 import lsst.daf.base as dafBase
16 import lsst.afw.math as afwMath
17 import lsst.afw.geom as afwGeom
18 import lsst.afw.detection as afwDet
19 import lsst.afw.image as afwImage
20 from lsst.afw.image import VisitInfo
21 import lsst.meas.algorithms as measAlg
22 from lsst.pipe.tasks.repair import RepairTask
23 from lsst.ip.isr import IsrTask
24 from lsst.afw.cameraGeom.utils import makeImageFromCamera
25 
26 from lsst.ctrl.pool.parallel import BatchPoolTask
27 from lsst.ctrl.pool.pool import Pool, NODE
28 from lsst.pipe.drivers.background import SkyMeasurementTask, FocalPlaneBackground, FocalPlaneBackgroundConfig
29 
30 from .checksum import checksum
31 from .utils import getDataRef
32 
33 
34 class CalibStatsConfig(Config):
35  """Parameters controlling the measurement of background statistics"""
36  stat = Field(doc="Statistic to use to estimate background (from lsst.afw.math)", dtype=int,
37  default=int(afwMath.MEANCLIP))
38  clip = Field(doc="Clipping threshold for background",
39  dtype=float, default=3.0)
40  nIter = Field(doc="Clipping iterations for background",
41  dtype=int, default=3)
42  maxVisitsToCalcErrorFromInputVariance = Field(
43  doc="Maximum number of visits to estimate variance from input variance, not per-pixel spread",
44  dtype=int, default=2)
45  mask = ListField(doc="Mask planes to reject",
46  dtype=str, default=["DETECTED", "BAD", "NO_DATA",])
47 
48 
49 class CalibStatsTask(Task):
50  """Measure statistics on the background
51 
52  This can be useful for scaling the background, e.g., for flats and fringe frames.
53  """
54  ConfigClass = CalibStatsConfig
55 
56  def run(self, exposureOrImage):
57  """!Measure a particular statistic on an image (of some sort).
58 
59  @param exposureOrImage Exposure, MaskedImage or Image.
60  @return Value of desired statistic
61  """
62  stats = afwMath.StatisticsControl(self.config.clip, self.config.nIter,
63  afwImage.Mask.getPlaneBitMask(self.config.mask))
64  try:
65  image = exposureOrImage.getMaskedImage()
66  except:
67  try:
68  image = exposureOrImage.getImage()
69  except:
70  image = exposureOrImage
71 
72  return afwMath.makeStatistics(image, self.config.stat, stats).getValue()
73 
74 
75 class CalibCombineConfig(Config):
76  """Configuration for combining calib images"""
77  rows = Field(doc="Number of rows to read at a time",
78  dtype=int, default=512)
79  mask = ListField(doc="Mask planes to respect", dtype=str,
80  default=["SAT", "DETECTED", "INTRP"])
81  combine = Field(doc="Statistic to use for combination (from lsst.afw.math)", dtype=int,
82  default=int(afwMath.MEANCLIP))
83  clip = Field(doc="Clipping threshold for combination",
84  dtype=float, default=3.0)
85  nIter = Field(doc="Clipping iterations for combination",
86  dtype=int, default=3)
87  stats = ConfigurableField(target=CalibStatsTask,
88  doc="Background statistics configuration")
89 
90 
91 class CalibCombineTask(Task):
92  """Task to combine calib images"""
93  ConfigClass = CalibCombineConfig
94 
95  def __init__(self, *args, **kwargs):
96  Task.__init__(self, *args, **kwargs)
97  self.makeSubtask("stats")
98 
99  def run(self, sensorRefList, expScales=None, finalScale=None, inputName="postISRCCD"):
100  """!Combine calib images for a single sensor
101 
102  @param sensorRefList List of data references to combine (for a single sensor)
103  @param expScales List of scales to apply for each exposure
104  @param finalScale Desired scale for final combined image
105  @param inputName Data set name for inputs
106  @return combined image
107  """
108  width, height = self.getDimensions(sensorRefList)
109  stats = afwMath.StatisticsControl(self.config.clip, self.config.nIter,
110  afwImage.Mask.getPlaneBitMask(self.config.mask))
111  numImages = len(sensorRefList)
112  if numImages < self.config.stats.maxVisitsToCalcErrorFromInputVariance:
113  stats.setCalcErrorFromInputVariance(True)
114 
115  # Combine images
116  combined = afwImage.MaskedImageF(width, height)
117  imageList = [None]*numImages
118  for start in range(0, height, self.config.rows):
119  rows = min(self.config.rows, height - start)
120  box = afwGeom.Box2I(afwGeom.Point2I(0, start),
121  afwGeom.Extent2I(width, rows))
122  subCombined = combined.Factory(combined, box)
123 
124  for i, sensorRef in enumerate(sensorRefList):
125  if sensorRef is None:
126  imageList[i] = None
127  continue
128  exposure = sensorRef.get(inputName + "_sub", bbox=box)
129  if expScales is not None:
130  self.applyScale(exposure, expScales[i])
131  imageList[i] = exposure.getMaskedImage()
132 
133  self.combine(subCombined, imageList, stats)
134 
135  if finalScale is not None:
136  background = self.stats.run(combined)
137  self.log.info("%s: Measured background of stack is %f; adjusting to %f" %
138  (NODE, background, finalScale))
139  combined *= finalScale / background
140 
141  return combined
142 
143  def getDimensions(self, sensorRefList, inputName="postISRCCD"):
144  """Get dimensions of the inputs"""
145  dimList = []
146  for sensorRef in sensorRefList:
147  if sensorRef is None:
148  continue
149  md = sensorRef.get(inputName + "_md")
150  dimList.append(afwImage.bboxFromMetadata(md).getDimensions())
151  return getSize(dimList)
152 
153  def applyScale(self, exposure, scale=None):
154  """Apply scale to input exposure
155 
156  This implementation applies a flux scaling: the input exposure is
157  divided by the provided scale.
158  """
159  if scale is not None:
160  mi = exposure.getMaskedImage()
161  mi /= scale
162 
163  def combine(self, target, imageList, stats):
164  """!Combine multiple images
165 
166  @param target Target image to receive the combined pixels
167  @param imageList List of input images
168  @param stats Statistics control
169  """
170  images = [img for img in imageList if img is not None]
171  afwMath.statisticsStack(target, images, afwMath.Property(self.config.combine), stats)
172 
173 
174 def getSize(dimList):
175  """Determine a consistent size, given a list of image sizes"""
176  dim = set((w, h) for w, h in dimList)
177  dim.discard(None)
178  if len(dim) != 1:
179  raise RuntimeError("Inconsistent dimensions: %s" % dim)
180  return dim.pop()
181 
182 
183 def dictToTuple(dict_, keys):
184  """!Return a tuple of specific values from a dict
185 
186  This provides a hashable representation of the dict from certain keywords.
187  This can be useful for creating e.g., a tuple of the values in the DataId
188  that identify the CCD.
189 
190  @param dict_ dict to parse
191  @param keys keys to extract (order is important)
192  @return tuple of values
193  """
194  return tuple(dict_[k] for k in keys)
195 
196 
197 def getCcdIdListFromExposures(expRefList, level="sensor", ccdKeys=["ccd"]):
198  """!Determine a list of CCDs from exposure references
199 
200  This essentially inverts the exposure-level references (which
201  provides a list of CCDs for each exposure), by providing
202  a dataId list for each CCD. Consider an input list of exposures
203  [e1, e2, e3], and each exposure has CCDs c1 and c2. Then this
204  function returns:
205 
206  {(c1,): [e1c1, e2c1, e3c1], (c2,): [e1c2, e2c2, e3c2]}
207 
208  This is a dict whose keys are tuples of the identifying values of a
209  CCD (usually just the CCD number) and the values are lists of dataIds
210  for that CCD in each exposure. A missing dataId is given the value
211  None.
212 
213  @param expRefList List of data references for exposures
214  @param level Level for the butler to generate CCDs
215  @param ccdKeys DataId keywords that identify a CCD
216  @return dict of data identifier lists for each CCD
217  """
218  expIdList = [[ccdRef.dataId for ccdRef in expRef.subItems(
219  level)] for expRef in expRefList]
220 
221  # Determine what additional keys make a CCD from an exposure
222  ccdKeys = set(ccdKeys) # Set of keywords in the dataId that identify a CCD
223  ccdNames = set() # Set of tuples which are values for each of the CCDs in an exposure
224  for ccdIdList in expIdList:
225  for ccdId in ccdIdList:
226  name = dictToTuple(ccdId, ccdKeys)
227  ccdNames.add(name)
228 
229  # Turn the list of CCDs for each exposure into a list of exposures for
230  # each CCD
231  ccdLists = {}
232  for n, ccdIdList in enumerate(expIdList):
233  for ccdId in ccdIdList:
234  name = dictToTuple(ccdId, ccdKeys)
235  if name not in ccdLists:
236  ccdLists[name] = []
237  ccdLists[name].append(ccdId)
238 
239  for ccd in ccdLists:
240  # Sort the list by the dataId values (ordered by key)
241  ccdLists[ccd] = sorted(ccdLists[ccd], key=lambda dd: dictToTuple(dd, sorted(dd.keys())))
242 
243  return ccdLists
244 
245 
246 def mapToMatrix(pool, func, ccdIdLists, *args, **kwargs):
247  """Generate a matrix of results using pool.map
248 
249  The function should have the call signature:
250  func(cache, dataId, *args, **kwargs)
251 
252  We return a dict mapping 'ccd name' to a list of values for
253  each exposure.
254 
255  @param pool Process pool
256  @param func Function to call for each dataId
257  @param ccdIdLists Dict of data identifier lists for each CCD name
258  @return matrix of results
259  """
260  dataIdList = sum(ccdIdLists.values(), [])
261  resultList = pool.map(func, dataIdList, *args, **kwargs)
262  # Piece everything back together
263  data = dict((ccdName, [None] * len(expList)) for ccdName, expList in ccdIdLists.items())
264  indices = dict(sum([[(tuple(dataId.values()) if dataId is not None else None, (ccdName, expNum))
265  for expNum, dataId in enumerate(expList)]
266  for ccdName, expList in ccdIdLists.items()], []))
267  for dataId, result in zip(dataIdList, resultList):
268  if dataId is None:
269  continue
270  ccdName, expNum = indices[tuple(dataId.values())]
271  data[ccdName][expNum] = result
272  return data
273 
274 
275 class CalibIdAction(argparse.Action):
276  """Split name=value pairs and put the result in a dict"""
277 
278  def __call__(self, parser, namespace, values, option_string):
279  output = getattr(namespace, self.dest, {})
280  for nameValue in values:
281  name, sep, valueStr = nameValue.partition("=")
282  if not valueStr:
283  parser.error("%s value %s must be in form name=value" %
284  (option_string, nameValue))
285  output[name] = valueStr
286  setattr(namespace, self.dest, output)
287 
288 
289 class CalibArgumentParser(ArgumentParser):
290  """ArgumentParser for calibration construction"""
291 
292  def __init__(self, calibName, *args, **kwargs):
293  """Add a --calibId argument to the standard pipe_base argument parser"""
294  ArgumentParser.__init__(self, *args, **kwargs)
295  self.calibName = calibName
296  self.add_id_argument("--id", datasetType="raw",
297  help="input identifiers, e.g., --id visit=123 ccd=4")
298  self.add_argument("--calibId", nargs="*", action=CalibIdAction, default={},
299  help="identifiers for calib, e.g., --calibId version=1",
300  metavar="KEY=VALUE1[^VALUE2[^VALUE3...]")
301 
302  def parse_args(self, *args, **kwargs):
303  """Parse arguments
304 
305  Checks that the "--calibId" provided works.
306  """
307  namespace = ArgumentParser.parse_args(self, *args, **kwargs)
308 
309  keys = namespace.butler.getKeys(self.calibName)
310  parsed = {}
311  for name, value in namespace.calibId.items():
312  if name not in keys:
313  self.error(
314  "%s is not a relevant calib identifier key (%s)" % (name, keys))
315  parsed[name] = keys[name](value)
316  namespace.calibId = parsed
317 
318  return namespace
319 
320 
321 class CalibConfig(Config):
322  """Configuration for constructing calibs"""
323  clobber = Field(dtype=bool, default=True,
324  doc="Clobber existing processed images?")
325  isr = ConfigurableField(target=IsrTask, doc="ISR configuration")
326  dateObs = Field(dtype=str, default="dateObs",
327  doc="Key for observation date in exposure registry")
328  dateCalib = Field(dtype=str, default="calibDate",
329  doc="Key for calib date in calib registry")
330  filter = Field(dtype=str, default="filter",
331  doc="Key for filter name in exposure/calib registries")
332  combination = ConfigurableField(
333  target=CalibCombineTask, doc="Calib combination configuration")
334  ccdKeys = ListField(dtype=str, default=["ccd"],
335  doc="DataId keywords specifying a CCD")
336  visitKeys = ListField(dtype=str, default=["visit"],
337  doc="DataId keywords specifying a visit")
338  calibKeys = ListField(dtype=str, default=[],
339  doc="DataId keywords specifying a calibration")
340  doCameraImage = Field(dtype=bool, default=True, doc="Create camera overview image?")
341  binning = Field(dtype=int, default=64, doc="Binning to apply for camera image")
342 
343  def setDefaults(self):
344  self.isr.doWrite = False
345 
346 
347 class CalibTaskRunner(TaskRunner):
348  """Get parsed values into the CalibTask.run"""
349  @staticmethod
350  def getTargetList(parsedCmd, **kwargs):
351  return [dict(expRefList=parsedCmd.id.refList, butler=parsedCmd.butler, calibId=parsedCmd.calibId)]
352 
353  def __call__(self, args):
354  """Call the Task with the kwargs from getTargetList"""
355  task = self.TaskClass(config=self.config, log=self.log)
356  exitStatus = 0 # exit status for the shell
357  if self.doRaise:
358  result = task.run(**args)
359  else:
360  try:
361  result = task.run(**args)
362  except Exception as e:
363  exitStatus = 1 # n.b. The shell exit value is the number of dataRefs returning
364  # non-zero, so the actual value used here is lost
365  task.log.fatal("Failed: %s" % e)
366  traceback.print_exc(file=sys.stderr)
367 
368  if self.doReturnResults:
369  return Struct(
370  exitStatus=exitStatus,
371  args=args,
372  metadata=task.metadata,
373  result=result,
374  )
375  else:
376  return Struct(
377  exitStatus=exitStatus,
378  )
379 
381  """!Base class for constructing calibs.
382 
383  This should be subclassed for each of the required calib types.
384  The subclass should be sure to define the following class variables:
385  * _DefaultName: default name of the task, used by CmdLineTask
386  * calibName: name of the calibration data set in the butler
387  The subclass may optionally set:
388  * filterName: filter name to give the resultant calib
389  """
390  ConfigClass = CalibConfig
391  RunnerClass = CalibTaskRunner
392  filterName = None
393  calibName = None
394  exposureTime = 1.0 # sets this exposureTime in the output
395 
396  def __init__(self, *args, **kwargs):
397  """Constructor"""
398  BatchPoolTask.__init__(self, *args, **kwargs)
399  self.makeSubtask("isr")
400  self.makeSubtask("combination")
401 
402  @classmethod
403  def batchWallTime(cls, time, parsedCmd, numCores):
404  numCcds = len(parsedCmd.butler.get("camera"))
405  numExps = len(cls.RunnerClass.getTargetList(
406  parsedCmd)[0]['expRefList'])
407  numCycles = int(numCcds/float(numCores) + 0.5)
408  return time*numExps*numCycles
409 
410  @classmethod
411  def _makeArgumentParser(cls, *args, **kwargs):
412  kwargs.pop("doBatch", False)
413  return CalibArgumentParser(calibName=cls.calibName, name=cls._DefaultName, *args, **kwargs)
414 
415  def run(self, expRefList, butler, calibId):
416  """!Construct a calib from a list of exposure references
417 
418  This is the entry point, called by the TaskRunner.__call__
419 
420  Only the master node executes this method.
421 
422  @param expRefList List of data references at the exposure level
423  @param butler Data butler
424  @param calibId Identifier dict for calib
425  """
426  for expRef in expRefList:
427  self.addMissingKeys(expRef.dataId, butler, self.config.ccdKeys, 'raw')
428 
429  outputId = self.getOutputId(expRefList, calibId)
430  ccdIdLists = getCcdIdListFromExposures(
431  expRefList, level="sensor", ccdKeys=self.config.ccdKeys)
432 
433  # Ensure we can generate filenames for each output
434  outputIdItemList = list(outputId.items())
435  for ccdName in ccdIdLists:
436  dataId = dict([(k, ccdName[i]) for i, k in enumerate(self.config.ccdKeys)])
437  dataId.update(outputIdItemList)
438  self.addMissingKeys(dataId, butler)
439  dataId.update(outputIdItemList)
440 
441  try:
442  butler.get(self.calibName + "_filename", dataId)
443  except Exception as e:
444  raise RuntimeError(
445  "Unable to determine output filename \"%s_filename\" from %s: %s" %
446  (self.calibName, dataId, e))
447 
448  processPool = Pool("process")
449  processPool.storeSet(butler=butler)
450 
451  # Scatter: process CCDs independently
452  data = self.scatterProcess(processPool, ccdIdLists)
453 
454  # Gather: determine scalings
455  scales = self.scale(ccdIdLists, data)
456 
457  combinePool = Pool("combine")
458  combinePool.storeSet(butler=butler)
459 
460  # Scatter: combine
461  calibs = self.scatterCombine(combinePool, outputId, ccdIdLists, scales)
462 
463  if self.config.doCameraImage:
464  camera = butler.get("camera")
465 
466  try:
467  cameraImage = self.makeCameraImage(camera, outputId, calibs)
468  butler.put(cameraImage, self.calibName + "_camera", dataId)
469  except Exception as exc:
470  self.log.warn("Unable to create camera image: %s" % (exc,))
471 
472  return Struct(
473  outputId = outputId,
474  ccdIdLists = ccdIdLists,
475  scales = scales,
476  calibs = calibs,
477  processPool = processPool,
478  combinePool = combinePool,
479  )
480 
481  def getOutputId(self, expRefList, calibId):
482  """!Generate the data identifier for the output calib
483 
484  The mean date and the common filter are included, using keywords
485  from the configuration. The CCD-specific part is not included
486  in the data identifier.
487 
488  @param expRefList List of data references at exposure level
489  @param calibId Data identifier elements for the calib provided by the user
490  @return data identifier
491  """
492  midTime = 0
493  filterName = None
494  for expRef in expRefList:
495  butler = expRef.getButler()
496  dataId = expRef.dataId
497 
498  midTime += self.getMjd(butler, dataId)
499  thisFilter = self.getFilter(
500  butler, dataId) if self.filterName is None else self.filterName
501  if filterName is None:
502  filterName = thisFilter
503  elif filterName != thisFilter:
504  raise RuntimeError("Filter mismatch for %s: %s vs %s" % (
505  dataId, thisFilter, filterName))
506 
507  midTime /= len(expRefList)
508  date = str(dafBase.DateTime(
509  midTime, dafBase.DateTime.MJD).toPython().date())
510 
511  outputId = {self.config.filter: filterName,
512  self.config.dateCalib: date}
513  outputId.update(calibId)
514  return outputId
515 
516  def getMjd(self, butler, dataId, timescale=dafBase.DateTime.UTC):
517  """Determine the Modified Julian Date (MJD; in TAI) from a data identifier"""
518  if self.config.dateObs in dataId:
519  dateObs = dataId[self.config.dateObs]
520  else:
521  dateObs = butler.queryMetadata('raw', [self.config.dateObs], dataId)[0]
522  if "T" not in dateObs:
523  dateObs = dateObs + "T12:00:00.0Z"
524  elif not dateObs.endswith("Z"):
525  dateObs += "Z"
526 
527  return dafBase.DateTime(dateObs, timescale).get(dafBase.DateTime.MJD)
528 
529  def getFilter(self, butler, dataId):
530  """Determine the filter from a data identifier"""
531  filt = butler.queryMetadata('raw', [self.config.filter], dataId)[0]
532  return filt
533 
534  def addMissingKeys(self, dataId, butler, missingKeys=None, calibName=None):
535  if calibName is None:
536  calibName = self.calibName
537 
538  if missingKeys is None:
539  missingKeys = set(butler.getKeys(calibName).keys()) - set(dataId.keys())
540 
541  for k in missingKeys:
542  try:
543  v = butler.queryMetadata('raw', [k], dataId) # n.b. --id refers to 'raw'
544  except Exception as e:
545  continue
546 
547  if len(v) == 0: # failed to lookup value
548  continue
549 
550  if len(v) == 1:
551  dataId[k] = v[0]
552  else:
553  raise RuntimeError("No unique lookup for %s: %s" % (k, v))
554 
555  def updateMetadata(self, calibImage, exposureTime, darkTime=None, **kwargs):
556  """!Update the metadata from the VisitInfo
557 
558  \param calibImage The image whose metadata is to be set
559  \param exposureTime The exposure time for the image
560  \param darkTime The time since the last read (default: exposureTime)
561  """
562 
563  if darkTime is None:
564  darkTime = exposureTime # avoid warning messages when using calibration products
565 
566  visitInfo = afwImage.makeVisitInfo(exposureTime=exposureTime, darkTime=darkTime, **kwargs)
567  md = calibImage.getMetadata()
568 
569  afwImage.setVisitInfoMetadata(md, visitInfo)
570 
571  def scatterProcess(self, pool, ccdIdLists):
572  """!Scatter the processing among the nodes
573 
574  We scatter each CCD independently (exposures aren't grouped together),
575  to make full use of all available processors. This necessitates piecing
576  everything back together in the same format as ccdIdLists afterwards.
577 
578  Only the master node executes this method.
579 
580  @param pool Process pool
581  @param ccdIdLists Dict of data identifier lists for each CCD name
582  @return Dict of lists of returned data for each CCD name
583  """
584  self.log.info("Scatter processing")
585  return mapToMatrix(pool, self.process, ccdIdLists)
586 
587  def process(self, cache, ccdId, outputName="postISRCCD", **kwargs):
588  """!Process a CCD, specified by a data identifier
589 
590  After processing, optionally returns a result (produced by
591  the 'processResult' method) calculated from the processed
592  exposure. These results will be gathered by the master node,
593  and is a means for coordinated scaling of all CCDs for flats,
594  etc.
595 
596  Only slave nodes execute this method.
597 
598  @param cache Process pool cache
599  @param ccdId Data identifier for CCD
600  @param outputName Output dataset name for butler
601  @return result from 'processResult'
602  """
603  if ccdId is None:
604  self.log.warn("Null identifier received on %s" % NODE)
605  return None
606  sensorRef = getDataRef(cache.butler, ccdId)
607  if self.config.clobber or not sensorRef.datasetExists(outputName):
608  self.log.info("Processing %s on %s" % (ccdId, NODE))
609  try:
610  exposure = self.processSingle(sensorRef, **kwargs)
611  except Exception as e:
612  self.log.warn("Unable to process %s: %s" % (ccdId, e))
613  raise
614  return None
615  self.processWrite(sensorRef, exposure)
616  else:
617  self.log.info(
618  "Using previously persisted processed exposure for %s" % (sensorRef.dataId,))
619  exposure = sensorRef.get(outputName)
620  return self.processResult(exposure)
621 
622  def processSingle(self, dataRef):
623  """Process a single CCD, specified by a data reference
624 
625  Generally, this simply means doing ISR.
626 
627  Only slave nodes execute this method.
628  """
629  return self.isr.runDataRef(dataRef).exposure
630 
631  def processWrite(self, dataRef, exposure, outputName="postISRCCD"):
632  """!Write the processed CCD
633 
634  We need to write these out because we can't hold them all in
635  memory at once.
636 
637  Only slave nodes execute this method.
638 
639  @param dataRef Data reference
640  @param exposure CCD exposure to write
641  @param outputName Output dataset name for butler.
642  """
643  dataRef.put(exposure, outputName)
644 
645  def processResult(self, exposure):
646  """Extract processing results from a processed exposure
647 
648  This method generates what is gathered by the master node.
649  This can be a background measurement or similar for scaling
650  flat-fields. It must be picklable!
651 
652  Only slave nodes execute this method.
653  """
654  return None
655 
656  def scale(self, ccdIdLists, data):
657  """!Determine scaling across CCDs and exposures
658 
659  This is necessary mainly for flats, so as to determine a
660  consistent scaling across the entire focal plane. This
661  implementation is simply a placeholder.
662 
663  Only the master node executes this method.
664 
665  @param ccdIdLists Dict of data identifier lists for each CCD tuple
666  @param data Dict of lists of returned data for each CCD tuple
667  @return dict of Struct(ccdScale: scaling for CCD,
668  expScales: scaling for each exposure
669  ) for each CCD tuple
670  """
671  self.log.info("Scale on %s" % NODE)
672  return dict((name, Struct(ccdScale=None, expScales=[None] * len(ccdIdLists[name])))
673  for name in ccdIdLists)
674 
675  def scatterCombine(self, pool, outputId, ccdIdLists, scales):
676  """!Scatter the combination of exposures across multiple nodes
677 
678  In this case, we can only scatter across as many nodes as
679  there are CCDs.
680 
681  Only the master node executes this method.
682 
683  @param pool Process pool
684  @param outputId Output identifier (exposure part only)
685  @param ccdIdLists Dict of data identifier lists for each CCD name
686  @param scales Dict of structs with scales, for each CCD name
687  @param dict of binned images
688  """
689  self.log.info("Scatter combination")
690  data = [Struct(ccdName=ccdName, ccdIdList=ccdIdLists[ccdName], scales=scales[ccdName]) for
691  ccdName in ccdIdLists]
692  images = pool.map(self.combine, data, outputId)
693  return dict(zip(ccdIdLists.keys(), images))
694 
695  def getFullyQualifiedOutputId(self, ccdName, butler, outputId):
696  """Get fully-qualified output data identifier
697 
698  We may need to look up keys that aren't in the output dataId.
699 
700  @param ccdName Name tuple for CCD
701  @param butler Data butler
702  @param outputId Data identifier for combined image (exposure part only)
703  @return fully-qualified output dataId
704  """
705  fullOutputId = {k: ccdName[i] for i, k in enumerate(self.config.ccdKeys)}
706  fullOutputId.update(outputId)
707  self.addMissingKeys(fullOutputId, butler)
708  fullOutputId.update(outputId) # must be after the call to queryMetadata in 'addMissingKeys'
709  return fullOutputId
710 
711  def combine(self, cache, struct, outputId):
712  """!Combine multiple exposures of a particular CCD and write the output
713 
714  Only the slave nodes execute this method.
715 
716  @param cache Process pool cache
717  @param struct Parameters for the combination, which has the following components:
718  * ccdName Name tuple for CCD
719  * ccdIdList List of data identifiers for combination
720  * scales Scales to apply (expScales are scalings for each exposure,
721  ccdScale is final scale for combined image)
722  @param outputId Data identifier for combined image (exposure part only)
723  @return binned calib image
724  """
725  outputId = self.getFullyQualifiedOutputId(struct.ccdName, cache.butler, outputId)
726  dataRefList = [getDataRef(cache.butler, dataId) if dataId is not None else None for
727  dataId in struct.ccdIdList]
728  self.log.info("Combining %s on %s" % (outputId, NODE))
729  calib = self.combination.run(dataRefList, expScales=struct.scales.expScales,
730  finalScale=struct.scales.ccdScale)
731 
732  if not hasattr(calib, "getMetadata"):
733  if hasattr(calib, "getVariance"):
734  calib = afwImage.makeExposure(calib)
735  else:
736  calib = afwImage.DecoratedImageF(calib.getImage()) # n.b. hardwires "F" for the output type
737 
738  self.updateMetadata(calib, self.exposureTime)
739 
740  self.recordCalibInputs(cache.butler, calib,
741  struct.ccdIdList, outputId)
742 
743  self.interpolateNans(calib)
744 
745  self.write(cache.butler, calib, outputId)
746 
747  return afwMath.binImage(calib.getImage(), self.config.binning)
748 
749  def recordCalibInputs(self, butler, calib, dataIdList, outputId):
750  """!Record metadata including the inputs and creation details
751 
752  This metadata will go into the FITS header.
753 
754  @param butler Data butler
755  @param calib Combined calib exposure.
756  @param dataIdList List of data identifiers for calibration inputs
757  @param outputId Data identifier for output
758  """
759  header = calib.getMetadata()
760  header.add("OBSTYPE", self.calibName) # Used by ingestCalibs.py
761 
762  # date, time, host, and root
763  now = time.localtime()
764  header.add("CALIB_CREATION_DATE", time.strftime("%Y-%m-%d", now))
765  header.add("CALIB_CREATION_TIME", time.strftime("%X %Z", now))
766  # add date-obs as its absence upsets ExposureInfo; use the mean date that the calibs were taken
767  header.add("DATE-OBS", "%sT00:00:00.00" % outputId[self.config.dateCalib])
768 
769  # Inputs
770  visits = [str(dictToTuple(dataId, self.config.visitKeys)) for dataId in dataIdList if
771  dataId is not None]
772  for i, v in enumerate(sorted(set(visits))):
773  header.add("CALIB_INPUT_%d" % (i,), v)
774 
775  header.add("CALIB_ID", " ".join("%s=%s" % (key, value)
776  for key, value in outputId.items()))
777  checksum(calib, header)
778 
779  def interpolateNans(self, image):
780  """Interpolate over NANs in the combined image
781 
782  NANs can result from masked areas on the CCD. We don't want them getting
783  into our science images, so we replace them with the median of the image.
784  """
785  if hasattr(image, "getMaskedImage"): # Deal with Exposure vs Image
786  self.interpolateNans(image.getMaskedImage().getVariance())
787  image = image.getMaskedImage().getImage()
788  if hasattr(image, "getImage"): # Deal with DecoratedImage or MaskedImage vs Image
789  image = image.getImage()
790  array = image.getArray()
791  bad = np.isnan(array)
792  array[bad] = np.median(array[np.logical_not(bad)])
793 
794  def write(self, butler, exposure, dataId):
795  """!Write the final combined calib
796 
797  Only the slave nodes execute this method
798 
799  @param butler Data butler
800  @param exposure CCD exposure to write
801  @param dataId Data identifier for output
802  """
803  self.log.info("Writing %s on %s" % (dataId, NODE))
804  butler.put(exposure, self.calibName, dataId)
805 
806  def makeCameraImage(self, camera, dataId, calibs):
807  """!Create and write an image of the entire camera
808 
809  This is useful for judging the quality or getting an overview of
810  the features of the calib.
811 
812  This requires that the 'ccd name' is a tuple containing only the
813  detector ID. If that is not the case, change CalibConfig.ccdKeys
814  or set CalibConfig.doCameraImage=False to disable this.
815 
816  @param camera Camera object
817  @param dataId Data identifier for output
818  @param calibs Dict mapping 'ccd name' to calib image
819  """
820 
821  class ImageSource(object):
822  """Source of images for makeImageFromCamera
823 
824  This assumes that the 'ccd name' is a tuple containing
825  only the detector ID.
826  """
827  def __init__(self, images):
828  self.isTrimmed = True
829  self.images = images
830  self.background = np.nan
831 
832  def getCcdImage(self, detector, imageFactory, binSize):
833  detId = (detector.getId(),)
834  if detId not in self.images:
835  return imageFactory(1, 1), detId
836  return self.images[detId], detId
837 
838  image = makeImageFromCamera(camera, imageSource=ImageSource(calibs), imageFactory=afwImage.ImageF,
839  binSize=self.config.binning)
840  return image
841 
843  """Configuration for bias construction.
844 
845  No changes required compared to the base class, but
846  subclassed for distinction.
847  """
848  pass
849 
850 
851 class BiasTask(CalibTask):
852  """Bias construction"""
853  ConfigClass = BiasConfig
854  _DefaultName = "bias"
855  calibName = "bias"
856  filterName = "NONE" # Sets this filter name in the output
857  exposureTime = 0.0 # sets this exposureTime in the output
858 
859  @classmethod
860  def applyOverrides(cls, config):
861  """Overrides to apply for bias construction"""
862  config.isr.doBias = False
863  config.isr.doDark = False
864  config.isr.doFlat = False
865  config.isr.doFringe = False
866 
867 
869  """Configuration for dark construction"""
870  doRepair = Field(dtype=bool, default=True, doc="Repair artifacts?")
871  psfFwhm = Field(dtype=float, default=3.0, doc="Repair PSF FWHM (pixels)")
872  psfSize = Field(dtype=int, default=21, doc="Repair PSF size (pixels)")
873  crGrow = Field(dtype=int, default=2, doc="Grow radius for CR (pixels)")
874  repair = ConfigurableField(
875  target=RepairTask, doc="Task to repair artifacts")
876 
877  def setDefaults(self):
878  CalibConfig.setDefaults(self)
879  self.combination.mask.append("CR")
880 
881 
883  """Dark construction
884 
885  The only major difference from the base class is a cosmic-ray
886  identification stage, and dividing each image by the dark time
887  to generate images of the dark rate.
888  """
889  ConfigClass = DarkConfig
890  _DefaultName = "dark"
891  calibName = "dark"
892  filterName = "NONE" # Sets this filter name in the output
893 
894  def __init__(self, *args, **kwargs):
895  CalibTask.__init__(self, *args, **kwargs)
896  self.makeSubtask("repair")
897 
898  @classmethod
899  def applyOverrides(cls, config):
900  """Overrides to apply for dark construction"""
901  config.isr.doDark = False
902  config.isr.doFlat = False
903  config.isr.doFringe = False
904 
905  def processSingle(self, sensorRef):
906  """Process a single CCD
907 
908  Besides the regular ISR, also masks cosmic-rays and divides each
909  processed image by the dark time to generate images of the dark rate.
910  The dark time is provided by the 'getDarkTime' method.
911  """
912  exposure = CalibTask.processSingle(self, sensorRef)
913 
914  if self.config.doRepair:
915  psf = measAlg.DoubleGaussianPsf(self.config.psfSize, self.config.psfSize,
916  self.config.psfFwhm/(2*math.sqrt(2*math.log(2))))
917  exposure.setPsf(psf)
918  self.repair.run(exposure, keepCRs=False)
919  if self.config.crGrow > 0:
920  mask = exposure.getMaskedImage().getMask().clone()
921  mask &= mask.getPlaneBitMask("CR")
922  fpSet = afwDet.FootprintSet(
923  mask, afwDet.Threshold(0.5))
924  fpSet = afwDet.FootprintSet(fpSet, self.config.crGrow, True)
925  fpSet.setMask(exposure.getMaskedImage().getMask(), "CR")
926 
927  mi = exposure.getMaskedImage()
928  mi /= self.getDarkTime(exposure)
929  return exposure
930 
931  def getDarkTime(self, exposure):
932  """Retrieve the dark time for an exposure"""
933  darkTime = exposure.getInfo().getVisitInfo().getDarkTime()
934  if not np.isfinite(darkTime):
935  raise RuntimeError("Non-finite darkTime")
936  return darkTime
937 
938 
940  """Configuration for flat construction"""
941  iterations = Field(dtype=int, default=10,
942  doc="Number of iterations for scale determination")
943  stats = ConfigurableField(target=CalibStatsTask,
944  doc="Background statistics configuration")
945 
946 
948  """Flat construction
949 
950  The principal change from the base class involves gathering the background
951  values from each image and using them to determine the scalings for the final
952  combination.
953  """
954  ConfigClass = FlatConfig
955  _DefaultName = "flat"
956  calibName = "flat"
957 
958  @classmethod
959  def applyOverrides(cls, config):
960  """Overrides for flat construction"""
961  config.isr.doFlat = False
962  config.isr.doFringe = False
963 
964  def __init__(self, *args, **kwargs):
965  CalibTask.__init__(self, *args, **kwargs)
966  self.makeSubtask("stats")
967 
968  def processResult(self, exposure):
969  return self.stats.run(exposure)
970 
971  def scale(self, ccdIdLists, data):
972  """Determine the scalings for the final combination
973 
974  We have a matrix B_ij = C_i E_j, where C_i is the relative scaling
975  of one CCD to all the others in an exposure, and E_j is the scaling
976  of the exposure. We convert everything to logarithms so we can work
977  with a linear system. We determine the C_i and E_j from B_ij by iteration,
978  under the additional constraint that the average CCD scale is unity.
979 
980  This algorithm comes from Eugene Magnier and Pan-STARRS.
981  """
982  assert len(ccdIdLists.values()) > 0, "No successful CCDs"
983  lengths = set([len(expList) for expList in ccdIdLists.values()])
984  assert len(
985  lengths) == 1, "Number of successful exposures for each CCD differs"
986  assert tuple(lengths)[0] > 0, "No successful exposures"
987  # Format background measurements into a matrix
988  indices = dict((name, i) for i, name in enumerate(ccdIdLists))
989  bgMatrix = np.array([[0.0] * len(expList)
990  for expList in ccdIdLists.values()])
991  for name in ccdIdLists:
992  i = indices[name]
993  bgMatrix[i] = [
994  d if d is not None else np.nan for d in data[name]]
995 
996  numpyPrint = np.get_printoptions()
997  np.set_printoptions(threshold=np.inf)
998  self.log.info("Input backgrounds: %s" % bgMatrix)
999 
1000  # Flat-field scaling
1001  numCcds = len(ccdIdLists)
1002  numExps = bgMatrix.shape[1]
1003  # log(Background) for each exposure/component
1004  bgMatrix = np.log(bgMatrix)
1005  bgMatrix = np.ma.masked_array(bgMatrix, np.isnan(bgMatrix))
1006  # Initial guess at log(scale) for each component
1007  compScales = np.zeros(numCcds)
1008  expScales = np.array(
1009  [(bgMatrix[:, i0] - compScales).mean() for i0 in range(numExps)])
1010 
1011  for iterate in range(self.config.iterations):
1012  compScales = np.array(
1013  [(bgMatrix[i1, :] - expScales).mean() for i1 in range(numCcds)])
1014  expScales = np.array(
1015  [(bgMatrix[:, i2] - compScales).mean() for i2 in range(numExps)])
1016 
1017  avgScale = np.average(np.exp(compScales))
1018  compScales -= np.log(avgScale)
1019  self.log.debug("Iteration %d exposure scales: %s",
1020  iterate, np.exp(expScales))
1021  self.log.debug("Iteration %d component scales: %s",
1022  iterate, np.exp(compScales))
1023 
1024  expScales = np.array(
1025  [(bgMatrix[:, i3] - compScales).mean() for i3 in range(numExps)])
1026 
1027  if np.any(np.isnan(expScales)):
1028  raise RuntimeError("Bad exposure scales: %s --> %s" %
1029  (bgMatrix, expScales))
1030 
1031  expScales = np.exp(expScales)
1032  compScales = np.exp(compScales)
1033 
1034  self.log.info("Exposure scales: %s" % expScales)
1035  self.log.info("Component relative scaling: %s" % compScales)
1036  np.set_printoptions(**numpyPrint)
1037 
1038  return dict((ccdName, Struct(ccdScale=compScales[indices[ccdName]], expScales=expScales))
1039  for ccdName in ccdIdLists)
1040 
1041 
1043  """Configuration for fringe construction"""
1044  stats = ConfigurableField(target=CalibStatsTask,
1045  doc="Background statistics configuration")
1046  subtractBackground = ConfigurableField(target=measAlg.SubtractBackgroundTask,
1047  doc="Background configuration")
1048  detection = ConfigurableField(
1049  target=measAlg.SourceDetectionTask, doc="Detection configuration")
1050  detectSigma = Field(dtype=float, default=1.0,
1051  doc="Detection PSF gaussian sigma")
1052 
1053 
1055  """Fringe construction task
1056 
1057  The principal change from the base class is that the images are
1058  background-subtracted and rescaled by the background.
1059 
1060  XXX This is probably not right for a straight-up combination, as we
1061  are currently doing, since the fringe amplitudes need not scale with
1062  the continuum.
1063 
1064  XXX Would like to have this do PCA and generate multiple images, but
1065  that will take a bit of work with the persistence code.
1066  """
1067  ConfigClass = FringeConfig
1068  _DefaultName = "fringe"
1069  calibName = "fringe"
1070 
1071  @classmethod
1072  def applyOverrides(cls, config):
1073  """Overrides for fringe construction"""
1074  config.isr.doFringe = False
1075 
1076  def __init__(self, *args, **kwargs):
1077  CalibTask.__init__(self, *args, **kwargs)
1078  self.makeSubtask("detection")
1079  self.makeSubtask("stats")
1080  self.makeSubtask("subtractBackground")
1081 
1082  def processSingle(self, sensorRef):
1083  """Subtract the background and normalise by the background level"""
1084  exposure = CalibTask.processSingle(self, sensorRef)
1085  bgLevel = self.stats.run(exposure)
1086  self.subtractBackground.run(exposure)
1087  mi = exposure.getMaskedImage()
1088  mi /= bgLevel
1089  footprintSets = self.detection.detectFootprints(
1090  exposure, sigma=self.config.detectSigma)
1091  mask = exposure.getMaskedImage().getMask()
1092  detected = 1 << mask.addMaskPlane("DETECTED")
1093  for fpSet in (footprintSets.positive, footprintSets.negative):
1094  if fpSet is not None:
1095  afwDet.setMaskFromFootprintList(
1096  mask, fpSet.getFootprints(), detected)
1097  return exposure
1098 
1099 
1101  """Configuration for sky frame construction"""
1102  detection = ConfigurableField(target=measAlg.SourceDetectionTask, doc="Detection configuration")
1103  detectSigma = Field(dtype=float, default=2.0, doc="Detection PSF gaussian sigma")
1104  subtractBackground = ConfigurableField(target=measAlg.SubtractBackgroundTask,
1105  doc="Regular-scale background configuration, for object detection")
1106  largeScaleBackground = ConfigField(dtype=FocalPlaneBackgroundConfig,
1107  doc="Large-scale background configuration")
1108  sky = ConfigurableField(target=SkyMeasurementTask, doc="Sky measurement")
1109  maskThresh = Field(dtype=float, default=3.0, doc="k-sigma threshold for masking pixels")
1110  mask = ListField(dtype=str, default=["BAD", "SAT", "DETECTED", "NO_DATA"],
1111  doc="Mask planes to consider as contaminated")
1112 
1113 
1115  """Task for sky frame construction
1116 
1117  The sky frame is a (relatively) small-scale background
1118  model, the response of the camera to the sky.
1119 
1120  To construct, we first remove a large-scale background (e.g., caused
1121  by moonlight) which may vary from image to image. Then we construct a
1122  model of the sky, which is essentially a binned version of the image
1123  (important configuration parameters: sky.background.[xy]BinSize).
1124  It is these models which are coadded to yield the sky frame.
1125  """
1126  ConfigClass = SkyConfig
1127  _DefaultName = "sky"
1128  calibName = "sky"
1129 
1130  def __init__(self, *args, **kwargs):
1131  CalibTask.__init__(self, *args, **kwargs)
1132  self.makeSubtask("detection")
1133  self.makeSubtask("subtractBackground")
1134  self.makeSubtask("sky")
1135 
1136  def scatterProcess(self, pool, ccdIdLists):
1137  """!Scatter the processing among the nodes
1138 
1139  Only the master node executes this method, assigning work to the
1140  slaves.
1141 
1142  We measure and subtract off a large-scale background model across
1143  all CCDs, which requires a scatter/gather. Then we process the
1144  individual CCDs, subtracting the large-scale background model and
1145  the residual background model measured. These residuals will be
1146  combined for the sky frame.
1147 
1148  @param pool Process pool
1149  @param ccdIdLists Dict of data identifier lists for each CCD name
1150  @return Dict of lists of returned data for each CCD name
1151  """
1152  self.log.info("Scatter processing")
1153 
1154  numExps = set(len(expList) for expList in ccdIdLists.values())
1155  assert len(numExps) == 1
1156  numExps = numExps.pop()
1157 
1158  # First subtract off general gradients to make all the exposures look similar.
1159  # We want to preserve the common small-scale structure, which we will coadd.
1160  bgModelList = mapToMatrix(pool, self.measureBackground, ccdIdLists)
1161 
1162  backgrounds = {}
1163  scales = {}
1164  for exp in range(numExps):
1165  bgModels = [bgModelList[ccdName][exp] for ccdName in ccdIdLists]
1166  visit = set(tuple(ccdIdLists[ccdName][exp][key] for key in sorted(self.config.visitKeys)) for
1167  ccdName in ccdIdLists)
1168  assert len(visit) == 1
1169  visit = visit.pop()
1170  bgModel = bgModels[0]
1171  for bg in bgModels[1:]:
1172  bgModel.merge(bg)
1173  self.log.info("Background model min/max for visit %s: %f %f", visit,
1174  np.min(bgModel.getStatsImage().getArray()),
1175  np.max(bgModel.getStatsImage().getArray()))
1176  backgrounds[visit] = bgModel
1177  scales[visit] = np.median(bgModel.getStatsImage().getArray())
1178 
1179  return mapToMatrix(pool, self.process, ccdIdLists, backgrounds=backgrounds, scales=scales)
1180 
1181  def measureBackground(self, cache, dataId):
1182  """!Measure background model for CCD
1183 
1184  This method is executed by the slaves.
1185 
1186  The background models for all CCDs in an exposure will be
1187  combined to form a full focal-plane background model.
1188 
1189  @param cache Process pool cache
1190  @param dataId Data identifier
1191  @return Bcakground model
1192  """
1193  dataRef = getDataRef(cache.butler, dataId)
1194  exposure = self.processSingleBackground(dataRef)
1195 
1196  # NAOJ prototype smoothed and then combined the entire image, but it shouldn't be any different
1197  # to bin and combine the binned images except that there's fewer pixels to worry about.
1198  config = self.config.largeScaleBackground
1199  camera = dataRef.get("camera")
1200  bgModel = FocalPlaneBackground.fromCamera(config, camera)
1201  bgModel.addCcd(exposure)
1202  return bgModel
1203 
1204  def processSingleBackground(self, dataRef):
1205  """!Process a single CCD for the background
1206 
1207  This method is executed by the slaves.
1208 
1209  Because we're interested in the background, we detect and mask astrophysical
1210  sources, and pixels above the noise level.
1211 
1212  @param dataRef Data reference for CCD.
1213  @return processed exposure
1214  """
1215  if not self.config.clobber and dataRef.datasetExists("postISRCCD"):
1216  return dataRef.get("postISRCCD")
1217  exposure = CalibTask.processSingle(self, dataRef)
1218 
1219  # Detect sources. Requires us to remove the background; we'll restore it later.
1220  bgTemp = self.subtractBackground.run(exposure).background
1221  footprints = self.detection.detectFootprints(exposure, sigma=self.config.detectSigma)
1222  image = exposure.getMaskedImage()
1223  if footprints.background is not None:
1224  image += footprints.background.getImageF()
1225 
1226  # Mask high pixels
1227  variance = image.getVariance()
1228  noise = np.sqrt(np.median(variance.getArray()))
1229  isHigh = image.getImage().getArray() > self.config.maskThresh*noise
1230  image.getMask().getArray()[isHigh] |= image.getMask().getPlaneBitMask("DETECTED")
1231 
1232  # Restore the background: it's what we want!
1233  image += bgTemp.getImage()
1234 
1235  # Set detected/bad pixels to background to ensure they don't corrupt the background
1236  maskVal = image.getMask().getPlaneBitMask(self.config.mask)
1237  isBad = image.getMask().getArray() & maskVal > 0
1238  bgLevel = np.median(image.getImage().getArray()[~isBad])
1239  image.getImage().getArray()[isBad] = bgLevel
1240  dataRef.put(exposure, "postISRCCD")
1241  return exposure
1242 
1243  def processSingle(self, dataRef, backgrounds, scales):
1244  """Process a single CCD, specified by a data reference
1245 
1246  We subtract the appropriate focal plane background model,
1247  divide by the appropriate scale and measure the background.
1248 
1249  Only slave nodes execute this method.
1250 
1251  @param dataRef Data reference for single CCD
1252  @param backgrounds Background model for each visit
1253  @param scales Scales for each visit
1254  @return Processed exposure
1255  """
1256  visit = tuple(dataRef.dataId[key] for key in sorted(self.config.visitKeys))
1257  exposure = dataRef.get("postISRCCD", immediate=True)
1258  image = exposure.getMaskedImage()
1259  detector = exposure.getDetector()
1260  bbox = image.getBBox()
1261 
1262  bgModel = backgrounds[visit]
1263  bg = bgModel.toCcdBackground(detector, bbox)
1264  image -= bg.getImage()
1265  image /= scales[visit]
1266 
1267  bg = self.sky.measureBackground(exposure.getMaskedImage())
1268  dataRef.put(bg, "icExpBackground")
1269  return exposure
1270 
1271  def combine(self, cache, struct, outputId):
1272  """!Combine multiple background models of a particular CCD and write the output
1273 
1274  Only the slave nodes execute this method.
1275 
1276  @param cache Process pool cache
1277  @param struct Parameters for the combination, which has the following components:
1278  * ccdName Name tuple for CCD
1279  * ccdIdList List of data identifiers for combination
1280  @param outputId Data identifier for combined image (exposure part only)
1281  @return binned calib image
1282  """
1283  outputId = self.getFullyQualifiedOutputId(struct.ccdName, cache.butler, outputId)
1284  dataRefList = [getDataRef(cache.butler, dataId) if dataId is not None else None for
1285  dataId in struct.ccdIdList]
1286  self.log.info("Combining %s on %s" % (outputId, NODE))
1287  bgList = [dataRef.get("icExpBackground", immediate=True).clone() for dataRef in dataRefList]
1288 
1289  bgExp = self.sky.averageBackgrounds(bgList)
1290 
1291  self.recordCalibInputs(cache.butler, bgExp, struct.ccdIdList, outputId)
1292  cache.butler.put(bgExp, "sky", outputId)
1293  return afwMath.binImage(self.sky.exposureToBackground(bgExp).getImage(), self.config.binning)
def processWrite(self, dataRef, exposure, outputName="postISRCCD")
Write the processed CCD.
def scatterCombine(self, pool, outputId, ccdIdLists, scales)
Scatter the combination of exposures across multiple nodes.
def run(self, exposureOrImage)
Measure a particular statistic on an image (of some sort).
def checksum(obj, header=None, sumType="MD5")
Calculate a checksum of an object.
Definition: checksum.py:29
def getCcdIdListFromExposures(expRefList, level="sensor", ccdKeys=["ccd"])
Determine a list of CCDs from exposure references.
def run(self, expRefList, butler, calibId)
Construct a calib from a list of exposure references.
def getFullyQualifiedOutputId(self, ccdName, butler, outputId)
def __call__(self, parser, namespace, values, option_string)
def dictToTuple(dict_, keys)
Return a tuple of specific values from a dict.
def getDataRef(butler, dataId, datasetType="raw")
Definition: utils.py:17
def updateMetadata(self, calibImage, exposureTime, darkTime=None, kwargs)
Update the metadata from the VisitInfo.
def process(self, cache, ccdId, outputName="postISRCCD", kwargs)
Process a CCD, specified by a data identifier.
def processSingle(self, dataRef, backgrounds, scales)
def makeCameraImage(self, camera, dataId, calibs)
Create and write an image of the entire camera.
def combine(self, target, imageList, stats)
Combine multiple images.
def getMjd(self, butler, dataId, timescale=dafBase.DateTime.UTC)
def combine(self, cache, struct, outputId)
Combine multiple exposures of a particular CCD and write the output.
def write(self, butler, exposure, dataId)
Write the final combined calib.
def combine(self, cache, struct, outputId)
Combine multiple background models of a particular CCD and write the output.
def measureBackground(self, cache, dataId)
Measure background model for CCD.
def recordCalibInputs(self, butler, calib, dataIdList, outputId)
Record metadata including the inputs and creation details.
def scatterProcess(self, pool, ccdIdLists)
Scatter the processing among the nodes.
def run(self, sensorRefList, expScales=None, finalScale=None, inputName="postISRCCD")
Combine calib images for a single sensor.
def getOutputId(self, expRefList, calibId)
Generate the data identifier for the output calib.
def mapToMatrix(pool, func, ccdIdLists, args, kwargs)
def scale(self, ccdIdLists, data)
Determine scaling across CCDs and exposures.
def addMissingKeys(self, dataId, butler, missingKeys=None, calibName=None)
def getDimensions(self, sensorRefList, inputName="postISRCCD")
Base class for constructing calibs.
def processSingleBackground(self, dataRef)
Process a single CCD for the background.
def scatterProcess(self, pool, ccdIdLists)
Scatter the processing among the nodes.
def batchWallTime(cls, time, parsedCmd, numCores)