lsst.pipe.drivers  14.0-7-g3001e72
constructCalibs.py
Go to the documentation of this file.
1 from __future__ import absolute_import, division, print_function
2 
3 import sys
4 import math
5 import time
6 import argparse
7 import traceback
8 
9 import numpy as np
10 from builtins import zip
11 from builtins import range
12 
13 from lsst.pex.config import Config, ConfigurableField, Field, ListField, ConfigField
14 from lsst.pipe.base import Task, Struct, TaskRunner, ArgumentParser
15 import lsst.daf.base as dafBase
16 import lsst.afw.math as afwMath
17 import lsst.afw.geom as afwGeom
18 import lsst.afw.detection as afwDet
19 import lsst.afw.image as afwImage
20 from lsst.afw.image import VisitInfo
21 import lsst.meas.algorithms as measAlg
22 from lsst.pipe.tasks.repair import RepairTask
23 from lsst.ip.isr import IsrTask
24 from lsst.afw.cameraGeom.utils import makeImageFromCamera
25 
26 from lsst.ctrl.pool.parallel import BatchPoolTask
27 from lsst.ctrl.pool.pool import Pool, NODE
28 from lsst.pipe.drivers.background import SkyMeasurementTask, FocalPlaneBackground, FocalPlaneBackgroundConfig
29 
30 from .checksum import checksum
31 from .utils import getDataRef
32 
33 
34 class CalibStatsConfig(Config):
35  """Parameters controlling the measurement of background statistics"""
36  stat = Field(doc="Statistic to use to estimate background (from lsst.afw.math)", dtype=int,
37  default=int(afwMath.MEANCLIP))
38  clip = Field(doc="Clipping threshold for background",
39  dtype=float, default=3.0)
40  nIter = Field(doc="Clipping iterations for background",
41  dtype=int, default=3)
42  mask = ListField(doc="Mask planes to reject",
43  dtype=str, default=["DETECTED", "BAD", "NO_DATA",])
44 
45 
46 class CalibStatsTask(Task):
47  """Measure statistics on the background
48 
49  This can be useful for scaling the background, e.g., for flats and fringe frames.
50  """
51  ConfigClass = CalibStatsConfig
52 
53  def run(self, exposureOrImage):
54  """!Measure a particular statistic on an image (of some sort).
55 
56  @param exposureOrImage Exposure, MaskedImage or Image.
57  @return Value of desired statistic
58  """
59  stats = afwMath.StatisticsControl(self.config.clip, self.config.nIter,
60  afwImage.Mask.getPlaneBitMask(self.config.mask))
61  try:
62  image = exposureOrImage.getMaskedImage()
63  except:
64  try:
65  image = exposureOrImage.getImage()
66  except:
67  image = exposureOrImage
68 
69  return afwMath.makeStatistics(image, self.config.stat, stats).getValue()
70 
71 
72 class CalibCombineConfig(Config):
73  """Configuration for combining calib images"""
74  rows = Field(doc="Number of rows to read at a time",
75  dtype=int, default=512)
76  mask = ListField(doc="Mask planes to respect", dtype=str,
77  default=["SAT", "DETECTED", "INTRP"])
78  combine = Field(doc="Statistic to use for combination (from lsst.afw.math)", dtype=int,
79  default=int(afwMath.MEANCLIP))
80  clip = Field(doc="Clipping threshold for combination",
81  dtype=float, default=3.0)
82  nIter = Field(doc="Clipping iterations for combination",
83  dtype=int, default=3)
84  stats = ConfigurableField(target=CalibStatsTask,
85  doc="Background statistics configuration")
86 
87 
88 class CalibCombineTask(Task):
89  """Task to combine calib images"""
90  ConfigClass = CalibCombineConfig
91 
92  def __init__(self, *args, **kwargs):
93  Task.__init__(self, *args, **kwargs)
94  self.makeSubtask("stats")
95 
96  def run(self, sensorRefList, expScales=None, finalScale=None, inputName="postISRCCD"):
97  """!Combine calib images for a single sensor
98 
99  @param sensorRefList List of data references to combine (for a single sensor)
100  @param expScales List of scales to apply for each exposure
101  @param finalScale Desired scale for final combined image
102  @param inputName Data set name for inputs
103  @return combined image
104  """
105  width, height = self.getDimensions(sensorRefList)
106  stats = afwMath.StatisticsControl(self.config.clip, self.config.nIter,
107  afwImage.Mask.getPlaneBitMask(self.config.mask))
108 
109  # Combine images
110  combined = afwImage.MaskedImageF(width, height)
111  numImages = len(sensorRefList)
112  imageList = [None]*numImages
113  for start in range(0, height, self.config.rows):
114  rows = min(self.config.rows, height - start)
115  box = afwGeom.Box2I(afwGeom.Point2I(0, start),
116  afwGeom.Extent2I(width, rows))
117  subCombined = combined.Factory(combined, box)
118 
119  for i, sensorRef in enumerate(sensorRefList):
120  if sensorRef is None:
121  imageList[i] = None
122  continue
123  exposure = sensorRef.get(inputName + "_sub", bbox=box)
124  if expScales is not None:
125  self.applyScale(exposure, expScales[i])
126  imageList[i] = exposure.getMaskedImage()
127 
128  self.combine(subCombined, imageList, stats)
129 
130  if finalScale is not None:
131  background = self.stats.run(combined)
132  self.log.info("%s: Measured background of stack is %f; adjusting to %f" %
133  (NODE, background, finalScale))
134  combined *= finalScale / background
135 
136  return combined
137 
138  def getDimensions(self, sensorRefList, inputName="postISRCCD"):
139  """Get dimensions of the inputs"""
140  dimList = []
141  for sensorRef in sensorRefList:
142  if sensorRef is None:
143  continue
144  md = sensorRef.get(inputName + "_md")
145  dimList.append(afwImage.bboxFromMetadata(md).getDimensions())
146  return getSize(dimList)
147 
148  def applyScale(self, exposure, scale=None):
149  """Apply scale to input exposure
150 
151  This implementation applies a flux scaling: the input exposure is
152  divided by the provided scale.
153  """
154  if scale is not None:
155  mi = exposure.getMaskedImage()
156  mi /= scale
157 
158  def combine(self, target, imageList, stats):
159  """!Combine multiple images
160 
161  @param target Target image to receive the combined pixels
162  @param imageList List of input images
163  @param stats Statistics control
164  """
165  images = [img for img in imageList if img is not None]
166  afwMath.statisticsStack(target, images, afwMath.Property(self.config.combine), stats)
167 
168 
169 def getSize(dimList):
170  """Determine a consistent size, given a list of image sizes"""
171  dim = set((w, h) for w, h in dimList)
172  dim.discard(None)
173  if len(dim) != 1:
174  raise RuntimeError("Inconsistent dimensions: %s" % dim)
175  return dim.pop()
176 
177 
178 def dictToTuple(dict_, keys):
179  """!Return a tuple of specific values from a dict
180 
181  This provides a hashable representation of the dict from certain keywords.
182  This can be useful for creating e.g., a tuple of the values in the DataId
183  that identify the CCD.
184 
185  @param dict_ dict to parse
186  @param keys keys to extract (order is important)
187  @return tuple of values
188  """
189  return tuple(dict_[k] for k in keys)
190 
191 
192 def getCcdIdListFromExposures(expRefList, level="sensor", ccdKeys=["ccd"]):
193  """!Determine a list of CCDs from exposure references
194 
195  This essentially inverts the exposure-level references (which
196  provides a list of CCDs for each exposure), by providing
197  a dataId list for each CCD. Consider an input list of exposures
198  [e1, e2, e3], and each exposure has CCDs c1 and c2. Then this
199  function returns:
200 
201  {(c1,): [e1c1, e2c1, e3c1], (c2,): [e1c2, e2c2, e3c2]}
202 
203  This is a dict whose keys are tuples of the identifying values of a
204  CCD (usually just the CCD number) and the values are lists of dataIds
205  for that CCD in each exposure. A missing dataId is given the value
206  None.
207 
208  @param expRefList List of data references for exposures
209  @param level Level for the butler to generate CCDs
210  @param ccdKeys DataId keywords that identify a CCD
211  @return dict of data identifier lists for each CCD
212  """
213  expIdList = [[ccdRef.dataId for ccdRef in expRef.subItems(
214  level)] for expRef in expRefList]
215 
216  # Determine what additional keys make a CCD from an exposure
217  ccdKeys = set(ccdKeys) # Set of keywords in the dataId that identify a CCD
218  ccdNames = set() # Set of tuples which are values for each of the CCDs in an exposure
219  for ccdIdList in expIdList:
220  for ccdId in ccdIdList:
221  name = dictToTuple(ccdId, ccdKeys)
222  ccdNames.add(name)
223 
224  # Turn the list of CCDs for each exposure into a list of exposures for
225  # each CCD
226  ccdLists = {}
227  for n, ccdIdList in enumerate(expIdList):
228  for ccdId in ccdIdList:
229  name = dictToTuple(ccdId, ccdKeys)
230  if name not in ccdLists:
231  ccdLists[name] = []
232  ccdLists[name].append(ccdId)
233 
234  for ccd in ccdLists:
235  # Sort the list by the dataId values (ordered by key)
236  ccdLists[ccd] = sorted(ccdLists[ccd], key=lambda dd: dictToTuple(dd, sorted(dd.keys())))
237 
238  return ccdLists
239 
240 
241 def mapToMatrix(pool, func, ccdIdLists, *args, **kwargs):
242  """Generate a matrix of results using pool.map
243 
244  The function should have the call signature:
245  func(cache, dataId, *args, **kwargs)
246 
247  We return a dict mapping 'ccd name' to a list of values for
248  each exposure.
249 
250  @param pool Process pool
251  @param func Function to call for each dataId
252  @param ccdIdLists Dict of data identifier lists for each CCD name
253  @return matrix of results
254  """
255  dataIdList = sum(ccdIdLists.values(), [])
256  resultList = pool.map(func, dataIdList, *args, **kwargs)
257  # Piece everything back together
258  data = dict((ccdName, [None] * len(expList)) for ccdName, expList in ccdIdLists.items())
259  indices = dict(sum([[(tuple(dataId.values()) if dataId is not None else None, (ccdName, expNum))
260  for expNum, dataId in enumerate(expList)]
261  for ccdName, expList in ccdIdLists.items()], []))
262  for dataId, result in zip(dataIdList, resultList):
263  if dataId is None:
264  continue
265  ccdName, expNum = indices[tuple(dataId.values())]
266  data[ccdName][expNum] = result
267  return data
268 
269 
270 class CalibIdAction(argparse.Action):
271  """Split name=value pairs and put the result in a dict"""
272 
273  def __call__(self, parser, namespace, values, option_string):
274  output = getattr(namespace, self.dest, {})
275  for nameValue in values:
276  name, sep, valueStr = nameValue.partition("=")
277  if not valueStr:
278  parser.error("%s value %s must be in form name=value" %
279  (option_string, nameValue))
280  output[name] = valueStr
281  setattr(namespace, self.dest, output)
282 
283 
284 class CalibArgumentParser(ArgumentParser):
285  """ArgumentParser for calibration construction"""
286 
287  def __init__(self, calibName, *args, **kwargs):
288  """Add a --calibId argument to the standard pipe_base argument parser"""
289  ArgumentParser.__init__(self, *args, **kwargs)
290  self.calibName = calibName
291  self.add_id_argument("--id", datasetType="raw",
292  help="input identifiers, e.g., --id visit=123 ccd=4")
293  self.add_argument("--calibId", nargs="*", action=CalibIdAction, default={},
294  help="identifiers for calib, e.g., --calibId version=1",
295  metavar="KEY=VALUE1[^VALUE2[^VALUE3...]")
296 
297  def parse_args(self, *args, **kwargs):
298  """Parse arguments
299 
300  Checks that the "--calibId" provided works.
301  """
302  namespace = ArgumentParser.parse_args(self, *args, **kwargs)
303 
304  keys = namespace.butler.getKeys(self.calibName)
305  parsed = {}
306  for name, value in namespace.calibId.items():
307  if name not in keys:
308  self.error(
309  "%s is not a relevant calib identifier key (%s)" % (name, keys))
310  parsed[name] = keys[name](value)
311  namespace.calibId = parsed
312 
313  return namespace
314 
315 
316 class CalibConfig(Config):
317  """Configuration for constructing calibs"""
318  clobber = Field(dtype=bool, default=True,
319  doc="Clobber existing processed images?")
320  isr = ConfigurableField(target=IsrTask, doc="ISR configuration")
321  dateObs = Field(dtype=str, default="dateObs",
322  doc="Key for observation date in exposure registry")
323  dateCalib = Field(dtype=str, default="calibDate",
324  doc="Key for calib date in calib registry")
325  filter = Field(dtype=str, default="filter",
326  doc="Key for filter name in exposure/calib registries")
327  combination = ConfigurableField(
328  target=CalibCombineTask, doc="Calib combination configuration")
329  ccdKeys = ListField(dtype=str, default=["ccd"],
330  doc="DataId keywords specifying a CCD")
331  visitKeys = ListField(dtype=str, default=["visit"],
332  doc="DataId keywords specifying a visit")
333  calibKeys = ListField(dtype=str, default=[],
334  doc="DataId keywords specifying a calibration")
335  doCameraImage = Field(dtype=bool, default=True, doc="Create camera overview image?")
336  binning = Field(dtype=int, default=64, doc="Binning to apply for camera image")
337 
338  def setDefaults(self):
339  self.isr.doWrite = False
340 
341 
342 class CalibTaskRunner(TaskRunner):
343  """Get parsed values into the CalibTask.run"""
344  @staticmethod
345  def getTargetList(parsedCmd, **kwargs):
346  return [dict(expRefList=parsedCmd.id.refList, butler=parsedCmd.butler, calibId=parsedCmd.calibId)]
347 
348  def __call__(self, args):
349  """Call the Task with the kwargs from getTargetList"""
350  task = self.TaskClass(config=self.config, log=self.log)
351  exitStatus = 0 # exit status for the shell
352  if self.doRaise:
353  result = task.run(**args)
354  else:
355  try:
356  result = task.run(**args)
357  except Exception as e:
358  exitStatus = 1 # n.b. The shell exit value is the number of dataRefs returning
359  # non-zero, so the actual value used here is lost
360  task.log.fatal("Failed: %s" % e)
361  traceback.print_exc(file=sys.stderr)
362 
363  if self.doReturnResults:
364  return Struct(
365  exitStatus=exitStatus,
366  args=args,
367  metadata=task.metadata,
368  result=result,
369  )
370  else:
371  return Struct(
372  exitStatus=exitStatus,
373  )
374 
376  """!Base class for constructing calibs.
377 
378  This should be subclassed for each of the required calib types.
379  The subclass should be sure to define the following class variables:
380  * _DefaultName: default name of the task, used by CmdLineTask
381  * calibName: name of the calibration data set in the butler
382  The subclass may optionally set:
383  * filterName: filter name to give the resultant calib
384  """
385  ConfigClass = CalibConfig
386  RunnerClass = CalibTaskRunner
387  filterName = None
388  calibName = None
389  exposureTime = 1.0 # sets this exposureTime in the output
390 
391  def __init__(self, *args, **kwargs):
392  """Constructor"""
393  BatchPoolTask.__init__(self, *args, **kwargs)
394  self.makeSubtask("isr")
395  self.makeSubtask("combination")
396 
397  @classmethod
398  def batchWallTime(cls, time, parsedCmd, numCores):
399  numCcds = len(parsedCmd.butler.get("camera"))
400  numExps = len(cls.RunnerClass.getTargetList(
401  parsedCmd)[0]['expRefList'])
402  numCycles = int(numCcds/float(numCores) + 0.5)
403  return time*numExps*numCycles
404 
405  @classmethod
406  def _makeArgumentParser(cls, *args, **kwargs):
407  kwargs.pop("doBatch", False)
408  return CalibArgumentParser(calibName=cls.calibName, name=cls._DefaultName, *args, **kwargs)
409 
410  def run(self, expRefList, butler, calibId):
411  """!Construct a calib from a list of exposure references
412 
413  This is the entry point, called by the TaskRunner.__call__
414 
415  Only the master node executes this method.
416 
417  @param expRefList List of data references at the exposure level
418  @param butler Data butler
419  @param calibId Identifier dict for calib
420  """
421  for expRef in expRefList:
422  self.addMissingKeys(expRef.dataId, butler, self.config.ccdKeys, 'raw')
423 
424  outputId = self.getOutputId(expRefList, calibId)
425  ccdIdLists = getCcdIdListFromExposures(
426  expRefList, level="sensor", ccdKeys=self.config.ccdKeys)
427 
428  # Ensure we can generate filenames for each output
429  outputIdItemList = list(outputId.items())
430  for ccdName in ccdIdLists:
431  dataId = dict([(k, ccdName[i]) for i, k in enumerate(self.config.ccdKeys)])
432  dataId.update(outputIdItemList)
433  self.addMissingKeys(dataId, butler)
434  dataId.update(outputIdItemList)
435 
436  try:
437  butler.get(self.calibName + "_filename", dataId)
438  except Exception as e:
439  raise RuntimeError(
440  "Unable to determine output filename \"%s_filename\" from %s: %s" %
441  (self.calibName, dataId, e))
442 
443  processPool = Pool("process")
444  processPool.storeSet(butler=butler)
445 
446  # Scatter: process CCDs independently
447  data = self.scatterProcess(processPool, ccdIdLists)
448 
449  # Gather: determine scalings
450  scales = self.scale(ccdIdLists, data)
451 
452  combinePool = Pool("combine")
453  combinePool.storeSet(butler=butler)
454 
455  # Scatter: combine
456  calibs = self.scatterCombine(combinePool, outputId, ccdIdLists, scales)
457 
458  if self.config.doCameraImage:
459  camera = butler.get("camera")
460 
461  try:
462  cameraImage = self.makeCameraImage(camera, outputId, calibs)
463  butler.put(cameraImage, self.calibName + "_camera", dataId)
464  except Exception as exc:
465  self.log.warn("Unable to create camera image: %s" % (exc,))
466 
467  return Struct(
468  outputId = outputId,
469  ccdIdLists = ccdIdLists,
470  scales = scales,
471  calibs = calibs,
472  processPool = processPool,
473  combinePool = combinePool,
474  )
475 
476  def getOutputId(self, expRefList, calibId):
477  """!Generate the data identifier for the output calib
478 
479  The mean date and the common filter are included, using keywords
480  from the configuration. The CCD-specific part is not included
481  in the data identifier.
482 
483  @param expRefList List of data references at exposure level
484  @param calibId Data identifier elements for the calib provided by the user
485  @return data identifier
486  """
487  midTime = 0
488  filterName = None
489  for expRef in expRefList:
490  butler = expRef.getButler()
491  dataId = expRef.dataId
492 
493  midTime += self.getMjd(butler, dataId)
494  thisFilter = self.getFilter(
495  butler, dataId) if self.filterName is None else self.filterName
496  if filterName is None:
497  filterName = thisFilter
498  elif filterName != thisFilter:
499  raise RuntimeError("Filter mismatch for %s: %s vs %s" % (
500  dataId, thisFilter, filterName))
501 
502  midTime /= len(expRefList)
503  date = str(dafBase.DateTime(
504  midTime, dafBase.DateTime.MJD).toPython().date())
505 
506  outputId = {self.config.filter: filterName,
507  self.config.dateCalib: date}
508  outputId.update(calibId)
509  return outputId
510 
511  def getMjd(self, butler, dataId, timescale=dafBase.DateTime.UTC):
512  """Determine the Modified Julian Date (MJD; in TAI) from a data identifier"""
513  if self.config.dateObs in dataId:
514  dateObs = dataId[self.config.dateObs]
515  else:
516  dateObs = butler.queryMetadata('raw', [self.config.dateObs], dataId)[0]
517  if "T" not in dateObs:
518  dateObs = dateObs + "T12:00:00.0Z"
519  elif not dateObs.endswith("Z"):
520  dateObs += "Z"
521 
522  return dafBase.DateTime(dateObs, timescale).get(dafBase.DateTime.MJD)
523 
524  def getFilter(self, butler, dataId):
525  """Determine the filter from a data identifier"""
526  filt = butler.queryMetadata('raw', [self.config.filter], dataId)[0]
527  return filt
528 
529  def addMissingKeys(self, dataId, butler, missingKeys=None, calibName=None):
530  if calibName is None:
531  calibName = self.calibName
532 
533  if missingKeys is None:
534  missingKeys = set(butler.getKeys(calibName).keys()) - set(dataId.keys())
535 
536  for k in missingKeys:
537  try:
538  v = butler.queryMetadata('raw', [k], dataId) # n.b. --id refers to 'raw'
539  except Exception as e:
540  continue
541 
542  if len(v) == 0: # failed to lookup value
543  continue
544 
545  if len(v) == 1:
546  dataId[k] = v[0]
547  else:
548  raise RuntimeError("No unique lookup for %s: %s" % (k, v))
549 
550  def updateMetadata(self, calibImage, exposureTime, darkTime=None, **kwargs):
551  """!Update the metadata from the VisitInfo
552 
553  \param calibImage The image whose metadata is to be set
554  \param exposureTime The exposure time for the image
555  \param darkTime The time since the last read (default: exposureTime)
556  """
557 
558  if darkTime is None:
559  darkTime = exposureTime # avoid warning messages when using calibration products
560 
561  visitInfo = afwImage.makeVisitInfo(exposureTime=exposureTime, darkTime=darkTime, **kwargs)
562  md = calibImage.getMetadata()
563 
564  afwImage.setVisitInfoMetadata(md, visitInfo)
565 
566  def scatterProcess(self, pool, ccdIdLists):
567  """!Scatter the processing among the nodes
568 
569  We scatter each CCD independently (exposures aren't grouped together),
570  to make full use of all available processors. This necessitates piecing
571  everything back together in the same format as ccdIdLists afterwards.
572 
573  Only the master node executes this method.
574 
575  @param pool Process pool
576  @param ccdIdLists Dict of data identifier lists for each CCD name
577  @return Dict of lists of returned data for each CCD name
578  """
579  self.log.info("Scatter processing")
580  return mapToMatrix(pool, self.process, ccdIdLists)
581 
582  def process(self, cache, ccdId, outputName="postISRCCD", **kwargs):
583  """!Process a CCD, specified by a data identifier
584 
585  After processing, optionally returns a result (produced by
586  the 'processResult' method) calculated from the processed
587  exposure. These results will be gathered by the master node,
588  and is a means for coordinated scaling of all CCDs for flats,
589  etc.
590 
591  Only slave nodes execute this method.
592 
593  @param cache Process pool cache
594  @param ccdId Data identifier for CCD
595  @param outputName Output dataset name for butler
596  @return result from 'processResult'
597  """
598  if ccdId is None:
599  self.log.warn("Null identifier received on %s" % NODE)
600  return None
601  sensorRef = getDataRef(cache.butler, ccdId)
602  if self.config.clobber or not sensorRef.datasetExists(outputName):
603  self.log.info("Processing %s on %s" % (ccdId, NODE))
604  try:
605  exposure = self.processSingle(sensorRef, **kwargs)
606  except Exception as e:
607  self.log.warn("Unable to process %s: %s" % (ccdId, e))
608  raise
609  return None
610  self.processWrite(sensorRef, exposure)
611  else:
612  self.log.info(
613  "Using previously persisted processed exposure for %s" % (sensorRef.dataId,))
614  exposure = sensorRef.get(outputName)
615  return self.processResult(exposure)
616 
617  def processSingle(self, dataRef):
618  """Process a single CCD, specified by a data reference
619 
620  Generally, this simply means doing ISR.
621 
622  Only slave nodes execute this method.
623  """
624  return self.isr.runDataRef(dataRef).exposure
625 
626  def processWrite(self, dataRef, exposure, outputName="postISRCCD"):
627  """!Write the processed CCD
628 
629  We need to write these out because we can't hold them all in
630  memory at once.
631 
632  Only slave nodes execute this method.
633 
634  @param dataRef Data reference
635  @param exposure CCD exposure to write
636  @param outputName Output dataset name for butler.
637  """
638  dataRef.put(exposure, outputName)
639 
640  def processResult(self, exposure):
641  """Extract processing results from a processed exposure
642 
643  This method generates what is gathered by the master node.
644  This can be a background measurement or similar for scaling
645  flat-fields. It must be picklable!
646 
647  Only slave nodes execute this method.
648  """
649  return None
650 
651  def scale(self, ccdIdLists, data):
652  """!Determine scaling across CCDs and exposures
653 
654  This is necessary mainly for flats, so as to determine a
655  consistent scaling across the entire focal plane. This
656  implementation is simply a placeholder.
657 
658  Only the master node executes this method.
659 
660  @param ccdIdLists Dict of data identifier lists for each CCD tuple
661  @param data Dict of lists of returned data for each CCD tuple
662  @return dict of Struct(ccdScale: scaling for CCD,
663  expScales: scaling for each exposure
664  ) for each CCD tuple
665  """
666  self.log.info("Scale on %s" % NODE)
667  return dict((name, Struct(ccdScale=None, expScales=[None] * len(ccdIdLists[name])))
668  for name in ccdIdLists)
669 
670  def scatterCombine(self, pool, outputId, ccdIdLists, scales):
671  """!Scatter the combination of exposures across multiple nodes
672 
673  In this case, we can only scatter across as many nodes as
674  there are CCDs.
675 
676  Only the master node executes this method.
677 
678  @param pool Process pool
679  @param outputId Output identifier (exposure part only)
680  @param ccdIdLists Dict of data identifier lists for each CCD name
681  @param scales Dict of structs with scales, for each CCD name
682  @param dict of binned images
683  """
684  self.log.info("Scatter combination")
685  data = [Struct(ccdName=ccdName, ccdIdList=ccdIdLists[ccdName], scales=scales[ccdName]) for
686  ccdName in ccdIdLists]
687  images = pool.map(self.combine, data, outputId)
688  return dict(zip(ccdIdLists.keys(), images))
689 
690  def getFullyQualifiedOutputId(self, ccdName, butler, outputId):
691  """Get fully-qualified output data identifier
692 
693  We may need to look up keys that aren't in the output dataId.
694 
695  @param ccdName Name tuple for CCD
696  @param butler Data butler
697  @param outputId Data identifier for combined image (exposure part only)
698  @return fully-qualified output dataId
699  """
700  fullOutputId = {k: ccdName[i] for i, k in enumerate(self.config.ccdKeys)}
701  fullOutputId.update(outputId)
702  self.addMissingKeys(fullOutputId, butler)
703  fullOutputId.update(outputId) # must be after the call to queryMetadata in 'addMissingKeys'
704  return fullOutputId
705 
706  def combine(self, cache, struct, outputId):
707  """!Combine multiple exposures of a particular CCD and write the output
708 
709  Only the slave nodes execute this method.
710 
711  @param cache Process pool cache
712  @param struct Parameters for the combination, which has the following components:
713  * ccdName Name tuple for CCD
714  * ccdIdList List of data identifiers for combination
715  * scales Scales to apply (expScales are scalings for each exposure,
716  ccdScale is final scale for combined image)
717  @param outputId Data identifier for combined image (exposure part only)
718  @return binned calib image
719  """
720  outputId = self.getFullyQualifiedOutputId(struct.ccdName, cache.butler, outputId)
721  dataRefList = [getDataRef(cache.butler, dataId) if dataId is not None else None for
722  dataId in struct.ccdIdList]
723  self.log.info("Combining %s on %s" % (outputId, NODE))
724  calib = self.combination.run(dataRefList, expScales=struct.scales.expScales,
725  finalScale=struct.scales.ccdScale)
726 
727  if not hasattr(calib, "getMetadata"):
728  if hasattr(calib, "getVariance"):
729  calib = afwImage.makeExposure(calib)
730  else:
731  calib = afwImage.DecoratedImageF(calib.getImage()) # n.b. hardwires "F" for the output type
732 
733  self.updateMetadata(calib, self.exposureTime)
734 
735  self.recordCalibInputs(cache.butler, calib,
736  struct.ccdIdList, outputId)
737 
738  self.interpolateNans(calib)
739 
740  self.write(cache.butler, calib, outputId)
741 
742  return afwMath.binImage(calib.getImage(), self.config.binning)
743 
744  def recordCalibInputs(self, butler, calib, dataIdList, outputId):
745  """!Record metadata including the inputs and creation details
746 
747  This metadata will go into the FITS header.
748 
749  @param butler Data butler
750  @param calib Combined calib exposure.
751  @param dataIdList List of data identifiers for calibration inputs
752  @param outputId Data identifier for output
753  """
754  header = calib.getMetadata()
755  header.add("OBSTYPE", self.calibName) # Used by ingestCalibs.py
756 
757  # date, time, host, and root
758  now = time.localtime()
759  header.add("CALIB_CREATION_DATE", time.strftime("%Y-%m-%d", now))
760  header.add("CALIB_CREATION_TIME", time.strftime("%X %Z", now))
761  # add date-obs as its absence upsets ExposureInfo; use the mean date that the calibs were taken
762  header.add("DATE-OBS", "%sT00:00:00.00" % outputId[self.config.dateCalib])
763 
764  # Inputs
765  visits = [str(dictToTuple(dataId, self.config.visitKeys)) for dataId in dataIdList if
766  dataId is not None]
767  for i, v in enumerate(sorted(set(visits))):
768  header.add("CALIB_INPUT_%d" % (i,), v)
769 
770  header.add("CALIB_ID", " ".join("%s=%s" % (key, value)
771  for key, value in outputId.items()))
772  checksum(calib, header)
773 
774  def interpolateNans(self, image):
775  """Interpolate over NANs in the combined image
776 
777  NANs can result from masked areas on the CCD. We don't want them getting
778  into our science images, so we replace them with the median of the image.
779  """
780  if hasattr(image, "getMaskedImage"): # Deal with Exposure vs Image
781  self.interpolateNans(image.getMaskedImage().getVariance())
782  image = image.getMaskedImage().getImage()
783  if hasattr(image, "getImage"): # Deal with DecoratedImage or MaskedImage vs Image
784  image = image.getImage()
785  array = image.getArray()
786  bad = np.isnan(array)
787  array[bad] = np.median(array[np.logical_not(bad)])
788 
789  def write(self, butler, exposure, dataId):
790  """!Write the final combined calib
791 
792  Only the slave nodes execute this method
793 
794  @param butler Data butler
795  @param exposure CCD exposure to write
796  @param dataId Data identifier for output
797  """
798  self.log.info("Writing %s on %s" % (dataId, NODE))
799  butler.put(exposure, self.calibName, dataId)
800 
801  def makeCameraImage(self, camera, dataId, calibs):
802  """!Create and write an image of the entire camera
803 
804  This is useful for judging the quality or getting an overview of
805  the features of the calib.
806 
807  This requires that the 'ccd name' is a tuple containing only the
808  detector ID. If that is not the case, change CalibConfig.ccdKeys
809  or set CalibConfig.doCameraImage=False to disable this.
810 
811  @param camera Camera object
812  @param dataId Data identifier for output
813  @param calibs Dict mapping 'ccd name' to calib image
814  """
815 
816  class ImageSource(object):
817  """Source of images for makeImageFromCamera
818 
819  This assumes that the 'ccd name' is a tuple containing
820  only the detector ID.
821  """
822  def __init__(self, images):
823  self.isTrimmed = True
824  self.images = images
825  self.background = np.nan
826 
827  def getCcdImage(self, detector, imageFactory, binSize):
828  detId = (detector.getId(),)
829  if detId not in self.images:
830  return imageFactory(1, 1), detId
831  return self.images[detId], detId
832 
833  image = makeImageFromCamera(camera, imageSource=ImageSource(calibs), imageFactory=afwImage.ImageF,
834  binSize=self.config.binning)
835  return image
836 
838  """Configuration for bias construction.
839 
840  No changes required compared to the base class, but
841  subclassed for distinction.
842  """
843  pass
844 
845 
846 class BiasTask(CalibTask):
847  """Bias construction"""
848  ConfigClass = BiasConfig
849  _DefaultName = "bias"
850  calibName = "bias"
851  filterName = "NONE" # Sets this filter name in the output
852  exposureTime = 0.0 # sets this exposureTime in the output
853 
854  @classmethod
855  def applyOverrides(cls, config):
856  """Overrides to apply for bias construction"""
857  config.isr.doBias = False
858  config.isr.doDark = False
859  config.isr.doFlat = False
860  config.isr.doFringe = False
861 
862 
864  """Configuration for dark construction"""
865  doRepair = Field(dtype=bool, default=True, doc="Repair artifacts?")
866  psfFwhm = Field(dtype=float, default=3.0, doc="Repair PSF FWHM (pixels)")
867  psfSize = Field(dtype=int, default=21, doc="Repair PSF size (pixels)")
868  crGrow = Field(dtype=int, default=2, doc="Grow radius for CR (pixels)")
869  repair = ConfigurableField(
870  target=RepairTask, doc="Task to repair artifacts")
871 
872  def setDefaults(self):
873  CalibConfig.setDefaults(self)
874  self.combination.mask.append("CR")
875 
876 
878  """Dark construction
879 
880  The only major difference from the base class is a cosmic-ray
881  identification stage, and dividing each image by the dark time
882  to generate images of the dark rate.
883  """
884  ConfigClass = DarkConfig
885  _DefaultName = "dark"
886  calibName = "dark"
887  filterName = "NONE" # Sets this filter name in the output
888 
889  def __init__(self, *args, **kwargs):
890  CalibTask.__init__(self, *args, **kwargs)
891  self.makeSubtask("repair")
892 
893  @classmethod
894  def applyOverrides(cls, config):
895  """Overrides to apply for dark construction"""
896  config.isr.doDark = False
897  config.isr.doFlat = False
898  config.isr.doFringe = False
899 
900  def processSingle(self, sensorRef):
901  """Process a single CCD
902 
903  Besides the regular ISR, also masks cosmic-rays and divides each
904  processed image by the dark time to generate images of the dark rate.
905  The dark time is provided by the 'getDarkTime' method.
906  """
907  exposure = CalibTask.processSingle(self, sensorRef)
908 
909  if self.config.doRepair:
910  psf = measAlg.DoubleGaussianPsf(self.config.psfSize, self.config.psfSize,
911  self.config.psfFwhm/(2*math.sqrt(2*math.log(2))))
912  exposure.setPsf(psf)
913  self.repair.run(exposure, keepCRs=False)
914  if self.config.crGrow > 0:
915  mask = exposure.getMaskedImage().getMask().clone()
916  mask &= mask.getPlaneBitMask("CR")
917  fpSet = afwDet.FootprintSet(
918  mask, afwDet.Threshold(0.5))
919  fpSet = afwDet.FootprintSet(fpSet, self.config.crGrow, True)
920  fpSet.setMask(exposure.getMaskedImage().getMask(), "CR")
921 
922  mi = exposure.getMaskedImage()
923  mi /= self.getDarkTime(exposure)
924  return exposure
925 
926  def getDarkTime(self, exposure):
927  """Retrieve the dark time for an exposure"""
928  darkTime = exposure.getInfo().getVisitInfo().getDarkTime()
929  if not np.isfinite(darkTime):
930  raise RuntimeError("Non-finite darkTime")
931  return darkTime
932 
933 
935  """Configuration for flat construction"""
936  iterations = Field(dtype=int, default=10,
937  doc="Number of iterations for scale determination")
938  stats = ConfigurableField(target=CalibStatsTask,
939  doc="Background statistics configuration")
940 
941 
943  """Flat construction
944 
945  The principal change from the base class involves gathering the background
946  values from each image and using them to determine the scalings for the final
947  combination.
948  """
949  ConfigClass = FlatConfig
950  _DefaultName = "flat"
951  calibName = "flat"
952 
953  @classmethod
954  def applyOverrides(cls, config):
955  """Overrides for flat construction"""
956  config.isr.doFlat = False
957  config.isr.doFringe = False
958 
959  def __init__(self, *args, **kwargs):
960  CalibTask.__init__(self, *args, **kwargs)
961  self.makeSubtask("stats")
962 
963  def processResult(self, exposure):
964  return self.stats.run(exposure)
965 
966  def scale(self, ccdIdLists, data):
967  """Determine the scalings for the final combination
968 
969  We have a matrix B_ij = C_i E_j, where C_i is the relative scaling
970  of one CCD to all the others in an exposure, and E_j is the scaling
971  of the exposure. We convert everything to logarithms so we can work
972  with a linear system. We determine the C_i and E_j from B_ij by iteration,
973  under the additional constraint that the average CCD scale is unity.
974 
975  This algorithm comes from Eugene Magnier and Pan-STARRS.
976  """
977  assert len(ccdIdLists.values()) > 0, "No successful CCDs"
978  lengths = set([len(expList) for expList in ccdIdLists.values()])
979  assert len(
980  lengths) == 1, "Number of successful exposures for each CCD differs"
981  assert tuple(lengths)[0] > 0, "No successful exposures"
982  # Format background measurements into a matrix
983  indices = dict((name, i) for i, name in enumerate(ccdIdLists))
984  bgMatrix = np.array([[0.0] * len(expList)
985  for expList in ccdIdLists.values()])
986  for name in ccdIdLists:
987  i = indices[name]
988  bgMatrix[i] = [
989  d if d is not None else np.nan for d in data[name]]
990 
991  numpyPrint = np.get_printoptions()
992  np.set_printoptions(threshold=np.inf)
993  self.log.info("Input backgrounds: %s" % bgMatrix)
994 
995  # Flat-field scaling
996  numCcds = len(ccdIdLists)
997  numExps = bgMatrix.shape[1]
998  # log(Background) for each exposure/component
999  bgMatrix = np.log(bgMatrix)
1000  bgMatrix = np.ma.masked_array(bgMatrix, np.isnan(bgMatrix))
1001  # Initial guess at log(scale) for each component
1002  compScales = np.zeros(numCcds)
1003  expScales = np.array(
1004  [(bgMatrix[:, i0] - compScales).mean() for i0 in range(numExps)])
1005 
1006  for iterate in range(self.config.iterations):
1007  compScales = np.array(
1008  [(bgMatrix[i1, :] - expScales).mean() for i1 in range(numCcds)])
1009  expScales = np.array(
1010  [(bgMatrix[:, i2] - compScales).mean() for i2 in range(numExps)])
1011 
1012  avgScale = np.average(np.exp(compScales))
1013  compScales -= np.log(avgScale)
1014  self.log.debug("Iteration %d exposure scales: %s",
1015  iterate, np.exp(expScales))
1016  self.log.debug("Iteration %d component scales: %s",
1017  iterate, np.exp(compScales))
1018 
1019  expScales = np.array(
1020  [(bgMatrix[:, i3] - compScales).mean() for i3 in range(numExps)])
1021 
1022  if np.any(np.isnan(expScales)):
1023  raise RuntimeError("Bad exposure scales: %s --> %s" %
1024  (bgMatrix, expScales))
1025 
1026  expScales = np.exp(expScales)
1027  compScales = np.exp(compScales)
1028 
1029  self.log.info("Exposure scales: %s" % expScales)
1030  self.log.info("Component relative scaling: %s" % compScales)
1031  np.set_printoptions(**numpyPrint)
1032 
1033  return dict((ccdName, Struct(ccdScale=compScales[indices[ccdName]], expScales=expScales))
1034  for ccdName in ccdIdLists)
1035 
1036 
1038  """Configuration for fringe construction"""
1039  stats = ConfigurableField(target=CalibStatsTask,
1040  doc="Background statistics configuration")
1041  subtractBackground = ConfigurableField(target=measAlg.SubtractBackgroundTask,
1042  doc="Background configuration")
1043  detection = ConfigurableField(
1044  target=measAlg.SourceDetectionTask, doc="Detection configuration")
1045  detectSigma = Field(dtype=float, default=1.0,
1046  doc="Detection PSF gaussian sigma")
1047 
1048 
1050  """Fringe construction task
1051 
1052  The principal change from the base class is that the images are
1053  background-subtracted and rescaled by the background.
1054 
1055  XXX This is probably not right for a straight-up combination, as we
1056  are currently doing, since the fringe amplitudes need not scale with
1057  the continuum.
1058 
1059  XXX Would like to have this do PCA and generate multiple images, but
1060  that will take a bit of work with the persistence code.
1061  """
1062  ConfigClass = FringeConfig
1063  _DefaultName = "fringe"
1064  calibName = "fringe"
1065 
1066  @classmethod
1067  def applyOverrides(cls, config):
1068  """Overrides for fringe construction"""
1069  config.isr.doFringe = False
1070 
1071  def __init__(self, *args, **kwargs):
1072  CalibTask.__init__(self, *args, **kwargs)
1073  self.makeSubtask("detection")
1074  self.makeSubtask("stats")
1075  self.makeSubtask("subtractBackground")
1076 
1077  def processSingle(self, sensorRef):
1078  """Subtract the background and normalise by the background level"""
1079  exposure = CalibTask.processSingle(self, sensorRef)
1080  bgLevel = self.stats.run(exposure)
1081  self.subtractBackground.run(exposure)
1082  mi = exposure.getMaskedImage()
1083  mi /= bgLevel
1084  footprintSets = self.detection.detectFootprints(
1085  exposure, sigma=self.config.detectSigma)
1086  mask = exposure.getMaskedImage().getMask()
1087  detected = 1 << mask.addMaskPlane("DETECTED")
1088  for fpSet in (footprintSets.positive, footprintSets.negative):
1089  if fpSet is not None:
1090  afwDet.setMaskFromFootprintList(
1091  mask, fpSet.getFootprints(), detected)
1092  return exposure
1093 
1094 
1096  """Configuration for sky frame construction"""
1097  detection = ConfigurableField(target=measAlg.SourceDetectionTask, doc="Detection configuration")
1098  detectSigma = Field(dtype=float, default=2.0, doc="Detection PSF gaussian sigma")
1099  subtractBackground = ConfigurableField(target=measAlg.SubtractBackgroundTask,
1100  doc="Regular-scale background configuration, for object detection")
1101  largeScaleBackground = ConfigField(dtype=FocalPlaneBackgroundConfig,
1102  doc="Large-scale background configuration")
1103  sky = ConfigurableField(target=SkyMeasurementTask, doc="Sky measurement")
1104  maskThresh = Field(dtype=float, default=3.0, doc="k-sigma threshold for masking pixels")
1105  mask = ListField(dtype=str, default=["BAD", "SAT", "DETECTED", "NO_DATA"],
1106  doc="Mask planes to consider as contaminated")
1107 
1108 
1110  """Task for sky frame construction
1111 
1112  The sky frame is a (relatively) small-scale background
1113  model, the response of the camera to the sky.
1114 
1115  To construct, we first remove a large-scale background (e.g., caused
1116  by moonlight) which may vary from image to image. Then we construct a
1117  model of the sky, which is essentially a binned version of the image
1118  (important configuration parameters: sky.background.[xy]BinSize).
1119  It is these models which are coadded to yield the sky frame.
1120  """
1121  ConfigClass = SkyConfig
1122  _DefaultName = "sky"
1123  calibName = "sky"
1124 
1125  def __init__(self, *args, **kwargs):
1126  CalibTask.__init__(self, *args, **kwargs)
1127  self.makeSubtask("detection")
1128  self.makeSubtask("subtractBackground")
1129  self.makeSubtask("sky")
1130 
1131  def scatterProcess(self, pool, ccdIdLists):
1132  """!Scatter the processing among the nodes
1133 
1134  Only the master node executes this method, assigning work to the
1135  slaves.
1136 
1137  We measure and subtract off a large-scale background model across
1138  all CCDs, which requires a scatter/gather. Then we process the
1139  individual CCDs, subtracting the large-scale background model and
1140  the residual background model measured. These residuals will be
1141  combined for the sky frame.
1142 
1143  @param pool Process pool
1144  @param ccdIdLists Dict of data identifier lists for each CCD name
1145  @return Dict of lists of returned data for each CCD name
1146  """
1147  self.log.info("Scatter processing")
1148 
1149  numExps = set(len(expList) for expList in ccdIdLists.values())
1150  assert len(numExps) == 1
1151  numExps = numExps.pop()
1152 
1153  # First subtract off general gradients to make all the exposures look similar.
1154  # We want to preserve the common small-scale structure, which we will coadd.
1155  bgModelList = mapToMatrix(pool, self.measureBackground, ccdIdLists)
1156 
1157  backgrounds = {}
1158  scales = {}
1159  for exp in range(numExps):
1160  bgModels = [bgModelList[ccdName][exp] for ccdName in ccdIdLists]
1161  visit = set(tuple(ccdIdLists[ccdName][exp][key] for key in sorted(self.config.visitKeys)) for
1162  ccdName in ccdIdLists)
1163  assert len(visit) == 1
1164  visit = visit.pop()
1165  bgModel = bgModels[0]
1166  for bg in bgModels[1:]:
1167  bgModel.merge(bg)
1168  self.log.info("Background model min/max for visit %s: %f %f", visit,
1169  np.min(bgModel.getStatsImage().getArray()),
1170  np.max(bgModel.getStatsImage().getArray()))
1171  backgrounds[visit] = bgModel
1172  scales[visit] = np.median(bgModel.getStatsImage().getArray())
1173 
1174  return mapToMatrix(pool, self.process, ccdIdLists, backgrounds=backgrounds, scales=scales)
1175 
1176  def measureBackground(self, cache, dataId):
1177  """!Measure background model for CCD
1178 
1179  This method is executed by the slaves.
1180 
1181  The background models for all CCDs in an exposure will be
1182  combined to form a full focal-plane background model.
1183 
1184  @param cache Process pool cache
1185  @param dataId Data identifier
1186  @return Bcakground model
1187  """
1188  dataRef = getDataRef(cache.butler, dataId)
1189  exposure = self.processSingleBackground(dataRef)
1190 
1191  # NAOJ prototype smoothed and then combined the entire image, but it shouldn't be any different
1192  # to bin and combine the binned images except that there's fewer pixels to worry about.
1193  config = self.config.largeScaleBackground
1194  camera = dataRef.get("camera")
1195  bgModel = FocalPlaneBackground.fromCamera(config, camera)
1196  bgModel.addCcd(exposure)
1197  return bgModel
1198 
1199  def processSingleBackground(self, dataRef):
1200  """!Process a single CCD for the background
1201 
1202  This method is executed by the slaves.
1203 
1204  Because we're interested in the background, we detect and mask astrophysical
1205  sources, and pixels above the noise level.
1206 
1207  @param dataRef Data reference for CCD.
1208  @return processed exposure
1209  """
1210  if not self.config.clobber and dataRef.datasetExists("postISRCCD"):
1211  return dataRef.get("postISRCCD")
1212  exposure = CalibTask.processSingle(self, dataRef)
1213 
1214  # Detect sources. Requires us to remove the background; we'll restore it later.
1215  bgTemp = self.subtractBackground.run(exposure).background
1216  footprints = self.detection.detectFootprints(exposure, sigma=self.config.detectSigma)
1217  image = exposure.getMaskedImage()
1218  if footprints.background is not None:
1219  image += footprints.background.getImageF()
1220 
1221  # Mask high pixels
1222  variance = image.getVariance()
1223  noise = np.sqrt(np.median(variance.getArray()))
1224  isHigh = image.getImage().getArray() > self.config.maskThresh*noise
1225  image.getMask().getArray()[isHigh] |= image.getMask().getPlaneBitMask("DETECTED")
1226 
1227  # Restore the background: it's what we want!
1228  image += bgTemp.getImage()
1229 
1230  # Set detected/bad pixels to background to ensure they don't corrupt the background
1231  maskVal = image.getMask().getPlaneBitMask(self.config.mask)
1232  isBad = image.getMask().getArray() & maskVal > 0
1233  bgLevel = np.median(image.getImage().getArray()[~isBad])
1234  image.getImage().getArray()[isBad] = bgLevel
1235  dataRef.put(exposure, "postISRCCD")
1236  return exposure
1237 
1238  def processSingle(self, dataRef, backgrounds, scales):
1239  """Process a single CCD, specified by a data reference
1240 
1241  We subtract the appropriate focal plane background model,
1242  divide by the appropriate scale and measure the background.
1243 
1244  Only slave nodes execute this method.
1245 
1246  @param dataRef Data reference for single CCD
1247  @param backgrounds Background model for each visit
1248  @param scales Scales for each visit
1249  @return Processed exposure
1250  """
1251  visit = tuple(dataRef.dataId[key] for key in sorted(self.config.visitKeys))
1252  exposure = dataRef.get("postISRCCD", immediate=True)
1253  image = exposure.getMaskedImage()
1254  detector = exposure.getDetector()
1255  bbox = image.getBBox()
1256 
1257  bgModel = backgrounds[visit]
1258  bg = bgModel.toCcdBackground(detector, bbox)
1259  image -= bg.getImage()
1260  image /= scales[visit]
1261 
1262  bg = self.sky.measureBackground(exposure.getMaskedImage())
1263  dataRef.put(bg, "icExpBackground")
1264  return exposure
1265 
1266  def combine(self, cache, struct, outputId):
1267  """!Combine multiple background models of a particular CCD and write the output
1268 
1269  Only the slave nodes execute this method.
1270 
1271  @param cache Process pool cache
1272  @param struct Parameters for the combination, which has the following components:
1273  * ccdName Name tuple for CCD
1274  * ccdIdList List of data identifiers for combination
1275  @param outputId Data identifier for combined image (exposure part only)
1276  @return binned calib image
1277  """
1278  outputId = self.getFullyQualifiedOutputId(struct.ccdName, cache.butler, outputId)
1279  dataRefList = [getDataRef(cache.butler, dataId) if dataId is not None else None for
1280  dataId in struct.ccdIdList]
1281  self.log.info("Combining %s on %s" % (outputId, NODE))
1282  bgList = [dataRef.get("icExpBackground", immediate=True).clone() for dataRef in dataRefList]
1283 
1284  bgExp = self.sky.averageBackgrounds(bgList)
1285 
1286  self.recordCalibInputs(cache.butler, bgExp, struct.ccdIdList, outputId)
1287  cache.butler.put(bgExp, "sky", outputId)
1288  return afwMath.binImage(self.sky.exposureToBackground(bgExp).getImage(), self.config.binning)
def processWrite(self, dataRef, exposure, outputName="postISRCCD")
Write the processed CCD.
def scatterCombine(self, pool, outputId, ccdIdLists, scales)
Scatter the combination of exposures across multiple nodes.
def run(self, exposureOrImage)
Measure a particular statistic on an image (of some sort).
def checksum(obj, header=None, sumType="MD5")
Calculate a checksum of an object.
Definition: checksum.py:29
def getCcdIdListFromExposures(expRefList, level="sensor", ccdKeys=["ccd"])
Determine a list of CCDs from exposure references.
def run(self, expRefList, butler, calibId)
Construct a calib from a list of exposure references.
def getFullyQualifiedOutputId(self, ccdName, butler, outputId)
def __call__(self, parser, namespace, values, option_string)
def dictToTuple(dict_, keys)
Return a tuple of specific values from a dict.
def getDataRef(butler, dataId, datasetType="raw")
Definition: utils.py:17
def updateMetadata(self, calibImage, exposureTime, darkTime=None, kwargs)
Update the metadata from the VisitInfo.
def process(self, cache, ccdId, outputName="postISRCCD", kwargs)
Process a CCD, specified by a data identifier.
def processSingle(self, dataRef, backgrounds, scales)
def makeCameraImage(self, camera, dataId, calibs)
Create and write an image of the entire camera.
def combine(self, target, imageList, stats)
Combine multiple images.
def getMjd(self, butler, dataId, timescale=dafBase.DateTime.UTC)
def combine(self, cache, struct, outputId)
Combine multiple exposures of a particular CCD and write the output.
def write(self, butler, exposure, dataId)
Write the final combined calib.
def combine(self, cache, struct, outputId)
Combine multiple background models of a particular CCD and write the output.
def measureBackground(self, cache, dataId)
Measure background model for CCD.
def recordCalibInputs(self, butler, calib, dataIdList, outputId)
Record metadata including the inputs and creation details.
def scatterProcess(self, pool, ccdIdLists)
Scatter the processing among the nodes.
def run(self, sensorRefList, expScales=None, finalScale=None, inputName="postISRCCD")
Combine calib images for a single sensor.
def getOutputId(self, expRefList, calibId)
Generate the data identifier for the output calib.
def mapToMatrix(pool, func, ccdIdLists, args, kwargs)
def scale(self, ccdIdLists, data)
Determine scaling across CCDs and exposures.
def addMissingKeys(self, dataId, butler, missingKeys=None, calibName=None)
def getDimensions(self, sensorRefList, inputName="postISRCCD")
Base class for constructing calibs.
def processSingleBackground(self, dataRef)
Process a single CCD for the background.
def scatterProcess(self, pool, ccdIdLists)
Scatter the processing among the nodes.
def batchWallTime(cls, time, parsedCmd, numCores)