lsst.pipe.drivers  6.0b0-hsc-13-g219e95d
constructCalibs.py
Go to the documentation of this file.
1 from __future__ import absolute_import, division, print_function
2 
3 import sys
4 import math
5 import time
6 import argparse
7 import traceback
8 
9 import numpy as np
10 from builtins import zip
11 from builtins import range
12 
13 from lsst.pex.config import Config, ConfigurableField, Field, ListField, ConfigField
14 from lsst.pipe.base import Task, Struct, TaskRunner, ArgumentParser
15 import lsst.daf.base as dafBase
16 import lsst.afw.math as afwMath
17 import lsst.afw.geom as afwGeom
18 import lsst.afw.detection as afwDet
19 import lsst.afw.image as afwImage
20 from lsst.afw.image import VisitInfo
21 import lsst.meas.algorithms as measAlg
22 from lsst.pipe.tasks.repair import RepairTask
23 from lsst.ip.isr import IsrTask
24 from lsst.afw.cameraGeom.utils import makeImageFromCamera
25 
26 from lsst.ctrl.pool.parallel import BatchPoolTask
27 from lsst.ctrl.pool.pool import Pool, NODE
28 from lsst.pipe.drivers.background import SkyMeasurementTask, FocalPlaneBackground, FocalPlaneBackgroundConfig
29 
30 from .checksum import checksum
31 from .utils import getDataRef
32 
33 
34 class CalibStatsConfig(Config):
35  """Parameters controlling the measurement of background statistics"""
36  stat = Field(doc="Statistic to use to estimate background (from lsst.afw.math)", dtype=int,
37  default=int(afwMath.MEANCLIP))
38  clip = Field(doc="Clipping threshold for background",
39  dtype=float, default=3.0)
40  nIter = Field(doc="Clipping iterations for background",
41  dtype=int, default=3)
42  maxVisitsToCalcErrorFromInputVariance = Field(
43  doc="Maximum number of visits to estimate variance from input variance, not per-pixel spread",
44  dtype=int, default=2)
45  mask = ListField(doc="Mask planes to reject",
46  dtype=str, default=["DETECTED", "BAD", "NO_DATA",])
47 
48 
49 class CalibStatsTask(Task):
50  """Measure statistics on the background
51 
52  This can be useful for scaling the background, e.g., for flats and fringe frames.
53  """
54  ConfigClass = CalibStatsConfig
55 
56  def run(self, exposureOrImage):
57  """!Measure a particular statistic on an image (of some sort).
58 
59  @param exposureOrImage Exposure, MaskedImage or Image.
60  @return Value of desired statistic
61  """
62  stats = afwMath.StatisticsControl(self.config.clip, self.config.nIter,
63  afwImage.Mask.getPlaneBitMask(self.config.mask))
64  try:
65  image = exposureOrImage.getMaskedImage()
66  except:
67  try:
68  image = exposureOrImage.getImage()
69  except:
70  image = exposureOrImage
71 
72  return afwMath.makeStatistics(image, self.config.stat, stats).getValue()
73 
74 
75 class CalibCombineConfig(Config):
76  """Configuration for combining calib images"""
77  rows = Field(doc="Number of rows to read at a time",
78  dtype=int, default=512)
79  mask = ListField(doc="Mask planes to respect", dtype=str,
80  default=["SAT", "DETECTED", "INTRP"])
81  combine = Field(doc="Statistic to use for combination (from lsst.afw.math)", dtype=int,
82  default=int(afwMath.MEANCLIP))
83  clip = Field(doc="Clipping threshold for combination",
84  dtype=float, default=3.0)
85  nIter = Field(doc="Clipping iterations for combination",
86  dtype=int, default=3)
87  stats = ConfigurableField(target=CalibStatsTask,
88  doc="Background statistics configuration")
89 
90 
91 class CalibCombineTask(Task):
92  """Task to combine calib images"""
93  ConfigClass = CalibCombineConfig
94 
95  def __init__(self, *args, **kwargs):
96  Task.__init__(self, *args, **kwargs)
97  self.makeSubtask("stats")
98 
99  def run(self, sensorRefList, expScales=None, finalScale=None, inputName="postISRCCD"):
100  """!Combine calib images for a single sensor
101 
102  @param sensorRefList List of data references to combine (for a single sensor)
103  @param expScales List of scales to apply for each exposure
104  @param finalScale Desired scale for final combined image
105  @param inputName Data set name for inputs
106  @return combined image
107  """
108  width, height = self.getDimensions(sensorRefList)
109  stats = afwMath.StatisticsControl(self.config.clip, self.config.nIter,
110  afwImage.Mask.getPlaneBitMask(self.config.mask))
111  numImages = len(sensorRefList)
112  if numImages < self.config.stats.maxVisitsToCalcErrorFromInputVariance:
113  stats.setCalcErrorFromInputVariance(True)
114 
115  # Combine images
116  combined = afwImage.MaskedImageF(width, height)
117  imageList = [None]*numImages
118  for start in range(0, height, self.config.rows):
119  rows = min(self.config.rows, height - start)
120  box = afwGeom.Box2I(afwGeom.Point2I(0, start),
121  afwGeom.Extent2I(width, rows))
122  subCombined = combined.Factory(combined, box)
123 
124  for i, sensorRef in enumerate(sensorRefList):
125  if sensorRef is None:
126  imageList[i] = None
127  continue
128  exposure = sensorRef.get(inputName + "_sub", bbox=box)
129  if expScales is not None:
130  self.applyScale(exposure, expScales[i])
131  imageList[i] = exposure.getMaskedImage()
132 
133  self.combine(subCombined, imageList, stats)
134 
135  if finalScale is not None:
136  background = self.stats.run(combined)
137  self.log.info("%s: Measured background of stack is %f; adjusting to %f" %
138  (NODE, background, finalScale))
139  combined *= finalScale / background
140 
141  return combined
142 
143  def getDimensions(self, sensorRefList, inputName="postISRCCD"):
144  """Get dimensions of the inputs"""
145  dimList = []
146  for sensorRef in sensorRefList:
147  if sensorRef is None:
148  continue
149  md = sensorRef.get(inputName + "_md")
150  dimList.append(afwImage.bboxFromMetadata(md).getDimensions())
151  return getSize(dimList)
152 
153  def applyScale(self, exposure, scale=None):
154  """Apply scale to input exposure
155 
156  This implementation applies a flux scaling: the input exposure is
157  divided by the provided scale.
158  """
159  if scale is not None:
160  mi = exposure.getMaskedImage()
161  mi /= scale
162 
163  def combine(self, target, imageList, stats):
164  """!Combine multiple images
165 
166  @param target Target image to receive the combined pixels
167  @param imageList List of input images
168  @param stats Statistics control
169  """
170  images = [img for img in imageList if img is not None]
171  afwMath.statisticsStack(target, images, afwMath.Property(self.config.combine), stats)
172 
173 
174 def getSize(dimList):
175  """Determine a consistent size, given a list of image sizes"""
176  dim = set((w, h) for w, h in dimList)
177  dim.discard(None)
178  if len(dim) != 1:
179  raise RuntimeError("Inconsistent dimensions: %s" % dim)
180  return dim.pop()
181 
182 
183 def dictToTuple(dict_, keys):
184  """!Return a tuple of specific values from a dict
185 
186  This provides a hashable representation of the dict from certain keywords.
187  This can be useful for creating e.g., a tuple of the values in the DataId
188  that identify the CCD.
189 
190  @param dict_ dict to parse
191  @param keys keys to extract (order is important)
192  @return tuple of values
193  """
194  return tuple(dict_[k] for k in keys)
195 
196 
197 def getCcdIdListFromExposures(expRefList, level="sensor", ccdKeys=["ccd"]):
198  """!Determine a list of CCDs from exposure references
199 
200  This essentially inverts the exposure-level references (which
201  provides a list of CCDs for each exposure), by providing
202  a dataId list for each CCD. Consider an input list of exposures
203  [e1, e2, e3], and each exposure has CCDs c1 and c2. Then this
204  function returns:
205 
206  {(c1,): [e1c1, e2c1, e3c1], (c2,): [e1c2, e2c2, e3c2]}
207 
208  This is a dict whose keys are tuples of the identifying values of a
209  CCD (usually just the CCD number) and the values are lists of dataIds
210  for that CCD in each exposure. A missing dataId is given the value
211  None.
212 
213  @param expRefList List of data references for exposures
214  @param level Level for the butler to generate CCDs
215  @param ccdKeys DataId keywords that identify a CCD
216  @return dict of data identifier lists for each CCD;
217  keys are values of ccdKeys in order
218  """
219  expIdList = [[ccdRef.dataId for ccdRef in expRef.subItems(
220  level)] for expRef in expRefList]
221 
222  # Determine what additional keys make a CCD from an exposure
223  if len(ccdKeys) != len(set(ccdKeys)):
224  raise RuntimeError("Duplicate keys found in ccdKeys: %s" % ccdKeys)
225  ccdNames = set() # Set of tuples which are values for each of the CCDs in an exposure
226  for ccdIdList in expIdList:
227  for ccdId in ccdIdList:
228  name = dictToTuple(ccdId, ccdKeys)
229  ccdNames.add(name)
230 
231  # Turn the list of CCDs for each exposure into a list of exposures for
232  # each CCD
233  ccdLists = {}
234  for n, ccdIdList in enumerate(expIdList):
235  for ccdId in ccdIdList:
236  name = dictToTuple(ccdId, ccdKeys)
237  if name not in ccdLists:
238  ccdLists[name] = []
239  ccdLists[name].append(ccdId)
240 
241  for ccd in ccdLists:
242  # Sort the list by the dataId values (ordered by key)
243  ccdLists[ccd] = sorted(ccdLists[ccd], key=lambda dd: dictToTuple(dd, sorted(dd.keys())))
244 
245  return ccdLists
246 
247 
248 def mapToMatrix(pool, func, ccdIdLists, *args, **kwargs):
249  """Generate a matrix of results using pool.map
250 
251  The function should have the call signature:
252  func(cache, dataId, *args, **kwargs)
253 
254  We return a dict mapping 'ccd name' to a list of values for
255  each exposure.
256 
257  @param pool Process pool
258  @param func Function to call for each dataId
259  @param ccdIdLists Dict of data identifier lists for each CCD name
260  @return matrix of results
261  """
262  dataIdList = sum(ccdIdLists.values(), [])
263  resultList = pool.map(func, dataIdList, *args, **kwargs)
264  # Piece everything back together
265  data = dict((ccdName, [None] * len(expList)) for ccdName, expList in ccdIdLists.items())
266  indices = dict(sum([[(tuple(dataId.values()) if dataId is not None else None, (ccdName, expNum))
267  for expNum, dataId in enumerate(expList)]
268  for ccdName, expList in ccdIdLists.items()], []))
269  for dataId, result in zip(dataIdList, resultList):
270  if dataId is None:
271  continue
272  ccdName, expNum = indices[tuple(dataId.values())]
273  data[ccdName][expNum] = result
274  return data
275 
276 
277 class CalibIdAction(argparse.Action):
278  """Split name=value pairs and put the result in a dict"""
279 
280  def __call__(self, parser, namespace, values, option_string):
281  output = getattr(namespace, self.dest, {})
282  for nameValue in values:
283  name, sep, valueStr = nameValue.partition("=")
284  if not valueStr:
285  parser.error("%s value %s must be in form name=value" %
286  (option_string, nameValue))
287  output[name] = valueStr
288  setattr(namespace, self.dest, output)
289 
290 
291 class CalibArgumentParser(ArgumentParser):
292  """ArgumentParser for calibration construction"""
293 
294  def __init__(self, calibName, *args, **kwargs):
295  """Add a --calibId argument to the standard pipe_base argument parser"""
296  ArgumentParser.__init__(self, *args, **kwargs)
297  self.calibName = calibName
298  self.add_id_argument("--id", datasetType="raw",
299  help="input identifiers, e.g., --id visit=123 ccd=4")
300  self.add_argument("--calibId", nargs="*", action=CalibIdAction, default={},
301  help="identifiers for calib, e.g., --calibId version=1",
302  metavar="KEY=VALUE1[^VALUE2[^VALUE3...]")
303 
304  def parse_args(self, *args, **kwargs):
305  """Parse arguments
306 
307  Checks that the "--calibId" provided works.
308  """
309  namespace = ArgumentParser.parse_args(self, *args, **kwargs)
310 
311  keys = namespace.butler.getKeys(self.calibName)
312  parsed = {}
313  for name, value in namespace.calibId.items():
314  if name not in keys:
315  self.error(
316  "%s is not a relevant calib identifier key (%s)" % (name, keys))
317  parsed[name] = keys[name](value)
318  namespace.calibId = parsed
319 
320  return namespace
321 
322 
323 class CalibConfig(Config):
324  """Configuration for constructing calibs"""
325  clobber = Field(dtype=bool, default=True,
326  doc="Clobber existing processed images?")
327  isr = ConfigurableField(target=IsrTask, doc="ISR configuration")
328  dateObs = Field(dtype=str, default="dateObs",
329  doc="Key for observation date in exposure registry")
330  dateCalib = Field(dtype=str, default="calibDate",
331  doc="Key for calib date in calib registry")
332  filter = Field(dtype=str, default="filter",
333  doc="Key for filter name in exposure/calib registries")
334  combination = ConfigurableField(
335  target=CalibCombineTask, doc="Calib combination configuration")
336  ccdKeys = ListField(dtype=str, default=["ccd"],
337  doc="DataId keywords specifying a CCD")
338  visitKeys = ListField(dtype=str, default=["visit"],
339  doc="DataId keywords specifying a visit")
340  calibKeys = ListField(dtype=str, default=[],
341  doc="DataId keywords specifying a calibration")
342  doCameraImage = Field(dtype=bool, default=True, doc="Create camera overview image?")
343  binning = Field(dtype=int, default=64, doc="Binning to apply for camera image")
344 
345  def setDefaults(self):
346  self.isr.doWrite = False
347 
348 
349 class CalibTaskRunner(TaskRunner):
350  """Get parsed values into the CalibTask.run"""
351  @staticmethod
352  def getTargetList(parsedCmd, **kwargs):
353  return [dict(expRefList=parsedCmd.id.refList, butler=parsedCmd.butler, calibId=parsedCmd.calibId)]
354 
355  def __call__(self, args):
356  """Call the Task with the kwargs from getTargetList"""
357  task = self.TaskClass(config=self.config, log=self.log)
358  exitStatus = 0 # exit status for the shell
359  if self.doRaise:
360  result = task.run(**args)
361  else:
362  try:
363  result = task.run(**args)
364  except Exception as e:
365  exitStatus = 1 # n.b. The shell exit value is the number of dataRefs returning
366  # non-zero, so the actual value used here is lost
367  task.log.fatal("Failed: %s" % e)
368  traceback.print_exc(file=sys.stderr)
369 
370  if self.doReturnResults:
371  return Struct(
372  exitStatus=exitStatus,
373  args=args,
374  metadata=task.metadata,
375  result=result,
376  )
377  else:
378  return Struct(
379  exitStatus=exitStatus,
380  )
381 
383  """!Base class for constructing calibs.
384 
385  This should be subclassed for each of the required calib types.
386  The subclass should be sure to define the following class variables:
387  * _DefaultName: default name of the task, used by CmdLineTask
388  * calibName: name of the calibration data set in the butler
389  The subclass may optionally set:
390  * filterName: filter name to give the resultant calib
391  """
392  ConfigClass = CalibConfig
393  RunnerClass = CalibTaskRunner
394  filterName = None
395  calibName = None
396  exposureTime = 1.0 # sets this exposureTime in the output
397 
398  def __init__(self, *args, **kwargs):
399  """Constructor"""
400  BatchPoolTask.__init__(self, *args, **kwargs)
401  self.makeSubtask("isr")
402  self.makeSubtask("combination")
403 
404  @classmethod
405  def batchWallTime(cls, time, parsedCmd, numCores):
406  numCcds = len(parsedCmd.butler.get("camera"))
407  numExps = len(cls.RunnerClass.getTargetList(
408  parsedCmd)[0]['expRefList'])
409  numCycles = int(numCcds/float(numCores) + 0.5)
410  return time*numExps*numCycles
411 
412  @classmethod
413  def _makeArgumentParser(cls, *args, **kwargs):
414  kwargs.pop("doBatch", False)
415  return CalibArgumentParser(calibName=cls.calibName, name=cls._DefaultName, *args, **kwargs)
416 
417  def run(self, expRefList, butler, calibId):
418  """!Construct a calib from a list of exposure references
419 
420  This is the entry point, called by the TaskRunner.__call__
421 
422  Only the master node executes this method.
423 
424  @param expRefList List of data references at the exposure level
425  @param butler Data butler
426  @param calibId Identifier dict for calib
427  """
428  for expRef in expRefList:
429  self.addMissingKeys(expRef.dataId, butler, self.config.ccdKeys, 'raw')
430 
431  outputId = self.getOutputId(expRefList, calibId)
432  ccdIdLists = getCcdIdListFromExposures(
433  expRefList, level="sensor", ccdKeys=self.config.ccdKeys)
434 
435  # Ensure we can generate filenames for each output
436  outputIdItemList = list(outputId.items())
437  for ccdName in ccdIdLists:
438  dataId = dict([(k, ccdName[i]) for i, k in enumerate(self.config.ccdKeys)])
439  dataId.update(outputIdItemList)
440  self.addMissingKeys(dataId, butler)
441  dataId.update(outputIdItemList)
442 
443  try:
444  butler.get(self.calibName + "_filename", dataId)
445  except Exception as e:
446  raise RuntimeError(
447  "Unable to determine output filename \"%s_filename\" from %s: %s" %
448  (self.calibName, dataId, e))
449 
450  processPool = Pool("process")
451  processPool.storeSet(butler=butler)
452 
453  # Scatter: process CCDs independently
454  data = self.scatterProcess(processPool, ccdIdLists)
455 
456  # Gather: determine scalings
457  scales = self.scale(ccdIdLists, data)
458 
459  combinePool = Pool("combine")
460  combinePool.storeSet(butler=butler)
461 
462  # Scatter: combine
463  calibs = self.scatterCombine(combinePool, outputId, ccdIdLists, scales)
464 
465  if self.config.doCameraImage:
466  camera = butler.get("camera")
467 
468  try:
469  cameraImage = self.makeCameraImage(camera, outputId, calibs)
470  butler.put(cameraImage, self.calibName + "_camera", dataId)
471  except Exception as exc:
472  self.log.warn("Unable to create camera image: %s" % (exc,))
473 
474  return Struct(
475  outputId = outputId,
476  ccdIdLists = ccdIdLists,
477  scales = scales,
478  calibs = calibs,
479  processPool = processPool,
480  combinePool = combinePool,
481  )
482 
483  def getOutputId(self, expRefList, calibId):
484  """!Generate the data identifier for the output calib
485 
486  The mean date and the common filter are included, using keywords
487  from the configuration. The CCD-specific part is not included
488  in the data identifier.
489 
490  @param expRefList List of data references at exposure level
491  @param calibId Data identifier elements for the calib provided by the user
492  @return data identifier
493  """
494  midTime = 0
495  filterName = None
496  for expRef in expRefList:
497  butler = expRef.getButler()
498  dataId = expRef.dataId
499 
500  midTime += self.getMjd(butler, dataId)
501  thisFilter = self.getFilter(
502  butler, dataId) if self.filterName is None else self.filterName
503  if filterName is None:
504  filterName = thisFilter
505  elif filterName != thisFilter:
506  raise RuntimeError("Filter mismatch for %s: %s vs %s" % (
507  dataId, thisFilter, filterName))
508 
509  midTime /= len(expRefList)
510  date = str(dafBase.DateTime(
511  midTime, dafBase.DateTime.MJD).toPython().date())
512 
513  outputId = {self.config.filter: filterName,
514  self.config.dateCalib: date}
515  outputId.update(calibId)
516  return outputId
517 
518  def getMjd(self, butler, dataId, timescale=dafBase.DateTime.UTC):
519  """Determine the Modified Julian Date (MJD; in TAI) from a data identifier"""
520  if self.config.dateObs in dataId:
521  dateObs = dataId[self.config.dateObs]
522  else:
523  dateObs = butler.queryMetadata('raw', [self.config.dateObs], dataId)[0]
524  if "T" not in dateObs:
525  dateObs = dateObs + "T12:00:00.0Z"
526  elif not dateObs.endswith("Z"):
527  dateObs += "Z"
528 
529  return dafBase.DateTime(dateObs, timescale).get(dafBase.DateTime.MJD)
530 
531  def getFilter(self, butler, dataId):
532  """Determine the filter from a data identifier"""
533  filt = butler.queryMetadata('raw', [self.config.filter], dataId)[0]
534  return filt
535 
536  def addMissingKeys(self, dataId, butler, missingKeys=None, calibName=None):
537  if calibName is None:
538  calibName = self.calibName
539 
540  if missingKeys is None:
541  missingKeys = set(butler.getKeys(calibName).keys()) - set(dataId.keys())
542 
543  for k in missingKeys:
544  try:
545  v = butler.queryMetadata('raw', [k], dataId) # n.b. --id refers to 'raw'
546  except Exception as e:
547  continue
548 
549  if len(v) == 0: # failed to lookup value
550  continue
551 
552  if len(v) == 1:
553  dataId[k] = v[0]
554  else:
555  raise RuntimeError("No unique lookup for %s: %s" % (k, v))
556 
557  def updateMetadata(self, calibImage, exposureTime, darkTime=None, **kwargs):
558  """!Update the metadata from the VisitInfo
559 
560  \param calibImage The image whose metadata is to be set
561  \param exposureTime The exposure time for the image
562  \param darkTime The time since the last read (default: exposureTime)
563  """
564 
565  if darkTime is None:
566  darkTime = exposureTime # avoid warning messages when using calibration products
567 
568  visitInfo = afwImage.makeVisitInfo(exposureTime=exposureTime, darkTime=darkTime, **kwargs)
569  md = calibImage.getMetadata()
570 
571  afwImage.setVisitInfoMetadata(md, visitInfo)
572 
573  def scatterProcess(self, pool, ccdIdLists):
574  """!Scatter the processing among the nodes
575 
576  We scatter each CCD independently (exposures aren't grouped together),
577  to make full use of all available processors. This necessitates piecing
578  everything back together in the same format as ccdIdLists afterwards.
579 
580  Only the master node executes this method.
581 
582  @param pool Process pool
583  @param ccdIdLists Dict of data identifier lists for each CCD name
584  @return Dict of lists of returned data for each CCD name
585  """
586  self.log.info("Scatter processing")
587  return mapToMatrix(pool, self.process, ccdIdLists)
588 
589  def process(self, cache, ccdId, outputName="postISRCCD", **kwargs):
590  """!Process a CCD, specified by a data identifier
591 
592  After processing, optionally returns a result (produced by
593  the 'processResult' method) calculated from the processed
594  exposure. These results will be gathered by the master node,
595  and is a means for coordinated scaling of all CCDs for flats,
596  etc.
597 
598  Only slave nodes execute this method.
599 
600  @param cache Process pool cache
601  @param ccdId Data identifier for CCD
602  @param outputName Output dataset name for butler
603  @return result from 'processResult'
604  """
605  if ccdId is None:
606  self.log.warn("Null identifier received on %s" % NODE)
607  return None
608  sensorRef = getDataRef(cache.butler, ccdId)
609  if self.config.clobber or not sensorRef.datasetExists(outputName):
610  self.log.info("Processing %s on %s" % (ccdId, NODE))
611  try:
612  exposure = self.processSingle(sensorRef, **kwargs)
613  except Exception as e:
614  self.log.warn("Unable to process %s: %s" % (ccdId, e))
615  raise
616  return None
617  self.processWrite(sensorRef, exposure)
618  else:
619  self.log.info(
620  "Using previously persisted processed exposure for %s" % (sensorRef.dataId,))
621  exposure = sensorRef.get(outputName)
622  return self.processResult(exposure)
623 
624  def processSingle(self, dataRef):
625  """Process a single CCD, specified by a data reference
626 
627  Generally, this simply means doing ISR.
628 
629  Only slave nodes execute this method.
630  """
631  return self.isr.runDataRef(dataRef).exposure
632 
633  def processWrite(self, dataRef, exposure, outputName="postISRCCD"):
634  """!Write the processed CCD
635 
636  We need to write these out because we can't hold them all in
637  memory at once.
638 
639  Only slave nodes execute this method.
640 
641  @param dataRef Data reference
642  @param exposure CCD exposure to write
643  @param outputName Output dataset name for butler.
644  """
645  dataRef.put(exposure, outputName)
646 
647  def processResult(self, exposure):
648  """Extract processing results from a processed exposure
649 
650  This method generates what is gathered by the master node.
651  This can be a background measurement or similar for scaling
652  flat-fields. It must be picklable!
653 
654  Only slave nodes execute this method.
655  """
656  return None
657 
658  def scale(self, ccdIdLists, data):
659  """!Determine scaling across CCDs and exposures
660 
661  This is necessary mainly for flats, so as to determine a
662  consistent scaling across the entire focal plane. This
663  implementation is simply a placeholder.
664 
665  Only the master node executes this method.
666 
667  @param ccdIdLists Dict of data identifier lists for each CCD tuple
668  @param data Dict of lists of returned data for each CCD tuple
669  @return dict of Struct(ccdScale: scaling for CCD,
670  expScales: scaling for each exposure
671  ) for each CCD tuple
672  """
673  self.log.info("Scale on %s" % NODE)
674  return dict((name, Struct(ccdScale=None, expScales=[None] * len(ccdIdLists[name])))
675  for name in ccdIdLists)
676 
677  def scatterCombine(self, pool, outputId, ccdIdLists, scales):
678  """!Scatter the combination of exposures across multiple nodes
679 
680  In this case, we can only scatter across as many nodes as
681  there are CCDs.
682 
683  Only the master node executes this method.
684 
685  @param pool Process pool
686  @param outputId Output identifier (exposure part only)
687  @param ccdIdLists Dict of data identifier lists for each CCD name
688  @param scales Dict of structs with scales, for each CCD name
689  @param dict of binned images
690  """
691  self.log.info("Scatter combination")
692  data = [Struct(ccdName=ccdName, ccdIdList=ccdIdLists[ccdName], scales=scales[ccdName]) for
693  ccdName in ccdIdLists]
694  images = pool.map(self.combine, data, outputId)
695  return dict(zip(ccdIdLists.keys(), images))
696 
697  def getFullyQualifiedOutputId(self, ccdName, butler, outputId):
698  """Get fully-qualified output data identifier
699 
700  We may need to look up keys that aren't in the output dataId.
701 
702  @param ccdName Name tuple for CCD
703  @param butler Data butler
704  @param outputId Data identifier for combined image (exposure part only)
705  @return fully-qualified output dataId
706  """
707  fullOutputId = {k: ccdName[i] for i, k in enumerate(self.config.ccdKeys)}
708  fullOutputId.update(outputId)
709  self.addMissingKeys(fullOutputId, butler)
710  fullOutputId.update(outputId) # must be after the call to queryMetadata in 'addMissingKeys'
711  return fullOutputId
712 
713  def combine(self, cache, struct, outputId):
714  """!Combine multiple exposures of a particular CCD and write the output
715 
716  Only the slave nodes execute this method.
717 
718  @param cache Process pool cache
719  @param struct Parameters for the combination, which has the following components:
720  * ccdName Name tuple for CCD
721  * ccdIdList List of data identifiers for combination
722  * scales Scales to apply (expScales are scalings for each exposure,
723  ccdScale is final scale for combined image)
724  @param outputId Data identifier for combined image (exposure part only)
725  @return binned calib image
726  """
727  outputId = self.getFullyQualifiedOutputId(struct.ccdName, cache.butler, outputId)
728  dataRefList = [getDataRef(cache.butler, dataId) if dataId is not None else None for
729  dataId in struct.ccdIdList]
730  self.log.info("Combining %s on %s" % (outputId, NODE))
731  calib = self.combination.run(dataRefList, expScales=struct.scales.expScales,
732  finalScale=struct.scales.ccdScale)
733 
734  if not hasattr(calib, "getMetadata"):
735  if hasattr(calib, "getVariance"):
736  calib = afwImage.makeExposure(calib)
737  else:
738  calib = afwImage.DecoratedImageF(calib.getImage()) # n.b. hardwires "F" for the output type
739 
740  self.updateMetadata(calib, self.exposureTime)
741 
742  self.recordCalibInputs(cache.butler, calib,
743  struct.ccdIdList, outputId)
744 
745  self.interpolateNans(calib)
746 
747  self.write(cache.butler, calib, outputId)
748 
749  return afwMath.binImage(calib.getImage(), self.config.binning)
750 
751  def recordCalibInputs(self, butler, calib, dataIdList, outputId):
752  """!Record metadata including the inputs and creation details
753 
754  This metadata will go into the FITS header.
755 
756  @param butler Data butler
757  @param calib Combined calib exposure.
758  @param dataIdList List of data identifiers for calibration inputs
759  @param outputId Data identifier for output
760  """
761  header = calib.getMetadata()
762  header.add("OBSTYPE", self.calibName) # Used by ingestCalibs.py
763 
764  # date, time, host, and root
765  now = time.localtime()
766  header.add("CALIB_CREATION_DATE", time.strftime("%Y-%m-%d", now))
767  header.add("CALIB_CREATION_TIME", time.strftime("%X %Z", now))
768  # add date-obs as its absence upsets ExposureInfo; use the mean date that the calibs were taken
769  header.add("DATE-OBS", "%sT00:00:00.00" % outputId[self.config.dateCalib])
770 
771  # Inputs
772  visits = [str(dictToTuple(dataId, self.config.visitKeys)) for dataId in dataIdList if
773  dataId is not None]
774  for i, v in enumerate(sorted(set(visits))):
775  header.add("CALIB_INPUT_%d" % (i,), v)
776 
777  header.add("CALIB_ID", " ".join("%s=%s" % (key, value)
778  for key, value in outputId.items()))
779  checksum(calib, header)
780 
781  def interpolateNans(self, image):
782  """Interpolate over NANs in the combined image
783 
784  NANs can result from masked areas on the CCD. We don't want them getting
785  into our science images, so we replace them with the median of the image.
786  """
787  if hasattr(image, "getMaskedImage"): # Deal with Exposure vs Image
788  self.interpolateNans(image.getMaskedImage().getVariance())
789  image = image.getMaskedImage().getImage()
790  if hasattr(image, "getImage"): # Deal with DecoratedImage or MaskedImage vs Image
791  image = image.getImage()
792  array = image.getArray()
793  bad = np.isnan(array)
794  array[bad] = np.median(array[np.logical_not(bad)])
795 
796  def write(self, butler, exposure, dataId):
797  """!Write the final combined calib
798 
799  Only the slave nodes execute this method
800 
801  @param butler Data butler
802  @param exposure CCD exposure to write
803  @param dataId Data identifier for output
804  """
805  self.log.info("Writing %s on %s" % (dataId, NODE))
806  butler.put(exposure, self.calibName, dataId)
807 
808  def makeCameraImage(self, camera, dataId, calibs):
809  """!Create and write an image of the entire camera
810 
811  This is useful for judging the quality or getting an overview of
812  the features of the calib.
813 
814  This requires that the 'ccd name' is a tuple containing only the
815  detector ID. If that is not the case, change CalibConfig.ccdKeys
816  or set CalibConfig.doCameraImage=False to disable this.
817 
818  @param camera Camera object
819  @param dataId Data identifier for output
820  @param calibs Dict mapping 'ccd name' to calib image
821  """
822 
823  class ImageSource(object):
824  """Source of images for makeImageFromCamera
825 
826  This assumes that the 'ccd name' is a tuple containing
827  only the detector ID.
828  """
829  def __init__(self, images):
830  self.isTrimmed = True
831  self.images = images
832  self.background = np.nan
833 
834  def getCcdImage(self, detector, imageFactory, binSize):
835  detId = (detector.getId(),)
836  if detId not in self.images:
837  return imageFactory(1, 1), detId
838  return self.images[detId], detId
839 
840  image = makeImageFromCamera(camera, imageSource=ImageSource(calibs), imageFactory=afwImage.ImageF,
841  binSize=self.config.binning)
842  return image
843 
845  """Configuration for bias construction.
846 
847  No changes required compared to the base class, but
848  subclassed for distinction.
849  """
850  pass
851 
852 
853 class BiasTask(CalibTask):
854  """Bias construction"""
855  ConfigClass = BiasConfig
856  _DefaultName = "bias"
857  calibName = "bias"
858  filterName = "NONE" # Sets this filter name in the output
859  exposureTime = 0.0 # sets this exposureTime in the output
860 
861  @classmethod
862  def applyOverrides(cls, config):
863  """Overrides to apply for bias construction"""
864  config.isr.doBias = False
865  config.isr.doDark = False
866  config.isr.doFlat = False
867  config.isr.doFringe = False
868 
869 
871  """Configuration for dark construction"""
872  doRepair = Field(dtype=bool, default=True, doc="Repair artifacts?")
873  psfFwhm = Field(dtype=float, default=3.0, doc="Repair PSF FWHM (pixels)")
874  psfSize = Field(dtype=int, default=21, doc="Repair PSF size (pixels)")
875  crGrow = Field(dtype=int, default=2, doc="Grow radius for CR (pixels)")
876  repair = ConfigurableField(
877  target=RepairTask, doc="Task to repair artifacts")
878 
879  def setDefaults(self):
880  CalibConfig.setDefaults(self)
881  self.combination.mask.append("CR")
882 
883 
885  """Dark construction
886 
887  The only major difference from the base class is a cosmic-ray
888  identification stage, and dividing each image by the dark time
889  to generate images of the dark rate.
890  """
891  ConfigClass = DarkConfig
892  _DefaultName = "dark"
893  calibName = "dark"
894  filterName = "NONE" # Sets this filter name in the output
895 
896  def __init__(self, *args, **kwargs):
897  CalibTask.__init__(self, *args, **kwargs)
898  self.makeSubtask("repair")
899 
900  @classmethod
901  def applyOverrides(cls, config):
902  """Overrides to apply for dark construction"""
903  config.isr.doDark = False
904  config.isr.doFlat = False
905  config.isr.doFringe = False
906 
907  def processSingle(self, sensorRef):
908  """Process a single CCD
909 
910  Besides the regular ISR, also masks cosmic-rays and divides each
911  processed image by the dark time to generate images of the dark rate.
912  The dark time is provided by the 'getDarkTime' method.
913  """
914  exposure = CalibTask.processSingle(self, sensorRef)
915 
916  if self.config.doRepair:
917  psf = measAlg.DoubleGaussianPsf(self.config.psfSize, self.config.psfSize,
918  self.config.psfFwhm/(2*math.sqrt(2*math.log(2))))
919  exposure.setPsf(psf)
920  self.repair.run(exposure, keepCRs=False)
921  if self.config.crGrow > 0:
922  mask = exposure.getMaskedImage().getMask().clone()
923  mask &= mask.getPlaneBitMask("CR")
924  fpSet = afwDet.FootprintSet(
925  mask, afwDet.Threshold(0.5))
926  fpSet = afwDet.FootprintSet(fpSet, self.config.crGrow, True)
927  fpSet.setMask(exposure.getMaskedImage().getMask(), "CR")
928 
929  mi = exposure.getMaskedImage()
930  mi /= self.getDarkTime(exposure)
931  return exposure
932 
933  def getDarkTime(self, exposure):
934  """Retrieve the dark time for an exposure"""
935  darkTime = exposure.getInfo().getVisitInfo().getDarkTime()
936  if not np.isfinite(darkTime):
937  raise RuntimeError("Non-finite darkTime")
938  return darkTime
939 
940 
942  """Configuration for flat construction"""
943  iterations = Field(dtype=int, default=10,
944  doc="Number of iterations for scale determination")
945  stats = ConfigurableField(target=CalibStatsTask,
946  doc="Background statistics configuration")
947 
948 
950  """Flat construction
951 
952  The principal change from the base class involves gathering the background
953  values from each image and using them to determine the scalings for the final
954  combination.
955  """
956  ConfigClass = FlatConfig
957  _DefaultName = "flat"
958  calibName = "flat"
959 
960  @classmethod
961  def applyOverrides(cls, config):
962  """Overrides for flat construction"""
963  config.isr.doFlat = False
964  config.isr.doFringe = False
965 
966  def __init__(self, *args, **kwargs):
967  CalibTask.__init__(self, *args, **kwargs)
968  self.makeSubtask("stats")
969 
970  def processResult(self, exposure):
971  return self.stats.run(exposure)
972 
973  def scale(self, ccdIdLists, data):
974  """Determine the scalings for the final combination
975 
976  We have a matrix B_ij = C_i E_j, where C_i is the relative scaling
977  of one CCD to all the others in an exposure, and E_j is the scaling
978  of the exposure. We convert everything to logarithms so we can work
979  with a linear system. We determine the C_i and E_j from B_ij by iteration,
980  under the additional constraint that the average CCD scale is unity.
981 
982  This algorithm comes from Eugene Magnier and Pan-STARRS.
983  """
984  assert len(ccdIdLists.values()) > 0, "No successful CCDs"
985  lengths = set([len(expList) for expList in ccdIdLists.values()])
986  assert len(
987  lengths) == 1, "Number of successful exposures for each CCD differs"
988  assert tuple(lengths)[0] > 0, "No successful exposures"
989  # Format background measurements into a matrix
990  indices = dict((name, i) for i, name in enumerate(ccdIdLists))
991  bgMatrix = np.array([[0.0] * len(expList)
992  for expList in ccdIdLists.values()])
993  for name in ccdIdLists:
994  i = indices[name]
995  bgMatrix[i] = [
996  d if d is not None else np.nan for d in data[name]]
997 
998  numpyPrint = np.get_printoptions()
999  np.set_printoptions(threshold=np.inf)
1000  self.log.info("Input backgrounds: %s" % bgMatrix)
1001 
1002  # Flat-field scaling
1003  numCcds = len(ccdIdLists)
1004  numExps = bgMatrix.shape[1]
1005  # log(Background) for each exposure/component
1006  bgMatrix = np.log(bgMatrix)
1007  bgMatrix = np.ma.masked_array(bgMatrix, np.isnan(bgMatrix))
1008  # Initial guess at log(scale) for each component
1009  compScales = np.zeros(numCcds)
1010  expScales = np.array(
1011  [(bgMatrix[:, i0] - compScales).mean() for i0 in range(numExps)])
1012 
1013  for iterate in range(self.config.iterations):
1014  compScales = np.array(
1015  [(bgMatrix[i1, :] - expScales).mean() for i1 in range(numCcds)])
1016  expScales = np.array(
1017  [(bgMatrix[:, i2] - compScales).mean() for i2 in range(numExps)])
1018 
1019  avgScale = np.average(np.exp(compScales))
1020  compScales -= np.log(avgScale)
1021  self.log.debug("Iteration %d exposure scales: %s",
1022  iterate, np.exp(expScales))
1023  self.log.debug("Iteration %d component scales: %s",
1024  iterate, np.exp(compScales))
1025 
1026  expScales = np.array(
1027  [(bgMatrix[:, i3] - compScales).mean() for i3 in range(numExps)])
1028 
1029  if np.any(np.isnan(expScales)):
1030  raise RuntimeError("Bad exposure scales: %s --> %s" %
1031  (bgMatrix, expScales))
1032 
1033  expScales = np.exp(expScales)
1034  compScales = np.exp(compScales)
1035 
1036  self.log.info("Exposure scales: %s" % expScales)
1037  self.log.info("Component relative scaling: %s" % compScales)
1038  np.set_printoptions(**numpyPrint)
1039 
1040  return dict((ccdName, Struct(ccdScale=compScales[indices[ccdName]], expScales=expScales))
1041  for ccdName in ccdIdLists)
1042 
1043 
1045  """Configuration for fringe construction"""
1046  stats = ConfigurableField(target=CalibStatsTask,
1047  doc="Background statistics configuration")
1048  subtractBackground = ConfigurableField(target=measAlg.SubtractBackgroundTask,
1049  doc="Background configuration")
1050  detection = ConfigurableField(
1051  target=measAlg.SourceDetectionTask, doc="Detection configuration")
1052  detectSigma = Field(dtype=float, default=1.0,
1053  doc="Detection PSF gaussian sigma")
1054 
1055 
1057  """Fringe construction task
1058 
1059  The principal change from the base class is that the images are
1060  background-subtracted and rescaled by the background.
1061 
1062  XXX This is probably not right for a straight-up combination, as we
1063  are currently doing, since the fringe amplitudes need not scale with
1064  the continuum.
1065 
1066  XXX Would like to have this do PCA and generate multiple images, but
1067  that will take a bit of work with the persistence code.
1068  """
1069  ConfigClass = FringeConfig
1070  _DefaultName = "fringe"
1071  calibName = "fringe"
1072 
1073  @classmethod
1074  def applyOverrides(cls, config):
1075  """Overrides for fringe construction"""
1076  config.isr.doFringe = False
1077 
1078  def __init__(self, *args, **kwargs):
1079  CalibTask.__init__(self, *args, **kwargs)
1080  self.makeSubtask("detection")
1081  self.makeSubtask("stats")
1082  self.makeSubtask("subtractBackground")
1083 
1084  def processSingle(self, sensorRef):
1085  """Subtract the background and normalise by the background level"""
1086  exposure = CalibTask.processSingle(self, sensorRef)
1087  bgLevel = self.stats.run(exposure)
1088  self.subtractBackground.run(exposure)
1089  mi = exposure.getMaskedImage()
1090  mi /= bgLevel
1091  footprintSets = self.detection.detectFootprints(
1092  exposure, sigma=self.config.detectSigma)
1093  mask = exposure.getMaskedImage().getMask()
1094  detected = 1 << mask.addMaskPlane("DETECTED")
1095  for fpSet in (footprintSets.positive, footprintSets.negative):
1096  if fpSet is not None:
1097  afwDet.setMaskFromFootprintList(
1098  mask, fpSet.getFootprints(), detected)
1099  return exposure
1100 
1101 
1103  """Configuration for sky frame construction"""
1104  detection = ConfigurableField(target=measAlg.SourceDetectionTask, doc="Detection configuration")
1105  detectSigma = Field(dtype=float, default=2.0, doc="Detection PSF gaussian sigma")
1106  subtractBackground = ConfigurableField(target=measAlg.SubtractBackgroundTask,
1107  doc="Regular-scale background configuration, for object detection")
1108  largeScaleBackground = ConfigField(dtype=FocalPlaneBackgroundConfig,
1109  doc="Large-scale background configuration")
1110  sky = ConfigurableField(target=SkyMeasurementTask, doc="Sky measurement")
1111  maskThresh = Field(dtype=float, default=3.0, doc="k-sigma threshold for masking pixels")
1112  mask = ListField(dtype=str, default=["BAD", "SAT", "DETECTED", "NO_DATA"],
1113  doc="Mask planes to consider as contaminated")
1114 
1115 
1117  """Task for sky frame construction
1118 
1119  The sky frame is a (relatively) small-scale background
1120  model, the response of the camera to the sky.
1121 
1122  To construct, we first remove a large-scale background (e.g., caused
1123  by moonlight) which may vary from image to image. Then we construct a
1124  model of the sky, which is essentially a binned version of the image
1125  (important configuration parameters: sky.background.[xy]BinSize).
1126  It is these models which are coadded to yield the sky frame.
1127  """
1128  ConfigClass = SkyConfig
1129  _DefaultName = "sky"
1130  calibName = "sky"
1131 
1132  def __init__(self, *args, **kwargs):
1133  CalibTask.__init__(self, *args, **kwargs)
1134  self.makeSubtask("detection")
1135  self.makeSubtask("subtractBackground")
1136  self.makeSubtask("sky")
1137 
1138  def scatterProcess(self, pool, ccdIdLists):
1139  """!Scatter the processing among the nodes
1140 
1141  Only the master node executes this method, assigning work to the
1142  slaves.
1143 
1144  We measure and subtract off a large-scale background model across
1145  all CCDs, which requires a scatter/gather. Then we process the
1146  individual CCDs, subtracting the large-scale background model and
1147  the residual background model measured. These residuals will be
1148  combined for the sky frame.
1149 
1150  @param pool Process pool
1151  @param ccdIdLists Dict of data identifier lists for each CCD name
1152  @return Dict of lists of returned data for each CCD name
1153  """
1154  self.log.info("Scatter processing")
1155 
1156  numExps = set(len(expList) for expList in ccdIdLists.values())
1157  assert len(numExps) == 1
1158  numExps = numExps.pop()
1159 
1160  # First subtract off general gradients to make all the exposures look similar.
1161  # We want to preserve the common small-scale structure, which we will coadd.
1162  bgModelList = mapToMatrix(pool, self.measureBackground, ccdIdLists)
1163 
1164  backgrounds = {}
1165  scales = {}
1166  for exp in range(numExps):
1167  bgModels = [bgModelList[ccdName][exp] for ccdName in ccdIdLists]
1168  visit = set(tuple(ccdIdLists[ccdName][exp][key] for key in sorted(self.config.visitKeys)) for
1169  ccdName in ccdIdLists)
1170  assert len(visit) == 1
1171  visit = visit.pop()
1172  bgModel = bgModels[0]
1173  for bg in bgModels[1:]:
1174  bgModel.merge(bg)
1175  self.log.info("Background model min/max for visit %s: %f %f", visit,
1176  np.min(bgModel.getStatsImage().getArray()),
1177  np.max(bgModel.getStatsImage().getArray()))
1178  backgrounds[visit] = bgModel
1179  scales[visit] = np.median(bgModel.getStatsImage().getArray())
1180 
1181  return mapToMatrix(pool, self.process, ccdIdLists, backgrounds=backgrounds, scales=scales)
1182 
1183  def measureBackground(self, cache, dataId):
1184  """!Measure background model for CCD
1185 
1186  This method is executed by the slaves.
1187 
1188  The background models for all CCDs in an exposure will be
1189  combined to form a full focal-plane background model.
1190 
1191  @param cache Process pool cache
1192  @param dataId Data identifier
1193  @return Bcakground model
1194  """
1195  dataRef = getDataRef(cache.butler, dataId)
1196  exposure = self.processSingleBackground(dataRef)
1197 
1198  # NAOJ prototype smoothed and then combined the entire image, but it shouldn't be any different
1199  # to bin and combine the binned images except that there's fewer pixels to worry about.
1200  config = self.config.largeScaleBackground
1201  camera = dataRef.get("camera")
1202  bgModel = FocalPlaneBackground.fromCamera(config, camera)
1203  bgModel.addCcd(exposure)
1204  return bgModel
1205 
1206  def processSingleBackground(self, dataRef):
1207  """!Process a single CCD for the background
1208 
1209  This method is executed by the slaves.
1210 
1211  Because we're interested in the background, we detect and mask astrophysical
1212  sources, and pixels above the noise level.
1213 
1214  @param dataRef Data reference for CCD.
1215  @return processed exposure
1216  """
1217  if not self.config.clobber and dataRef.datasetExists("postISRCCD"):
1218  return dataRef.get("postISRCCD")
1219  exposure = CalibTask.processSingle(self, dataRef)
1220 
1221  # Detect sources. Requires us to remove the background; we'll restore it later.
1222  bgTemp = self.subtractBackground.run(exposure).background
1223  footprints = self.detection.detectFootprints(exposure, sigma=self.config.detectSigma)
1224  image = exposure.getMaskedImage()
1225  if footprints.background is not None:
1226  image += footprints.background.getImageF()
1227 
1228  # Mask high pixels
1229  variance = image.getVariance()
1230  noise = np.sqrt(np.median(variance.getArray()))
1231  isHigh = image.getImage().getArray() > self.config.maskThresh*noise
1232  image.getMask().getArray()[isHigh] |= image.getMask().getPlaneBitMask("DETECTED")
1233 
1234  # Restore the background: it's what we want!
1235  image += bgTemp.getImage()
1236 
1237  # Set detected/bad pixels to background to ensure they don't corrupt the background
1238  maskVal = image.getMask().getPlaneBitMask(self.config.mask)
1239  isBad = image.getMask().getArray() & maskVal > 0
1240  bgLevel = np.median(image.getImage().getArray()[~isBad])
1241  image.getImage().getArray()[isBad] = bgLevel
1242  dataRef.put(exposure, "postISRCCD")
1243  return exposure
1244 
1245  def processSingle(self, dataRef, backgrounds, scales):
1246  """Process a single CCD, specified by a data reference
1247 
1248  We subtract the appropriate focal plane background model,
1249  divide by the appropriate scale and measure the background.
1250 
1251  Only slave nodes execute this method.
1252 
1253  @param dataRef Data reference for single CCD
1254  @param backgrounds Background model for each visit
1255  @param scales Scales for each visit
1256  @return Processed exposure
1257  """
1258  visit = tuple(dataRef.dataId[key] for key in sorted(self.config.visitKeys))
1259  exposure = dataRef.get("postISRCCD", immediate=True)
1260  image = exposure.getMaskedImage()
1261  detector = exposure.getDetector()
1262  bbox = image.getBBox()
1263 
1264  bgModel = backgrounds[visit]
1265  bg = bgModel.toCcdBackground(detector, bbox)
1266  image -= bg.getImage()
1267  image /= scales[visit]
1268 
1269  bg = self.sky.measureBackground(exposure.getMaskedImage())
1270  dataRef.put(bg, "icExpBackground")
1271  return exposure
1272 
1273  def combine(self, cache, struct, outputId):
1274  """!Combine multiple background models of a particular CCD and write the output
1275 
1276  Only the slave nodes execute this method.
1277 
1278  @param cache Process pool cache
1279  @param struct Parameters for the combination, which has the following components:
1280  * ccdName Name tuple for CCD
1281  * ccdIdList List of data identifiers for combination
1282  @param outputId Data identifier for combined image (exposure part only)
1283  @return binned calib image
1284  """
1285  outputId = self.getFullyQualifiedOutputId(struct.ccdName, cache.butler, outputId)
1286  dataRefList = [getDataRef(cache.butler, dataId) if dataId is not None else None for
1287  dataId in struct.ccdIdList]
1288  self.log.info("Combining %s on %s" % (outputId, NODE))
1289  bgList = [dataRef.get("icExpBackground", immediate=True).clone() for dataRef in dataRefList]
1290 
1291  bgExp = self.sky.averageBackgrounds(bgList)
1292 
1293  self.recordCalibInputs(cache.butler, bgExp, struct.ccdIdList, outputId)
1294  cache.butler.put(bgExp, "sky", outputId)
1295  return afwMath.binImage(self.sky.exposureToBackground(bgExp).getImage(), self.config.binning)
def processWrite(self, dataRef, exposure, outputName="postISRCCD")
Write the processed CCD.
def scatterCombine(self, pool, outputId, ccdIdLists, scales)
Scatter the combination of exposures across multiple nodes.
def run(self, exposureOrImage)
Measure a particular statistic on an image (of some sort).
def checksum(obj, header=None, sumType="MD5")
Calculate a checksum of an object.
Definition: checksum.py:29
def getCcdIdListFromExposures(expRefList, level="sensor", ccdKeys=["ccd"])
Determine a list of CCDs from exposure references.
def run(self, expRefList, butler, calibId)
Construct a calib from a list of exposure references.
def getFullyQualifiedOutputId(self, ccdName, butler, outputId)
def __call__(self, parser, namespace, values, option_string)
def dictToTuple(dict_, keys)
Return a tuple of specific values from a dict.
def getDataRef(butler, dataId, datasetType="raw")
Definition: utils.py:17
def updateMetadata(self, calibImage, exposureTime, darkTime=None, kwargs)
Update the metadata from the VisitInfo.
def process(self, cache, ccdId, outputName="postISRCCD", kwargs)
Process a CCD, specified by a data identifier.
def processSingle(self, dataRef, backgrounds, scales)
def makeCameraImage(self, camera, dataId, calibs)
Create and write an image of the entire camera.
def combine(self, target, imageList, stats)
Combine multiple images.
def getMjd(self, butler, dataId, timescale=dafBase.DateTime.UTC)
def combine(self, cache, struct, outputId)
Combine multiple exposures of a particular CCD and write the output.
def write(self, butler, exposure, dataId)
Write the final combined calib.
def combine(self, cache, struct, outputId)
Combine multiple background models of a particular CCD and write the output.
def measureBackground(self, cache, dataId)
Measure background model for CCD.
def recordCalibInputs(self, butler, calib, dataIdList, outputId)
Record metadata including the inputs and creation details.
def scatterProcess(self, pool, ccdIdLists)
Scatter the processing among the nodes.
def run(self, sensorRefList, expScales=None, finalScale=None, inputName="postISRCCD")
Combine calib images for a single sensor.
def getOutputId(self, expRefList, calibId)
Generate the data identifier for the output calib.
def mapToMatrix(pool, func, ccdIdLists, args, kwargs)
def scale(self, ccdIdLists, data)
Determine scaling across CCDs and exposures.
def addMissingKeys(self, dataId, butler, missingKeys=None, calibName=None)
def getDimensions(self, sensorRefList, inputName="postISRCCD")
Base class for constructing calibs.
def processSingleBackground(self, dataRef)
Process a single CCD for the background.
def scatterProcess(self, pool, ccdIdLists)
Scatter the processing among the nodes.
def batchWallTime(cls, time, parsedCmd, numCores)