lsst.pipe.drivers  16.0-7-gd2eeba5+14
constructCalibs.py
Go to the documentation of this file.
1 from __future__ import absolute_import, division, print_function
2 
3 import sys
4 import math
5 import time
6 import argparse
7 import traceback
8 import collections
9 
10 import numpy as np
11 from builtins import zip
12 from builtins import range
13 
14 from lsst.pex.config import Config, ConfigurableField, Field, ListField, ConfigField
15 from lsst.pipe.base import Task, Struct, TaskRunner, ArgumentParser
16 import lsst.daf.base as dafBase
17 import lsst.afw.math as afwMath
18 import lsst.afw.geom as afwGeom
19 import lsst.afw.detection as afwDet
20 import lsst.afw.image as afwImage
21 from lsst.afw.image import VisitInfo
22 import lsst.meas.algorithms as measAlg
23 from lsst.pipe.tasks.repair import RepairTask
24 from lsst.ip.isr import IsrTask
25 from lsst.afw.cameraGeom.utils import makeImageFromCamera
26 
27 from lsst.ctrl.pool.parallel import BatchPoolTask
28 from lsst.ctrl.pool.pool import Pool, NODE
29 from lsst.pipe.drivers.background import SkyMeasurementTask, FocalPlaneBackground, FocalPlaneBackgroundConfig
30 from lsst.pipe.drivers.visualizeVisit import makeCameraImage
31 
32 from .checksum import checksum
33 from .utils import getDataRef
34 
35 
36 class CalibStatsConfig(Config):
37  """Parameters controlling the measurement of background statistics"""
38  stat = Field(doc="Statistic to use to estimate background (from lsst.afw.math)", dtype=int,
39  default=int(afwMath.MEANCLIP))
40  clip = Field(doc="Clipping threshold for background",
41  dtype=float, default=3.0)
42  nIter = Field(doc="Clipping iterations for background",
43  dtype=int, default=3)
44  maxVisitsToCalcErrorFromInputVariance = Field(
45  doc="Maximum number of visits to estimate variance from input variance, not per-pixel spread",
46  dtype=int, default=2)
47  mask = ListField(doc="Mask planes to reject",
48  dtype=str, default=["DETECTED", "BAD", "NO_DATA",])
49 
50 
51 class CalibStatsTask(Task):
52  """Measure statistics on the background
53 
54  This can be useful for scaling the background, e.g., for flats and fringe frames.
55  """
56  ConfigClass = CalibStatsConfig
57 
58  def run(self, exposureOrImage):
59  """!Measure a particular statistic on an image (of some sort).
60 
61  @param exposureOrImage Exposure, MaskedImage or Image.
62  @return Value of desired statistic
63  """
64  stats = afwMath.StatisticsControl(self.config.clip, self.config.nIter,
65  afwImage.Mask.getPlaneBitMask(self.config.mask))
66  try:
67  image = exposureOrImage.getMaskedImage()
68  except:
69  try:
70  image = exposureOrImage.getImage()
71  except:
72  image = exposureOrImage
73 
74  return afwMath.makeStatistics(image, self.config.stat, stats).getValue()
75 
76 
77 class CalibCombineConfig(Config):
78  """Configuration for combining calib images"""
79  rows = Field(doc="Number of rows to read at a time",
80  dtype=int, default=512)
81  mask = ListField(doc="Mask planes to respect", dtype=str,
82  default=["SAT", "DETECTED", "INTRP"])
83  combine = Field(doc="Statistic to use for combination (from lsst.afw.math)", dtype=int,
84  default=int(afwMath.MEANCLIP))
85  clip = Field(doc="Clipping threshold for combination",
86  dtype=float, default=3.0)
87  nIter = Field(doc="Clipping iterations for combination",
88  dtype=int, default=3)
89  stats = ConfigurableField(target=CalibStatsTask,
90  doc="Background statistics configuration")
91 
92 
93 class CalibCombineTask(Task):
94  """Task to combine calib images"""
95  ConfigClass = CalibCombineConfig
96 
97  def __init__(self, *args, **kwargs):
98  Task.__init__(self, *args, **kwargs)
99  self.makeSubtask("stats")
100 
101  def run(self, sensorRefList, expScales=None, finalScale=None, inputName="postISRCCD"):
102  """!Combine calib images for a single sensor
103 
104  @param sensorRefList List of data references to combine (for a single sensor)
105  @param expScales List of scales to apply for each exposure
106  @param finalScale Desired scale for final combined image
107  @param inputName Data set name for inputs
108  @return combined image
109  """
110  width, height = self.getDimensions(sensorRefList)
111  stats = afwMath.StatisticsControl(self.config.clip, self.config.nIter,
112  afwImage.Mask.getPlaneBitMask(self.config.mask))
113  numImages = len(sensorRefList)
114  if numImages < 1:
115  raise RuntimeError("No valid input data")
116  if numImages < self.config.stats.maxVisitsToCalcErrorFromInputVariance:
117  stats.setCalcErrorFromInputVariance(True)
118 
119  # Combine images
120  combined = afwImage.MaskedImageF(width, height)
121  imageList = [None]*numImages
122  for start in range(0, height, self.config.rows):
123  rows = min(self.config.rows, height - start)
124  box = afwGeom.Box2I(afwGeom.Point2I(0, start),
125  afwGeom.Extent2I(width, rows))
126  subCombined = combined.Factory(combined, box)
127 
128  for i, sensorRef in enumerate(sensorRefList):
129  if sensorRef is None:
130  imageList[i] = None
131  continue
132  exposure = sensorRef.get(inputName + "_sub", bbox=box)
133  if expScales is not None:
134  self.applyScale(exposure, expScales[i])
135  imageList[i] = exposure.getMaskedImage()
136 
137  self.combine(subCombined, imageList, stats)
138 
139  if finalScale is not None:
140  background = self.stats.run(combined)
141  self.log.info("%s: Measured background of stack is %f; adjusting to %f" %
142  (NODE, background, finalScale))
143  combined *= finalScale / background
144 
145  return combined
146 
147  def getDimensions(self, sensorRefList, inputName="postISRCCD"):
148  """Get dimensions of the inputs"""
149  dimList = []
150  for sensorRef in sensorRefList:
151  if sensorRef is None:
152  continue
153  md = sensorRef.get(inputName + "_md")
154  dimList.append(afwImage.bboxFromMetadata(md).getDimensions())
155  return getSize(dimList)
156 
157  def applyScale(self, exposure, scale=None):
158  """Apply scale to input exposure
159 
160  This implementation applies a flux scaling: the input exposure is
161  divided by the provided scale.
162  """
163  if scale is not None:
164  mi = exposure.getMaskedImage()
165  mi /= scale
166 
167  def combine(self, target, imageList, stats):
168  """!Combine multiple images
169 
170  @param target Target image to receive the combined pixels
171  @param imageList List of input images
172  @param stats Statistics control
173  """
174  images = [img for img in imageList if img is not None]
175  afwMath.statisticsStack(target, images, afwMath.Property(self.config.combine), stats)
176 
177 
178 def getSize(dimList):
179  """Determine a consistent size, given a list of image sizes"""
180  dim = set((w, h) for w, h in dimList)
181  dim.discard(None)
182  if len(dim) != 1:
183  raise RuntimeError("Inconsistent dimensions: %s" % dim)
184  return dim.pop()
185 
186 
187 def dictToTuple(dict_, keys):
188  """!Return a tuple of specific values from a dict
189 
190  This provides a hashable representation of the dict from certain keywords.
191  This can be useful for creating e.g., a tuple of the values in the DataId
192  that identify the CCD.
193 
194  @param dict_ dict to parse
195  @param keys keys to extract (order is important)
196  @return tuple of values
197  """
198  return tuple(dict_[k] for k in keys)
199 
200 
201 def getCcdIdListFromExposures(expRefList, level="sensor", ccdKeys=["ccd"]):
202  """!Determine a list of CCDs from exposure references
203 
204  This essentially inverts the exposure-level references (which
205  provides a list of CCDs for each exposure), by providing
206  a dataId list for each CCD. Consider an input list of exposures
207  [e1, e2, e3], and each exposure has CCDs c1 and c2. Then this
208  function returns:
209 
210  {(c1,): [e1c1, e2c1, e3c1], (c2,): [e1c2, e2c2, e3c2]}
211 
212  This is a dict whose keys are tuples of the identifying values of a
213  CCD (usually just the CCD number) and the values are lists of dataIds
214  for that CCD in each exposure. A missing dataId is given the value
215  None.
216 
217  @param expRefList List of data references for exposures
218  @param level Level for the butler to generate CCDs
219  @param ccdKeys DataId keywords that identify a CCD
220  @return dict of data identifier lists for each CCD;
221  keys are values of ccdKeys in order
222  """
223  expIdList = [[ccdRef.dataId for ccdRef in expRef.subItems(
224  level)] for expRef in expRefList]
225 
226  # Determine what additional keys make a CCD from an exposure
227  if len(ccdKeys) != len(set(ccdKeys)):
228  raise RuntimeError("Duplicate keys found in ccdKeys: %s" % ccdKeys)
229  ccdNames = set() # Set of tuples which are values for each of the CCDs in an exposure
230  for ccdIdList in expIdList:
231  for ccdId in ccdIdList:
232  name = dictToTuple(ccdId, ccdKeys)
233  ccdNames.add(name)
234 
235  # Turn the list of CCDs for each exposure into a list of exposures for
236  # each CCD
237  ccdLists = {}
238  for n, ccdIdList in enumerate(expIdList):
239  for ccdId in ccdIdList:
240  name = dictToTuple(ccdId, ccdKeys)
241  if name not in ccdLists:
242  ccdLists[name] = []
243  ccdLists[name].append(ccdId)
244 
245  for ccd in ccdLists:
246  # Sort the list by the dataId values (ordered by key)
247  ccdLists[ccd] = sorted(ccdLists[ccd], key=lambda dd: dictToTuple(dd, sorted(dd.keys())))
248 
249  return ccdLists
250 
251 
252 def mapToMatrix(pool, func, ccdIdLists, *args, **kwargs):
253  """Generate a matrix of results using pool.map
254 
255  The function should have the call signature:
256  func(cache, dataId, *args, **kwargs)
257 
258  We return a dict mapping 'ccd name' to a list of values for
259  each exposure.
260 
261  @param pool Process pool
262  @param func Function to call for each dataId
263  @param ccdIdLists Dict of data identifier lists for each CCD name
264  @return matrix of results
265  """
266  dataIdList = sum(ccdIdLists.values(), [])
267  resultList = pool.map(func, dataIdList, *args, **kwargs)
268  # Piece everything back together
269  data = dict((ccdName, [None] * len(expList)) for ccdName, expList in ccdIdLists.items())
270  indices = dict(sum([[(tuple(dataId.values()) if dataId is not None else None, (ccdName, expNum))
271  for expNum, dataId in enumerate(expList)]
272  for ccdName, expList in ccdIdLists.items()], []))
273  for dataId, result in zip(dataIdList, resultList):
274  if dataId is None:
275  continue
276  ccdName, expNum = indices[tuple(dataId.values())]
277  data[ccdName][expNum] = result
278  return data
279 
280 
281 class CalibIdAction(argparse.Action):
282  """Split name=value pairs and put the result in a dict"""
283 
284  def __call__(self, parser, namespace, values, option_string):
285  output = getattr(namespace, self.dest, {})
286  for nameValue in values:
287  name, sep, valueStr = nameValue.partition("=")
288  if not valueStr:
289  parser.error("%s value %s must be in form name=value" %
290  (option_string, nameValue))
291  output[name] = valueStr
292  setattr(namespace, self.dest, output)
293 
294 
295 class CalibArgumentParser(ArgumentParser):
296  """ArgumentParser for calibration construction"""
297 
298  def __init__(self, calibName, *args, **kwargs):
299  """Add a --calibId argument to the standard pipe_base argument parser"""
300  ArgumentParser.__init__(self, *args, **kwargs)
301  self.calibName = calibName
302  self.add_id_argument("--id", datasetType="raw",
303  help="input identifiers, e.g., --id visit=123 ccd=4")
304  self.add_argument("--calibId", nargs="*", action=CalibIdAction, default={},
305  help="identifiers for calib, e.g., --calibId version=1",
306  metavar="KEY=VALUE1[^VALUE2[^VALUE3...]")
307 
308  def parse_args(self, *args, **kwargs):
309  """Parse arguments
310 
311  Checks that the "--calibId" provided works.
312  """
313  namespace = ArgumentParser.parse_args(self, *args, **kwargs)
314 
315  keys = namespace.butler.getKeys(self.calibName)
316  parsed = {}
317  for name, value in namespace.calibId.items():
318  if name not in keys:
319  self.error(
320  "%s is not a relevant calib identifier key (%s)" % (name, keys))
321  parsed[name] = keys[name](value)
322  namespace.calibId = parsed
323 
324  return namespace
325 
326 
327 class CalibConfig(Config):
328  """Configuration for constructing calibs"""
329  clobber = Field(dtype=bool, default=True,
330  doc="Clobber existing processed images?")
331  isr = ConfigurableField(target=IsrTask, doc="ISR configuration")
332  dateObs = Field(dtype=str, default="dateObs",
333  doc="Key for observation date in exposure registry")
334  dateCalib = Field(dtype=str, default="calibDate",
335  doc="Key for calib date in calib registry")
336  filter = Field(dtype=str, default="filter",
337  doc="Key for filter name in exposure/calib registries")
338  combination = ConfigurableField(
339  target=CalibCombineTask, doc="Calib combination configuration")
340  ccdKeys = ListField(dtype=str, default=["ccd"],
341  doc="DataId keywords specifying a CCD")
342  visitKeys = ListField(dtype=str, default=["visit"],
343  doc="DataId keywords specifying a visit")
344  calibKeys = ListField(dtype=str, default=[],
345  doc="DataId keywords specifying a calibration")
346  doCameraImage = Field(dtype=bool, default=True, doc="Create camera overview image?")
347  binning = Field(dtype=int, default=64, doc="Binning to apply for camera image")
348 
349  def setDefaults(self):
350  self.isr.doWrite = False
351 
352 
353 class CalibTaskRunner(TaskRunner):
354  """Get parsed values into the CalibTask.run"""
355  @staticmethod
356  def getTargetList(parsedCmd, **kwargs):
357  return [dict(expRefList=parsedCmd.id.refList, butler=parsedCmd.butler, calibId=parsedCmd.calibId)]
358 
359  def __call__(self, args):
360  """Call the Task with the kwargs from getTargetList"""
361  task = self.TaskClass(config=self.config, log=self.log)
362  exitStatus = 0 # exit status for the shell
363  if self.doRaise:
364  result = task.runDataRef(**args)
365  else:
366  try:
367  result = task.runDataRef(**args)
368  except Exception as e:
369  exitStatus = 1 # n.b. The shell exit value is the number of dataRefs returning
370  # non-zero, so the actual value used here is lost
371  task.log.fatal("Failed: %s" % e)
372  traceback.print_exc(file=sys.stderr)
373 
374  if self.doReturnResults:
375  return Struct(
376  exitStatus=exitStatus,
377  args=args,
378  metadata=task.metadata,
379  result=result,
380  )
381  else:
382  return Struct(
383  exitStatus=exitStatus,
384  )
385 
387  """!Base class for constructing calibs.
388 
389  This should be subclassed for each of the required calib types.
390  The subclass should be sure to define the following class variables:
391  * _DefaultName: default name of the task, used by CmdLineTask
392  * calibName: name of the calibration data set in the butler
393  The subclass may optionally set:
394  * filterName: filter name to give the resultant calib
395  """
396  ConfigClass = CalibConfig
397  RunnerClass = CalibTaskRunner
398  filterName = None
399  calibName = None
400  exposureTime = 1.0 # sets this exposureTime in the output
401 
402  def __init__(self, *args, **kwargs):
403  """Constructor"""
404  BatchPoolTask.__init__(self, *args, **kwargs)
405  self.makeSubtask("isr")
406  self.makeSubtask("combination")
407 
408  @classmethod
409  def batchWallTime(cls, time, parsedCmd, numCores):
410  numCcds = len(parsedCmd.butler.get("camera"))
411  numExps = len(cls.RunnerClass.getTargetList(
412  parsedCmd)[0]['expRefList'])
413  numCycles = int(numCcds/float(numCores) + 0.5)
414  return time*numExps*numCycles
415 
416  @classmethod
417  def _makeArgumentParser(cls, *args, **kwargs):
418  kwargs.pop("doBatch", False)
419  return CalibArgumentParser(calibName=cls.calibName, name=cls._DefaultName, *args, **kwargs)
420 
421  def runDataRef(self, expRefList, butler, calibId):
422  """!Construct a calib from a list of exposure references
423 
424  This is the entry point, called by the TaskRunner.__call__
425 
426  Only the master node executes this method.
427 
428  @param expRefList List of data references at the exposure level
429  @param butler Data butler
430  @param calibId Identifier dict for calib
431  """
432  if len(expRefList) < 1:
433  raise RuntimeError("No valid input data")
434 
435  for expRef in expRefList:
436  self.addMissingKeys(expRef.dataId, butler, self.config.ccdKeys, 'raw')
437 
438  outputId = self.getOutputId(expRefList, calibId)
439  ccdIdLists = getCcdIdListFromExposures(
440  expRefList, level="sensor", ccdKeys=self.config.ccdKeys)
441  self.checkCcdIdLists(ccdIdLists)
442 
443  # Ensure we can generate filenames for each output
444  outputIdItemList = list(outputId.items())
445  for ccdName in ccdIdLists:
446  dataId = dict([(k, ccdName[i]) for i, k in enumerate(self.config.ccdKeys)])
447  dataId.update(outputIdItemList)
448  self.addMissingKeys(dataId, butler)
449  dataId.update(outputIdItemList)
450 
451  try:
452  butler.get(self.calibName + "_filename", dataId)
453  except Exception as e:
454  raise RuntimeError(
455  "Unable to determine output filename \"%s_filename\" from %s: %s" %
456  (self.calibName, dataId, e))
457 
458  processPool = Pool("process")
459  processPool.storeSet(butler=butler)
460 
461  # Scatter: process CCDs independently
462  data = self.scatterProcess(processPool, ccdIdLists)
463 
464  # Gather: determine scalings
465  scales = self.scale(ccdIdLists, data)
466 
467  combinePool = Pool("combine")
468  combinePool.storeSet(butler=butler)
469 
470  # Scatter: combine
471  calibs = self.scatterCombine(combinePool, outputId, ccdIdLists, scales)
472 
473  if self.config.doCameraImage:
474  camera = butler.get("camera")
475  # Convert indexing of calibs from "ccdName" to detector ID (as used by makeImageFromCamera)
476  calibs = {butler.get("postISRCCD_detector",
477  dict(zip(self.config.ccdKeys, ccdName))).getId(): calibs[ccdName]
478  for ccdName in ccdIdLists}
479 
480  try:
481  cameraImage = self.makeCameraImage(camera, outputId, calibs)
482  butler.put(cameraImage, self.calibName + "_camera", dataId)
483  except Exception as exc:
484  self.log.warn("Unable to create camera image: %s" % (exc,))
485 
486  return Struct(
487  outputId = outputId,
488  ccdIdLists = ccdIdLists,
489  scales = scales,
490  calibs = calibs,
491  processPool = processPool,
492  combinePool = combinePool,
493  )
494 
495  def getOutputId(self, expRefList, calibId):
496  """!Generate the data identifier for the output calib
497 
498  The mean date and the common filter are included, using keywords
499  from the configuration. The CCD-specific part is not included
500  in the data identifier.
501 
502  @param expRefList List of data references at exposure level
503  @param calibId Data identifier elements for the calib provided by the user
504  @return data identifier
505  """
506  midTime = 0
507  filterName = None
508  for expRef in expRefList:
509  butler = expRef.getButler()
510  dataId = expRef.dataId
511 
512  midTime += self.getMjd(butler, dataId)
513  thisFilter = self.getFilter(
514  butler, dataId) if self.filterName is None else self.filterName
515  if filterName is None:
516  filterName = thisFilter
517  elif filterName != thisFilter:
518  raise RuntimeError("Filter mismatch for %s: %s vs %s" % (
519  dataId, thisFilter, filterName))
520 
521  midTime /= len(expRefList)
522  date = str(dafBase.DateTime(
523  midTime, dafBase.DateTime.MJD).toPython().date())
524 
525  outputId = {self.config.filter: filterName,
526  self.config.dateCalib: date}
527  outputId.update(calibId)
528  return outputId
529 
530  def getMjd(self, butler, dataId, timescale=dafBase.DateTime.UTC):
531  """Determine the Modified Julian Date (MJD; in TAI) from a data identifier"""
532  if self.config.dateObs in dataId:
533  dateObs = dataId[self.config.dateObs]
534  else:
535  dateObs = butler.queryMetadata('raw', [self.config.dateObs], dataId)[0]
536  if "T" not in dateObs:
537  dateObs = dateObs + "T12:00:00.0Z"
538  elif not dateObs.endswith("Z"):
539  dateObs += "Z"
540 
541  return dafBase.DateTime(dateObs, timescale).get(dafBase.DateTime.MJD)
542 
543  def getFilter(self, butler, dataId):
544  """Determine the filter from a data identifier"""
545  filt = butler.queryMetadata('raw', [self.config.filter], dataId)[0]
546  return filt
547 
548  def addMissingKeys(self, dataId, butler, missingKeys=None, calibName=None):
549  if calibName is None:
550  calibName = self.calibName
551 
552  if missingKeys is None:
553  missingKeys = set(butler.getKeys(calibName).keys()) - set(dataId.keys())
554 
555  for k in missingKeys:
556  try:
557  v = butler.queryMetadata('raw', [k], dataId) # n.b. --id refers to 'raw'
558  except Exception as e:
559  continue
560 
561  if len(v) == 0: # failed to lookup value
562  continue
563 
564  if len(v) == 1:
565  dataId[k] = v[0]
566  else:
567  raise RuntimeError("No unique lookup for %s: %s" % (k, v))
568 
569  def updateMetadata(self, calibImage, exposureTime, darkTime=None, **kwargs):
570  """!Update the metadata from the VisitInfo
571 
572  \param calibImage The image whose metadata is to be set
573  \param exposureTime The exposure time for the image
574  \param darkTime The time since the last read (default: exposureTime)
575  """
576 
577  if darkTime is None:
578  darkTime = exposureTime # avoid warning messages when using calibration products
579 
580  visitInfo = afwImage.makeVisitInfo(exposureTime=exposureTime, darkTime=darkTime, **kwargs)
581  md = calibImage.getMetadata()
582 
583  afwImage.setVisitInfoMetadata(md, visitInfo)
584 
585  def scatterProcess(self, pool, ccdIdLists):
586  """!Scatter the processing among the nodes
587 
588  We scatter each CCD independently (exposures aren't grouped together),
589  to make full use of all available processors. This necessitates piecing
590  everything back together in the same format as ccdIdLists afterwards.
591 
592  Only the master node executes this method.
593 
594  @param pool Process pool
595  @param ccdIdLists Dict of data identifier lists for each CCD name
596  @return Dict of lists of returned data for each CCD name
597  """
598  self.log.info("Scatter processing")
599  return mapToMatrix(pool, self.process, ccdIdLists)
600 
601  def process(self, cache, ccdId, outputName="postISRCCD", **kwargs):
602  """!Process a CCD, specified by a data identifier
603 
604  After processing, optionally returns a result (produced by
605  the 'processResult' method) calculated from the processed
606  exposure. These results will be gathered by the master node,
607  and is a means for coordinated scaling of all CCDs for flats,
608  etc.
609 
610  Only slave nodes execute this method.
611 
612  @param cache Process pool cache
613  @param ccdId Data identifier for CCD
614  @param outputName Output dataset name for butler
615  @return result from 'processResult'
616  """
617  if ccdId is None:
618  self.log.warn("Null identifier received on %s" % NODE)
619  return None
620  sensorRef = getDataRef(cache.butler, ccdId)
621  if self.config.clobber or not sensorRef.datasetExists(outputName):
622  self.log.info("Processing %s on %s" % (ccdId, NODE))
623  try:
624  exposure = self.processSingle(sensorRef, **kwargs)
625  except Exception as e:
626  self.log.warn("Unable to process %s: %s" % (ccdId, e))
627  raise
628  return None
629  self.processWrite(sensorRef, exposure)
630  else:
631  self.log.info(
632  "Using previously persisted processed exposure for %s" % (sensorRef.dataId,))
633  exposure = sensorRef.get(outputName)
634  return self.processResult(exposure)
635 
636  def processSingle(self, dataRef):
637  """Process a single CCD, specified by a data reference
638 
639  Generally, this simply means doing ISR.
640 
641  Only slave nodes execute this method.
642  """
643  return self.isr.runDataRef(dataRef).exposure
644 
645  def processWrite(self, dataRef, exposure, outputName="postISRCCD"):
646  """!Write the processed CCD
647 
648  We need to write these out because we can't hold them all in
649  memory at once.
650 
651  Only slave nodes execute this method.
652 
653  @param dataRef Data reference
654  @param exposure CCD exposure to write
655  @param outputName Output dataset name for butler.
656  """
657  dataRef.put(exposure, outputName)
658 
659  def processResult(self, exposure):
660  """Extract processing results from a processed exposure
661 
662  This method generates what is gathered by the master node.
663  This can be a background measurement or similar for scaling
664  flat-fields. It must be picklable!
665 
666  Only slave nodes execute this method.
667  """
668  return None
669 
670  def scale(self, ccdIdLists, data):
671  """!Determine scaling across CCDs and exposures
672 
673  This is necessary mainly for flats, so as to determine a
674  consistent scaling across the entire focal plane. This
675  implementation is simply a placeholder.
676 
677  Only the master node executes this method.
678 
679  @param ccdIdLists Dict of data identifier lists for each CCD tuple
680  @param data Dict of lists of returned data for each CCD tuple
681  @return dict of Struct(ccdScale: scaling for CCD,
682  expScales: scaling for each exposure
683  ) for each CCD tuple
684  """
685  self.log.info("Scale on %s" % NODE)
686  return dict((name, Struct(ccdScale=None, expScales=[None] * len(ccdIdLists[name])))
687  for name in ccdIdLists)
688 
689  def scatterCombine(self, pool, outputId, ccdIdLists, scales):
690  """!Scatter the combination of exposures across multiple nodes
691 
692  In this case, we can only scatter across as many nodes as
693  there are CCDs.
694 
695  Only the master node executes this method.
696 
697  @param pool Process pool
698  @param outputId Output identifier (exposure part only)
699  @param ccdIdLists Dict of data identifier lists for each CCD name
700  @param scales Dict of structs with scales, for each CCD name
701  @param dict of binned images
702  """
703  self.log.info("Scatter combination")
704  data = [Struct(ccdName=ccdName, ccdIdList=ccdIdLists[ccdName], scales=scales[ccdName]) for
705  ccdName in ccdIdLists]
706  images = pool.map(self.combine, data, outputId)
707  return dict(zip(ccdIdLists.keys(), images))
708 
709  def getFullyQualifiedOutputId(self, ccdName, butler, outputId):
710  """Get fully-qualified output data identifier
711 
712  We may need to look up keys that aren't in the output dataId.
713 
714  @param ccdName Name tuple for CCD
715  @param butler Data butler
716  @param outputId Data identifier for combined image (exposure part only)
717  @return fully-qualified output dataId
718  """
719  fullOutputId = {k: ccdName[i] for i, k in enumerate(self.config.ccdKeys)}
720  fullOutputId.update(outputId)
721  self.addMissingKeys(fullOutputId, butler)
722  fullOutputId.update(outputId) # must be after the call to queryMetadata in 'addMissingKeys'
723  return fullOutputId
724 
725  def combine(self, cache, struct, outputId):
726  """!Combine multiple exposures of a particular CCD and write the output
727 
728  Only the slave nodes execute this method.
729 
730  @param cache Process pool cache
731  @param struct Parameters for the combination, which has the following components:
732  * ccdName Name tuple for CCD
733  * ccdIdList List of data identifiers for combination
734  * scales Scales to apply (expScales are scalings for each exposure,
735  ccdScale is final scale for combined image)
736  @param outputId Data identifier for combined image (exposure part only)
737  @return binned calib image
738  """
739  outputId = self.getFullyQualifiedOutputId(struct.ccdName, cache.butler, outputId)
740  dataRefList = [getDataRef(cache.butler, dataId) if dataId is not None else None for
741  dataId in struct.ccdIdList]
742  self.log.info("Combining %s on %s" % (outputId, NODE))
743  calib = self.combination.run(dataRefList, expScales=struct.scales.expScales,
744  finalScale=struct.scales.ccdScale)
745 
746  if not hasattr(calib, "getMetadata"):
747  if hasattr(calib, "getVariance"):
748  calib = afwImage.makeExposure(calib)
749  else:
750  calib = afwImage.DecoratedImageF(calib.getImage()) # n.b. hardwires "F" for the output type
751 
752  self.updateMetadata(calib, self.exposureTime)
753 
754  self.recordCalibInputs(cache.butler, calib,
755  struct.ccdIdList, outputId)
756 
757  self.interpolateNans(calib)
758 
759  self.write(cache.butler, calib, outputId)
760 
761  return afwMath.binImage(calib.getImage(), self.config.binning)
762 
763  def recordCalibInputs(self, butler, calib, dataIdList, outputId):
764  """!Record metadata including the inputs and creation details
765 
766  This metadata will go into the FITS header.
767 
768  @param butler Data butler
769  @param calib Combined calib exposure.
770  @param dataIdList List of data identifiers for calibration inputs
771  @param outputId Data identifier for output
772  """
773  header = calib.getMetadata()
774  header.add("OBSTYPE", self.calibName) # Used by ingestCalibs.py
775 
776  # date, time, host, and root
777  now = time.localtime()
778  header.add("CALIB_CREATION_DATE", time.strftime("%Y-%m-%d", now))
779  header.add("CALIB_CREATION_TIME", time.strftime("%X %Z", now))
780  # add date-obs as its absence upsets ExposureInfo; use the mean date that the calibs were taken
781  header.add("DATE-OBS", "%sT00:00:00.00" % outputId[self.config.dateCalib])
782 
783  # Inputs
784  visits = [str(dictToTuple(dataId, self.config.visitKeys)) for dataId in dataIdList if
785  dataId is not None]
786  for i, v in enumerate(sorted(set(visits))):
787  header.add("CALIB_INPUT_%d" % (i,), v)
788 
789  header.add("CALIB_ID", " ".join("%s=%s" % (key, value)
790  for key, value in outputId.items()))
791  checksum(calib, header)
792 
793  def interpolateNans(self, image):
794  """Interpolate over NANs in the combined image
795 
796  NANs can result from masked areas on the CCD. We don't want them getting
797  into our science images, so we replace them with the median of the image.
798  """
799  if hasattr(image, "getMaskedImage"): # Deal with Exposure vs Image
800  self.interpolateNans(image.getMaskedImage().getVariance())
801  image = image.getMaskedImage().getImage()
802  if hasattr(image, "getImage"): # Deal with DecoratedImage or MaskedImage vs Image
803  image = image.getImage()
804  array = image.getArray()
805  bad = np.isnan(array)
806  array[bad] = np.median(array[np.logical_not(bad)])
807 
808  def write(self, butler, exposure, dataId):
809  """!Write the final combined calib
810 
811  Only the slave nodes execute this method
812 
813  @param butler Data butler
814  @param exposure CCD exposure to write
815  @param dataId Data identifier for output
816  """
817  self.log.info("Writing %s on %s" % (dataId, NODE))
818  butler.put(exposure, self.calibName, dataId)
819 
820  def makeCameraImage(self, camera, dataId, calibs):
821  """!Create and write an image of the entire camera
822 
823  This is useful for judging the quality or getting an overview of
824  the features of the calib.
825 
826  @param camera Camera object
827  @param dataId Data identifier for output
828  @param calibs Dict mapping CCD detector ID to calib image
829  """
830  return makeCameraImage(camera, calibs, self.config.binning)
831 
832  def checkCcdIdLists(self, ccdIdLists):
833  """Check that the list of CCD dataIds is consistent
834 
835  @param ccdIdLists Dict of data identifier lists for each CCD name
836  @return Number of exposures, number of CCDs
837  """
838  visitIdLists = collections.defaultdict(list)
839  for ccdName in ccdIdLists:
840  for dataId in ccdIdLists[ccdName]:
841  visitName = dictToTuple(dataId, self.config.visitKeys)
842  visitIdLists[visitName].append(dataId)
843 
844  numExps = set(len(expList) for expList in ccdIdLists.values())
845  numCcds = set(len(ccdList) for ccdList in visitIdLists.values())
846 
847  if len(numExps) != 1 or len(numCcds) != 1:
848  # Presumably a visit somewhere doesn't have the full complement available.
849  # Dump the information so the user can figure it out.
850  self.log.warn("Number of visits for each CCD: %s",
851  {ccdName: len(ccdIdLists[ccdName]) for ccdName in ccdIdLists})
852  self.log.warn("Number of CCDs for each visit: %s",
853  {vv: len(visitIdLists[vv]) for vv in visitIdLists})
854  raise RuntimeError("Inconsistent number of exposures/CCDs")
855 
856  return numExps.pop(), numCcds.pop()
857 
858 
860  """Configuration for bias construction.
861 
862  No changes required compared to the base class, but
863  subclassed for distinction.
864  """
865  pass
866 
867 
868 class BiasTask(CalibTask):
869  """Bias construction"""
870  ConfigClass = BiasConfig
871  _DefaultName = "bias"
872  calibName = "bias"
873  filterName = "NONE" # Sets this filter name in the output
874  exposureTime = 0.0 # sets this exposureTime in the output
875 
876  @classmethod
877  def applyOverrides(cls, config):
878  """Overrides to apply for bias construction"""
879  config.isr.doBias = False
880  config.isr.doDark = False
881  config.isr.doFlat = False
882  config.isr.doFringe = False
883 
884 
886  """Configuration for dark construction"""
887  doRepair = Field(dtype=bool, default=True, doc="Repair artifacts?")
888  psfFwhm = Field(dtype=float, default=3.0, doc="Repair PSF FWHM (pixels)")
889  psfSize = Field(dtype=int, default=21, doc="Repair PSF size (pixels)")
890  crGrow = Field(dtype=int, default=2, doc="Grow radius for CR (pixels)")
891  repair = ConfigurableField(
892  target=RepairTask, doc="Task to repair artifacts")
893 
894  def setDefaults(self):
895  CalibConfig.setDefaults(self)
896  self.combination.mask.append("CR")
897 
898 
900  """Dark construction
901 
902  The only major difference from the base class is a cosmic-ray
903  identification stage, and dividing each image by the dark time
904  to generate images of the dark rate.
905  """
906  ConfigClass = DarkConfig
907  _DefaultName = "dark"
908  calibName = "dark"
909  filterName = "NONE" # Sets this filter name in the output
910 
911  def __init__(self, *args, **kwargs):
912  CalibTask.__init__(self, *args, **kwargs)
913  self.makeSubtask("repair")
914 
915  @classmethod
916  def applyOverrides(cls, config):
917  """Overrides to apply for dark construction"""
918  config.isr.doDark = False
919  config.isr.doFlat = False
920  config.isr.doFringe = False
921 
922  def processSingle(self, sensorRef):
923  """Process a single CCD
924 
925  Besides the regular ISR, also masks cosmic-rays and divides each
926  processed image by the dark time to generate images of the dark rate.
927  The dark time is provided by the 'getDarkTime' method.
928  """
929  exposure = CalibTask.processSingle(self, sensorRef)
930 
931  if self.config.doRepair:
932  psf = measAlg.DoubleGaussianPsf(self.config.psfSize, self.config.psfSize,
933  self.config.psfFwhm/(2*math.sqrt(2*math.log(2))))
934  exposure.setPsf(psf)
935  self.repair.run(exposure, keepCRs=False)
936  if self.config.crGrow > 0:
937  mask = exposure.getMaskedImage().getMask().clone()
938  mask &= mask.getPlaneBitMask("CR")
939  fpSet = afwDet.FootprintSet(
940  mask, afwDet.Threshold(0.5))
941  fpSet = afwDet.FootprintSet(fpSet, self.config.crGrow, True)
942  fpSet.setMask(exposure.getMaskedImage().getMask(), "CR")
943 
944  mi = exposure.getMaskedImage()
945  mi /= self.getDarkTime(exposure)
946  return exposure
947 
948  def getDarkTime(self, exposure):
949  """Retrieve the dark time for an exposure"""
950  darkTime = exposure.getInfo().getVisitInfo().getDarkTime()
951  if not np.isfinite(darkTime):
952  raise RuntimeError("Non-finite darkTime")
953  return darkTime
954 
955 
957  """Configuration for flat construction"""
958  iterations = Field(dtype=int, default=10,
959  doc="Number of iterations for scale determination")
960  stats = ConfigurableField(target=CalibStatsTask,
961  doc="Background statistics configuration")
962 
963 
965  """Flat construction
966 
967  The principal change from the base class involves gathering the background
968  values from each image and using them to determine the scalings for the final
969  combination.
970  """
971  ConfigClass = FlatConfig
972  _DefaultName = "flat"
973  calibName = "flat"
974 
975  @classmethod
976  def applyOverrides(cls, config):
977  """Overrides for flat construction"""
978  config.isr.doFlat = False
979  config.isr.doFringe = False
980 
981  def __init__(self, *args, **kwargs):
982  CalibTask.__init__(self, *args, **kwargs)
983  self.makeSubtask("stats")
984 
985  def processResult(self, exposure):
986  return self.stats.run(exposure)
987 
988  def scale(self, ccdIdLists, data):
989  """Determine the scalings for the final combination
990 
991  We have a matrix B_ij = C_i E_j, where C_i is the relative scaling
992  of one CCD to all the others in an exposure, and E_j is the scaling
993  of the exposure. We convert everything to logarithms so we can work
994  with a linear system. We determine the C_i and E_j from B_ij by iteration,
995  under the additional constraint that the average CCD scale is unity.
996 
997  This algorithm comes from Eugene Magnier and Pan-STARRS.
998  """
999  assert len(ccdIdLists.values()) > 0, "No successful CCDs"
1000  lengths = set([len(expList) for expList in ccdIdLists.values()])
1001  assert len(
1002  lengths) == 1, "Number of successful exposures for each CCD differs"
1003  assert tuple(lengths)[0] > 0, "No successful exposures"
1004  # Format background measurements into a matrix
1005  indices = dict((name, i) for i, name in enumerate(ccdIdLists))
1006  bgMatrix = np.array([[0.0] * len(expList)
1007  for expList in ccdIdLists.values()])
1008  for name in ccdIdLists:
1009  i = indices[name]
1010  bgMatrix[i] = [
1011  d if d is not None else np.nan for d in data[name]]
1012 
1013  numpyPrint = np.get_printoptions()
1014  np.set_printoptions(threshold=np.inf)
1015  self.log.info("Input backgrounds: %s" % bgMatrix)
1016 
1017  # Flat-field scaling
1018  numCcds = len(ccdIdLists)
1019  numExps = bgMatrix.shape[1]
1020  # log(Background) for each exposure/component
1021  bgMatrix = np.log(bgMatrix)
1022  bgMatrix = np.ma.masked_array(bgMatrix, np.isnan(bgMatrix))
1023  # Initial guess at log(scale) for each component
1024  compScales = np.zeros(numCcds)
1025  expScales = np.array(
1026  [(bgMatrix[:, i0] - compScales).mean() for i0 in range(numExps)])
1027 
1028  for iterate in range(self.config.iterations):
1029  compScales = np.array(
1030  [(bgMatrix[i1, :] - expScales).mean() for i1 in range(numCcds)])
1031  expScales = np.array(
1032  [(bgMatrix[:, i2] - compScales).mean() for i2 in range(numExps)])
1033 
1034  avgScale = np.average(np.exp(compScales))
1035  compScales -= np.log(avgScale)
1036  self.log.debug("Iteration %d exposure scales: %s",
1037  iterate, np.exp(expScales))
1038  self.log.debug("Iteration %d component scales: %s",
1039  iterate, np.exp(compScales))
1040 
1041  expScales = np.array(
1042  [(bgMatrix[:, i3] - compScales).mean() for i3 in range(numExps)])
1043 
1044  if np.any(np.isnan(expScales)):
1045  raise RuntimeError("Bad exposure scales: %s --> %s" %
1046  (bgMatrix, expScales))
1047 
1048  expScales = np.exp(expScales)
1049  compScales = np.exp(compScales)
1050 
1051  self.log.info("Exposure scales: %s" % expScales)
1052  self.log.info("Component relative scaling: %s" % compScales)
1053  np.set_printoptions(**numpyPrint)
1054 
1055  return dict((ccdName, Struct(ccdScale=compScales[indices[ccdName]], expScales=expScales))
1056  for ccdName in ccdIdLists)
1057 
1058 
1060  """Configuration for fringe construction"""
1061  stats = ConfigurableField(target=CalibStatsTask,
1062  doc="Background statistics configuration")
1063  subtractBackground = ConfigurableField(target=measAlg.SubtractBackgroundTask,
1064  doc="Background configuration")
1065  detection = ConfigurableField(
1066  target=measAlg.SourceDetectionTask, doc="Detection configuration")
1067  detectSigma = Field(dtype=float, default=1.0,
1068  doc="Detection PSF gaussian sigma")
1069 
1070  def setDefaults(self):
1071  CalibConfig.setDefaults(self)
1072  self.detection.reEstimateBackground = False
1073 
1074 
1076  """Fringe construction task
1077 
1078  The principal change from the base class is that the images are
1079  background-subtracted and rescaled by the background.
1080 
1081  XXX This is probably not right for a straight-up combination, as we
1082  are currently doing, since the fringe amplitudes need not scale with
1083  the continuum.
1084 
1085  XXX Would like to have this do PCA and generate multiple images, but
1086  that will take a bit of work with the persistence code.
1087  """
1088  ConfigClass = FringeConfig
1089  _DefaultName = "fringe"
1090  calibName = "fringe"
1091 
1092  @classmethod
1093  def applyOverrides(cls, config):
1094  """Overrides for fringe construction"""
1095  config.isr.doFringe = False
1096 
1097  def __init__(self, *args, **kwargs):
1098  CalibTask.__init__(self, *args, **kwargs)
1099  self.makeSubtask("detection")
1100  self.makeSubtask("stats")
1101  self.makeSubtask("subtractBackground")
1102 
1103  def processSingle(self, sensorRef):
1104  """Subtract the background and normalise by the background level"""
1105  exposure = CalibTask.processSingle(self, sensorRef)
1106  bgLevel = self.stats.run(exposure)
1107  self.subtractBackground.run(exposure)
1108  mi = exposure.getMaskedImage()
1109  mi /= bgLevel
1110  footprintSets = self.detection.detectFootprints(
1111  exposure, sigma=self.config.detectSigma)
1112  mask = exposure.getMaskedImage().getMask()
1113  detected = 1 << mask.addMaskPlane("DETECTED")
1114  for fpSet in (footprintSets.positive, footprintSets.negative):
1115  if fpSet is not None:
1116  afwDet.setMaskFromFootprintList(
1117  mask, fpSet.getFootprints(), detected)
1118  return exposure
1119 
1120 
1122  """Configuration for sky frame construction"""
1123  detection = ConfigurableField(target=measAlg.SourceDetectionTask, doc="Detection configuration")
1124  detectSigma = Field(dtype=float, default=2.0, doc="Detection PSF gaussian sigma")
1125  subtractBackground = ConfigurableField(target=measAlg.SubtractBackgroundTask,
1126  doc="Regular-scale background configuration, for object detection")
1127  largeScaleBackground = ConfigField(dtype=FocalPlaneBackgroundConfig,
1128  doc="Large-scale background configuration")
1129  sky = ConfigurableField(target=SkyMeasurementTask, doc="Sky measurement")
1130  maskThresh = Field(dtype=float, default=3.0, doc="k-sigma threshold for masking pixels")
1131  mask = ListField(dtype=str, default=["BAD", "SAT", "DETECTED", "NO_DATA"],
1132  doc="Mask planes to consider as contaminated")
1133 
1134 
1136  """Task for sky frame construction
1137 
1138  The sky frame is a (relatively) small-scale background
1139  model, the response of the camera to the sky.
1140 
1141  To construct, we first remove a large-scale background (e.g., caused
1142  by moonlight) which may vary from image to image. Then we construct a
1143  model of the sky, which is essentially a binned version of the image
1144  (important configuration parameters: sky.background.[xy]BinSize).
1145  It is these models which are coadded to yield the sky frame.
1146  """
1147  ConfigClass = SkyConfig
1148  _DefaultName = "sky"
1149  calibName = "sky"
1150 
1151  def __init__(self, *args, **kwargs):
1152  CalibTask.__init__(self, *args, **kwargs)
1153  self.makeSubtask("detection")
1154  self.makeSubtask("subtractBackground")
1155  self.makeSubtask("sky")
1156 
1157  def scatterProcess(self, pool, ccdIdLists):
1158  """!Scatter the processing among the nodes
1159 
1160  Only the master node executes this method, assigning work to the
1161  slaves.
1162 
1163  We measure and subtract off a large-scale background model across
1164  all CCDs, which requires a scatter/gather. Then we process the
1165  individual CCDs, subtracting the large-scale background model and
1166  the residual background model measured. These residuals will be
1167  combined for the sky frame.
1168 
1169  @param pool Process pool
1170  @param ccdIdLists Dict of data identifier lists for each CCD name
1171  @return Dict of lists of returned data for each CCD name
1172  """
1173  self.log.info("Scatter processing")
1174 
1175  numExps = set(len(expList) for expList in ccdIdLists.values()).pop()
1176 
1177  # First subtract off general gradients to make all the exposures look similar.
1178  # We want to preserve the common small-scale structure, which we will coadd.
1179  bgModelList = mapToMatrix(pool, self.measureBackground, ccdIdLists)
1180 
1181  backgrounds = {}
1182  scales = {}
1183  for exp in range(numExps):
1184  bgModels = [bgModelList[ccdName][exp] for ccdName in ccdIdLists]
1185  visit = set(tuple(ccdIdLists[ccdName][exp][key] for key in sorted(self.config.visitKeys)) for
1186  ccdName in ccdIdLists)
1187  assert len(visit) == 1
1188  visit = visit.pop()
1189  bgModel = bgModels[0]
1190  for bg in bgModels[1:]:
1191  bgModel.merge(bg)
1192  self.log.info("Background model min/max for visit %s: %f %f", visit,
1193  np.min(bgModel.getStatsImage().getArray()),
1194  np.max(bgModel.getStatsImage().getArray()))
1195  backgrounds[visit] = bgModel
1196  scales[visit] = np.median(bgModel.getStatsImage().getArray())
1197 
1198  return mapToMatrix(pool, self.process, ccdIdLists, backgrounds=backgrounds, scales=scales)
1199 
1200  def measureBackground(self, cache, dataId):
1201  """!Measure background model for CCD
1202 
1203  This method is executed by the slaves.
1204 
1205  The background models for all CCDs in an exposure will be
1206  combined to form a full focal-plane background model.
1207 
1208  @param cache Process pool cache
1209  @param dataId Data identifier
1210  @return Bcakground model
1211  """
1212  dataRef = getDataRef(cache.butler, dataId)
1213  exposure = self.processSingleBackground(dataRef)
1214 
1215  # NAOJ prototype smoothed and then combined the entire image, but it shouldn't be any different
1216  # to bin and combine the binned images except that there's fewer pixels to worry about.
1217  config = self.config.largeScaleBackground
1218  camera = dataRef.get("camera")
1219  bgModel = FocalPlaneBackground.fromCamera(config, camera)
1220  bgModel.addCcd(exposure)
1221  return bgModel
1222 
1223  def processSingleBackground(self, dataRef):
1224  """!Process a single CCD for the background
1225 
1226  This method is executed by the slaves.
1227 
1228  Because we're interested in the background, we detect and mask astrophysical
1229  sources, and pixels above the noise level.
1230 
1231  @param dataRef Data reference for CCD.
1232  @return processed exposure
1233  """
1234  if not self.config.clobber and dataRef.datasetExists("postISRCCD"):
1235  return dataRef.get("postISRCCD")
1236  exposure = CalibTask.processSingle(self, dataRef)
1237 
1238  # Detect sources. Requires us to remove the background; we'll restore it later.
1239  bgTemp = self.subtractBackground.run(exposure).background
1240  footprints = self.detection.detectFootprints(exposure, sigma=self.config.detectSigma)
1241  image = exposure.getMaskedImage()
1242  if footprints.background is not None:
1243  image += footprints.background.getImage()
1244 
1245  # Mask high pixels
1246  variance = image.getVariance()
1247  noise = np.sqrt(np.median(variance.getArray()))
1248  isHigh = image.getImage().getArray() > self.config.maskThresh*noise
1249  image.getMask().getArray()[isHigh] |= image.getMask().getPlaneBitMask("DETECTED")
1250 
1251  # Restore the background: it's what we want!
1252  image += bgTemp.getImage()
1253 
1254  # Set detected/bad pixels to background to ensure they don't corrupt the background
1255  maskVal = image.getMask().getPlaneBitMask(self.config.mask)
1256  isBad = image.getMask().getArray() & maskVal > 0
1257  bgLevel = np.median(image.getImage().getArray()[~isBad])
1258  image.getImage().getArray()[isBad] = bgLevel
1259  dataRef.put(exposure, "postISRCCD")
1260  return exposure
1261 
1262  def processSingle(self, dataRef, backgrounds, scales):
1263  """Process a single CCD, specified by a data reference
1264 
1265  We subtract the appropriate focal plane background model,
1266  divide by the appropriate scale and measure the background.
1267 
1268  Only slave nodes execute this method.
1269 
1270  @param dataRef Data reference for single CCD
1271  @param backgrounds Background model for each visit
1272  @param scales Scales for each visit
1273  @return Processed exposure
1274  """
1275  visit = tuple(dataRef.dataId[key] for key in sorted(self.config.visitKeys))
1276  exposure = dataRef.get("postISRCCD", immediate=True)
1277  image = exposure.getMaskedImage()
1278  detector = exposure.getDetector()
1279  bbox = image.getBBox()
1280 
1281  bgModel = backgrounds[visit]
1282  bg = bgModel.toCcdBackground(detector, bbox)
1283  image -= bg.getImage()
1284  image /= scales[visit]
1285 
1286  bg = self.sky.measureBackground(exposure.getMaskedImage())
1287  dataRef.put(bg, "icExpBackground")
1288  return exposure
1289 
1290  def combine(self, cache, struct, outputId):
1291  """!Combine multiple background models of a particular CCD and write the output
1292 
1293  Only the slave nodes execute this method.
1294 
1295  @param cache Process pool cache
1296  @param struct Parameters for the combination, which has the following components:
1297  * ccdName Name tuple for CCD
1298  * ccdIdList List of data identifiers for combination
1299  @param outputId Data identifier for combined image (exposure part only)
1300  @return binned calib image
1301  """
1302  outputId = self.getFullyQualifiedOutputId(struct.ccdName, cache.butler, outputId)
1303  dataRefList = [getDataRef(cache.butler, dataId) if dataId is not None else None for
1304  dataId in struct.ccdIdList]
1305  self.log.info("Combining %s on %s" % (outputId, NODE))
1306  bgList = [dataRef.get("icExpBackground", immediate=True).clone() for dataRef in dataRefList]
1307 
1308  bgExp = self.sky.averageBackgrounds(bgList)
1309 
1310  self.recordCalibInputs(cache.butler, bgExp, struct.ccdIdList, outputId)
1311  cache.butler.put(bgExp, "sky", outputId)
1312  return afwMath.binImage(self.sky.exposureToBackground(bgExp).getImage(), self.config.binning)
def processWrite(self, dataRef, exposure, outputName="postISRCCD")
Write the processed CCD.
def scatterCombine(self, pool, outputId, ccdIdLists, scales)
Scatter the combination of exposures across multiple nodes.
def run(self, exposureOrImage)
Measure a particular statistic on an image (of some sort).
def checksum(obj, header=None, sumType="MD5")
Calculate a checksum of an object.
Definition: checksum.py:29
def getCcdIdListFromExposures(expRefList, level="sensor", ccdKeys=["ccd"])
Determine a list of CCDs from exposure references.
def getFullyQualifiedOutputId(self, ccdName, butler, outputId)
def __call__(self, parser, namespace, values, option_string)
def dictToTuple(dict_, keys)
Return a tuple of specific values from a dict.
def getDataRef(butler, dataId, datasetType="raw")
Definition: utils.py:17
def updateMetadata(self, calibImage, exposureTime, darkTime=None, kwargs)
Update the metadata from the VisitInfo.
def process(self, cache, ccdId, outputName="postISRCCD", kwargs)
Process a CCD, specified by a data identifier.
def processSingle(self, dataRef, backgrounds, scales)
def makeCameraImage(self, camera, dataId, calibs)
Create and write an image of the entire camera.
def combine(self, target, imageList, stats)
Combine multiple images.
def getMjd(self, butler, dataId, timescale=dafBase.DateTime.UTC)
def combine(self, cache, struct, outputId)
Combine multiple exposures of a particular CCD and write the output.
def runDataRef(self, expRefList, butler, calibId)
Construct a calib from a list of exposure references.
def write(self, butler, exposure, dataId)
Write the final combined calib.
def combine(self, cache, struct, outputId)
Combine multiple background models of a particular CCD and write the output.
def measureBackground(self, cache, dataId)
Measure background model for CCD.
def recordCalibInputs(self, butler, calib, dataIdList, outputId)
Record metadata including the inputs and creation details.
def scatterProcess(self, pool, ccdIdLists)
Scatter the processing among the nodes.
def run(self, sensorRefList, expScales=None, finalScale=None, inputName="postISRCCD")
Combine calib images for a single sensor.
def getOutputId(self, expRefList, calibId)
Generate the data identifier for the output calib.
def mapToMatrix(pool, func, ccdIdLists, args, kwargs)
def scale(self, ccdIdLists, data)
Determine scaling across CCDs and exposures.
def addMissingKeys(self, dataId, butler, missingKeys=None, calibName=None)
def getDimensions(self, sensorRefList, inputName="postISRCCD")
Base class for constructing calibs.
def processSingleBackground(self, dataRef)
Process a single CCD for the background.
def scatterProcess(self, pool, ccdIdLists)
Scatter the processing among the nodes.
def batchWallTime(cls, time, parsedCmd, numCores)