lsst.pipe.drivers  6.0b0-hsc-16-geaac007+3
constructCalibs.py
Go to the documentation of this file.
1 from __future__ import absolute_import, division, print_function
2 
3 import sys
4 import math
5 import time
6 import argparse
7 import traceback
8 import collections
9 
10 import numpy as np
11 from builtins import zip
12 from builtins import range
13 
14 from lsst.pex.config import Config, ConfigurableField, Field, ListField, ConfigField
15 from lsst.pipe.base import Task, Struct, TaskRunner, ArgumentParser
16 import lsst.daf.base as dafBase
17 import lsst.afw.math as afwMath
18 import lsst.afw.geom as afwGeom
19 import lsst.afw.detection as afwDet
20 import lsst.afw.image as afwImage
21 from lsst.afw.image import VisitInfo
22 import lsst.meas.algorithms as measAlg
23 from lsst.pipe.tasks.repair import RepairTask
24 from lsst.ip.isr import IsrTask
25 from lsst.afw.cameraGeom.utils import makeImageFromCamera
26 
27 from lsst.ctrl.pool.parallel import BatchPoolTask
28 from lsst.ctrl.pool.pool import Pool, NODE
29 from lsst.pipe.drivers.background import SkyMeasurementTask, FocalPlaneBackground, FocalPlaneBackgroundConfig
30 
31 from .checksum import checksum
32 from .utils import getDataRef
33 
34 
35 class CalibStatsConfig(Config):
36  """Parameters controlling the measurement of background statistics"""
37  stat = Field(doc="Statistic to use to estimate background (from lsst.afw.math)", dtype=int,
38  default=int(afwMath.MEANCLIP))
39  clip = Field(doc="Clipping threshold for background",
40  dtype=float, default=3.0)
41  nIter = Field(doc="Clipping iterations for background",
42  dtype=int, default=3)
43  maxVisitsToCalcErrorFromInputVariance = Field(
44  doc="Maximum number of visits to estimate variance from input variance, not per-pixel spread",
45  dtype=int, default=2)
46  mask = ListField(doc="Mask planes to reject",
47  dtype=str, default=["DETECTED", "BAD", "NO_DATA",])
48 
49 
50 class CalibStatsTask(Task):
51  """Measure statistics on the background
52 
53  This can be useful for scaling the background, e.g., for flats and fringe frames.
54  """
55  ConfigClass = CalibStatsConfig
56 
57  def run(self, exposureOrImage):
58  """!Measure a particular statistic on an image (of some sort).
59 
60  @param exposureOrImage Exposure, MaskedImage or Image.
61  @return Value of desired statistic
62  """
63  stats = afwMath.StatisticsControl(self.config.clip, self.config.nIter,
64  afwImage.Mask.getPlaneBitMask(self.config.mask))
65  try:
66  image = exposureOrImage.getMaskedImage()
67  except:
68  try:
69  image = exposureOrImage.getImage()
70  except:
71  image = exposureOrImage
72 
73  return afwMath.makeStatistics(image, self.config.stat, stats).getValue()
74 
75 
76 class CalibCombineConfig(Config):
77  """Configuration for combining calib images"""
78  rows = Field(doc="Number of rows to read at a time",
79  dtype=int, default=512)
80  mask = ListField(doc="Mask planes to respect", dtype=str,
81  default=["SAT", "DETECTED", "INTRP"])
82  combine = Field(doc="Statistic to use for combination (from lsst.afw.math)", dtype=int,
83  default=int(afwMath.MEANCLIP))
84  clip = Field(doc="Clipping threshold for combination",
85  dtype=float, default=3.0)
86  nIter = Field(doc="Clipping iterations for combination",
87  dtype=int, default=3)
88  stats = ConfigurableField(target=CalibStatsTask,
89  doc="Background statistics configuration")
90 
91 
92 class CalibCombineTask(Task):
93  """Task to combine calib images"""
94  ConfigClass = CalibCombineConfig
95 
96  def __init__(self, *args, **kwargs):
97  Task.__init__(self, *args, **kwargs)
98  self.makeSubtask("stats")
99 
100  def run(self, sensorRefList, expScales=None, finalScale=None, inputName="postISRCCD"):
101  """!Combine calib images for a single sensor
102 
103  @param sensorRefList List of data references to combine (for a single sensor)
104  @param expScales List of scales to apply for each exposure
105  @param finalScale Desired scale for final combined image
106  @param inputName Data set name for inputs
107  @return combined image
108  """
109  width, height = self.getDimensions(sensorRefList)
110  stats = afwMath.StatisticsControl(self.config.clip, self.config.nIter,
111  afwImage.Mask.getPlaneBitMask(self.config.mask))
112  numImages = len(sensorRefList)
113  if numImages < self.config.stats.maxVisitsToCalcErrorFromInputVariance:
114  stats.setCalcErrorFromInputVariance(True)
115 
116  # Combine images
117  combined = afwImage.MaskedImageF(width, height)
118  imageList = [None]*numImages
119  for start in range(0, height, self.config.rows):
120  rows = min(self.config.rows, height - start)
121  box = afwGeom.Box2I(afwGeom.Point2I(0, start),
122  afwGeom.Extent2I(width, rows))
123  subCombined = combined.Factory(combined, box)
124 
125  for i, sensorRef in enumerate(sensorRefList):
126  if sensorRef is None:
127  imageList[i] = None
128  continue
129  exposure = sensorRef.get(inputName + "_sub", bbox=box)
130  if expScales is not None:
131  self.applyScale(exposure, expScales[i])
132  imageList[i] = exposure.getMaskedImage()
133 
134  self.combine(subCombined, imageList, stats)
135 
136  if finalScale is not None:
137  background = self.stats.run(combined)
138  self.log.info("%s: Measured background of stack is %f; adjusting to %f" %
139  (NODE, background, finalScale))
140  combined *= finalScale / background
141 
142  return combined
143 
144  def getDimensions(self, sensorRefList, inputName="postISRCCD"):
145  """Get dimensions of the inputs"""
146  dimList = []
147  for sensorRef in sensorRefList:
148  if sensorRef is None:
149  continue
150  md = sensorRef.get(inputName + "_md")
151  dimList.append(afwImage.bboxFromMetadata(md).getDimensions())
152  return getSize(dimList)
153 
154  def applyScale(self, exposure, scale=None):
155  """Apply scale to input exposure
156 
157  This implementation applies a flux scaling: the input exposure is
158  divided by the provided scale.
159  """
160  if scale is not None:
161  mi = exposure.getMaskedImage()
162  mi /= scale
163 
164  def combine(self, target, imageList, stats):
165  """!Combine multiple images
166 
167  @param target Target image to receive the combined pixels
168  @param imageList List of input images
169  @param stats Statistics control
170  """
171  images = [img for img in imageList if img is not None]
172  afwMath.statisticsStack(target, images, afwMath.Property(self.config.combine), stats)
173 
174 
175 def getSize(dimList):
176  """Determine a consistent size, given a list of image sizes"""
177  dim = set((w, h) for w, h in dimList)
178  dim.discard(None)
179  if len(dim) != 1:
180  raise RuntimeError("Inconsistent dimensions: %s" % dim)
181  return dim.pop()
182 
183 
184 def dictToTuple(dict_, keys):
185  """!Return a tuple of specific values from a dict
186 
187  This provides a hashable representation of the dict from certain keywords.
188  This can be useful for creating e.g., a tuple of the values in the DataId
189  that identify the CCD.
190 
191  @param dict_ dict to parse
192  @param keys keys to extract (order is important)
193  @return tuple of values
194  """
195  return tuple(dict_[k] for k in keys)
196 
197 
198 def getCcdIdListFromExposures(expRefList, level="sensor", ccdKeys=["ccd"]):
199  """!Determine a list of CCDs from exposure references
200 
201  This essentially inverts the exposure-level references (which
202  provides a list of CCDs for each exposure), by providing
203  a dataId list for each CCD. Consider an input list of exposures
204  [e1, e2, e3], and each exposure has CCDs c1 and c2. Then this
205  function returns:
206 
207  {(c1,): [e1c1, e2c1, e3c1], (c2,): [e1c2, e2c2, e3c2]}
208 
209  This is a dict whose keys are tuples of the identifying values of a
210  CCD (usually just the CCD number) and the values are lists of dataIds
211  for that CCD in each exposure. A missing dataId is given the value
212  None.
213 
214  @param expRefList List of data references for exposures
215  @param level Level for the butler to generate CCDs
216  @param ccdKeys DataId keywords that identify a CCD
217  @return dict of data identifier lists for each CCD;
218  keys are values of ccdKeys in order
219  """
220  expIdList = [[ccdRef.dataId for ccdRef in expRef.subItems(
221  level)] for expRef in expRefList]
222 
223  # Determine what additional keys make a CCD from an exposure
224  if len(ccdKeys) != len(set(ccdKeys)):
225  raise RuntimeError("Duplicate keys found in ccdKeys: %s" % ccdKeys)
226  ccdNames = set() # Set of tuples which are values for each of the CCDs in an exposure
227  for ccdIdList in expIdList:
228  for ccdId in ccdIdList:
229  name = dictToTuple(ccdId, ccdKeys)
230  ccdNames.add(name)
231 
232  # Turn the list of CCDs for each exposure into a list of exposures for
233  # each CCD
234  ccdLists = {}
235  for n, ccdIdList in enumerate(expIdList):
236  for ccdId in ccdIdList:
237  name = dictToTuple(ccdId, ccdKeys)
238  if name not in ccdLists:
239  ccdLists[name] = []
240  ccdLists[name].append(ccdId)
241 
242  for ccd in ccdLists:
243  # Sort the list by the dataId values (ordered by key)
244  ccdLists[ccd] = sorted(ccdLists[ccd], key=lambda dd: dictToTuple(dd, sorted(dd.keys())))
245 
246  return ccdLists
247 
248 
249 def mapToMatrix(pool, func, ccdIdLists, *args, **kwargs):
250  """Generate a matrix of results using pool.map
251 
252  The function should have the call signature:
253  func(cache, dataId, *args, **kwargs)
254 
255  We return a dict mapping 'ccd name' to a list of values for
256  each exposure.
257 
258  @param pool Process pool
259  @param func Function to call for each dataId
260  @param ccdIdLists Dict of data identifier lists for each CCD name
261  @return matrix of results
262  """
263  dataIdList = sum(ccdIdLists.values(), [])
264  resultList = pool.map(func, dataIdList, *args, **kwargs)
265  # Piece everything back together
266  data = dict((ccdName, [None] * len(expList)) for ccdName, expList in ccdIdLists.items())
267  indices = dict(sum([[(tuple(dataId.values()) if dataId is not None else None, (ccdName, expNum))
268  for expNum, dataId in enumerate(expList)]
269  for ccdName, expList in ccdIdLists.items()], []))
270  for dataId, result in zip(dataIdList, resultList):
271  if dataId is None:
272  continue
273  ccdName, expNum = indices[tuple(dataId.values())]
274  data[ccdName][expNum] = result
275  return data
276 
277 
278 class CalibIdAction(argparse.Action):
279  """Split name=value pairs and put the result in a dict"""
280 
281  def __call__(self, parser, namespace, values, option_string):
282  output = getattr(namespace, self.dest, {})
283  for nameValue in values:
284  name, sep, valueStr = nameValue.partition("=")
285  if not valueStr:
286  parser.error("%s value %s must be in form name=value" %
287  (option_string, nameValue))
288  output[name] = valueStr
289  setattr(namespace, self.dest, output)
290 
291 
292 class CalibArgumentParser(ArgumentParser):
293  """ArgumentParser for calibration construction"""
294 
295  def __init__(self, calibName, *args, **kwargs):
296  """Add a --calibId argument to the standard pipe_base argument parser"""
297  ArgumentParser.__init__(self, *args, **kwargs)
298  self.calibName = calibName
299  self.add_id_argument("--id", datasetType="raw",
300  help="input identifiers, e.g., --id visit=123 ccd=4")
301  self.add_argument("--calibId", nargs="*", action=CalibIdAction, default={},
302  help="identifiers for calib, e.g., --calibId version=1",
303  metavar="KEY=VALUE1[^VALUE2[^VALUE3...]")
304 
305  def parse_args(self, *args, **kwargs):
306  """Parse arguments
307 
308  Checks that the "--calibId" provided works.
309  """
310  namespace = ArgumentParser.parse_args(self, *args, **kwargs)
311 
312  keys = namespace.butler.getKeys(self.calibName)
313  parsed = {}
314  for name, value in namespace.calibId.items():
315  if name not in keys:
316  self.error(
317  "%s is not a relevant calib identifier key (%s)" % (name, keys))
318  parsed[name] = keys[name](value)
319  namespace.calibId = parsed
320 
321  return namespace
322 
323 
324 class CalibConfig(Config):
325  """Configuration for constructing calibs"""
326  clobber = Field(dtype=bool, default=True,
327  doc="Clobber existing processed images?")
328  isr = ConfigurableField(target=IsrTask, doc="ISR configuration")
329  dateObs = Field(dtype=str, default="dateObs",
330  doc="Key for observation date in exposure registry")
331  dateCalib = Field(dtype=str, default="calibDate",
332  doc="Key for calib date in calib registry")
333  filter = Field(dtype=str, default="filter",
334  doc="Key for filter name in exposure/calib registries")
335  combination = ConfigurableField(
336  target=CalibCombineTask, doc="Calib combination configuration")
337  ccdKeys = ListField(dtype=str, default=["ccd"],
338  doc="DataId keywords specifying a CCD")
339  visitKeys = ListField(dtype=str, default=["visit"],
340  doc="DataId keywords specifying a visit")
341  calibKeys = ListField(dtype=str, default=[],
342  doc="DataId keywords specifying a calibration")
343  doCameraImage = Field(dtype=bool, default=True, doc="Create camera overview image?")
344  binning = Field(dtype=int, default=64, doc="Binning to apply for camera image")
345 
346  def setDefaults(self):
347  self.isr.doWrite = False
348 
349 
350 class CalibTaskRunner(TaskRunner):
351  """Get parsed values into the CalibTask.run"""
352  @staticmethod
353  def getTargetList(parsedCmd, **kwargs):
354  return [dict(expRefList=parsedCmd.id.refList, butler=parsedCmd.butler, calibId=parsedCmd.calibId)]
355 
356  def __call__(self, args):
357  """Call the Task with the kwargs from getTargetList"""
358  task = self.TaskClass(config=self.config, log=self.log)
359  exitStatus = 0 # exit status for the shell
360  if self.doRaise:
361  result = task.run(**args)
362  else:
363  try:
364  result = task.run(**args)
365  except Exception as e:
366  exitStatus = 1 # n.b. The shell exit value is the number of dataRefs returning
367  # non-zero, so the actual value used here is lost
368  task.log.fatal("Failed: %s" % e)
369  traceback.print_exc(file=sys.stderr)
370 
371  if self.doReturnResults:
372  return Struct(
373  exitStatus=exitStatus,
374  args=args,
375  metadata=task.metadata,
376  result=result,
377  )
378  else:
379  return Struct(
380  exitStatus=exitStatus,
381  )
382 
384  """!Base class for constructing calibs.
385 
386  This should be subclassed for each of the required calib types.
387  The subclass should be sure to define the following class variables:
388  * _DefaultName: default name of the task, used by CmdLineTask
389  * calibName: name of the calibration data set in the butler
390  The subclass may optionally set:
391  * filterName: filter name to give the resultant calib
392  """
393  ConfigClass = CalibConfig
394  RunnerClass = CalibTaskRunner
395  filterName = None
396  calibName = None
397  exposureTime = 1.0 # sets this exposureTime in the output
398 
399  def __init__(self, *args, **kwargs):
400  """Constructor"""
401  BatchPoolTask.__init__(self, *args, **kwargs)
402  self.makeSubtask("isr")
403  self.makeSubtask("combination")
404 
405  @classmethod
406  def batchWallTime(cls, time, parsedCmd, numCores):
407  numCcds = len(parsedCmd.butler.get("camera"))
408  numExps = len(cls.RunnerClass.getTargetList(
409  parsedCmd)[0]['expRefList'])
410  numCycles = int(numCcds/float(numCores) + 0.5)
411  return time*numExps*numCycles
412 
413  @classmethod
414  def _makeArgumentParser(cls, *args, **kwargs):
415  kwargs.pop("doBatch", False)
416  return CalibArgumentParser(calibName=cls.calibName, name=cls._DefaultName, *args, **kwargs)
417 
418  def run(self, expRefList, butler, calibId):
419  """!Construct a calib from a list of exposure references
420 
421  This is the entry point, called by the TaskRunner.__call__
422 
423  Only the master node executes this method.
424 
425  @param expRefList List of data references at the exposure level
426  @param butler Data butler
427  @param calibId Identifier dict for calib
428  """
429  for expRef in expRefList:
430  self.addMissingKeys(expRef.dataId, butler, self.config.ccdKeys, 'raw')
431 
432  outputId = self.getOutputId(expRefList, calibId)
433  ccdIdLists = getCcdIdListFromExposures(
434  expRefList, level="sensor", ccdKeys=self.config.ccdKeys)
435  self.checkCcdIdLists(ccdIdLists)
436 
437  # Ensure we can generate filenames for each output
438  outputIdItemList = list(outputId.items())
439  for ccdName in ccdIdLists:
440  dataId = dict([(k, ccdName[i]) for i, k in enumerate(self.config.ccdKeys)])
441  dataId.update(outputIdItemList)
442  self.addMissingKeys(dataId, butler)
443  dataId.update(outputIdItemList)
444 
445  try:
446  butler.get(self.calibName + "_filename", dataId)
447  except Exception as e:
448  raise RuntimeError(
449  "Unable to determine output filename \"%s_filename\" from %s: %s" %
450  (self.calibName, dataId, e))
451 
452  processPool = Pool("process")
453  processPool.storeSet(butler=butler)
454 
455  # Scatter: process CCDs independently
456  data = self.scatterProcess(processPool, ccdIdLists)
457 
458  # Gather: determine scalings
459  scales = self.scale(ccdIdLists, data)
460 
461  combinePool = Pool("combine")
462  combinePool.storeSet(butler=butler)
463 
464  # Scatter: combine
465  calibs = self.scatterCombine(combinePool, outputId, ccdIdLists, scales)
466 
467  if self.config.doCameraImage:
468  camera = butler.get("camera")
469  # Convert indexing of calibs from "ccdName" to detector ID (as used by makeImageFromCamera)
470  calibs = {butler.get("postISRCCD_detector",
471  dict(zip(self.config.ccdKeys, ccdName))).getId(): calibs[ccdName]
472  for ccdName in ccdIdLists}
473 
474  try:
475  cameraImage = self.makeCameraImage(camera, outputId, calibs)
476  butler.put(cameraImage, self.calibName + "_camera", dataId)
477  except Exception as exc:
478  self.log.warn("Unable to create camera image: %s" % (exc,))
479 
480  return Struct(
481  outputId = outputId,
482  ccdIdLists = ccdIdLists,
483  scales = scales,
484  calibs = calibs,
485  processPool = processPool,
486  combinePool = combinePool,
487  )
488 
489  def getOutputId(self, expRefList, calibId):
490  """!Generate the data identifier for the output calib
491 
492  The mean date and the common filter are included, using keywords
493  from the configuration. The CCD-specific part is not included
494  in the data identifier.
495 
496  @param expRefList List of data references at exposure level
497  @param calibId Data identifier elements for the calib provided by the user
498  @return data identifier
499  """
500  midTime = 0
501  filterName = None
502  for expRef in expRefList:
503  butler = expRef.getButler()
504  dataId = expRef.dataId
505 
506  midTime += self.getMjd(butler, dataId)
507  thisFilter = self.getFilter(
508  butler, dataId) if self.filterName is None else self.filterName
509  if filterName is None:
510  filterName = thisFilter
511  elif filterName != thisFilter:
512  raise RuntimeError("Filter mismatch for %s: %s vs %s" % (
513  dataId, thisFilter, filterName))
514 
515  midTime /= len(expRefList)
516  date = str(dafBase.DateTime(
517  midTime, dafBase.DateTime.MJD).toPython().date())
518 
519  outputId = {self.config.filter: filterName,
520  self.config.dateCalib: date}
521  outputId.update(calibId)
522  return outputId
523 
524  def getMjd(self, butler, dataId, timescale=dafBase.DateTime.UTC):
525  """Determine the Modified Julian Date (MJD; in TAI) from a data identifier"""
526  if self.config.dateObs in dataId:
527  dateObs = dataId[self.config.dateObs]
528  else:
529  dateObs = butler.queryMetadata('raw', [self.config.dateObs], dataId)[0]
530  if "T" not in dateObs:
531  dateObs = dateObs + "T12:00:00.0Z"
532  elif not dateObs.endswith("Z"):
533  dateObs += "Z"
534 
535  return dafBase.DateTime(dateObs, timescale).get(dafBase.DateTime.MJD)
536 
537  def getFilter(self, butler, dataId):
538  """Determine the filter from a data identifier"""
539  filt = butler.queryMetadata('raw', [self.config.filter], dataId)[0]
540  return filt
541 
542  def addMissingKeys(self, dataId, butler, missingKeys=None, calibName=None):
543  if calibName is None:
544  calibName = self.calibName
545 
546  if missingKeys is None:
547  missingKeys = set(butler.getKeys(calibName).keys()) - set(dataId.keys())
548 
549  for k in missingKeys:
550  try:
551  v = butler.queryMetadata('raw', [k], dataId) # n.b. --id refers to 'raw'
552  except Exception as e:
553  continue
554 
555  if len(v) == 0: # failed to lookup value
556  continue
557 
558  if len(v) == 1:
559  dataId[k] = v[0]
560  else:
561  raise RuntimeError("No unique lookup for %s: %s" % (k, v))
562 
563  def updateMetadata(self, calibImage, exposureTime, darkTime=None, **kwargs):
564  """!Update the metadata from the VisitInfo
565 
566  \param calibImage The image whose metadata is to be set
567  \param exposureTime The exposure time for the image
568  \param darkTime The time since the last read (default: exposureTime)
569  """
570 
571  if darkTime is None:
572  darkTime = exposureTime # avoid warning messages when using calibration products
573 
574  visitInfo = afwImage.makeVisitInfo(exposureTime=exposureTime, darkTime=darkTime, **kwargs)
575  md = calibImage.getMetadata()
576 
577  afwImage.setVisitInfoMetadata(md, visitInfo)
578 
579  def scatterProcess(self, pool, ccdIdLists):
580  """!Scatter the processing among the nodes
581 
582  We scatter each CCD independently (exposures aren't grouped together),
583  to make full use of all available processors. This necessitates piecing
584  everything back together in the same format as ccdIdLists afterwards.
585 
586  Only the master node executes this method.
587 
588  @param pool Process pool
589  @param ccdIdLists Dict of data identifier lists for each CCD name
590  @return Dict of lists of returned data for each CCD name
591  """
592  self.log.info("Scatter processing")
593  return mapToMatrix(pool, self.process, ccdIdLists)
594 
595  def process(self, cache, ccdId, outputName="postISRCCD", **kwargs):
596  """!Process a CCD, specified by a data identifier
597 
598  After processing, optionally returns a result (produced by
599  the 'processResult' method) calculated from the processed
600  exposure. These results will be gathered by the master node,
601  and is a means for coordinated scaling of all CCDs for flats,
602  etc.
603 
604  Only slave nodes execute this method.
605 
606  @param cache Process pool cache
607  @param ccdId Data identifier for CCD
608  @param outputName Output dataset name for butler
609  @return result from 'processResult'
610  """
611  if ccdId is None:
612  self.log.warn("Null identifier received on %s" % NODE)
613  return None
614  sensorRef = getDataRef(cache.butler, ccdId)
615  if self.config.clobber or not sensorRef.datasetExists(outputName):
616  self.log.info("Processing %s on %s" % (ccdId, NODE))
617  try:
618  exposure = self.processSingle(sensorRef, **kwargs)
619  except Exception as e:
620  self.log.warn("Unable to process %s: %s" % (ccdId, e))
621  raise
622  return None
623  self.processWrite(sensorRef, exposure)
624  else:
625  self.log.info(
626  "Using previously persisted processed exposure for %s" % (sensorRef.dataId,))
627  exposure = sensorRef.get(outputName)
628  return self.processResult(exposure)
629 
630  def processSingle(self, dataRef):
631  """Process a single CCD, specified by a data reference
632 
633  Generally, this simply means doing ISR.
634 
635  Only slave nodes execute this method.
636  """
637  return self.isr.runDataRef(dataRef).exposure
638 
639  def processWrite(self, dataRef, exposure, outputName="postISRCCD"):
640  """!Write the processed CCD
641 
642  We need to write these out because we can't hold them all in
643  memory at once.
644 
645  Only slave nodes execute this method.
646 
647  @param dataRef Data reference
648  @param exposure CCD exposure to write
649  @param outputName Output dataset name for butler.
650  """
651  dataRef.put(exposure, outputName)
652 
653  def processResult(self, exposure):
654  """Extract processing results from a processed exposure
655 
656  This method generates what is gathered by the master node.
657  This can be a background measurement or similar for scaling
658  flat-fields. It must be picklable!
659 
660  Only slave nodes execute this method.
661  """
662  return None
663 
664  def scale(self, ccdIdLists, data):
665  """!Determine scaling across CCDs and exposures
666 
667  This is necessary mainly for flats, so as to determine a
668  consistent scaling across the entire focal plane. This
669  implementation is simply a placeholder.
670 
671  Only the master node executes this method.
672 
673  @param ccdIdLists Dict of data identifier lists for each CCD tuple
674  @param data Dict of lists of returned data for each CCD tuple
675  @return dict of Struct(ccdScale: scaling for CCD,
676  expScales: scaling for each exposure
677  ) for each CCD tuple
678  """
679  self.log.info("Scale on %s" % NODE)
680  return dict((name, Struct(ccdScale=None, expScales=[None] * len(ccdIdLists[name])))
681  for name in ccdIdLists)
682 
683  def scatterCombine(self, pool, outputId, ccdIdLists, scales):
684  """!Scatter the combination of exposures across multiple nodes
685 
686  In this case, we can only scatter across as many nodes as
687  there are CCDs.
688 
689  Only the master node executes this method.
690 
691  @param pool Process pool
692  @param outputId Output identifier (exposure part only)
693  @param ccdIdLists Dict of data identifier lists for each CCD name
694  @param scales Dict of structs with scales, for each CCD name
695  @param dict of binned images
696  """
697  self.log.info("Scatter combination")
698  data = [Struct(ccdName=ccdName, ccdIdList=ccdIdLists[ccdName], scales=scales[ccdName]) for
699  ccdName in ccdIdLists]
700  images = pool.map(self.combine, data, outputId)
701  return dict(zip(ccdIdLists.keys(), images))
702 
703  def getFullyQualifiedOutputId(self, ccdName, butler, outputId):
704  """Get fully-qualified output data identifier
705 
706  We may need to look up keys that aren't in the output dataId.
707 
708  @param ccdName Name tuple for CCD
709  @param butler Data butler
710  @param outputId Data identifier for combined image (exposure part only)
711  @return fully-qualified output dataId
712  """
713  fullOutputId = {k: ccdName[i] for i, k in enumerate(self.config.ccdKeys)}
714  fullOutputId.update(outputId)
715  self.addMissingKeys(fullOutputId, butler)
716  fullOutputId.update(outputId) # must be after the call to queryMetadata in 'addMissingKeys'
717  return fullOutputId
718 
719  def combine(self, cache, struct, outputId):
720  """!Combine multiple exposures of a particular CCD and write the output
721 
722  Only the slave nodes execute this method.
723 
724  @param cache Process pool cache
725  @param struct Parameters for the combination, which has the following components:
726  * ccdName Name tuple for CCD
727  * ccdIdList List of data identifiers for combination
728  * scales Scales to apply (expScales are scalings for each exposure,
729  ccdScale is final scale for combined image)
730  @param outputId Data identifier for combined image (exposure part only)
731  @return binned calib image
732  """
733  outputId = self.getFullyQualifiedOutputId(struct.ccdName, cache.butler, outputId)
734  dataRefList = [getDataRef(cache.butler, dataId) if dataId is not None else None for
735  dataId in struct.ccdIdList]
736  self.log.info("Combining %s on %s" % (outputId, NODE))
737  calib = self.combination.run(dataRefList, expScales=struct.scales.expScales,
738  finalScale=struct.scales.ccdScale)
739 
740  if not hasattr(calib, "getMetadata"):
741  if hasattr(calib, "getVariance"):
742  calib = afwImage.makeExposure(calib)
743  else:
744  calib = afwImage.DecoratedImageF(calib.getImage()) # n.b. hardwires "F" for the output type
745 
746  self.updateMetadata(calib, self.exposureTime)
747 
748  self.recordCalibInputs(cache.butler, calib,
749  struct.ccdIdList, outputId)
750 
751  self.interpolateNans(calib)
752 
753  self.write(cache.butler, calib, outputId)
754 
755  return afwMath.binImage(calib.getImage(), self.config.binning)
756 
757  def recordCalibInputs(self, butler, calib, dataIdList, outputId):
758  """!Record metadata including the inputs and creation details
759 
760  This metadata will go into the FITS header.
761 
762  @param butler Data butler
763  @param calib Combined calib exposure.
764  @param dataIdList List of data identifiers for calibration inputs
765  @param outputId Data identifier for output
766  """
767  header = calib.getMetadata()
768  header.add("OBSTYPE", self.calibName) # Used by ingestCalibs.py
769 
770  # date, time, host, and root
771  now = time.localtime()
772  header.add("CALIB_CREATION_DATE", time.strftime("%Y-%m-%d", now))
773  header.add("CALIB_CREATION_TIME", time.strftime("%X %Z", now))
774  # add date-obs as its absence upsets ExposureInfo; use the mean date that the calibs were taken
775  header.add("DATE-OBS", "%sT00:00:00.00" % outputId[self.config.dateCalib])
776 
777  # Inputs
778  visits = [str(dictToTuple(dataId, self.config.visitKeys)) for dataId in dataIdList if
779  dataId is not None]
780  for i, v in enumerate(sorted(set(visits))):
781  header.add("CALIB_INPUT_%d" % (i,), v)
782 
783  header.add("CALIB_ID", " ".join("%s=%s" % (key, value)
784  for key, value in outputId.items()))
785  checksum(calib, header)
786 
787  def interpolateNans(self, image):
788  """Interpolate over NANs in the combined image
789 
790  NANs can result from masked areas on the CCD. We don't want them getting
791  into our science images, so we replace them with the median of the image.
792  """
793  if hasattr(image, "getMaskedImage"): # Deal with Exposure vs Image
794  self.interpolateNans(image.getMaskedImage().getVariance())
795  image = image.getMaskedImage().getImage()
796  if hasattr(image, "getImage"): # Deal with DecoratedImage or MaskedImage vs Image
797  image = image.getImage()
798  array = image.getArray()
799  bad = np.isnan(array)
800  array[bad] = np.median(array[np.logical_not(bad)])
801 
802  def write(self, butler, exposure, dataId):
803  """!Write the final combined calib
804 
805  Only the slave nodes execute this method
806 
807  @param butler Data butler
808  @param exposure CCD exposure to write
809  @param dataId Data identifier for output
810  """
811  self.log.info("Writing %s on %s" % (dataId, NODE))
812  butler.put(exposure, self.calibName, dataId)
813 
814  def makeCameraImage(self, camera, dataId, calibs):
815  """!Create and write an image of the entire camera
816 
817  This is useful for judging the quality or getting an overview of
818  the features of the calib.
819 
820  @param camera Camera object
821  @param dataId Data identifier for output
822  @param calibs Dict mapping CCD detector ID to calib image
823  """
824  class ImageSource(object):
825  """Source of images for makeImageFromCamera"""
826  def __init__(self, images):
827  self.isTrimmed = True
828  self.images = images
829  self.background = np.nan
830 
831  def getCcdImage(self, detector, imageFactory, binSize):
832  detId = detector.getId()
833  if detId not in self.images:
834  dims = detector.getBBox().getDimensions()/binSize
835  image = imageFactory(*[int(xx) for xx in dims])
836  image.set(self.background)
837  else:
838  image = self.images[detId]
839  return image, detId
840 
841  image = makeImageFromCamera(camera, imageSource=ImageSource(calibs), imageFactory=afwImage.ImageF,
842  binSize=self.config.binning)
843  return image
844 
845  def checkCcdIdLists(self, ccdIdLists):
846  """Check that the list of CCD dataIds is consistent
847 
848  @param ccdIdLists Dict of data identifier lists for each CCD name
849  @return Number of exposures, number of CCDs
850  """
851  visitIdLists = collections.defaultdict(list)
852  for ccdName in ccdIdLists:
853  for dataId in ccdIdLists[ccdName]:
854  visitName = dictToTuple(dataId, self.config.visitKeys)
855  visitIdLists[visitName].append(dataId)
856 
857  numExps = set(len(expList) for expList in ccdIdLists.values())
858  numCcds = set(len(ccdList) for ccdList in visitIdLists.values())
859 
860  if len(numExps) != 1 or len(numCcds) != 1:
861  # Presumably a visit somewhere doesn't have the full complement available.
862  # Dump the information so the user can figure it out.
863  self.log.warn("Number of visits for each CCD: %s",
864  {ccdName: len(ccdIdLists[ccdName]) for ccdName in ccdIdLists})
865  self.log.warn("Number of CCDs for each visit: %s",
866  {vv: len(visitIdLists[vv]) for vv in visitIdLists})
867  raise RuntimeError("Inconsistent number of exposures/CCDs")
868 
869  return numExps.pop(), numCcds.pop()
870 
871 
873  """Configuration for bias construction.
874 
875  No changes required compared to the base class, but
876  subclassed for distinction.
877  """
878  pass
879 
880 
881 class BiasTask(CalibTask):
882  """Bias construction"""
883  ConfigClass = BiasConfig
884  _DefaultName = "bias"
885  calibName = "bias"
886  filterName = "NONE" # Sets this filter name in the output
887  exposureTime = 0.0 # sets this exposureTime in the output
888 
889  @classmethod
890  def applyOverrides(cls, config):
891  """Overrides to apply for bias construction"""
892  config.isr.doBias = False
893  config.isr.doDark = False
894  config.isr.doFlat = False
895  config.isr.doFringe = False
896 
897 
899  """Configuration for dark construction"""
900  doRepair = Field(dtype=bool, default=True, doc="Repair artifacts?")
901  psfFwhm = Field(dtype=float, default=3.0, doc="Repair PSF FWHM (pixels)")
902  psfSize = Field(dtype=int, default=21, doc="Repair PSF size (pixels)")
903  crGrow = Field(dtype=int, default=2, doc="Grow radius for CR (pixels)")
904  repair = ConfigurableField(
905  target=RepairTask, doc="Task to repair artifacts")
906 
907  def setDefaults(self):
908  CalibConfig.setDefaults(self)
909  self.combination.mask.append("CR")
910 
911 
913  """Dark construction
914 
915  The only major difference from the base class is a cosmic-ray
916  identification stage, and dividing each image by the dark time
917  to generate images of the dark rate.
918  """
919  ConfigClass = DarkConfig
920  _DefaultName = "dark"
921  calibName = "dark"
922  filterName = "NONE" # Sets this filter name in the output
923 
924  def __init__(self, *args, **kwargs):
925  CalibTask.__init__(self, *args, **kwargs)
926  self.makeSubtask("repair")
927 
928  @classmethod
929  def applyOverrides(cls, config):
930  """Overrides to apply for dark construction"""
931  config.isr.doDark = False
932  config.isr.doFlat = False
933  config.isr.doFringe = False
934 
935  def processSingle(self, sensorRef):
936  """Process a single CCD
937 
938  Besides the regular ISR, also masks cosmic-rays and divides each
939  processed image by the dark time to generate images of the dark rate.
940  The dark time is provided by the 'getDarkTime' method.
941  """
942  exposure = CalibTask.processSingle(self, sensorRef)
943 
944  if self.config.doRepair:
945  psf = measAlg.DoubleGaussianPsf(self.config.psfSize, self.config.psfSize,
946  self.config.psfFwhm/(2*math.sqrt(2*math.log(2))))
947  exposure.setPsf(psf)
948  self.repair.run(exposure, keepCRs=False)
949  if self.config.crGrow > 0:
950  mask = exposure.getMaskedImage().getMask().clone()
951  mask &= mask.getPlaneBitMask("CR")
952  fpSet = afwDet.FootprintSet(
953  mask, afwDet.Threshold(0.5))
954  fpSet = afwDet.FootprintSet(fpSet, self.config.crGrow, True)
955  fpSet.setMask(exposure.getMaskedImage().getMask(), "CR")
956 
957  mi = exposure.getMaskedImage()
958  mi /= self.getDarkTime(exposure)
959  return exposure
960 
961  def getDarkTime(self, exposure):
962  """Retrieve the dark time for an exposure"""
963  darkTime = exposure.getInfo().getVisitInfo().getDarkTime()
964  if not np.isfinite(darkTime):
965  raise RuntimeError("Non-finite darkTime")
966  return darkTime
967 
968 
970  """Configuration for flat construction"""
971  iterations = Field(dtype=int, default=10,
972  doc="Number of iterations for scale determination")
973  stats = ConfigurableField(target=CalibStatsTask,
974  doc="Background statistics configuration")
975 
976 
978  """Flat construction
979 
980  The principal change from the base class involves gathering the background
981  values from each image and using them to determine the scalings for the final
982  combination.
983  """
984  ConfigClass = FlatConfig
985  _DefaultName = "flat"
986  calibName = "flat"
987 
988  @classmethod
989  def applyOverrides(cls, config):
990  """Overrides for flat construction"""
991  config.isr.doFlat = False
992  config.isr.doFringe = False
993 
994  def __init__(self, *args, **kwargs):
995  CalibTask.__init__(self, *args, **kwargs)
996  self.makeSubtask("stats")
997 
998  def processResult(self, exposure):
999  return self.stats.run(exposure)
1000 
1001  def scale(self, ccdIdLists, data):
1002  """Determine the scalings for the final combination
1003 
1004  We have a matrix B_ij = C_i E_j, where C_i is the relative scaling
1005  of one CCD to all the others in an exposure, and E_j is the scaling
1006  of the exposure. We convert everything to logarithms so we can work
1007  with a linear system. We determine the C_i and E_j from B_ij by iteration,
1008  under the additional constraint that the average CCD scale is unity.
1009 
1010  This algorithm comes from Eugene Magnier and Pan-STARRS.
1011  """
1012  assert len(ccdIdLists.values()) > 0, "No successful CCDs"
1013  lengths = set([len(expList) for expList in ccdIdLists.values()])
1014  assert len(
1015  lengths) == 1, "Number of successful exposures for each CCD differs"
1016  assert tuple(lengths)[0] > 0, "No successful exposures"
1017  # Format background measurements into a matrix
1018  indices = dict((name, i) for i, name in enumerate(ccdIdLists))
1019  bgMatrix = np.array([[0.0] * len(expList)
1020  for expList in ccdIdLists.values()])
1021  for name in ccdIdLists:
1022  i = indices[name]
1023  bgMatrix[i] = [
1024  d if d is not None else np.nan for d in data[name]]
1025 
1026  numpyPrint = np.get_printoptions()
1027  np.set_printoptions(threshold=np.inf)
1028  self.log.info("Input backgrounds: %s" % bgMatrix)
1029 
1030  # Flat-field scaling
1031  numCcds = len(ccdIdLists)
1032  numExps = bgMatrix.shape[1]
1033  # log(Background) for each exposure/component
1034  bgMatrix = np.log(bgMatrix)
1035  bgMatrix = np.ma.masked_array(bgMatrix, np.isnan(bgMatrix))
1036  # Initial guess at log(scale) for each component
1037  compScales = np.zeros(numCcds)
1038  expScales = np.array(
1039  [(bgMatrix[:, i0] - compScales).mean() for i0 in range(numExps)])
1040 
1041  for iterate in range(self.config.iterations):
1042  compScales = np.array(
1043  [(bgMatrix[i1, :] - expScales).mean() for i1 in range(numCcds)])
1044  expScales = np.array(
1045  [(bgMatrix[:, i2] - compScales).mean() for i2 in range(numExps)])
1046 
1047  avgScale = np.average(np.exp(compScales))
1048  compScales -= np.log(avgScale)
1049  self.log.debug("Iteration %d exposure scales: %s",
1050  iterate, np.exp(expScales))
1051  self.log.debug("Iteration %d component scales: %s",
1052  iterate, np.exp(compScales))
1053 
1054  expScales = np.array(
1055  [(bgMatrix[:, i3] - compScales).mean() for i3 in range(numExps)])
1056 
1057  if np.any(np.isnan(expScales)):
1058  raise RuntimeError("Bad exposure scales: %s --> %s" %
1059  (bgMatrix, expScales))
1060 
1061  expScales = np.exp(expScales)
1062  compScales = np.exp(compScales)
1063 
1064  self.log.info("Exposure scales: %s" % expScales)
1065  self.log.info("Component relative scaling: %s" % compScales)
1066  np.set_printoptions(**numpyPrint)
1067 
1068  return dict((ccdName, Struct(ccdScale=compScales[indices[ccdName]], expScales=expScales))
1069  for ccdName in ccdIdLists)
1070 
1071 
1073  """Configuration for fringe construction"""
1074  stats = ConfigurableField(target=CalibStatsTask,
1075  doc="Background statistics configuration")
1076  subtractBackground = ConfigurableField(target=measAlg.SubtractBackgroundTask,
1077  doc="Background configuration")
1078  detection = ConfigurableField(
1079  target=measAlg.SourceDetectionTask, doc="Detection configuration")
1080  detectSigma = Field(dtype=float, default=1.0,
1081  doc="Detection PSF gaussian sigma")
1082 
1083 
1085  """Fringe construction task
1086 
1087  The principal change from the base class is that the images are
1088  background-subtracted and rescaled by the background.
1089 
1090  XXX This is probably not right for a straight-up combination, as we
1091  are currently doing, since the fringe amplitudes need not scale with
1092  the continuum.
1093 
1094  XXX Would like to have this do PCA and generate multiple images, but
1095  that will take a bit of work with the persistence code.
1096  """
1097  ConfigClass = FringeConfig
1098  _DefaultName = "fringe"
1099  calibName = "fringe"
1100 
1101  @classmethod
1102  def applyOverrides(cls, config):
1103  """Overrides for fringe construction"""
1104  config.isr.doFringe = False
1105 
1106  def __init__(self, *args, **kwargs):
1107  CalibTask.__init__(self, *args, **kwargs)
1108  self.makeSubtask("detection")
1109  self.makeSubtask("stats")
1110  self.makeSubtask("subtractBackground")
1111 
1112  def processSingle(self, sensorRef):
1113  """Subtract the background and normalise by the background level"""
1114  exposure = CalibTask.processSingle(self, sensorRef)
1115  bgLevel = self.stats.run(exposure)
1116  self.subtractBackground.run(exposure)
1117  mi = exposure.getMaskedImage()
1118  mi /= bgLevel
1119  footprintSets = self.detection.detectFootprints(
1120  exposure, sigma=self.config.detectSigma)
1121  mask = exposure.getMaskedImage().getMask()
1122  detected = 1 << mask.addMaskPlane("DETECTED")
1123  for fpSet in (footprintSets.positive, footprintSets.negative):
1124  if fpSet is not None:
1125  afwDet.setMaskFromFootprintList(
1126  mask, fpSet.getFootprints(), detected)
1127  return exposure
1128 
1129 
1131  """Configuration for sky frame construction"""
1132  detection = ConfigurableField(target=measAlg.SourceDetectionTask, doc="Detection configuration")
1133  detectSigma = Field(dtype=float, default=2.0, doc="Detection PSF gaussian sigma")
1134  subtractBackground = ConfigurableField(target=measAlg.SubtractBackgroundTask,
1135  doc="Regular-scale background configuration, for object detection")
1136  largeScaleBackground = ConfigField(dtype=FocalPlaneBackgroundConfig,
1137  doc="Large-scale background configuration")
1138  sky = ConfigurableField(target=SkyMeasurementTask, doc="Sky measurement")
1139  maskThresh = Field(dtype=float, default=3.0, doc="k-sigma threshold for masking pixels")
1140  mask = ListField(dtype=str, default=["BAD", "SAT", "DETECTED", "NO_DATA"],
1141  doc="Mask planes to consider as contaminated")
1142 
1143 
1145  """Task for sky frame construction
1146 
1147  The sky frame is a (relatively) small-scale background
1148  model, the response of the camera to the sky.
1149 
1150  To construct, we first remove a large-scale background (e.g., caused
1151  by moonlight) which may vary from image to image. Then we construct a
1152  model of the sky, which is essentially a binned version of the image
1153  (important configuration parameters: sky.background.[xy]BinSize).
1154  It is these models which are coadded to yield the sky frame.
1155  """
1156  ConfigClass = SkyConfig
1157  _DefaultName = "sky"
1158  calibName = "sky"
1159 
1160  def __init__(self, *args, **kwargs):
1161  CalibTask.__init__(self, *args, **kwargs)
1162  self.makeSubtask("detection")
1163  self.makeSubtask("subtractBackground")
1164  self.makeSubtask("sky")
1165 
1166  def scatterProcess(self, pool, ccdIdLists):
1167  """!Scatter the processing among the nodes
1168 
1169  Only the master node executes this method, assigning work to the
1170  slaves.
1171 
1172  We measure and subtract off a large-scale background model across
1173  all CCDs, which requires a scatter/gather. Then we process the
1174  individual CCDs, subtracting the large-scale background model and
1175  the residual background model measured. These residuals will be
1176  combined for the sky frame.
1177 
1178  @param pool Process pool
1179  @param ccdIdLists Dict of data identifier lists for each CCD name
1180  @return Dict of lists of returned data for each CCD name
1181  """
1182  self.log.info("Scatter processing")
1183 
1184  numExps = set(len(expList) for expList in ccdIdLists.values()).pop()
1185 
1186  # First subtract off general gradients to make all the exposures look similar.
1187  # We want to preserve the common small-scale structure, which we will coadd.
1188  bgModelList = mapToMatrix(pool, self.measureBackground, ccdIdLists)
1189 
1190  backgrounds = {}
1191  scales = {}
1192  for exp in range(numExps):
1193  bgModels = [bgModelList[ccdName][exp] for ccdName in ccdIdLists]
1194  visit = set(tuple(ccdIdLists[ccdName][exp][key] for key in sorted(self.config.visitKeys)) for
1195  ccdName in ccdIdLists)
1196  assert len(visit) == 1
1197  visit = visit.pop()
1198  bgModel = bgModels[0]
1199  for bg in bgModels[1:]:
1200  bgModel.merge(bg)
1201  self.log.info("Background model min/max for visit %s: %f %f", visit,
1202  np.min(bgModel.getStatsImage().getArray()),
1203  np.max(bgModel.getStatsImage().getArray()))
1204  backgrounds[visit] = bgModel
1205  scales[visit] = np.median(bgModel.getStatsImage().getArray())
1206 
1207  return mapToMatrix(pool, self.process, ccdIdLists, backgrounds=backgrounds, scales=scales)
1208 
1209  def measureBackground(self, cache, dataId):
1210  """!Measure background model for CCD
1211 
1212  This method is executed by the slaves.
1213 
1214  The background models for all CCDs in an exposure will be
1215  combined to form a full focal-plane background model.
1216 
1217  @param cache Process pool cache
1218  @param dataId Data identifier
1219  @return Bcakground model
1220  """
1221  dataRef = getDataRef(cache.butler, dataId)
1222  exposure = self.processSingleBackground(dataRef)
1223 
1224  # NAOJ prototype smoothed and then combined the entire image, but it shouldn't be any different
1225  # to bin and combine the binned images except that there's fewer pixels to worry about.
1226  config = self.config.largeScaleBackground
1227  camera = dataRef.get("camera")
1228  bgModel = FocalPlaneBackground.fromCamera(config, camera)
1229  bgModel.addCcd(exposure)
1230  return bgModel
1231 
1232  def processSingleBackground(self, dataRef):
1233  """!Process a single CCD for the background
1234 
1235  This method is executed by the slaves.
1236 
1237  Because we're interested in the background, we detect and mask astrophysical
1238  sources, and pixels above the noise level.
1239 
1240  @param dataRef Data reference for CCD.
1241  @return processed exposure
1242  """
1243  if not self.config.clobber and dataRef.datasetExists("postISRCCD"):
1244  return dataRef.get("postISRCCD")
1245  exposure = CalibTask.processSingle(self, dataRef)
1246 
1247  # Detect sources. Requires us to remove the background; we'll restore it later.
1248  bgTemp = self.subtractBackground.run(exposure).background
1249  footprints = self.detection.detectFootprints(exposure, sigma=self.config.detectSigma)
1250  image = exposure.getMaskedImage()
1251  if footprints.background is not None:
1252  image += footprints.background.getImageF()
1253 
1254  # Mask high pixels
1255  variance = image.getVariance()
1256  noise = np.sqrt(np.median(variance.getArray()))
1257  isHigh = image.getImage().getArray() > self.config.maskThresh*noise
1258  image.getMask().getArray()[isHigh] |= image.getMask().getPlaneBitMask("DETECTED")
1259 
1260  # Restore the background: it's what we want!
1261  image += bgTemp.getImage()
1262 
1263  # Set detected/bad pixels to background to ensure they don't corrupt the background
1264  maskVal = image.getMask().getPlaneBitMask(self.config.mask)
1265  isBad = image.getMask().getArray() & maskVal > 0
1266  bgLevel = np.median(image.getImage().getArray()[~isBad])
1267  image.getImage().getArray()[isBad] = bgLevel
1268  dataRef.put(exposure, "postISRCCD")
1269  return exposure
1270 
1271  def processSingle(self, dataRef, backgrounds, scales):
1272  """Process a single CCD, specified by a data reference
1273 
1274  We subtract the appropriate focal plane background model,
1275  divide by the appropriate scale and measure the background.
1276 
1277  Only slave nodes execute this method.
1278 
1279  @param dataRef Data reference for single CCD
1280  @param backgrounds Background model for each visit
1281  @param scales Scales for each visit
1282  @return Processed exposure
1283  """
1284  visit = tuple(dataRef.dataId[key] for key in sorted(self.config.visitKeys))
1285  exposure = dataRef.get("postISRCCD", immediate=True)
1286  image = exposure.getMaskedImage()
1287  detector = exposure.getDetector()
1288  bbox = image.getBBox()
1289 
1290  bgModel = backgrounds[visit]
1291  bg = bgModel.toCcdBackground(detector, bbox)
1292  image -= bg.getImage()
1293  image /= scales[visit]
1294 
1295  bg = self.sky.measureBackground(exposure.getMaskedImage())
1296  dataRef.put(bg, "icExpBackground")
1297  return exposure
1298 
1299  def combine(self, cache, struct, outputId):
1300  """!Combine multiple background models of a particular CCD and write the output
1301 
1302  Only the slave nodes execute this method.
1303 
1304  @param cache Process pool cache
1305  @param struct Parameters for the combination, which has the following components:
1306  * ccdName Name tuple for CCD
1307  * ccdIdList List of data identifiers for combination
1308  @param outputId Data identifier for combined image (exposure part only)
1309  @return binned calib image
1310  """
1311  outputId = self.getFullyQualifiedOutputId(struct.ccdName, cache.butler, outputId)
1312  dataRefList = [getDataRef(cache.butler, dataId) if dataId is not None else None for
1313  dataId in struct.ccdIdList]
1314  self.log.info("Combining %s on %s" % (outputId, NODE))
1315  bgList = [dataRef.get("icExpBackground", immediate=True).clone() for dataRef in dataRefList]
1316 
1317  bgExp = self.sky.averageBackgrounds(bgList)
1318 
1319  self.recordCalibInputs(cache.butler, bgExp, struct.ccdIdList, outputId)
1320  cache.butler.put(bgExp, "sky", outputId)
1321  return afwMath.binImage(self.sky.exposureToBackground(bgExp).getImage(), self.config.binning)
def processWrite(self, dataRef, exposure, outputName="postISRCCD")
Write the processed CCD.
def scatterCombine(self, pool, outputId, ccdIdLists, scales)
Scatter the combination of exposures across multiple nodes.
def run(self, exposureOrImage)
Measure a particular statistic on an image (of some sort).
def checksum(obj, header=None, sumType="MD5")
Calculate a checksum of an object.
Definition: checksum.py:29
def getCcdIdListFromExposures(expRefList, level="sensor", ccdKeys=["ccd"])
Determine a list of CCDs from exposure references.
def run(self, expRefList, butler, calibId)
Construct a calib from a list of exposure references.
def getFullyQualifiedOutputId(self, ccdName, butler, outputId)
def __call__(self, parser, namespace, values, option_string)
def dictToTuple(dict_, keys)
Return a tuple of specific values from a dict.
def getDataRef(butler, dataId, datasetType="raw")
Definition: utils.py:17
def updateMetadata(self, calibImage, exposureTime, darkTime=None, kwargs)
Update the metadata from the VisitInfo.
def process(self, cache, ccdId, outputName="postISRCCD", kwargs)
Process a CCD, specified by a data identifier.
def processSingle(self, dataRef, backgrounds, scales)
def makeCameraImage(self, camera, dataId, calibs)
Create and write an image of the entire camera.
def combine(self, target, imageList, stats)
Combine multiple images.
def getMjd(self, butler, dataId, timescale=dafBase.DateTime.UTC)
def combine(self, cache, struct, outputId)
Combine multiple exposures of a particular CCD and write the output.
def write(self, butler, exposure, dataId)
Write the final combined calib.
def combine(self, cache, struct, outputId)
Combine multiple background models of a particular CCD and write the output.
def measureBackground(self, cache, dataId)
Measure background model for CCD.
def recordCalibInputs(self, butler, calib, dataIdList, outputId)
Record metadata including the inputs and creation details.
def scatterProcess(self, pool, ccdIdLists)
Scatter the processing among the nodes.
def run(self, sensorRefList, expScales=None, finalScale=None, inputName="postISRCCD")
Combine calib images for a single sensor.
def getOutputId(self, expRefList, calibId)
Generate the data identifier for the output calib.
def mapToMatrix(pool, func, ccdIdLists, args, kwargs)
def scale(self, ccdIdLists, data)
Determine scaling across CCDs and exposures.
def addMissingKeys(self, dataId, butler, missingKeys=None, calibName=None)
def getDimensions(self, sensorRefList, inputName="postISRCCD")
Base class for constructing calibs.
def processSingleBackground(self, dataRef)
Process a single CCD for the background.
def scatterProcess(self, pool, ccdIdLists)
Scatter the processing among the nodes.
def batchWallTime(cls, time, parsedCmd, numCores)