lsst.pipe.drivers  6.0b0-hsc-15-g8ba6355+1
constructCalibs.py
Go to the documentation of this file.
1 from __future__ import absolute_import, division, print_function
2 
3 import sys
4 import math
5 import time
6 import argparse
7 import traceback
8 import collections
9 
10 import numpy as np
11 from builtins import zip
12 from builtins import range
13 
14 from lsst.pex.config import Config, ConfigurableField, Field, ListField, ConfigField
15 from lsst.pipe.base import Task, Struct, TaskRunner, ArgumentParser
16 import lsst.daf.base as dafBase
17 import lsst.afw.math as afwMath
18 import lsst.afw.geom as afwGeom
19 import lsst.afw.detection as afwDet
20 import lsst.afw.image as afwImage
21 from lsst.afw.image import VisitInfo
22 import lsst.meas.algorithms as measAlg
23 from lsst.pipe.tasks.repair import RepairTask
24 from lsst.ip.isr import IsrTask
25 from lsst.afw.cameraGeom.utils import makeImageFromCamera
26 
27 from lsst.ctrl.pool.parallel import BatchPoolTask
28 from lsst.ctrl.pool.pool import Pool, NODE
29 from lsst.pipe.drivers.background import SkyMeasurementTask, FocalPlaneBackground, FocalPlaneBackgroundConfig
30 
31 from .checksum import checksum
32 from .utils import getDataRef
33 
34 
35 class CalibStatsConfig(Config):
36  """Parameters controlling the measurement of background statistics"""
37  stat = Field(doc="Statistic to use to estimate background (from lsst.afw.math)", dtype=int,
38  default=int(afwMath.MEANCLIP))
39  clip = Field(doc="Clipping threshold for background",
40  dtype=float, default=3.0)
41  nIter = Field(doc="Clipping iterations for background",
42  dtype=int, default=3)
43  maxVisitsToCalcErrorFromInputVariance = Field(
44  doc="Maximum number of visits to estimate variance from input variance, not per-pixel spread",
45  dtype=int, default=2)
46  mask = ListField(doc="Mask planes to reject",
47  dtype=str, default=["DETECTED", "BAD", "NO_DATA",])
48 
49 
50 class CalibStatsTask(Task):
51  """Measure statistics on the background
52 
53  This can be useful for scaling the background, e.g., for flats and fringe frames.
54  """
55  ConfigClass = CalibStatsConfig
56 
57  def run(self, exposureOrImage):
58  """!Measure a particular statistic on an image (of some sort).
59 
60  @param exposureOrImage Exposure, MaskedImage or Image.
61  @return Value of desired statistic
62  """
63  stats = afwMath.StatisticsControl(self.config.clip, self.config.nIter,
64  afwImage.Mask.getPlaneBitMask(self.config.mask))
65  try:
66  image = exposureOrImage.getMaskedImage()
67  except:
68  try:
69  image = exposureOrImage.getImage()
70  except:
71  image = exposureOrImage
72 
73  return afwMath.makeStatistics(image, self.config.stat, stats).getValue()
74 
75 
76 class CalibCombineConfig(Config):
77  """Configuration for combining calib images"""
78  rows = Field(doc="Number of rows to read at a time",
79  dtype=int, default=512)
80  mask = ListField(doc="Mask planes to respect", dtype=str,
81  default=["SAT", "DETECTED", "INTRP"])
82  combine = Field(doc="Statistic to use for combination (from lsst.afw.math)", dtype=int,
83  default=int(afwMath.MEANCLIP))
84  clip = Field(doc="Clipping threshold for combination",
85  dtype=float, default=3.0)
86  nIter = Field(doc="Clipping iterations for combination",
87  dtype=int, default=3)
88  stats = ConfigurableField(target=CalibStatsTask,
89  doc="Background statistics configuration")
90 
91 
92 class CalibCombineTask(Task):
93  """Task to combine calib images"""
94  ConfigClass = CalibCombineConfig
95 
96  def __init__(self, *args, **kwargs):
97  Task.__init__(self, *args, **kwargs)
98  self.makeSubtask("stats")
99 
100  def run(self, sensorRefList, expScales=None, finalScale=None, inputName="postISRCCD"):
101  """!Combine calib images for a single sensor
102 
103  @param sensorRefList List of data references to combine (for a single sensor)
104  @param expScales List of scales to apply for each exposure
105  @param finalScale Desired scale for final combined image
106  @param inputName Data set name for inputs
107  @return combined image
108  """
109  width, height = self.getDimensions(sensorRefList)
110  stats = afwMath.StatisticsControl(self.config.clip, self.config.nIter,
111  afwImage.Mask.getPlaneBitMask(self.config.mask))
112  numImages = len(sensorRefList)
113  if numImages < self.config.stats.maxVisitsToCalcErrorFromInputVariance:
114  stats.setCalcErrorFromInputVariance(True)
115 
116  # Combine images
117  combined = afwImage.MaskedImageF(width, height)
118  imageList = [None]*numImages
119  for start in range(0, height, self.config.rows):
120  rows = min(self.config.rows, height - start)
121  box = afwGeom.Box2I(afwGeom.Point2I(0, start),
122  afwGeom.Extent2I(width, rows))
123  subCombined = combined.Factory(combined, box)
124 
125  for i, sensorRef in enumerate(sensorRefList):
126  if sensorRef is None:
127  imageList[i] = None
128  continue
129  exposure = sensorRef.get(inputName + "_sub", bbox=box)
130  if expScales is not None:
131  self.applyScale(exposure, expScales[i])
132  imageList[i] = exposure.getMaskedImage()
133 
134  self.combine(subCombined, imageList, stats)
135 
136  if finalScale is not None:
137  background = self.stats.run(combined)
138  self.log.info("%s: Measured background of stack is %f; adjusting to %f" %
139  (NODE, background, finalScale))
140  combined *= finalScale / background
141 
142  return combined
143 
144  def getDimensions(self, sensorRefList, inputName="postISRCCD"):
145  """Get dimensions of the inputs"""
146  dimList = []
147  for sensorRef in sensorRefList:
148  if sensorRef is None:
149  continue
150  md = sensorRef.get(inputName + "_md")
151  dimList.append(afwImage.bboxFromMetadata(md).getDimensions())
152  return getSize(dimList)
153 
154  def applyScale(self, exposure, scale=None):
155  """Apply scale to input exposure
156 
157  This implementation applies a flux scaling: the input exposure is
158  divided by the provided scale.
159  """
160  if scale is not None:
161  mi = exposure.getMaskedImage()
162  mi /= scale
163 
164  def combine(self, target, imageList, stats):
165  """!Combine multiple images
166 
167  @param target Target image to receive the combined pixels
168  @param imageList List of input images
169  @param stats Statistics control
170  """
171  images = [img for img in imageList if img is not None]
172  afwMath.statisticsStack(target, images, afwMath.Property(self.config.combine), stats)
173 
174 
175 def getSize(dimList):
176  """Determine a consistent size, given a list of image sizes"""
177  dim = set((w, h) for w, h in dimList)
178  dim.discard(None)
179  if len(dim) != 1:
180  raise RuntimeError("Inconsistent dimensions: %s" % dim)
181  return dim.pop()
182 
183 
184 def dictToTuple(dict_, keys):
185  """!Return a tuple of specific values from a dict
186 
187  This provides a hashable representation of the dict from certain keywords.
188  This can be useful for creating e.g., a tuple of the values in the DataId
189  that identify the CCD.
190 
191  @param dict_ dict to parse
192  @param keys keys to extract (order is important)
193  @return tuple of values
194  """
195  return tuple(dict_[k] for k in keys)
196 
197 
198 def getCcdIdListFromExposures(expRefList, level="sensor", ccdKeys=["ccd"]):
199  """!Determine a list of CCDs from exposure references
200 
201  This essentially inverts the exposure-level references (which
202  provides a list of CCDs for each exposure), by providing
203  a dataId list for each CCD. Consider an input list of exposures
204  [e1, e2, e3], and each exposure has CCDs c1 and c2. Then this
205  function returns:
206 
207  {(c1,): [e1c1, e2c1, e3c1], (c2,): [e1c2, e2c2, e3c2]}
208 
209  This is a dict whose keys are tuples of the identifying values of a
210  CCD (usually just the CCD number) and the values are lists of dataIds
211  for that CCD in each exposure. A missing dataId is given the value
212  None.
213 
214  @param expRefList List of data references for exposures
215  @param level Level for the butler to generate CCDs
216  @param ccdKeys DataId keywords that identify a CCD
217  @return dict of data identifier lists for each CCD;
218  keys are values of ccdKeys in order
219  """
220  expIdList = [[ccdRef.dataId for ccdRef in expRef.subItems(
221  level)] for expRef in expRefList]
222 
223  # Determine what additional keys make a CCD from an exposure
224  if len(ccdKeys) != len(set(ccdKeys)):
225  raise RuntimeError("Duplicate keys found in ccdKeys: %s" % ccdKeys)
226  ccdNames = set() # Set of tuples which are values for each of the CCDs in an exposure
227  for ccdIdList in expIdList:
228  for ccdId in ccdIdList:
229  name = dictToTuple(ccdId, ccdKeys)
230  ccdNames.add(name)
231 
232  # Turn the list of CCDs for each exposure into a list of exposures for
233  # each CCD
234  ccdLists = {}
235  for n, ccdIdList in enumerate(expIdList):
236  for ccdId in ccdIdList:
237  name = dictToTuple(ccdId, ccdKeys)
238  if name not in ccdLists:
239  ccdLists[name] = []
240  ccdLists[name].append(ccdId)
241 
242  for ccd in ccdLists:
243  # Sort the list by the dataId values (ordered by key)
244  ccdLists[ccd] = sorted(ccdLists[ccd], key=lambda dd: dictToTuple(dd, sorted(dd.keys())))
245 
246  return ccdLists
247 
248 
249 def mapToMatrix(pool, func, ccdIdLists, *args, **kwargs):
250  """Generate a matrix of results using pool.map
251 
252  The function should have the call signature:
253  func(cache, dataId, *args, **kwargs)
254 
255  We return a dict mapping 'ccd name' to a list of values for
256  each exposure.
257 
258  @param pool Process pool
259  @param func Function to call for each dataId
260  @param ccdIdLists Dict of data identifier lists for each CCD name
261  @return matrix of results
262  """
263  dataIdList = sum(ccdIdLists.values(), [])
264  resultList = pool.map(func, dataIdList, *args, **kwargs)
265  # Piece everything back together
266  data = dict((ccdName, [None] * len(expList)) for ccdName, expList in ccdIdLists.items())
267  indices = dict(sum([[(tuple(dataId.values()) if dataId is not None else None, (ccdName, expNum))
268  for expNum, dataId in enumerate(expList)]
269  for ccdName, expList in ccdIdLists.items()], []))
270  for dataId, result in zip(dataIdList, resultList):
271  if dataId is None:
272  continue
273  ccdName, expNum = indices[tuple(dataId.values())]
274  data[ccdName][expNum] = result
275  return data
276 
277 
278 class CalibIdAction(argparse.Action):
279  """Split name=value pairs and put the result in a dict"""
280 
281  def __call__(self, parser, namespace, values, option_string):
282  output = getattr(namespace, self.dest, {})
283  for nameValue in values:
284  name, sep, valueStr = nameValue.partition("=")
285  if not valueStr:
286  parser.error("%s value %s must be in form name=value" %
287  (option_string, nameValue))
288  output[name] = valueStr
289  setattr(namespace, self.dest, output)
290 
291 
292 class CalibArgumentParser(ArgumentParser):
293  """ArgumentParser for calibration construction"""
294 
295  def __init__(self, calibName, *args, **kwargs):
296  """Add a --calibId argument to the standard pipe_base argument parser"""
297  ArgumentParser.__init__(self, *args, **kwargs)
298  self.calibName = calibName
299  self.add_id_argument("--id", datasetType="raw",
300  help="input identifiers, e.g., --id visit=123 ccd=4")
301  self.add_argument("--calibId", nargs="*", action=CalibIdAction, default={},
302  help="identifiers for calib, e.g., --calibId version=1",
303  metavar="KEY=VALUE1[^VALUE2[^VALUE3...]")
304 
305  def parse_args(self, *args, **kwargs):
306  """Parse arguments
307 
308  Checks that the "--calibId" provided works.
309  """
310  namespace = ArgumentParser.parse_args(self, *args, **kwargs)
311 
312  keys = namespace.butler.getKeys(self.calibName)
313  parsed = {}
314  for name, value in namespace.calibId.items():
315  if name not in keys:
316  self.error(
317  "%s is not a relevant calib identifier key (%s)" % (name, keys))
318  parsed[name] = keys[name](value)
319  namespace.calibId = parsed
320 
321  return namespace
322 
323 
324 class CalibConfig(Config):
325  """Configuration for constructing calibs"""
326  clobber = Field(dtype=bool, default=True,
327  doc="Clobber existing processed images?")
328  isr = ConfigurableField(target=IsrTask, doc="ISR configuration")
329  dateObs = Field(dtype=str, default="dateObs",
330  doc="Key for observation date in exposure registry")
331  dateCalib = Field(dtype=str, default="calibDate",
332  doc="Key for calib date in calib registry")
333  filter = Field(dtype=str, default="filter",
334  doc="Key for filter name in exposure/calib registries")
335  combination = ConfigurableField(
336  target=CalibCombineTask, doc="Calib combination configuration")
337  ccdKeys = ListField(dtype=str, default=["ccd"],
338  doc="DataId keywords specifying a CCD")
339  visitKeys = ListField(dtype=str, default=["visit"],
340  doc="DataId keywords specifying a visit")
341  calibKeys = ListField(dtype=str, default=[],
342  doc="DataId keywords specifying a calibration")
343  doCameraImage = Field(dtype=bool, default=True, doc="Create camera overview image?")
344  binning = Field(dtype=int, default=64, doc="Binning to apply for camera image")
345 
346  def setDefaults(self):
347  self.isr.doWrite = False
348 
349 
350 class CalibTaskRunner(TaskRunner):
351  """Get parsed values into the CalibTask.run"""
352  @staticmethod
353  def getTargetList(parsedCmd, **kwargs):
354  return [dict(expRefList=parsedCmd.id.refList, butler=parsedCmd.butler, calibId=parsedCmd.calibId)]
355 
356  def __call__(self, args):
357  """Call the Task with the kwargs from getTargetList"""
358  task = self.TaskClass(config=self.config, log=self.log)
359  exitStatus = 0 # exit status for the shell
360  if self.doRaise:
361  result = task.run(**args)
362  else:
363  try:
364  result = task.run(**args)
365  except Exception as e:
366  exitStatus = 1 # n.b. The shell exit value is the number of dataRefs returning
367  # non-zero, so the actual value used here is lost
368  task.log.fatal("Failed: %s" % e)
369  traceback.print_exc(file=sys.stderr)
370 
371  if self.doReturnResults:
372  return Struct(
373  exitStatus=exitStatus,
374  args=args,
375  metadata=task.metadata,
376  result=result,
377  )
378  else:
379  return Struct(
380  exitStatus=exitStatus,
381  )
382 
384  """!Base class for constructing calibs.
385 
386  This should be subclassed for each of the required calib types.
387  The subclass should be sure to define the following class variables:
388  * _DefaultName: default name of the task, used by CmdLineTask
389  * calibName: name of the calibration data set in the butler
390  The subclass may optionally set:
391  * filterName: filter name to give the resultant calib
392  """
393  ConfigClass = CalibConfig
394  RunnerClass = CalibTaskRunner
395  filterName = None
396  calibName = None
397  exposureTime = 1.0 # sets this exposureTime in the output
398 
399  def __init__(self, *args, **kwargs):
400  """Constructor"""
401  BatchPoolTask.__init__(self, *args, **kwargs)
402  self.makeSubtask("isr")
403  self.makeSubtask("combination")
404 
405  @classmethod
406  def batchWallTime(cls, time, parsedCmd, numCores):
407  numCcds = len(parsedCmd.butler.get("camera"))
408  numExps = len(cls.RunnerClass.getTargetList(
409  parsedCmd)[0]['expRefList'])
410  numCycles = int(numCcds/float(numCores) + 0.5)
411  return time*numExps*numCycles
412 
413  @classmethod
414  def _makeArgumentParser(cls, *args, **kwargs):
415  kwargs.pop("doBatch", False)
416  return CalibArgumentParser(calibName=cls.calibName, name=cls._DefaultName, *args, **kwargs)
417 
418  def run(self, expRefList, butler, calibId):
419  """!Construct a calib from a list of exposure references
420 
421  This is the entry point, called by the TaskRunner.__call__
422 
423  Only the master node executes this method.
424 
425  @param expRefList List of data references at the exposure level
426  @param butler Data butler
427  @param calibId Identifier dict for calib
428  """
429  for expRef in expRefList:
430  self.addMissingKeys(expRef.dataId, butler, self.config.ccdKeys, 'raw')
431 
432  outputId = self.getOutputId(expRefList, calibId)
433  ccdIdLists = getCcdIdListFromExposures(
434  expRefList, level="sensor", ccdKeys=self.config.ccdKeys)
435  self.checkCcdIdLists(ccdIdLists)
436 
437  # Ensure we can generate filenames for each output
438  outputIdItemList = list(outputId.items())
439  for ccdName in ccdIdLists:
440  dataId = dict([(k, ccdName[i]) for i, k in enumerate(self.config.ccdKeys)])
441  dataId.update(outputIdItemList)
442  self.addMissingKeys(dataId, butler)
443  dataId.update(outputIdItemList)
444 
445  try:
446  butler.get(self.calibName + "_filename", dataId)
447  except Exception as e:
448  raise RuntimeError(
449  "Unable to determine output filename \"%s_filename\" from %s: %s" %
450  (self.calibName, dataId, e))
451 
452  processPool = Pool("process")
453  processPool.storeSet(butler=butler)
454 
455  # Scatter: process CCDs independently
456  data = self.scatterProcess(processPool, ccdIdLists)
457 
458  # Gather: determine scalings
459  scales = self.scale(ccdIdLists, data)
460 
461  combinePool = Pool("combine")
462  combinePool.storeSet(butler=butler)
463 
464  # Scatter: combine
465  calibs = self.scatterCombine(combinePool, outputId, ccdIdLists, scales)
466 
467  if self.config.doCameraImage:
468  camera = butler.get("camera")
469 
470  try:
471  cameraImage = self.makeCameraImage(camera, outputId, calibs)
472  butler.put(cameraImage, self.calibName + "_camera", dataId)
473  except Exception as exc:
474  self.log.warn("Unable to create camera image: %s" % (exc,))
475 
476  return Struct(
477  outputId = outputId,
478  ccdIdLists = ccdIdLists,
479  scales = scales,
480  calibs = calibs,
481  processPool = processPool,
482  combinePool = combinePool,
483  )
484 
485  def getOutputId(self, expRefList, calibId):
486  """!Generate the data identifier for the output calib
487 
488  The mean date and the common filter are included, using keywords
489  from the configuration. The CCD-specific part is not included
490  in the data identifier.
491 
492  @param expRefList List of data references at exposure level
493  @param calibId Data identifier elements for the calib provided by the user
494  @return data identifier
495  """
496  midTime = 0
497  filterName = None
498  for expRef in expRefList:
499  butler = expRef.getButler()
500  dataId = expRef.dataId
501 
502  midTime += self.getMjd(butler, dataId)
503  thisFilter = self.getFilter(
504  butler, dataId) if self.filterName is None else self.filterName
505  if filterName is None:
506  filterName = thisFilter
507  elif filterName != thisFilter:
508  raise RuntimeError("Filter mismatch for %s: %s vs %s" % (
509  dataId, thisFilter, filterName))
510 
511  midTime /= len(expRefList)
512  date = str(dafBase.DateTime(
513  midTime, dafBase.DateTime.MJD).toPython().date())
514 
515  outputId = {self.config.filter: filterName,
516  self.config.dateCalib: date}
517  outputId.update(calibId)
518  return outputId
519 
520  def getMjd(self, butler, dataId, timescale=dafBase.DateTime.UTC):
521  """Determine the Modified Julian Date (MJD; in TAI) from a data identifier"""
522  if self.config.dateObs in dataId:
523  dateObs = dataId[self.config.dateObs]
524  else:
525  dateObs = butler.queryMetadata('raw', [self.config.dateObs], dataId)[0]
526  if "T" not in dateObs:
527  dateObs = dateObs + "T12:00:00.0Z"
528  elif not dateObs.endswith("Z"):
529  dateObs += "Z"
530 
531  return dafBase.DateTime(dateObs, timescale).get(dafBase.DateTime.MJD)
532 
533  def getFilter(self, butler, dataId):
534  """Determine the filter from a data identifier"""
535  filt = butler.queryMetadata('raw', [self.config.filter], dataId)[0]
536  return filt
537 
538  def addMissingKeys(self, dataId, butler, missingKeys=None, calibName=None):
539  if calibName is None:
540  calibName = self.calibName
541 
542  if missingKeys is None:
543  missingKeys = set(butler.getKeys(calibName).keys()) - set(dataId.keys())
544 
545  for k in missingKeys:
546  try:
547  v = butler.queryMetadata('raw', [k], dataId) # n.b. --id refers to 'raw'
548  except Exception as e:
549  continue
550 
551  if len(v) == 0: # failed to lookup value
552  continue
553 
554  if len(v) == 1:
555  dataId[k] = v[0]
556  else:
557  raise RuntimeError("No unique lookup for %s: %s" % (k, v))
558 
559  def updateMetadata(self, calibImage, exposureTime, darkTime=None, **kwargs):
560  """!Update the metadata from the VisitInfo
561 
562  \param calibImage The image whose metadata is to be set
563  \param exposureTime The exposure time for the image
564  \param darkTime The time since the last read (default: exposureTime)
565  """
566 
567  if darkTime is None:
568  darkTime = exposureTime # avoid warning messages when using calibration products
569 
570  visitInfo = afwImage.makeVisitInfo(exposureTime=exposureTime, darkTime=darkTime, **kwargs)
571  md = calibImage.getMetadata()
572 
573  afwImage.setVisitInfoMetadata(md, visitInfo)
574 
575  def scatterProcess(self, pool, ccdIdLists):
576  """!Scatter the processing among the nodes
577 
578  We scatter each CCD independently (exposures aren't grouped together),
579  to make full use of all available processors. This necessitates piecing
580  everything back together in the same format as ccdIdLists afterwards.
581 
582  Only the master node executes this method.
583 
584  @param pool Process pool
585  @param ccdIdLists Dict of data identifier lists for each CCD name
586  @return Dict of lists of returned data for each CCD name
587  """
588  self.log.info("Scatter processing")
589  return mapToMatrix(pool, self.process, ccdIdLists)
590 
591  def process(self, cache, ccdId, outputName="postISRCCD", **kwargs):
592  """!Process a CCD, specified by a data identifier
593 
594  After processing, optionally returns a result (produced by
595  the 'processResult' method) calculated from the processed
596  exposure. These results will be gathered by the master node,
597  and is a means for coordinated scaling of all CCDs for flats,
598  etc.
599 
600  Only slave nodes execute this method.
601 
602  @param cache Process pool cache
603  @param ccdId Data identifier for CCD
604  @param outputName Output dataset name for butler
605  @return result from 'processResult'
606  """
607  if ccdId is None:
608  self.log.warn("Null identifier received on %s" % NODE)
609  return None
610  sensorRef = getDataRef(cache.butler, ccdId)
611  if self.config.clobber or not sensorRef.datasetExists(outputName):
612  self.log.info("Processing %s on %s" % (ccdId, NODE))
613  try:
614  exposure = self.processSingle(sensorRef, **kwargs)
615  except Exception as e:
616  self.log.warn("Unable to process %s: %s" % (ccdId, e))
617  raise
618  return None
619  self.processWrite(sensorRef, exposure)
620  else:
621  self.log.info(
622  "Using previously persisted processed exposure for %s" % (sensorRef.dataId,))
623  exposure = sensorRef.get(outputName)
624  return self.processResult(exposure)
625 
626  def processSingle(self, dataRef):
627  """Process a single CCD, specified by a data reference
628 
629  Generally, this simply means doing ISR.
630 
631  Only slave nodes execute this method.
632  """
633  return self.isr.runDataRef(dataRef).exposure
634 
635  def processWrite(self, dataRef, exposure, outputName="postISRCCD"):
636  """!Write the processed CCD
637 
638  We need to write these out because we can't hold them all in
639  memory at once.
640 
641  Only slave nodes execute this method.
642 
643  @param dataRef Data reference
644  @param exposure CCD exposure to write
645  @param outputName Output dataset name for butler.
646  """
647  dataRef.put(exposure, outputName)
648 
649  def processResult(self, exposure):
650  """Extract processing results from a processed exposure
651 
652  This method generates what is gathered by the master node.
653  This can be a background measurement or similar for scaling
654  flat-fields. It must be picklable!
655 
656  Only slave nodes execute this method.
657  """
658  return None
659 
660  def scale(self, ccdIdLists, data):
661  """!Determine scaling across CCDs and exposures
662 
663  This is necessary mainly for flats, so as to determine a
664  consistent scaling across the entire focal plane. This
665  implementation is simply a placeholder.
666 
667  Only the master node executes this method.
668 
669  @param ccdIdLists Dict of data identifier lists for each CCD tuple
670  @param data Dict of lists of returned data for each CCD tuple
671  @return dict of Struct(ccdScale: scaling for CCD,
672  expScales: scaling for each exposure
673  ) for each CCD tuple
674  """
675  self.log.info("Scale on %s" % NODE)
676  return dict((name, Struct(ccdScale=None, expScales=[None] * len(ccdIdLists[name])))
677  for name in ccdIdLists)
678 
679  def scatterCombine(self, pool, outputId, ccdIdLists, scales):
680  """!Scatter the combination of exposures across multiple nodes
681 
682  In this case, we can only scatter across as many nodes as
683  there are CCDs.
684 
685  Only the master node executes this method.
686 
687  @param pool Process pool
688  @param outputId Output identifier (exposure part only)
689  @param ccdIdLists Dict of data identifier lists for each CCD name
690  @param scales Dict of structs with scales, for each CCD name
691  @param dict of binned images
692  """
693  self.log.info("Scatter combination")
694  data = [Struct(ccdName=ccdName, ccdIdList=ccdIdLists[ccdName], scales=scales[ccdName]) for
695  ccdName in ccdIdLists]
696  images = pool.map(self.combine, data, outputId)
697  return dict(zip(ccdIdLists.keys(), images))
698 
699  def getFullyQualifiedOutputId(self, ccdName, butler, outputId):
700  """Get fully-qualified output data identifier
701 
702  We may need to look up keys that aren't in the output dataId.
703 
704  @param ccdName Name tuple for CCD
705  @param butler Data butler
706  @param outputId Data identifier for combined image (exposure part only)
707  @return fully-qualified output dataId
708  """
709  fullOutputId = {k: ccdName[i] for i, k in enumerate(self.config.ccdKeys)}
710  fullOutputId.update(outputId)
711  self.addMissingKeys(fullOutputId, butler)
712  fullOutputId.update(outputId) # must be after the call to queryMetadata in 'addMissingKeys'
713  return fullOutputId
714 
715  def combine(self, cache, struct, outputId):
716  """!Combine multiple exposures of a particular CCD and write the output
717 
718  Only the slave nodes execute this method.
719 
720  @param cache Process pool cache
721  @param struct Parameters for the combination, which has the following components:
722  * ccdName Name tuple for CCD
723  * ccdIdList List of data identifiers for combination
724  * scales Scales to apply (expScales are scalings for each exposure,
725  ccdScale is final scale for combined image)
726  @param outputId Data identifier for combined image (exposure part only)
727  @return binned calib image
728  """
729  outputId = self.getFullyQualifiedOutputId(struct.ccdName, cache.butler, outputId)
730  dataRefList = [getDataRef(cache.butler, dataId) if dataId is not None else None for
731  dataId in struct.ccdIdList]
732  self.log.info("Combining %s on %s" % (outputId, NODE))
733  calib = self.combination.run(dataRefList, expScales=struct.scales.expScales,
734  finalScale=struct.scales.ccdScale)
735 
736  if not hasattr(calib, "getMetadata"):
737  if hasattr(calib, "getVariance"):
738  calib = afwImage.makeExposure(calib)
739  else:
740  calib = afwImage.DecoratedImageF(calib.getImage()) # n.b. hardwires "F" for the output type
741 
742  self.updateMetadata(calib, self.exposureTime)
743 
744  self.recordCalibInputs(cache.butler, calib,
745  struct.ccdIdList, outputId)
746 
747  self.interpolateNans(calib)
748 
749  self.write(cache.butler, calib, outputId)
750 
751  return afwMath.binImage(calib.getImage(), self.config.binning)
752 
753  def recordCalibInputs(self, butler, calib, dataIdList, outputId):
754  """!Record metadata including the inputs and creation details
755 
756  This metadata will go into the FITS header.
757 
758  @param butler Data butler
759  @param calib Combined calib exposure.
760  @param dataIdList List of data identifiers for calibration inputs
761  @param outputId Data identifier for output
762  """
763  header = calib.getMetadata()
764  header.add("OBSTYPE", self.calibName) # Used by ingestCalibs.py
765 
766  # date, time, host, and root
767  now = time.localtime()
768  header.add("CALIB_CREATION_DATE", time.strftime("%Y-%m-%d", now))
769  header.add("CALIB_CREATION_TIME", time.strftime("%X %Z", now))
770  # add date-obs as its absence upsets ExposureInfo; use the mean date that the calibs were taken
771  header.add("DATE-OBS", "%sT00:00:00.00" % outputId[self.config.dateCalib])
772 
773  # Inputs
774  visits = [str(dictToTuple(dataId, self.config.visitKeys)) for dataId in dataIdList if
775  dataId is not None]
776  for i, v in enumerate(sorted(set(visits))):
777  header.add("CALIB_INPUT_%d" % (i,), v)
778 
779  header.add("CALIB_ID", " ".join("%s=%s" % (key, value)
780  for key, value in outputId.items()))
781  checksum(calib, header)
782 
783  def interpolateNans(self, image):
784  """Interpolate over NANs in the combined image
785 
786  NANs can result from masked areas on the CCD. We don't want them getting
787  into our science images, so we replace them with the median of the image.
788  """
789  if hasattr(image, "getMaskedImage"): # Deal with Exposure vs Image
790  self.interpolateNans(image.getMaskedImage().getVariance())
791  image = image.getMaskedImage().getImage()
792  if hasattr(image, "getImage"): # Deal with DecoratedImage or MaskedImage vs Image
793  image = image.getImage()
794  array = image.getArray()
795  bad = np.isnan(array)
796  array[bad] = np.median(array[np.logical_not(bad)])
797 
798  def write(self, butler, exposure, dataId):
799  """!Write the final combined calib
800 
801  Only the slave nodes execute this method
802 
803  @param butler Data butler
804  @param exposure CCD exposure to write
805  @param dataId Data identifier for output
806  """
807  self.log.info("Writing %s on %s" % (dataId, NODE))
808  butler.put(exposure, self.calibName, dataId)
809 
810  def makeCameraImage(self, camera, dataId, calibs):
811  """!Create and write an image of the entire camera
812 
813  This is useful for judging the quality or getting an overview of
814  the features of the calib.
815 
816  This requires that the 'ccd name' is a tuple containing only the
817  detector ID. If that is not the case, change CalibConfig.ccdKeys
818  or set CalibConfig.doCameraImage=False to disable this.
819 
820  @param camera Camera object
821  @param dataId Data identifier for output
822  @param calibs Dict mapping 'ccd name' to calib image
823  """
824 
825  class ImageSource(object):
826  """Source of images for makeImageFromCamera
827 
828  This assumes that the 'ccd name' is a tuple containing
829  only the detector ID.
830  """
831  def __init__(self, images):
832  self.isTrimmed = True
833  self.images = images
834  self.background = np.nan
835 
836  def getCcdImage(self, detector, imageFactory, binSize):
837  detId = (detector.getId(),)
838  if detId not in self.images:
839  return imageFactory(1, 1), detId
840  return self.images[detId], detId
841 
842  image = makeImageFromCamera(camera, imageSource=ImageSource(calibs), imageFactory=afwImage.ImageF,
843  binSize=self.config.binning)
844  return image
845 
846  def checkCcdIdLists(self, ccdIdLists):
847  """Check that the list of CCD dataIds is consistent
848 
849  @param ccdIdLists Dict of data identifier lists for each CCD name
850  @return Number of exposures, number of CCDs
851  """
852  visitIdLists = collections.defaultdict(list)
853  for ccdName in ccdIdLists:
854  for dataId in ccdIdLists[ccdName]:
855  visitName = dictToTuple(dataId, self.config.visitKeys)
856  visitIdLists[visitName].append(dataId)
857 
858  numExps = set(len(expList) for expList in ccdIdLists.values())
859  numCcds = set(len(ccdList) for ccdList in visitIdLists.values())
860 
861  if len(numExps) != 1 or len(numCcds) != 1:
862  # Presumably a visit somewhere doesn't have the full complement available.
863  # Dump the information so the user can figure it out.
864  self.log.warn("Number of visits for each CCD: %s",
865  {ccdName: len(ccdIdLists[ccdName]) for ccdName in ccdIdLists})
866  self.log.warn("Number of CCDs for each visit: %s",
867  {vv: len(visitIdLists[vv]) for vv in visitIdLists})
868  raise RuntimeError("Inconsistent number of exposures/CCDs")
869 
870  return numExps.pop(), numCcds.pop()
871 
872 
874  """Configuration for bias construction.
875 
876  No changes required compared to the base class, but
877  subclassed for distinction.
878  """
879  pass
880 
881 
882 class BiasTask(CalibTask):
883  """Bias construction"""
884  ConfigClass = BiasConfig
885  _DefaultName = "bias"
886  calibName = "bias"
887  filterName = "NONE" # Sets this filter name in the output
888  exposureTime = 0.0 # sets this exposureTime in the output
889 
890  @classmethod
891  def applyOverrides(cls, config):
892  """Overrides to apply for bias construction"""
893  config.isr.doBias = False
894  config.isr.doDark = False
895  config.isr.doFlat = False
896  config.isr.doFringe = False
897 
898 
900  """Configuration for dark construction"""
901  doRepair = Field(dtype=bool, default=True, doc="Repair artifacts?")
902  psfFwhm = Field(dtype=float, default=3.0, doc="Repair PSF FWHM (pixels)")
903  psfSize = Field(dtype=int, default=21, doc="Repair PSF size (pixels)")
904  crGrow = Field(dtype=int, default=2, doc="Grow radius for CR (pixels)")
905  repair = ConfigurableField(
906  target=RepairTask, doc="Task to repair artifacts")
907 
908  def setDefaults(self):
909  CalibConfig.setDefaults(self)
910  self.combination.mask.append("CR")
911 
912 
914  """Dark construction
915 
916  The only major difference from the base class is a cosmic-ray
917  identification stage, and dividing each image by the dark time
918  to generate images of the dark rate.
919  """
920  ConfigClass = DarkConfig
921  _DefaultName = "dark"
922  calibName = "dark"
923  filterName = "NONE" # Sets this filter name in the output
924 
925  def __init__(self, *args, **kwargs):
926  CalibTask.__init__(self, *args, **kwargs)
927  self.makeSubtask("repair")
928 
929  @classmethod
930  def applyOverrides(cls, config):
931  """Overrides to apply for dark construction"""
932  config.isr.doDark = False
933  config.isr.doFlat = False
934  config.isr.doFringe = False
935 
936  def processSingle(self, sensorRef):
937  """Process a single CCD
938 
939  Besides the regular ISR, also masks cosmic-rays and divides each
940  processed image by the dark time to generate images of the dark rate.
941  The dark time is provided by the 'getDarkTime' method.
942  """
943  exposure = CalibTask.processSingle(self, sensorRef)
944 
945  if self.config.doRepair:
946  psf = measAlg.DoubleGaussianPsf(self.config.psfSize, self.config.psfSize,
947  self.config.psfFwhm/(2*math.sqrt(2*math.log(2))))
948  exposure.setPsf(psf)
949  self.repair.run(exposure, keepCRs=False)
950  if self.config.crGrow > 0:
951  mask = exposure.getMaskedImage().getMask().clone()
952  mask &= mask.getPlaneBitMask("CR")
953  fpSet = afwDet.FootprintSet(
954  mask, afwDet.Threshold(0.5))
955  fpSet = afwDet.FootprintSet(fpSet, self.config.crGrow, True)
956  fpSet.setMask(exposure.getMaskedImage().getMask(), "CR")
957 
958  mi = exposure.getMaskedImage()
959  mi /= self.getDarkTime(exposure)
960  return exposure
961 
962  def getDarkTime(self, exposure):
963  """Retrieve the dark time for an exposure"""
964  darkTime = exposure.getInfo().getVisitInfo().getDarkTime()
965  if not np.isfinite(darkTime):
966  raise RuntimeError("Non-finite darkTime")
967  return darkTime
968 
969 
971  """Configuration for flat construction"""
972  iterations = Field(dtype=int, default=10,
973  doc="Number of iterations for scale determination")
974  stats = ConfigurableField(target=CalibStatsTask,
975  doc="Background statistics configuration")
976 
977 
979  """Flat construction
980 
981  The principal change from the base class involves gathering the background
982  values from each image and using them to determine the scalings for the final
983  combination.
984  """
985  ConfigClass = FlatConfig
986  _DefaultName = "flat"
987  calibName = "flat"
988 
989  @classmethod
990  def applyOverrides(cls, config):
991  """Overrides for flat construction"""
992  config.isr.doFlat = False
993  config.isr.doFringe = False
994 
995  def __init__(self, *args, **kwargs):
996  CalibTask.__init__(self, *args, **kwargs)
997  self.makeSubtask("stats")
998 
999  def processResult(self, exposure):
1000  return self.stats.run(exposure)
1001 
1002  def scale(self, ccdIdLists, data):
1003  """Determine the scalings for the final combination
1004 
1005  We have a matrix B_ij = C_i E_j, where C_i is the relative scaling
1006  of one CCD to all the others in an exposure, and E_j is the scaling
1007  of the exposure. We convert everything to logarithms so we can work
1008  with a linear system. We determine the C_i and E_j from B_ij by iteration,
1009  under the additional constraint that the average CCD scale is unity.
1010 
1011  This algorithm comes from Eugene Magnier and Pan-STARRS.
1012  """
1013  assert len(ccdIdLists.values()) > 0, "No successful CCDs"
1014  lengths = set([len(expList) for expList in ccdIdLists.values()])
1015  assert len(
1016  lengths) == 1, "Number of successful exposures for each CCD differs"
1017  assert tuple(lengths)[0] > 0, "No successful exposures"
1018  # Format background measurements into a matrix
1019  indices = dict((name, i) for i, name in enumerate(ccdIdLists))
1020  bgMatrix = np.array([[0.0] * len(expList)
1021  for expList in ccdIdLists.values()])
1022  for name in ccdIdLists:
1023  i = indices[name]
1024  bgMatrix[i] = [
1025  d if d is not None else np.nan for d in data[name]]
1026 
1027  numpyPrint = np.get_printoptions()
1028  np.set_printoptions(threshold=np.inf)
1029  self.log.info("Input backgrounds: %s" % bgMatrix)
1030 
1031  # Flat-field scaling
1032  numCcds = len(ccdIdLists)
1033  numExps = bgMatrix.shape[1]
1034  # log(Background) for each exposure/component
1035  bgMatrix = np.log(bgMatrix)
1036  bgMatrix = np.ma.masked_array(bgMatrix, np.isnan(bgMatrix))
1037  # Initial guess at log(scale) for each component
1038  compScales = np.zeros(numCcds)
1039  expScales = np.array(
1040  [(bgMatrix[:, i0] - compScales).mean() for i0 in range(numExps)])
1041 
1042  for iterate in range(self.config.iterations):
1043  compScales = np.array(
1044  [(bgMatrix[i1, :] - expScales).mean() for i1 in range(numCcds)])
1045  expScales = np.array(
1046  [(bgMatrix[:, i2] - compScales).mean() for i2 in range(numExps)])
1047 
1048  avgScale = np.average(np.exp(compScales))
1049  compScales -= np.log(avgScale)
1050  self.log.debug("Iteration %d exposure scales: %s",
1051  iterate, np.exp(expScales))
1052  self.log.debug("Iteration %d component scales: %s",
1053  iterate, np.exp(compScales))
1054 
1055  expScales = np.array(
1056  [(bgMatrix[:, i3] - compScales).mean() for i3 in range(numExps)])
1057 
1058  if np.any(np.isnan(expScales)):
1059  raise RuntimeError("Bad exposure scales: %s --> %s" %
1060  (bgMatrix, expScales))
1061 
1062  expScales = np.exp(expScales)
1063  compScales = np.exp(compScales)
1064 
1065  self.log.info("Exposure scales: %s" % expScales)
1066  self.log.info("Component relative scaling: %s" % compScales)
1067  np.set_printoptions(**numpyPrint)
1068 
1069  return dict((ccdName, Struct(ccdScale=compScales[indices[ccdName]], expScales=expScales))
1070  for ccdName in ccdIdLists)
1071 
1072 
1074  """Configuration for fringe construction"""
1075  stats = ConfigurableField(target=CalibStatsTask,
1076  doc="Background statistics configuration")
1077  subtractBackground = ConfigurableField(target=measAlg.SubtractBackgroundTask,
1078  doc="Background configuration")
1079  detection = ConfigurableField(
1080  target=measAlg.SourceDetectionTask, doc="Detection configuration")
1081  detectSigma = Field(dtype=float, default=1.0,
1082  doc="Detection PSF gaussian sigma")
1083 
1084 
1086  """Fringe construction task
1087 
1088  The principal change from the base class is that the images are
1089  background-subtracted and rescaled by the background.
1090 
1091  XXX This is probably not right for a straight-up combination, as we
1092  are currently doing, since the fringe amplitudes need not scale with
1093  the continuum.
1094 
1095  XXX Would like to have this do PCA and generate multiple images, but
1096  that will take a bit of work with the persistence code.
1097  """
1098  ConfigClass = FringeConfig
1099  _DefaultName = "fringe"
1100  calibName = "fringe"
1101 
1102  @classmethod
1103  def applyOverrides(cls, config):
1104  """Overrides for fringe construction"""
1105  config.isr.doFringe = False
1106 
1107  def __init__(self, *args, **kwargs):
1108  CalibTask.__init__(self, *args, **kwargs)
1109  self.makeSubtask("detection")
1110  self.makeSubtask("stats")
1111  self.makeSubtask("subtractBackground")
1112 
1113  def processSingle(self, sensorRef):
1114  """Subtract the background and normalise by the background level"""
1115  exposure = CalibTask.processSingle(self, sensorRef)
1116  bgLevel = self.stats.run(exposure)
1117  self.subtractBackground.run(exposure)
1118  mi = exposure.getMaskedImage()
1119  mi /= bgLevel
1120  footprintSets = self.detection.detectFootprints(
1121  exposure, sigma=self.config.detectSigma)
1122  mask = exposure.getMaskedImage().getMask()
1123  detected = 1 << mask.addMaskPlane("DETECTED")
1124  for fpSet in (footprintSets.positive, footprintSets.negative):
1125  if fpSet is not None:
1126  afwDet.setMaskFromFootprintList(
1127  mask, fpSet.getFootprints(), detected)
1128  return exposure
1129 
1130 
1132  """Configuration for sky frame construction"""
1133  detection = ConfigurableField(target=measAlg.SourceDetectionTask, doc="Detection configuration")
1134  detectSigma = Field(dtype=float, default=2.0, doc="Detection PSF gaussian sigma")
1135  subtractBackground = ConfigurableField(target=measAlg.SubtractBackgroundTask,
1136  doc="Regular-scale background configuration, for object detection")
1137  largeScaleBackground = ConfigField(dtype=FocalPlaneBackgroundConfig,
1138  doc="Large-scale background configuration")
1139  sky = ConfigurableField(target=SkyMeasurementTask, doc="Sky measurement")
1140  maskThresh = Field(dtype=float, default=3.0, doc="k-sigma threshold for masking pixels")
1141  mask = ListField(dtype=str, default=["BAD", "SAT", "DETECTED", "NO_DATA"],
1142  doc="Mask planes to consider as contaminated")
1143 
1144 
1146  """Task for sky frame construction
1147 
1148  The sky frame is a (relatively) small-scale background
1149  model, the response of the camera to the sky.
1150 
1151  To construct, we first remove a large-scale background (e.g., caused
1152  by moonlight) which may vary from image to image. Then we construct a
1153  model of the sky, which is essentially a binned version of the image
1154  (important configuration parameters: sky.background.[xy]BinSize).
1155  It is these models which are coadded to yield the sky frame.
1156  """
1157  ConfigClass = SkyConfig
1158  _DefaultName = "sky"
1159  calibName = "sky"
1160 
1161  def __init__(self, *args, **kwargs):
1162  CalibTask.__init__(self, *args, **kwargs)
1163  self.makeSubtask("detection")
1164  self.makeSubtask("subtractBackground")
1165  self.makeSubtask("sky")
1166 
1167  def scatterProcess(self, pool, ccdIdLists):
1168  """!Scatter the processing among the nodes
1169 
1170  Only the master node executes this method, assigning work to the
1171  slaves.
1172 
1173  We measure and subtract off a large-scale background model across
1174  all CCDs, which requires a scatter/gather. Then we process the
1175  individual CCDs, subtracting the large-scale background model and
1176  the residual background model measured. These residuals will be
1177  combined for the sky frame.
1178 
1179  @param pool Process pool
1180  @param ccdIdLists Dict of data identifier lists for each CCD name
1181  @return Dict of lists of returned data for each CCD name
1182  """
1183  self.log.info("Scatter processing")
1184 
1185  numExps = set(len(expList) for expList in ccdIdLists.values()).pop()
1186 
1187  # First subtract off general gradients to make all the exposures look similar.
1188  # We want to preserve the common small-scale structure, which we will coadd.
1189  bgModelList = mapToMatrix(pool, self.measureBackground, ccdIdLists)
1190 
1191  backgrounds = {}
1192  scales = {}
1193  for exp in range(numExps):
1194  bgModels = [bgModelList[ccdName][exp] for ccdName in ccdIdLists]
1195  visit = set(tuple(ccdIdLists[ccdName][exp][key] for key in sorted(self.config.visitKeys)) for
1196  ccdName in ccdIdLists)
1197  assert len(visit) == 1
1198  visit = visit.pop()
1199  bgModel = bgModels[0]
1200  for bg in bgModels[1:]:
1201  bgModel.merge(bg)
1202  self.log.info("Background model min/max for visit %s: %f %f", visit,
1203  np.min(bgModel.getStatsImage().getArray()),
1204  np.max(bgModel.getStatsImage().getArray()))
1205  backgrounds[visit] = bgModel
1206  scales[visit] = np.median(bgModel.getStatsImage().getArray())
1207 
1208  return mapToMatrix(pool, self.process, ccdIdLists, backgrounds=backgrounds, scales=scales)
1209 
1210  def measureBackground(self, cache, dataId):
1211  """!Measure background model for CCD
1212 
1213  This method is executed by the slaves.
1214 
1215  The background models for all CCDs in an exposure will be
1216  combined to form a full focal-plane background model.
1217 
1218  @param cache Process pool cache
1219  @param dataId Data identifier
1220  @return Bcakground model
1221  """
1222  dataRef = getDataRef(cache.butler, dataId)
1223  exposure = self.processSingleBackground(dataRef)
1224 
1225  # NAOJ prototype smoothed and then combined the entire image, but it shouldn't be any different
1226  # to bin and combine the binned images except that there's fewer pixels to worry about.
1227  config = self.config.largeScaleBackground
1228  camera = dataRef.get("camera")
1229  bgModel = FocalPlaneBackground.fromCamera(config, camera)
1230  bgModel.addCcd(exposure)
1231  return bgModel
1232 
1233  def processSingleBackground(self, dataRef):
1234  """!Process a single CCD for the background
1235 
1236  This method is executed by the slaves.
1237 
1238  Because we're interested in the background, we detect and mask astrophysical
1239  sources, and pixels above the noise level.
1240 
1241  @param dataRef Data reference for CCD.
1242  @return processed exposure
1243  """
1244  if not self.config.clobber and dataRef.datasetExists("postISRCCD"):
1245  return dataRef.get("postISRCCD")
1246  exposure = CalibTask.processSingle(self, dataRef)
1247 
1248  # Detect sources. Requires us to remove the background; we'll restore it later.
1249  bgTemp = self.subtractBackground.run(exposure).background
1250  footprints = self.detection.detectFootprints(exposure, sigma=self.config.detectSigma)
1251  image = exposure.getMaskedImage()
1252  if footprints.background is not None:
1253  image += footprints.background.getImageF()
1254 
1255  # Mask high pixels
1256  variance = image.getVariance()
1257  noise = np.sqrt(np.median(variance.getArray()))
1258  isHigh = image.getImage().getArray() > self.config.maskThresh*noise
1259  image.getMask().getArray()[isHigh] |= image.getMask().getPlaneBitMask("DETECTED")
1260 
1261  # Restore the background: it's what we want!
1262  image += bgTemp.getImage()
1263 
1264  # Set detected/bad pixels to background to ensure they don't corrupt the background
1265  maskVal = image.getMask().getPlaneBitMask(self.config.mask)
1266  isBad = image.getMask().getArray() & maskVal > 0
1267  bgLevel = np.median(image.getImage().getArray()[~isBad])
1268  image.getImage().getArray()[isBad] = bgLevel
1269  dataRef.put(exposure, "postISRCCD")
1270  return exposure
1271 
1272  def processSingle(self, dataRef, backgrounds, scales):
1273  """Process a single CCD, specified by a data reference
1274 
1275  We subtract the appropriate focal plane background model,
1276  divide by the appropriate scale and measure the background.
1277 
1278  Only slave nodes execute this method.
1279 
1280  @param dataRef Data reference for single CCD
1281  @param backgrounds Background model for each visit
1282  @param scales Scales for each visit
1283  @return Processed exposure
1284  """
1285  visit = tuple(dataRef.dataId[key] for key in sorted(self.config.visitKeys))
1286  exposure = dataRef.get("postISRCCD", immediate=True)
1287  image = exposure.getMaskedImage()
1288  detector = exposure.getDetector()
1289  bbox = image.getBBox()
1290 
1291  bgModel = backgrounds[visit]
1292  bg = bgModel.toCcdBackground(detector, bbox)
1293  image -= bg.getImage()
1294  image /= scales[visit]
1295 
1296  bg = self.sky.measureBackground(exposure.getMaskedImage())
1297  dataRef.put(bg, "icExpBackground")
1298  return exposure
1299 
1300  def combine(self, cache, struct, outputId):
1301  """!Combine multiple background models of a particular CCD and write the output
1302 
1303  Only the slave nodes execute this method.
1304 
1305  @param cache Process pool cache
1306  @param struct Parameters for the combination, which has the following components:
1307  * ccdName Name tuple for CCD
1308  * ccdIdList List of data identifiers for combination
1309  @param outputId Data identifier for combined image (exposure part only)
1310  @return binned calib image
1311  """
1312  outputId = self.getFullyQualifiedOutputId(struct.ccdName, cache.butler, outputId)
1313  dataRefList = [getDataRef(cache.butler, dataId) if dataId is not None else None for
1314  dataId in struct.ccdIdList]
1315  self.log.info("Combining %s on %s" % (outputId, NODE))
1316  bgList = [dataRef.get("icExpBackground", immediate=True).clone() for dataRef in dataRefList]
1317 
1318  bgExp = self.sky.averageBackgrounds(bgList)
1319 
1320  self.recordCalibInputs(cache.butler, bgExp, struct.ccdIdList, outputId)
1321  cache.butler.put(bgExp, "sky", outputId)
1322  return afwMath.binImage(self.sky.exposureToBackground(bgExp).getImage(), self.config.binning)
def processWrite(self, dataRef, exposure, outputName="postISRCCD")
Write the processed CCD.
def scatterCombine(self, pool, outputId, ccdIdLists, scales)
Scatter the combination of exposures across multiple nodes.
def run(self, exposureOrImage)
Measure a particular statistic on an image (of some sort).
def checksum(obj, header=None, sumType="MD5")
Calculate a checksum of an object.
Definition: checksum.py:29
def getCcdIdListFromExposures(expRefList, level="sensor", ccdKeys=["ccd"])
Determine a list of CCDs from exposure references.
def run(self, expRefList, butler, calibId)
Construct a calib from a list of exposure references.
def getFullyQualifiedOutputId(self, ccdName, butler, outputId)
def __call__(self, parser, namespace, values, option_string)
def dictToTuple(dict_, keys)
Return a tuple of specific values from a dict.
def getDataRef(butler, dataId, datasetType="raw")
Definition: utils.py:17
def updateMetadata(self, calibImage, exposureTime, darkTime=None, kwargs)
Update the metadata from the VisitInfo.
def process(self, cache, ccdId, outputName="postISRCCD", kwargs)
Process a CCD, specified by a data identifier.
def processSingle(self, dataRef, backgrounds, scales)
def makeCameraImage(self, camera, dataId, calibs)
Create and write an image of the entire camera.
def combine(self, target, imageList, stats)
Combine multiple images.
def getMjd(self, butler, dataId, timescale=dafBase.DateTime.UTC)
def combine(self, cache, struct, outputId)
Combine multiple exposures of a particular CCD and write the output.
def write(self, butler, exposure, dataId)
Write the final combined calib.
def combine(self, cache, struct, outputId)
Combine multiple background models of a particular CCD and write the output.
def measureBackground(self, cache, dataId)
Measure background model for CCD.
def recordCalibInputs(self, butler, calib, dataIdList, outputId)
Record metadata including the inputs and creation details.
def scatterProcess(self, pool, ccdIdLists)
Scatter the processing among the nodes.
def run(self, sensorRefList, expScales=None, finalScale=None, inputName="postISRCCD")
Combine calib images for a single sensor.
def getOutputId(self, expRefList, calibId)
Generate the data identifier for the output calib.
def mapToMatrix(pool, func, ccdIdLists, args, kwargs)
def scale(self, ccdIdLists, data)
Determine scaling across CCDs and exposures.
def addMissingKeys(self, dataId, butler, missingKeys=None, calibName=None)
def getDimensions(self, sensorRefList, inputName="postISRCCD")
Base class for constructing calibs.
def processSingleBackground(self, dataRef)
Process a single CCD for the background.
def scatterProcess(self, pool, ccdIdLists)
Scatter the processing among the nodes.
def batchWallTime(cls, time, parsedCmd, numCores)