lsst.pipe.drivers  16.0-1-g98efed3+6
constructCalibs.py
Go to the documentation of this file.
1 from __future__ import absolute_import, division, print_function
2 
3 import sys
4 import math
5 import time
6 import argparse
7 import traceback
8 import collections
9 
10 import numpy as np
11 from builtins import zip
12 from builtins import range
13 
14 from lsst.pex.config import Config, ConfigurableField, Field, ListField, ConfigField
15 from lsst.pipe.base import Task, Struct, TaskRunner, ArgumentParser
16 import lsst.daf.base as dafBase
17 import lsst.afw.math as afwMath
18 import lsst.afw.geom as afwGeom
19 import lsst.afw.detection as afwDet
20 import lsst.afw.image as afwImage
21 from lsst.afw.image import VisitInfo
22 import lsst.meas.algorithms as measAlg
23 from lsst.pipe.tasks.repair import RepairTask
24 from lsst.ip.isr import IsrTask
25 from lsst.afw.cameraGeom.utils import makeImageFromCamera
26 
27 from lsst.ctrl.pool.parallel import BatchPoolTask
28 from lsst.ctrl.pool.pool import Pool, NODE
29 from lsst.pipe.drivers.background import SkyMeasurementTask, FocalPlaneBackground, FocalPlaneBackgroundConfig
30 from lsst.pipe.drivers.visualizeVisit import makeCameraImage
31 
32 from .checksum import checksum
33 from .utils import getDataRef
34 
35 
36 class CalibStatsConfig(Config):
37  """Parameters controlling the measurement of background statistics"""
38  stat = Field(doc="Statistic to use to estimate background (from lsst.afw.math)", dtype=int,
39  default=int(afwMath.MEANCLIP))
40  clip = Field(doc="Clipping threshold for background",
41  dtype=float, default=3.0)
42  nIter = Field(doc="Clipping iterations for background",
43  dtype=int, default=3)
44  maxVisitsToCalcErrorFromInputVariance = Field(
45  doc="Maximum number of visits to estimate variance from input variance, not per-pixel spread",
46  dtype=int, default=2)
47  mask = ListField(doc="Mask planes to reject",
48  dtype=str, default=["DETECTED", "BAD", "NO_DATA",])
49 
50 
51 class CalibStatsTask(Task):
52  """Measure statistics on the background
53 
54  This can be useful for scaling the background, e.g., for flats and fringe frames.
55  """
56  ConfigClass = CalibStatsConfig
57 
58  def run(self, exposureOrImage):
59  """!Measure a particular statistic on an image (of some sort).
60 
61  @param exposureOrImage Exposure, MaskedImage or Image.
62  @return Value of desired statistic
63  """
64  stats = afwMath.StatisticsControl(self.config.clip, self.config.nIter,
65  afwImage.Mask.getPlaneBitMask(self.config.mask))
66  try:
67  image = exposureOrImage.getMaskedImage()
68  except:
69  try:
70  image = exposureOrImage.getImage()
71  except:
72  image = exposureOrImage
73 
74  return afwMath.makeStatistics(image, self.config.stat, stats).getValue()
75 
76 
77 class CalibCombineConfig(Config):
78  """Configuration for combining calib images"""
79  rows = Field(doc="Number of rows to read at a time",
80  dtype=int, default=512)
81  mask = ListField(doc="Mask planes to respect", dtype=str,
82  default=["SAT", "DETECTED", "INTRP"])
83  combine = Field(doc="Statistic to use for combination (from lsst.afw.math)", dtype=int,
84  default=int(afwMath.MEANCLIP))
85  clip = Field(doc="Clipping threshold for combination",
86  dtype=float, default=3.0)
87  nIter = Field(doc="Clipping iterations for combination",
88  dtype=int, default=3)
89  stats = ConfigurableField(target=CalibStatsTask,
90  doc="Background statistics configuration")
91 
92 
93 class CalibCombineTask(Task):
94  """Task to combine calib images"""
95  ConfigClass = CalibCombineConfig
96 
97  def __init__(self, *args, **kwargs):
98  Task.__init__(self, *args, **kwargs)
99  self.makeSubtask("stats")
100 
101  def run(self, sensorRefList, expScales=None, finalScale=None, inputName="postISRCCD"):
102  """!Combine calib images for a single sensor
103 
104  @param sensorRefList List of data references to combine (for a single sensor)
105  @param expScales List of scales to apply for each exposure
106  @param finalScale Desired scale for final combined image
107  @param inputName Data set name for inputs
108  @return combined image
109  """
110  width, height = self.getDimensions(sensorRefList)
111  stats = afwMath.StatisticsControl(self.config.clip, self.config.nIter,
112  afwImage.Mask.getPlaneBitMask(self.config.mask))
113  numImages = len(sensorRefList)
114  if numImages < self.config.stats.maxVisitsToCalcErrorFromInputVariance:
115  stats.setCalcErrorFromInputVariance(True)
116 
117  # Combine images
118  combined = afwImage.MaskedImageF(width, height)
119  imageList = [None]*numImages
120  for start in range(0, height, self.config.rows):
121  rows = min(self.config.rows, height - start)
122  box = afwGeom.Box2I(afwGeom.Point2I(0, start),
123  afwGeom.Extent2I(width, rows))
124  subCombined = combined.Factory(combined, box)
125 
126  for i, sensorRef in enumerate(sensorRefList):
127  if sensorRef is None:
128  imageList[i] = None
129  continue
130  exposure = sensorRef.get(inputName + "_sub", bbox=box)
131  if expScales is not None:
132  self.applyScale(exposure, expScales[i])
133  imageList[i] = exposure.getMaskedImage()
134 
135  self.combine(subCombined, imageList, stats)
136 
137  if finalScale is not None:
138  background = self.stats.run(combined)
139  self.log.info("%s: Measured background of stack is %f; adjusting to %f" %
140  (NODE, background, finalScale))
141  combined *= finalScale / background
142 
143  return combined
144 
145  def getDimensions(self, sensorRefList, inputName="postISRCCD"):
146  """Get dimensions of the inputs"""
147  dimList = []
148  for sensorRef in sensorRefList:
149  if sensorRef is None:
150  continue
151  md = sensorRef.get(inputName + "_md")
152  dimList.append(afwImage.bboxFromMetadata(md).getDimensions())
153  return getSize(dimList)
154 
155  def applyScale(self, exposure, scale=None):
156  """Apply scale to input exposure
157 
158  This implementation applies a flux scaling: the input exposure is
159  divided by the provided scale.
160  """
161  if scale is not None:
162  mi = exposure.getMaskedImage()
163  mi /= scale
164 
165  def combine(self, target, imageList, stats):
166  """!Combine multiple images
167 
168  @param target Target image to receive the combined pixels
169  @param imageList List of input images
170  @param stats Statistics control
171  """
172  images = [img for img in imageList if img is not None]
173  afwMath.statisticsStack(target, images, afwMath.Property(self.config.combine), stats)
174 
175 
176 def getSize(dimList):
177  """Determine a consistent size, given a list of image sizes"""
178  dim = set((w, h) for w, h in dimList)
179  dim.discard(None)
180  if len(dim) != 1:
181  raise RuntimeError("Inconsistent dimensions: %s" % dim)
182  return dim.pop()
183 
184 
185 def dictToTuple(dict_, keys):
186  """!Return a tuple of specific values from a dict
187 
188  This provides a hashable representation of the dict from certain keywords.
189  This can be useful for creating e.g., a tuple of the values in the DataId
190  that identify the CCD.
191 
192  @param dict_ dict to parse
193  @param keys keys to extract (order is important)
194  @return tuple of values
195  """
196  return tuple(dict_[k] for k in keys)
197 
198 
199 def getCcdIdListFromExposures(expRefList, level="sensor", ccdKeys=["ccd"]):
200  """!Determine a list of CCDs from exposure references
201 
202  This essentially inverts the exposure-level references (which
203  provides a list of CCDs for each exposure), by providing
204  a dataId list for each CCD. Consider an input list of exposures
205  [e1, e2, e3], and each exposure has CCDs c1 and c2. Then this
206  function returns:
207 
208  {(c1,): [e1c1, e2c1, e3c1], (c2,): [e1c2, e2c2, e3c2]}
209 
210  This is a dict whose keys are tuples of the identifying values of a
211  CCD (usually just the CCD number) and the values are lists of dataIds
212  for that CCD in each exposure. A missing dataId is given the value
213  None.
214 
215  @param expRefList List of data references for exposures
216  @param level Level for the butler to generate CCDs
217  @param ccdKeys DataId keywords that identify a CCD
218  @return dict of data identifier lists for each CCD;
219  keys are values of ccdKeys in order
220  """
221  expIdList = [[ccdRef.dataId for ccdRef in expRef.subItems(
222  level)] for expRef in expRefList]
223 
224  # Determine what additional keys make a CCD from an exposure
225  if len(ccdKeys) != len(set(ccdKeys)):
226  raise RuntimeError("Duplicate keys found in ccdKeys: %s" % ccdKeys)
227  ccdNames = set() # Set of tuples which are values for each of the CCDs in an exposure
228  for ccdIdList in expIdList:
229  for ccdId in ccdIdList:
230  name = dictToTuple(ccdId, ccdKeys)
231  ccdNames.add(name)
232 
233  # Turn the list of CCDs for each exposure into a list of exposures for
234  # each CCD
235  ccdLists = {}
236  for n, ccdIdList in enumerate(expIdList):
237  for ccdId in ccdIdList:
238  name = dictToTuple(ccdId, ccdKeys)
239  if name not in ccdLists:
240  ccdLists[name] = []
241  ccdLists[name].append(ccdId)
242 
243  for ccd in ccdLists:
244  # Sort the list by the dataId values (ordered by key)
245  ccdLists[ccd] = sorted(ccdLists[ccd], key=lambda dd: dictToTuple(dd, sorted(dd.keys())))
246 
247  return ccdLists
248 
249 
250 def mapToMatrix(pool, func, ccdIdLists, *args, **kwargs):
251  """Generate a matrix of results using pool.map
252 
253  The function should have the call signature:
254  func(cache, dataId, *args, **kwargs)
255 
256  We return a dict mapping 'ccd name' to a list of values for
257  each exposure.
258 
259  @param pool Process pool
260  @param func Function to call for each dataId
261  @param ccdIdLists Dict of data identifier lists for each CCD name
262  @return matrix of results
263  """
264  dataIdList = sum(ccdIdLists.values(), [])
265  resultList = pool.map(func, dataIdList, *args, **kwargs)
266  # Piece everything back together
267  data = dict((ccdName, [None] * len(expList)) for ccdName, expList in ccdIdLists.items())
268  indices = dict(sum([[(tuple(dataId.values()) if dataId is not None else None, (ccdName, expNum))
269  for expNum, dataId in enumerate(expList)]
270  for ccdName, expList in ccdIdLists.items()], []))
271  for dataId, result in zip(dataIdList, resultList):
272  if dataId is None:
273  continue
274  ccdName, expNum = indices[tuple(dataId.values())]
275  data[ccdName][expNum] = result
276  return data
277 
278 
279 class CalibIdAction(argparse.Action):
280  """Split name=value pairs and put the result in a dict"""
281 
282  def __call__(self, parser, namespace, values, option_string):
283  output = getattr(namespace, self.dest, {})
284  for nameValue in values:
285  name, sep, valueStr = nameValue.partition("=")
286  if not valueStr:
287  parser.error("%s value %s must be in form name=value" %
288  (option_string, nameValue))
289  output[name] = valueStr
290  setattr(namespace, self.dest, output)
291 
292 
293 class CalibArgumentParser(ArgumentParser):
294  """ArgumentParser for calibration construction"""
295 
296  def __init__(self, calibName, *args, **kwargs):
297  """Add a --calibId argument to the standard pipe_base argument parser"""
298  ArgumentParser.__init__(self, *args, **kwargs)
299  self.calibName = calibName
300  self.add_id_argument("--id", datasetType="raw",
301  help="input identifiers, e.g., --id visit=123 ccd=4")
302  self.add_argument("--calibId", nargs="*", action=CalibIdAction, default={},
303  help="identifiers for calib, e.g., --calibId version=1",
304  metavar="KEY=VALUE1[^VALUE2[^VALUE3...]")
305 
306  def parse_args(self, *args, **kwargs):
307  """Parse arguments
308 
309  Checks that the "--calibId" provided works.
310  """
311  namespace = ArgumentParser.parse_args(self, *args, **kwargs)
312 
313  keys = namespace.butler.getKeys(self.calibName)
314  parsed = {}
315  for name, value in namespace.calibId.items():
316  if name not in keys:
317  self.error(
318  "%s is not a relevant calib identifier key (%s)" % (name, keys))
319  parsed[name] = keys[name](value)
320  namespace.calibId = parsed
321 
322  return namespace
323 
324 
325 class CalibConfig(Config):
326  """Configuration for constructing calibs"""
327  clobber = Field(dtype=bool, default=True,
328  doc="Clobber existing processed images?")
329  isr = ConfigurableField(target=IsrTask, doc="ISR configuration")
330  dateObs = Field(dtype=str, default="dateObs",
331  doc="Key for observation date in exposure registry")
332  dateCalib = Field(dtype=str, default="calibDate",
333  doc="Key for calib date in calib registry")
334  filter = Field(dtype=str, default="filter",
335  doc="Key for filter name in exposure/calib registries")
336  combination = ConfigurableField(
337  target=CalibCombineTask, doc="Calib combination configuration")
338  ccdKeys = ListField(dtype=str, default=["ccd"],
339  doc="DataId keywords specifying a CCD")
340  visitKeys = ListField(dtype=str, default=["visit"],
341  doc="DataId keywords specifying a visit")
342  calibKeys = ListField(dtype=str, default=[],
343  doc="DataId keywords specifying a calibration")
344  doCameraImage = Field(dtype=bool, default=True, doc="Create camera overview image?")
345  binning = Field(dtype=int, default=64, doc="Binning to apply for camera image")
346 
347  def setDefaults(self):
348  self.isr.doWrite = False
349 
350 
351 class CalibTaskRunner(TaskRunner):
352  """Get parsed values into the CalibTask.run"""
353  @staticmethod
354  def getTargetList(parsedCmd, **kwargs):
355  return [dict(expRefList=parsedCmd.id.refList, butler=parsedCmd.butler, calibId=parsedCmd.calibId)]
356 
357  def __call__(self, args):
358  """Call the Task with the kwargs from getTargetList"""
359  task = self.TaskClass(config=self.config, log=self.log)
360  exitStatus = 0 # exit status for the shell
361  if self.doRaise:
362  result = task.run(**args)
363  else:
364  try:
365  result = task.run(**args)
366  except Exception as e:
367  exitStatus = 1 # n.b. The shell exit value is the number of dataRefs returning
368  # non-zero, so the actual value used here is lost
369  task.log.fatal("Failed: %s" % e)
370  traceback.print_exc(file=sys.stderr)
371 
372  if self.doReturnResults:
373  return Struct(
374  exitStatus=exitStatus,
375  args=args,
376  metadata=task.metadata,
377  result=result,
378  )
379  else:
380  return Struct(
381  exitStatus=exitStatus,
382  )
383 
385  """!Base class for constructing calibs.
386 
387  This should be subclassed for each of the required calib types.
388  The subclass should be sure to define the following class variables:
389  * _DefaultName: default name of the task, used by CmdLineTask
390  * calibName: name of the calibration data set in the butler
391  The subclass may optionally set:
392  * filterName: filter name to give the resultant calib
393  """
394  ConfigClass = CalibConfig
395  RunnerClass = CalibTaskRunner
396  filterName = None
397  calibName = None
398  exposureTime = 1.0 # sets this exposureTime in the output
399 
400  def __init__(self, *args, **kwargs):
401  """Constructor"""
402  BatchPoolTask.__init__(self, *args, **kwargs)
403  self.makeSubtask("isr")
404  self.makeSubtask("combination")
405 
406  @classmethod
407  def batchWallTime(cls, time, parsedCmd, numCores):
408  numCcds = len(parsedCmd.butler.get("camera"))
409  numExps = len(cls.RunnerClass.getTargetList(
410  parsedCmd)[0]['expRefList'])
411  numCycles = int(numCcds/float(numCores) + 0.5)
412  return time*numExps*numCycles
413 
414  @classmethod
415  def _makeArgumentParser(cls, *args, **kwargs):
416  kwargs.pop("doBatch", False)
417  return CalibArgumentParser(calibName=cls.calibName, name=cls._DefaultName, *args, **kwargs)
418 
419  def run(self, expRefList, butler, calibId):
420  """!Construct a calib from a list of exposure references
421 
422  This is the entry point, called by the TaskRunner.__call__
423 
424  Only the master node executes this method.
425 
426  @param expRefList List of data references at the exposure level
427  @param butler Data butler
428  @param calibId Identifier dict for calib
429  """
430  for expRef in expRefList:
431  self.addMissingKeys(expRef.dataId, butler, self.config.ccdKeys, 'raw')
432 
433  outputId = self.getOutputId(expRefList, calibId)
434  ccdIdLists = getCcdIdListFromExposures(
435  expRefList, level="sensor", ccdKeys=self.config.ccdKeys)
436  self.checkCcdIdLists(ccdIdLists)
437 
438  # Ensure we can generate filenames for each output
439  outputIdItemList = list(outputId.items())
440  for ccdName in ccdIdLists:
441  dataId = dict([(k, ccdName[i]) for i, k in enumerate(self.config.ccdKeys)])
442  dataId.update(outputIdItemList)
443  self.addMissingKeys(dataId, butler)
444  dataId.update(outputIdItemList)
445 
446  try:
447  butler.get(self.calibName + "_filename", dataId)
448  except Exception as e:
449  raise RuntimeError(
450  "Unable to determine output filename \"%s_filename\" from %s: %s" %
451  (self.calibName, dataId, e))
452 
453  processPool = Pool("process")
454  processPool.storeSet(butler=butler)
455 
456  # Scatter: process CCDs independently
457  data = self.scatterProcess(processPool, ccdIdLists)
458 
459  # Gather: determine scalings
460  scales = self.scale(ccdIdLists, data)
461 
462  combinePool = Pool("combine")
463  combinePool.storeSet(butler=butler)
464 
465  # Scatter: combine
466  calibs = self.scatterCombine(combinePool, outputId, ccdIdLists, scales)
467 
468  if self.config.doCameraImage:
469  camera = butler.get("camera")
470  # Convert indexing of calibs from "ccdName" to detector ID (as used by makeImageFromCamera)
471  calibs = {butler.get("postISRCCD_detector",
472  dict(zip(self.config.ccdKeys, ccdName))).getId(): calibs[ccdName]
473  for ccdName in ccdIdLists}
474 
475  try:
476  cameraImage = self.makeCameraImage(camera, outputId, calibs)
477  butler.put(cameraImage, self.calibName + "_camera", dataId)
478  except Exception as exc:
479  self.log.warn("Unable to create camera image: %s" % (exc,))
480 
481  return Struct(
482  outputId = outputId,
483  ccdIdLists = ccdIdLists,
484  scales = scales,
485  calibs = calibs,
486  processPool = processPool,
487  combinePool = combinePool,
488  )
489 
490  def getOutputId(self, expRefList, calibId):
491  """!Generate the data identifier for the output calib
492 
493  The mean date and the common filter are included, using keywords
494  from the configuration. The CCD-specific part is not included
495  in the data identifier.
496 
497  @param expRefList List of data references at exposure level
498  @param calibId Data identifier elements for the calib provided by the user
499  @return data identifier
500  """
501  midTime = 0
502  filterName = None
503  for expRef in expRefList:
504  butler = expRef.getButler()
505  dataId = expRef.dataId
506 
507  midTime += self.getMjd(butler, dataId)
508  thisFilter = self.getFilter(
509  butler, dataId) if self.filterName is None else self.filterName
510  if filterName is None:
511  filterName = thisFilter
512  elif filterName != thisFilter:
513  raise RuntimeError("Filter mismatch for %s: %s vs %s" % (
514  dataId, thisFilter, filterName))
515 
516  midTime /= len(expRefList)
517  date = str(dafBase.DateTime(
518  midTime, dafBase.DateTime.MJD).toPython().date())
519 
520  outputId = {self.config.filter: filterName,
521  self.config.dateCalib: date}
522  outputId.update(calibId)
523  return outputId
524 
525  def getMjd(self, butler, dataId, timescale=dafBase.DateTime.UTC):
526  """Determine the Modified Julian Date (MJD; in TAI) from a data identifier"""
527  if self.config.dateObs in dataId:
528  dateObs = dataId[self.config.dateObs]
529  else:
530  dateObs = butler.queryMetadata('raw', [self.config.dateObs], dataId)[0]
531  if "T" not in dateObs:
532  dateObs = dateObs + "T12:00:00.0Z"
533  elif not dateObs.endswith("Z"):
534  dateObs += "Z"
535 
536  return dafBase.DateTime(dateObs, timescale).get(dafBase.DateTime.MJD)
537 
538  def getFilter(self, butler, dataId):
539  """Determine the filter from a data identifier"""
540  filt = butler.queryMetadata('raw', [self.config.filter], dataId)[0]
541  return filt
542 
543  def addMissingKeys(self, dataId, butler, missingKeys=None, calibName=None):
544  if calibName is None:
545  calibName = self.calibName
546 
547  if missingKeys is None:
548  missingKeys = set(butler.getKeys(calibName).keys()) - set(dataId.keys())
549 
550  for k in missingKeys:
551  try:
552  v = butler.queryMetadata('raw', [k], dataId) # n.b. --id refers to 'raw'
553  except Exception as e:
554  continue
555 
556  if len(v) == 0: # failed to lookup value
557  continue
558 
559  if len(v) == 1:
560  dataId[k] = v[0]
561  else:
562  raise RuntimeError("No unique lookup for %s: %s" % (k, v))
563 
564  def updateMetadata(self, calibImage, exposureTime, darkTime=None, **kwargs):
565  """!Update the metadata from the VisitInfo
566 
567  \param calibImage The image whose metadata is to be set
568  \param exposureTime The exposure time for the image
569  \param darkTime The time since the last read (default: exposureTime)
570  """
571 
572  if darkTime is None:
573  darkTime = exposureTime # avoid warning messages when using calibration products
574 
575  visitInfo = afwImage.makeVisitInfo(exposureTime=exposureTime, darkTime=darkTime, **kwargs)
576  md = calibImage.getMetadata()
577 
578  afwImage.setVisitInfoMetadata(md, visitInfo)
579 
580  def scatterProcess(self, pool, ccdIdLists):
581  """!Scatter the processing among the nodes
582 
583  We scatter each CCD independently (exposures aren't grouped together),
584  to make full use of all available processors. This necessitates piecing
585  everything back together in the same format as ccdIdLists afterwards.
586 
587  Only the master node executes this method.
588 
589  @param pool Process pool
590  @param ccdIdLists Dict of data identifier lists for each CCD name
591  @return Dict of lists of returned data for each CCD name
592  """
593  self.log.info("Scatter processing")
594  return mapToMatrix(pool, self.process, ccdIdLists)
595 
596  def process(self, cache, ccdId, outputName="postISRCCD", **kwargs):
597  """!Process a CCD, specified by a data identifier
598 
599  After processing, optionally returns a result (produced by
600  the 'processResult' method) calculated from the processed
601  exposure. These results will be gathered by the master node,
602  and is a means for coordinated scaling of all CCDs for flats,
603  etc.
604 
605  Only slave nodes execute this method.
606 
607  @param cache Process pool cache
608  @param ccdId Data identifier for CCD
609  @param outputName Output dataset name for butler
610  @return result from 'processResult'
611  """
612  if ccdId is None:
613  self.log.warn("Null identifier received on %s" % NODE)
614  return None
615  sensorRef = getDataRef(cache.butler, ccdId)
616  if self.config.clobber or not sensorRef.datasetExists(outputName):
617  self.log.info("Processing %s on %s" % (ccdId, NODE))
618  try:
619  exposure = self.processSingle(sensorRef, **kwargs)
620  except Exception as e:
621  self.log.warn("Unable to process %s: %s" % (ccdId, e))
622  raise
623  return None
624  self.processWrite(sensorRef, exposure)
625  else:
626  self.log.info(
627  "Using previously persisted processed exposure for %s" % (sensorRef.dataId,))
628  exposure = sensorRef.get(outputName)
629  return self.processResult(exposure)
630 
631  def processSingle(self, dataRef):
632  """Process a single CCD, specified by a data reference
633 
634  Generally, this simply means doing ISR.
635 
636  Only slave nodes execute this method.
637  """
638  return self.isr.runDataRef(dataRef).exposure
639 
640  def processWrite(self, dataRef, exposure, outputName="postISRCCD"):
641  """!Write the processed CCD
642 
643  We need to write these out because we can't hold them all in
644  memory at once.
645 
646  Only slave nodes execute this method.
647 
648  @param dataRef Data reference
649  @param exposure CCD exposure to write
650  @param outputName Output dataset name for butler.
651  """
652  dataRef.put(exposure, outputName)
653 
654  def processResult(self, exposure):
655  """Extract processing results from a processed exposure
656 
657  This method generates what is gathered by the master node.
658  This can be a background measurement or similar for scaling
659  flat-fields. It must be picklable!
660 
661  Only slave nodes execute this method.
662  """
663  return None
664 
665  def scale(self, ccdIdLists, data):
666  """!Determine scaling across CCDs and exposures
667 
668  This is necessary mainly for flats, so as to determine a
669  consistent scaling across the entire focal plane. This
670  implementation is simply a placeholder.
671 
672  Only the master node executes this method.
673 
674  @param ccdIdLists Dict of data identifier lists for each CCD tuple
675  @param data Dict of lists of returned data for each CCD tuple
676  @return dict of Struct(ccdScale: scaling for CCD,
677  expScales: scaling for each exposure
678  ) for each CCD tuple
679  """
680  self.log.info("Scale on %s" % NODE)
681  return dict((name, Struct(ccdScale=None, expScales=[None] * len(ccdIdLists[name])))
682  for name in ccdIdLists)
683 
684  def scatterCombine(self, pool, outputId, ccdIdLists, scales):
685  """!Scatter the combination of exposures across multiple nodes
686 
687  In this case, we can only scatter across as many nodes as
688  there are CCDs.
689 
690  Only the master node executes this method.
691 
692  @param pool Process pool
693  @param outputId Output identifier (exposure part only)
694  @param ccdIdLists Dict of data identifier lists for each CCD name
695  @param scales Dict of structs with scales, for each CCD name
696  @param dict of binned images
697  """
698  self.log.info("Scatter combination")
699  data = [Struct(ccdName=ccdName, ccdIdList=ccdIdLists[ccdName], scales=scales[ccdName]) for
700  ccdName in ccdIdLists]
701  images = pool.map(self.combine, data, outputId)
702  return dict(zip(ccdIdLists.keys(), images))
703 
704  def getFullyQualifiedOutputId(self, ccdName, butler, outputId):
705  """Get fully-qualified output data identifier
706 
707  We may need to look up keys that aren't in the output dataId.
708 
709  @param ccdName Name tuple for CCD
710  @param butler Data butler
711  @param outputId Data identifier for combined image (exposure part only)
712  @return fully-qualified output dataId
713  """
714  fullOutputId = {k: ccdName[i] for i, k in enumerate(self.config.ccdKeys)}
715  fullOutputId.update(outputId)
716  self.addMissingKeys(fullOutputId, butler)
717  fullOutputId.update(outputId) # must be after the call to queryMetadata in 'addMissingKeys'
718  return fullOutputId
719 
720  def combine(self, cache, struct, outputId):
721  """!Combine multiple exposures of a particular CCD and write the output
722 
723  Only the slave nodes execute this method.
724 
725  @param cache Process pool cache
726  @param struct Parameters for the combination, which has the following components:
727  * ccdName Name tuple for CCD
728  * ccdIdList List of data identifiers for combination
729  * scales Scales to apply (expScales are scalings for each exposure,
730  ccdScale is final scale for combined image)
731  @param outputId Data identifier for combined image (exposure part only)
732  @return binned calib image
733  """
734  outputId = self.getFullyQualifiedOutputId(struct.ccdName, cache.butler, outputId)
735  dataRefList = [getDataRef(cache.butler, dataId) if dataId is not None else None for
736  dataId in struct.ccdIdList]
737  self.log.info("Combining %s on %s" % (outputId, NODE))
738  calib = self.combination.run(dataRefList, expScales=struct.scales.expScales,
739  finalScale=struct.scales.ccdScale)
740 
741  if not hasattr(calib, "getMetadata"):
742  if hasattr(calib, "getVariance"):
743  calib = afwImage.makeExposure(calib)
744  else:
745  calib = afwImage.DecoratedImageF(calib.getImage()) # n.b. hardwires "F" for the output type
746 
747  self.updateMetadata(calib, self.exposureTime)
748 
749  self.recordCalibInputs(cache.butler, calib,
750  struct.ccdIdList, outputId)
751 
752  self.interpolateNans(calib)
753 
754  self.write(cache.butler, calib, outputId)
755 
756  return afwMath.binImage(calib.getImage(), self.config.binning)
757 
758  def recordCalibInputs(self, butler, calib, dataIdList, outputId):
759  """!Record metadata including the inputs and creation details
760 
761  This metadata will go into the FITS header.
762 
763  @param butler Data butler
764  @param calib Combined calib exposure.
765  @param dataIdList List of data identifiers for calibration inputs
766  @param outputId Data identifier for output
767  """
768  header = calib.getMetadata()
769  header.add("OBSTYPE", self.calibName) # Used by ingestCalibs.py
770 
771  # date, time, host, and root
772  now = time.localtime()
773  header.add("CALIB_CREATION_DATE", time.strftime("%Y-%m-%d", now))
774  header.add("CALIB_CREATION_TIME", time.strftime("%X %Z", now))
775  # add date-obs as its absence upsets ExposureInfo; use the mean date that the calibs were taken
776  header.add("DATE-OBS", "%sT00:00:00.00" % outputId[self.config.dateCalib])
777 
778  # Inputs
779  visits = [str(dictToTuple(dataId, self.config.visitKeys)) for dataId in dataIdList if
780  dataId is not None]
781  for i, v in enumerate(sorted(set(visits))):
782  header.add("CALIB_INPUT_%d" % (i,), v)
783 
784  header.add("CALIB_ID", " ".join("%s=%s" % (key, value)
785  for key, value in outputId.items()))
786  checksum(calib, header)
787 
788  def interpolateNans(self, image):
789  """Interpolate over NANs in the combined image
790 
791  NANs can result from masked areas on the CCD. We don't want them getting
792  into our science images, so we replace them with the median of the image.
793  """
794  if hasattr(image, "getMaskedImage"): # Deal with Exposure vs Image
795  self.interpolateNans(image.getMaskedImage().getVariance())
796  image = image.getMaskedImage().getImage()
797  if hasattr(image, "getImage"): # Deal with DecoratedImage or MaskedImage vs Image
798  image = image.getImage()
799  array = image.getArray()
800  bad = np.isnan(array)
801  array[bad] = np.median(array[np.logical_not(bad)])
802 
803  def write(self, butler, exposure, dataId):
804  """!Write the final combined calib
805 
806  Only the slave nodes execute this method
807 
808  @param butler Data butler
809  @param exposure CCD exposure to write
810  @param dataId Data identifier for output
811  """
812  self.log.info("Writing %s on %s" % (dataId, NODE))
813  butler.put(exposure, self.calibName, dataId)
814 
815  def makeCameraImage(self, camera, dataId, calibs):
816  """!Create and write an image of the entire camera
817 
818  This is useful for judging the quality or getting an overview of
819  the features of the calib.
820 
821  @param camera Camera object
822  @param dataId Data identifier for output
823  @param calibs Dict mapping CCD detector ID to calib image
824  """
825  return makeCameraImage(camera, calibs, self.config.binning)
826 
827  def checkCcdIdLists(self, ccdIdLists):
828  """Check that the list of CCD dataIds is consistent
829 
830  @param ccdIdLists Dict of data identifier lists for each CCD name
831  @return Number of exposures, number of CCDs
832  """
833  visitIdLists = collections.defaultdict(list)
834  for ccdName in ccdIdLists:
835  for dataId in ccdIdLists[ccdName]:
836  visitName = dictToTuple(dataId, self.config.visitKeys)
837  visitIdLists[visitName].append(dataId)
838 
839  numExps = set(len(expList) for expList in ccdIdLists.values())
840  numCcds = set(len(ccdList) for ccdList in visitIdLists.values())
841 
842  if len(numExps) != 1 or len(numCcds) != 1:
843  # Presumably a visit somewhere doesn't have the full complement available.
844  # Dump the information so the user can figure it out.
845  self.log.warn("Number of visits for each CCD: %s",
846  {ccdName: len(ccdIdLists[ccdName]) for ccdName in ccdIdLists})
847  self.log.warn("Number of CCDs for each visit: %s",
848  {vv: len(visitIdLists[vv]) for vv in visitIdLists})
849  raise RuntimeError("Inconsistent number of exposures/CCDs")
850 
851  return numExps.pop(), numCcds.pop()
852 
853 
855  """Configuration for bias construction.
856 
857  No changes required compared to the base class, but
858  subclassed for distinction.
859  """
860  pass
861 
862 
863 class BiasTask(CalibTask):
864  """Bias construction"""
865  ConfigClass = BiasConfig
866  _DefaultName = "bias"
867  calibName = "bias"
868  filterName = "NONE" # Sets this filter name in the output
869  exposureTime = 0.0 # sets this exposureTime in the output
870 
871  @classmethod
872  def applyOverrides(cls, config):
873  """Overrides to apply for bias construction"""
874  config.isr.doBias = False
875  config.isr.doDark = False
876  config.isr.doFlat = False
877  config.isr.doFringe = False
878 
879 
881  """Configuration for dark construction"""
882  doRepair = Field(dtype=bool, default=True, doc="Repair artifacts?")
883  psfFwhm = Field(dtype=float, default=3.0, doc="Repair PSF FWHM (pixels)")
884  psfSize = Field(dtype=int, default=21, doc="Repair PSF size (pixels)")
885  crGrow = Field(dtype=int, default=2, doc="Grow radius for CR (pixels)")
886  repair = ConfigurableField(
887  target=RepairTask, doc="Task to repair artifacts")
888 
889  def setDefaults(self):
890  CalibConfig.setDefaults(self)
891  self.combination.mask.append("CR")
892 
893 
895  """Dark construction
896 
897  The only major difference from the base class is a cosmic-ray
898  identification stage, and dividing each image by the dark time
899  to generate images of the dark rate.
900  """
901  ConfigClass = DarkConfig
902  _DefaultName = "dark"
903  calibName = "dark"
904  filterName = "NONE" # Sets this filter name in the output
905 
906  def __init__(self, *args, **kwargs):
907  CalibTask.__init__(self, *args, **kwargs)
908  self.makeSubtask("repair")
909 
910  @classmethod
911  def applyOverrides(cls, config):
912  """Overrides to apply for dark construction"""
913  config.isr.doDark = False
914  config.isr.doFlat = False
915  config.isr.doFringe = False
916 
917  def processSingle(self, sensorRef):
918  """Process a single CCD
919 
920  Besides the regular ISR, also masks cosmic-rays and divides each
921  processed image by the dark time to generate images of the dark rate.
922  The dark time is provided by the 'getDarkTime' method.
923  """
924  exposure = CalibTask.processSingle(self, sensorRef)
925 
926  if self.config.doRepair:
927  psf = measAlg.DoubleGaussianPsf(self.config.psfSize, self.config.psfSize,
928  self.config.psfFwhm/(2*math.sqrt(2*math.log(2))))
929  exposure.setPsf(psf)
930  self.repair.run(exposure, keepCRs=False)
931  if self.config.crGrow > 0:
932  mask = exposure.getMaskedImage().getMask().clone()
933  mask &= mask.getPlaneBitMask("CR")
934  fpSet = afwDet.FootprintSet(
935  mask, afwDet.Threshold(0.5))
936  fpSet = afwDet.FootprintSet(fpSet, self.config.crGrow, True)
937  fpSet.setMask(exposure.getMaskedImage().getMask(), "CR")
938 
939  mi = exposure.getMaskedImage()
940  mi /= self.getDarkTime(exposure)
941  return exposure
942 
943  def getDarkTime(self, exposure):
944  """Retrieve the dark time for an exposure"""
945  darkTime = exposure.getInfo().getVisitInfo().getDarkTime()
946  if not np.isfinite(darkTime):
947  raise RuntimeError("Non-finite darkTime")
948  return darkTime
949 
950 
952  """Configuration for flat construction"""
953  iterations = Field(dtype=int, default=10,
954  doc="Number of iterations for scale determination")
955  stats = ConfigurableField(target=CalibStatsTask,
956  doc="Background statistics configuration")
957 
958 
960  """Flat construction
961 
962  The principal change from the base class involves gathering the background
963  values from each image and using them to determine the scalings for the final
964  combination.
965  """
966  ConfigClass = FlatConfig
967  _DefaultName = "flat"
968  calibName = "flat"
969 
970  @classmethod
971  def applyOverrides(cls, config):
972  """Overrides for flat construction"""
973  config.isr.doFlat = False
974  config.isr.doFringe = False
975 
976  def __init__(self, *args, **kwargs):
977  CalibTask.__init__(self, *args, **kwargs)
978  self.makeSubtask("stats")
979 
980  def processResult(self, exposure):
981  return self.stats.run(exposure)
982 
983  def scale(self, ccdIdLists, data):
984  """Determine the scalings for the final combination
985 
986  We have a matrix B_ij = C_i E_j, where C_i is the relative scaling
987  of one CCD to all the others in an exposure, and E_j is the scaling
988  of the exposure. We convert everything to logarithms so we can work
989  with a linear system. We determine the C_i and E_j from B_ij by iteration,
990  under the additional constraint that the average CCD scale is unity.
991 
992  This algorithm comes from Eugene Magnier and Pan-STARRS.
993  """
994  assert len(ccdIdLists.values()) > 0, "No successful CCDs"
995  lengths = set([len(expList) for expList in ccdIdLists.values()])
996  assert len(
997  lengths) == 1, "Number of successful exposures for each CCD differs"
998  assert tuple(lengths)[0] > 0, "No successful exposures"
999  # Format background measurements into a matrix
1000  indices = dict((name, i) for i, name in enumerate(ccdIdLists))
1001  bgMatrix = np.array([[0.0] * len(expList)
1002  for expList in ccdIdLists.values()])
1003  for name in ccdIdLists:
1004  i = indices[name]
1005  bgMatrix[i] = [
1006  d if d is not None else np.nan for d in data[name]]
1007 
1008  numpyPrint = np.get_printoptions()
1009  np.set_printoptions(threshold=np.inf)
1010  self.log.info("Input backgrounds: %s" % bgMatrix)
1011 
1012  # Flat-field scaling
1013  numCcds = len(ccdIdLists)
1014  numExps = bgMatrix.shape[1]
1015  # log(Background) for each exposure/component
1016  bgMatrix = np.log(bgMatrix)
1017  bgMatrix = np.ma.masked_array(bgMatrix, np.isnan(bgMatrix))
1018  # Initial guess at log(scale) for each component
1019  compScales = np.zeros(numCcds)
1020  expScales = np.array(
1021  [(bgMatrix[:, i0] - compScales).mean() for i0 in range(numExps)])
1022 
1023  for iterate in range(self.config.iterations):
1024  compScales = np.array(
1025  [(bgMatrix[i1, :] - expScales).mean() for i1 in range(numCcds)])
1026  expScales = np.array(
1027  [(bgMatrix[:, i2] - compScales).mean() for i2 in range(numExps)])
1028 
1029  avgScale = np.average(np.exp(compScales))
1030  compScales -= np.log(avgScale)
1031  self.log.debug("Iteration %d exposure scales: %s",
1032  iterate, np.exp(expScales))
1033  self.log.debug("Iteration %d component scales: %s",
1034  iterate, np.exp(compScales))
1035 
1036  expScales = np.array(
1037  [(bgMatrix[:, i3] - compScales).mean() for i3 in range(numExps)])
1038 
1039  if np.any(np.isnan(expScales)):
1040  raise RuntimeError("Bad exposure scales: %s --> %s" %
1041  (bgMatrix, expScales))
1042 
1043  expScales = np.exp(expScales)
1044  compScales = np.exp(compScales)
1045 
1046  self.log.info("Exposure scales: %s" % expScales)
1047  self.log.info("Component relative scaling: %s" % compScales)
1048  np.set_printoptions(**numpyPrint)
1049 
1050  return dict((ccdName, Struct(ccdScale=compScales[indices[ccdName]], expScales=expScales))
1051  for ccdName in ccdIdLists)
1052 
1053 
1055  """Configuration for fringe construction"""
1056  stats = ConfigurableField(target=CalibStatsTask,
1057  doc="Background statistics configuration")
1058  subtractBackground = ConfigurableField(target=measAlg.SubtractBackgroundTask,
1059  doc="Background configuration")
1060  detection = ConfigurableField(
1061  target=measAlg.SourceDetectionTask, doc="Detection configuration")
1062  detectSigma = Field(dtype=float, default=1.0,
1063  doc="Detection PSF gaussian sigma")
1064 
1065  def setDefaults(self):
1066  CalibConfig.setDefaults(self)
1067  self.detection.reEstimateBackground = False
1068 
1069 
1071  """Fringe construction task
1072 
1073  The principal change from the base class is that the images are
1074  background-subtracted and rescaled by the background.
1075 
1076  XXX This is probably not right for a straight-up combination, as we
1077  are currently doing, since the fringe amplitudes need not scale with
1078  the continuum.
1079 
1080  XXX Would like to have this do PCA and generate multiple images, but
1081  that will take a bit of work with the persistence code.
1082  """
1083  ConfigClass = FringeConfig
1084  _DefaultName = "fringe"
1085  calibName = "fringe"
1086 
1087  @classmethod
1088  def applyOverrides(cls, config):
1089  """Overrides for fringe construction"""
1090  config.isr.doFringe = False
1091 
1092  def __init__(self, *args, **kwargs):
1093  CalibTask.__init__(self, *args, **kwargs)
1094  self.makeSubtask("detection")
1095  self.makeSubtask("stats")
1096  self.makeSubtask("subtractBackground")
1097 
1098  def processSingle(self, sensorRef):
1099  """Subtract the background and normalise by the background level"""
1100  exposure = CalibTask.processSingle(self, sensorRef)
1101  bgLevel = self.stats.run(exposure)
1102  self.subtractBackground.run(exposure)
1103  mi = exposure.getMaskedImage()
1104  mi /= bgLevel
1105  footprintSets = self.detection.detectFootprints(
1106  exposure, sigma=self.config.detectSigma)
1107  mask = exposure.getMaskedImage().getMask()
1108  detected = 1 << mask.addMaskPlane("DETECTED")
1109  for fpSet in (footprintSets.positive, footprintSets.negative):
1110  if fpSet is not None:
1111  afwDet.setMaskFromFootprintList(
1112  mask, fpSet.getFootprints(), detected)
1113  return exposure
1114 
1115 
1117  """Configuration for sky frame construction"""
1118  detection = ConfigurableField(target=measAlg.SourceDetectionTask, doc="Detection configuration")
1119  detectSigma = Field(dtype=float, default=2.0, doc="Detection PSF gaussian sigma")
1120  subtractBackground = ConfigurableField(target=measAlg.SubtractBackgroundTask,
1121  doc="Regular-scale background configuration, for object detection")
1122  largeScaleBackground = ConfigField(dtype=FocalPlaneBackgroundConfig,
1123  doc="Large-scale background configuration")
1124  sky = ConfigurableField(target=SkyMeasurementTask, doc="Sky measurement")
1125  maskThresh = Field(dtype=float, default=3.0, doc="k-sigma threshold for masking pixels")
1126  mask = ListField(dtype=str, default=["BAD", "SAT", "DETECTED", "NO_DATA"],
1127  doc="Mask planes to consider as contaminated")
1128 
1129 
1131  """Task for sky frame construction
1132 
1133  The sky frame is a (relatively) small-scale background
1134  model, the response of the camera to the sky.
1135 
1136  To construct, we first remove a large-scale background (e.g., caused
1137  by moonlight) which may vary from image to image. Then we construct a
1138  model of the sky, which is essentially a binned version of the image
1139  (important configuration parameters: sky.background.[xy]BinSize).
1140  It is these models which are coadded to yield the sky frame.
1141  """
1142  ConfigClass = SkyConfig
1143  _DefaultName = "sky"
1144  calibName = "sky"
1145 
1146  def __init__(self, *args, **kwargs):
1147  CalibTask.__init__(self, *args, **kwargs)
1148  self.makeSubtask("detection")
1149  self.makeSubtask("subtractBackground")
1150  self.makeSubtask("sky")
1151 
1152  def scatterProcess(self, pool, ccdIdLists):
1153  """!Scatter the processing among the nodes
1154 
1155  Only the master node executes this method, assigning work to the
1156  slaves.
1157 
1158  We measure and subtract off a large-scale background model across
1159  all CCDs, which requires a scatter/gather. Then we process the
1160  individual CCDs, subtracting the large-scale background model and
1161  the residual background model measured. These residuals will be
1162  combined for the sky frame.
1163 
1164  @param pool Process pool
1165  @param ccdIdLists Dict of data identifier lists for each CCD name
1166  @return Dict of lists of returned data for each CCD name
1167  """
1168  self.log.info("Scatter processing")
1169 
1170  numExps = set(len(expList) for expList in ccdIdLists.values()).pop()
1171 
1172  # First subtract off general gradients to make all the exposures look similar.
1173  # We want to preserve the common small-scale structure, which we will coadd.
1174  bgModelList = mapToMatrix(pool, self.measureBackground, ccdIdLists)
1175 
1176  backgrounds = {}
1177  scales = {}
1178  for exp in range(numExps):
1179  bgModels = [bgModelList[ccdName][exp] for ccdName in ccdIdLists]
1180  visit = set(tuple(ccdIdLists[ccdName][exp][key] for key in sorted(self.config.visitKeys)) for
1181  ccdName in ccdIdLists)
1182  assert len(visit) == 1
1183  visit = visit.pop()
1184  bgModel = bgModels[0]
1185  for bg in bgModels[1:]:
1186  bgModel.merge(bg)
1187  self.log.info("Background model min/max for visit %s: %f %f", visit,
1188  np.min(bgModel.getStatsImage().getArray()),
1189  np.max(bgModel.getStatsImage().getArray()))
1190  backgrounds[visit] = bgModel
1191  scales[visit] = np.median(bgModel.getStatsImage().getArray())
1192 
1193  return mapToMatrix(pool, self.process, ccdIdLists, backgrounds=backgrounds, scales=scales)
1194 
1195  def measureBackground(self, cache, dataId):
1196  """!Measure background model for CCD
1197 
1198  This method is executed by the slaves.
1199 
1200  The background models for all CCDs in an exposure will be
1201  combined to form a full focal-plane background model.
1202 
1203  @param cache Process pool cache
1204  @param dataId Data identifier
1205  @return Bcakground model
1206  """
1207  dataRef = getDataRef(cache.butler, dataId)
1208  exposure = self.processSingleBackground(dataRef)
1209 
1210  # NAOJ prototype smoothed and then combined the entire image, but it shouldn't be any different
1211  # to bin and combine the binned images except that there's fewer pixels to worry about.
1212  config = self.config.largeScaleBackground
1213  camera = dataRef.get("camera")
1214  bgModel = FocalPlaneBackground.fromCamera(config, camera)
1215  bgModel.addCcd(exposure)
1216  return bgModel
1217 
1218  def processSingleBackground(self, dataRef):
1219  """!Process a single CCD for the background
1220 
1221  This method is executed by the slaves.
1222 
1223  Because we're interested in the background, we detect and mask astrophysical
1224  sources, and pixels above the noise level.
1225 
1226  @param dataRef Data reference for CCD.
1227  @return processed exposure
1228  """
1229  if not self.config.clobber and dataRef.datasetExists("postISRCCD"):
1230  return dataRef.get("postISRCCD")
1231  exposure = CalibTask.processSingle(self, dataRef)
1232 
1233  # Detect sources. Requires us to remove the background; we'll restore it later.
1234  bgTemp = self.subtractBackground.run(exposure).background
1235  footprints = self.detection.detectFootprints(exposure, sigma=self.config.detectSigma)
1236  image = exposure.getMaskedImage()
1237  if footprints.background is not None:
1238  image += footprints.background.getImage()
1239 
1240  # Mask high pixels
1241  variance = image.getVariance()
1242  noise = np.sqrt(np.median(variance.getArray()))
1243  isHigh = image.getImage().getArray() > self.config.maskThresh*noise
1244  image.getMask().getArray()[isHigh] |= image.getMask().getPlaneBitMask("DETECTED")
1245 
1246  # Restore the background: it's what we want!
1247  image += bgTemp.getImage()
1248 
1249  # Set detected/bad pixels to background to ensure they don't corrupt the background
1250  maskVal = image.getMask().getPlaneBitMask(self.config.mask)
1251  isBad = image.getMask().getArray() & maskVal > 0
1252  bgLevel = np.median(image.getImage().getArray()[~isBad])
1253  image.getImage().getArray()[isBad] = bgLevel
1254  dataRef.put(exposure, "postISRCCD")
1255  return exposure
1256 
1257  def processSingle(self, dataRef, backgrounds, scales):
1258  """Process a single CCD, specified by a data reference
1259 
1260  We subtract the appropriate focal plane background model,
1261  divide by the appropriate scale and measure the background.
1262 
1263  Only slave nodes execute this method.
1264 
1265  @param dataRef Data reference for single CCD
1266  @param backgrounds Background model for each visit
1267  @param scales Scales for each visit
1268  @return Processed exposure
1269  """
1270  visit = tuple(dataRef.dataId[key] for key in sorted(self.config.visitKeys))
1271  exposure = dataRef.get("postISRCCD", immediate=True)
1272  image = exposure.getMaskedImage()
1273  detector = exposure.getDetector()
1274  bbox = image.getBBox()
1275 
1276  bgModel = backgrounds[visit]
1277  bg = bgModel.toCcdBackground(detector, bbox)
1278  image -= bg.getImage()
1279  image /= scales[visit]
1280 
1281  bg = self.sky.measureBackground(exposure.getMaskedImage())
1282  dataRef.put(bg, "icExpBackground")
1283  return exposure
1284 
1285  def combine(self, cache, struct, outputId):
1286  """!Combine multiple background models of a particular CCD and write the output
1287 
1288  Only the slave nodes execute this method.
1289 
1290  @param cache Process pool cache
1291  @param struct Parameters for the combination, which has the following components:
1292  * ccdName Name tuple for CCD
1293  * ccdIdList List of data identifiers for combination
1294  @param outputId Data identifier for combined image (exposure part only)
1295  @return binned calib image
1296  """
1297  outputId = self.getFullyQualifiedOutputId(struct.ccdName, cache.butler, outputId)
1298  dataRefList = [getDataRef(cache.butler, dataId) if dataId is not None else None for
1299  dataId in struct.ccdIdList]
1300  self.log.info("Combining %s on %s" % (outputId, NODE))
1301  bgList = [dataRef.get("icExpBackground", immediate=True).clone() for dataRef in dataRefList]
1302 
1303  bgExp = self.sky.averageBackgrounds(bgList)
1304 
1305  self.recordCalibInputs(cache.butler, bgExp, struct.ccdIdList, outputId)
1306  cache.butler.put(bgExp, "sky", outputId)
1307  return afwMath.binImage(self.sky.exposureToBackground(bgExp).getImage(), self.config.binning)
def processWrite(self, dataRef, exposure, outputName="postISRCCD")
Write the processed CCD.
def scatterCombine(self, pool, outputId, ccdIdLists, scales)
Scatter the combination of exposures across multiple nodes.
def run(self, exposureOrImage)
Measure a particular statistic on an image (of some sort).
def checksum(obj, header=None, sumType="MD5")
Calculate a checksum of an object.
Definition: checksum.py:29
def getCcdIdListFromExposures(expRefList, level="sensor", ccdKeys=["ccd"])
Determine a list of CCDs from exposure references.
def run(self, expRefList, butler, calibId)
Construct a calib from a list of exposure references.
def getFullyQualifiedOutputId(self, ccdName, butler, outputId)
def __call__(self, parser, namespace, values, option_string)
def dictToTuple(dict_, keys)
Return a tuple of specific values from a dict.
def getDataRef(butler, dataId, datasetType="raw")
Definition: utils.py:17
def updateMetadata(self, calibImage, exposureTime, darkTime=None, kwargs)
Update the metadata from the VisitInfo.
def process(self, cache, ccdId, outputName="postISRCCD", kwargs)
Process a CCD, specified by a data identifier.
def processSingle(self, dataRef, backgrounds, scales)
def makeCameraImage(self, camera, dataId, calibs)
Create and write an image of the entire camera.
def combine(self, target, imageList, stats)
Combine multiple images.
def getMjd(self, butler, dataId, timescale=dafBase.DateTime.UTC)
def combine(self, cache, struct, outputId)
Combine multiple exposures of a particular CCD and write the output.
def write(self, butler, exposure, dataId)
Write the final combined calib.
def combine(self, cache, struct, outputId)
Combine multiple background models of a particular CCD and write the output.
def measureBackground(self, cache, dataId)
Measure background model for CCD.
def recordCalibInputs(self, butler, calib, dataIdList, outputId)
Record metadata including the inputs and creation details.
def scatterProcess(self, pool, ccdIdLists)
Scatter the processing among the nodes.
def run(self, sensorRefList, expScales=None, finalScale=None, inputName="postISRCCD")
Combine calib images for a single sensor.
def getOutputId(self, expRefList, calibId)
Generate the data identifier for the output calib.
def mapToMatrix(pool, func, ccdIdLists, args, kwargs)
def scale(self, ccdIdLists, data)
Determine scaling across CCDs and exposures.
def addMissingKeys(self, dataId, butler, missingKeys=None, calibName=None)
def getDimensions(self, sensorRefList, inputName="postISRCCD")
Base class for constructing calibs.
def processSingleBackground(self, dataRef)
Process a single CCD for the background.
def scatterProcess(self, pool, ccdIdLists)
Scatter the processing among the nodes.
def batchWallTime(cls, time, parsedCmd, numCores)