lsst.pipe.drivers  14.0-6-gdf47936+2
constructCalibs.py
Go to the documentation of this file.
1 from __future__ import absolute_import, division, print_function
2 import sys
3 import math
4 import time
5 import argparse
6 import traceback
7 
8 import numpy as np
9 from builtins import zip
10 from builtins import range
11 
12 from lsst.pex.config import Config, ConfigurableField, Field, ListField
13 from lsst.pipe.base import Task, Struct, TaskRunner, ArgumentParser
14 import lsst.daf.base as dafBase
15 import lsst.afw.math as afwMath
16 import lsst.afw.geom as afwGeom
17 import lsst.afw.detection as afwDet
18 import lsst.afw.image as afwImage
19 from lsst.afw.image import VisitInfo
20 import lsst.meas.algorithms as measAlg
21 from lsst.pipe.tasks.repair import RepairTask
22 from lsst.ip.isr import IsrTask
23 
24 from lsst.ctrl.pool.parallel import BatchPoolTask
25 from lsst.ctrl.pool.pool import Pool, NODE
26 
27 from .checksum import checksum
28 from .utils import getDataRef
29 
30 
31 class CalibStatsConfig(Config):
32  """Parameters controlling the measurement of background statistics"""
33  stat = Field(doc="Statistic to use to estimate background (from lsst.afw.math)", dtype=int,
34  default=int(afwMath.MEANCLIP))
35  clip = Field(doc="Clipping threshold for background",
36  dtype=float, default=3.0)
37  nIter = Field(doc="Clipping iterations for background",
38  dtype=int, default=3)
39  mask = ListField(doc="Mask planes to reject",
40  dtype=str, default=["DETECTED", "BAD"])
41 
42 
43 class CalibStatsTask(Task):
44  """Measure statistics on the background
45 
46  This can be useful for scaling the background, e.g., for flats and fringe frames.
47  """
48  ConfigClass = CalibStatsConfig
49 
50  def run(self, exposureOrImage):
51  """!Measure a particular statistic on an image (of some sort).
52 
53  @param exposureOrImage Exposure, MaskedImage or Image.
54  @return Value of desired statistic
55  """
56  stats = afwMath.StatisticsControl(self.config.clip, self.config.nIter,
57  afwImage.Mask.getPlaneBitMask(self.config.mask))
58  try:
59  image = exposureOrImage.getMaskedImage()
60  except:
61  try:
62  image = exposureOrImage.getImage()
63  except:
64  image = exposureOrImage
65 
66  return afwMath.makeStatistics(image, self.config.stat, stats).getValue()
67 
68 
69 class CalibCombineConfig(Config):
70  """Configuration for combining calib images"""
71  rows = Field(doc="Number of rows to read at a time",
72  dtype=int, default=512)
73  mask = ListField(doc="Mask planes to respect", dtype=str,
74  default=["SAT", "DETECTED", "INTRP"])
75  combine = Field(doc="Statistic to use for combination (from lsst.afw.math)", dtype=int,
76  default=int(afwMath.MEANCLIP))
77  clip = Field(doc="Clipping threshold for combination",
78  dtype=float, default=3.0)
79  nIter = Field(doc="Clipping iterations for combination",
80  dtype=int, default=3)
81  stats = ConfigurableField(target=CalibStatsTask,
82  doc="Background statistics configuration")
83 
84 
85 class CalibCombineTask(Task):
86  """Task to combine calib images"""
87  ConfigClass = CalibCombineConfig
88 
89  def __init__(self, *args, **kwargs):
90  Task.__init__(self, *args, **kwargs)
91  self.makeSubtask("stats")
92 
93  def run(self, sensorRefList, expScales=None, finalScale=None, inputName="postISRCCD"):
94  """!Combine calib images for a single sensor
95 
96  @param sensorRefList List of data references to combine (for a single sensor)
97  @param expScales List of scales to apply for each exposure
98  @param finalScale Desired scale for final combined image
99  @param inputName Data set name for inputs
100  @return combined image
101  """
102  width, height = self.getDimensions(sensorRefList)
103  stats = afwMath.StatisticsControl(self.config.clip, self.config.nIter,
104  afwImage.Mask.getPlaneBitMask(self.config.mask))
105 
106  # Combine images
107  combined = afwImage.MaskedImageF(width, height)
108  numImages = len(sensorRefList)
109  imageList = [None]*numImages
110  for start in range(0, height, self.config.rows):
111  rows = min(self.config.rows, height - start)
112  box = afwGeom.Box2I(afwGeom.Point2I(0, start),
113  afwGeom.Extent2I(width, rows))
114  subCombined = combined.Factory(combined, box)
115 
116  for i, sensorRef in enumerate(sensorRefList):
117  if sensorRef is None:
118  imageList[i] = None
119  continue
120  exposure = sensorRef.get(inputName + "_sub", bbox=box)
121  if expScales is not None:
122  self.applyScale(exposure, expScales[i])
123  imageList[i] = exposure.getMaskedImage()
124 
125  self.combine(subCombined, imageList, stats)
126 
127  if finalScale is not None:
128  background = self.stats.run(combined)
129  self.log.info("%s: Measured background of stack is %f; adjusting to %f" %
130  (NODE, background, finalScale))
131  combined *= finalScale / background
132 
133  return combined
134 
135  def getDimensions(self, sensorRefList, inputName="postISRCCD"):
136  """Get dimensions of the inputs"""
137  dimList = []
138  for sensorRef in sensorRefList:
139  if sensorRef is None:
140  continue
141  md = sensorRef.get(inputName + "_md")
142  dimList.append(afwImage.bboxFromMetadata(md).getDimensions())
143  return getSize(dimList)
144 
145  def applyScale(self, exposure, scale=None):
146  """Apply scale to input exposure
147 
148  This implementation applies a flux scaling: the input exposure is
149  divided by the provided scale.
150  """
151  if scale is not None:
152  mi = exposure.getMaskedImage()
153  mi /= scale
154 
155  def combine(self, target, imageList, stats):
156  """!Combine multiple images
157 
158  @param target Target image to receive the combined pixels
159  @param imageList List of input images
160  @param stats Statistics control
161  """
162  images = [img for img in imageList if img is not None]
163  afwMath.statisticsStack(target, images, afwMath.Property(self.config.combine), stats)
164 
165 
166 def getSize(dimList):
167  """Determine a consistent size, given a list of image sizes"""
168  dim = set((w, h) for w, h in dimList)
169  dim.discard(None)
170  if len(dim) != 1:
171  raise RuntimeError("Inconsistent dimensions: %s" % dim)
172  return dim.pop()
173 
174 
175 def dictToTuple(dict_, keys):
176  """!Return a tuple of specific values from a dict
177 
178  This provides a hashable representation of the dict from certain keywords.
179  This can be useful for creating e.g., a tuple of the values in the DataId
180  that identify the CCD.
181 
182  @param dict_ dict to parse
183  @param keys keys to extract (order is important)
184  @return tuple of values
185  """
186  return tuple(dict_[k] for k in keys)
187 
188 
189 def getCcdIdListFromExposures(expRefList, level="sensor", ccdKeys=["ccd"]):
190  """!Determine a list of CCDs from exposure references
191 
192  This essentially inverts the exposure-level references (which
193  provides a list of CCDs for each exposure), by providing
194  a dataId list for each CCD. Consider an input list of exposures
195  [e1, e2, e3], and each exposure has CCDs c1 and c2. Then this
196  function returns:
197 
198  {(c1,): [e1c1, e2c1, e3c1], (c2,): [e1c2, e2c2, e3c2]}
199 
200  This is a dict whose keys are tuples of the identifying values of a
201  CCD (usually just the CCD number) and the values are lists of dataIds
202  for that CCD in each exposure. A missing dataId is given the value
203  None.
204 
205  @param expRefList List of data references for exposures
206  @param level Level for the butler to generate CCDs
207  @param ccdKeys DataId keywords that identify a CCD
208  @return dict of data identifier lists for each CCD
209  """
210  expIdList = [[ccdRef.dataId for ccdRef in expRef.subItems(
211  level)] for expRef in expRefList]
212 
213  # Determine what additional keys make a CCD from an exposure
214  ccdKeys = set(ccdKeys) # Set of keywords in the dataId that identify a CCD
215  ccdNames = set() # Set of tuples which are values for each of the CCDs in an exposure
216  for ccdIdList in expIdList:
217  for ccdId in ccdIdList:
218  name = dictToTuple(ccdId, ccdKeys)
219  ccdNames.add(name)
220 
221  # Turn the list of CCDs for each exposure into a list of exposures for
222  # each CCD
223  ccdLists = {}
224  for n, ccdIdList in enumerate(expIdList):
225  for ccdId in ccdIdList:
226  name = dictToTuple(ccdId, ccdKeys)
227  if name not in ccdLists:
228  ccdLists[name] = []
229  ccdLists[name].append(ccdId)
230 
231  return ccdLists
232 
233 
234 class CalibIdAction(argparse.Action):
235  """Split name=value pairs and put the result in a dict"""
236 
237  def __call__(self, parser, namespace, values, option_string):
238  output = getattr(namespace, self.dest, {})
239  for nameValue in values:
240  name, sep, valueStr = nameValue.partition("=")
241  if not valueStr:
242  parser.error("%s value %s must be in form name=value" %
243  (option_string, nameValue))
244  output[name] = valueStr
245  setattr(namespace, self.dest, output)
246 
247 
248 class CalibArgumentParser(ArgumentParser):
249  """ArgumentParser for calibration construction"""
250 
251  def __init__(self, calibName, *args, **kwargs):
252  """Add a --calibId argument to the standard pipe_base argument parser"""
253  ArgumentParser.__init__(self, *args, **kwargs)
254  self.calibName = calibName
255  self.add_id_argument("--id", datasetType="raw",
256  help="input identifiers, e.g., --id visit=123 ccd=4")
257  self.add_argument("--calibId", nargs="*", action=CalibIdAction, default={},
258  help="identifiers for calib, e.g., --calibId version=1",
259  metavar="KEY=VALUE1[^VALUE2[^VALUE3...]")
260 
261  def parse_args(self, *args, **kwargs):
262  """Parse arguments
263 
264  Checks that the "--calibId" provided works.
265  """
266  namespace = ArgumentParser.parse_args(self, *args, **kwargs)
267 
268  keys = namespace.butler.getKeys(self.calibName)
269  parsed = {}
270  for name, value in namespace.calibId.items():
271  if name not in keys:
272  self.error(
273  "%s is not a relevant calib identifier key (%s)" % (name, keys))
274  parsed[name] = keys[name](value)
275  namespace.calibId = parsed
276 
277  return namespace
278 
279 
280 class CalibConfig(Config):
281  """Configuration for constructing calibs"""
282  clobber = Field(dtype=bool, default=True,
283  doc="Clobber existing processed images?")
284  isr = ConfigurableField(target=IsrTask, doc="ISR configuration")
285  dateObs = Field(dtype=str, default="dateObs",
286  doc="Key for observation date in exposure registry")
287  dateCalib = Field(dtype=str, default="calibDate",
288  doc="Key for calib date in calib registry")
289  filter = Field(dtype=str, default="filter",
290  doc="Key for filter name in exposure/calib registries")
291  combination = ConfigurableField(
292  target=CalibCombineTask, doc="Calib combination configuration")
293  ccdKeys = ListField(dtype=str, default=[
294  "ccd"], doc="DataId keywords specifying a CCD")
295  visitKeys = ListField(dtype=str, default=[
296  "visit"], doc="DataId keywords specifying a visit")
297  calibKeys = ListField(dtype=str, default=[],
298  doc="DataId keywords specifying a calibration")
299 
300  def setDefaults(self):
301  self.isr.doWrite = False
302 
303 
304 class CalibTaskRunner(TaskRunner):
305  """Get parsed values into the CalibTask.run"""
306  @staticmethod
307  def getTargetList(parsedCmd, **kwargs):
308  return [dict(expRefList=parsedCmd.id.refList, butler=parsedCmd.butler, calibId=parsedCmd.calibId)]
309 
310  def __call__(self, args):
311  """Call the Task with the kwargs from getTargetList"""
312  task = self.TaskClass(config=self.config, log=self.log)
313  exitStatus = 0 # exit status for the shell
314  if self.doRaise:
315  result = task.run(**args)
316  else:
317  try:
318  result = task.run(**args)
319  except Exception as e:
320  exitStatus = 1 # n.b. The shell exit value is the number of dataRefs returning
321  # non-zero, so the actual value used here is lost
322  task.log.fatal("Failed: %s" % e)
323  traceback.print_exc(file=sys.stderr)
324 
325  if self.doReturnResults:
326  return Struct(
327  exitStatus=exitStatus,
328  args=args,
329  metadata=task.metadata,
330  result=result,
331  )
332  else:
333  return Struct(
334  exitStatus=exitStatus,
335  )
336 
338  """!Base class for constructing calibs.
339 
340  This should be subclassed for each of the required calib types.
341  The subclass should be sure to define the following class variables:
342  * _DefaultName: default name of the task, used by CmdLineTask
343  * calibName: name of the calibration data set in the butler
344  The subclass may optionally set:
345  * filterName: filter name to give the resultant calib
346  """
347  ConfigClass = CalibConfig
348  RunnerClass = CalibTaskRunner
349  filterName = None
350  calibName = None
351  exposureTime = 1.0 # sets this exposureTime in the output
352 
353  def __init__(self, *args, **kwargs):
354  """Constructor"""
355  BatchPoolTask.__init__(self, *args, **kwargs)
356  self.makeSubtask("isr")
357  self.makeSubtask("combination")
358 
359  @classmethod
360  def batchWallTime(cls, time, parsedCmd, numCores):
361  numCcds = len(parsedCmd.butler.get("camera"))
362  numExps = len(cls.RunnerClass.getTargetList(
363  parsedCmd)[0]['expRefList'])
364  numCycles = int(numCcds/float(numCores) + 0.5)
365  return time*numExps*numCycles
366 
367  @classmethod
368  def _makeArgumentParser(cls, *args, **kwargs):
369  kwargs.pop("doBatch", False)
370  return CalibArgumentParser(calibName=cls.calibName, name=cls._DefaultName, *args, **kwargs)
371 
372  def run(self, expRefList, butler, calibId):
373  """!Construct a calib from a list of exposure references
374 
375  This is the entry point, called by the TaskRunner.__call__
376 
377  Only the master node executes this method.
378 
379  @param expRefList List of data references at the exposure level
380  @param butler Data butler
381  @param calibId Identifier dict for calib
382  """
383  for expRef in expRefList:
384  self.addMissingKeys(expRef.dataId, butler, self.config.ccdKeys, 'raw')
385 
386  outputId = self.getOutputId(expRefList, calibId)
387  ccdIdLists = getCcdIdListFromExposures(
388  expRefList, level="sensor", ccdKeys=self.config.ccdKeys)
389 
390  # Ensure we can generate filenames for each output
391  outputIdItemList = list(outputId.items())
392  for ccdName in ccdIdLists:
393  dataId = dict([(k, ccdName[i]) for i, k in enumerate(self.config.ccdKeys)])
394  dataId.update(outputIdItemList)
395  self.addMissingKeys(dataId, butler)
396  dataId.update(outputIdItemList)
397 
398  try:
399  butler.get(self.calibName + "_filename", dataId)
400  except Exception as e:
401  raise RuntimeError(
402  "Unable to determine output filename \"%s_filename\" from %s: %s" %
403  (self.calibName, dataId, e))
404 
405  pool = Pool()
406  pool.storeSet(butler=butler)
407 
408  # Scatter: process CCDs independently
409  data = self.scatterProcess(pool, ccdIdLists)
410 
411  # Gather: determine scalings
412  scales = self.scale(ccdIdLists, data)
413 
414  # Scatter: combine
415  self.scatterCombine(pool, outputId, ccdIdLists, scales)
416 
417  def getOutputId(self, expRefList, calibId):
418  """!Generate the data identifier for the output calib
419 
420  The mean date and the common filter are included, using keywords
421  from the configuration. The CCD-specific part is not included
422  in the data identifier.
423 
424  @param expRefList List of data references at exposure level
425  @param calibId Data identifier elements for the calib provided by the user
426  @return data identifier
427  """
428  midTime = 0
429  filterName = None
430  for expRef in expRefList:
431  butler = expRef.getButler()
432  dataId = expRef.dataId
433 
434  midTime += self.getMjd(butler, dataId)
435  thisFilter = self.getFilter(
436  butler, dataId) if self.filterName is None else self.filterName
437  if filterName is None:
438  filterName = thisFilter
439  elif filterName != thisFilter:
440  raise RuntimeError("Filter mismatch for %s: %s vs %s" % (
441  dataId, thisFilter, filterName))
442 
443  midTime /= len(expRefList)
444  date = str(dafBase.DateTime(
445  midTime, dafBase.DateTime.MJD).toPython().date())
446 
447  outputId = {self.config.filter: filterName,
448  self.config.dateCalib: date}
449  outputId.update(calibId)
450  return outputId
451 
452  def getMjd(self, butler, dataId, timescale=dafBase.DateTime.UTC):
453  """Determine the Modified Julian Date (MJD; in TAI) from a data identifier"""
454  if self.config.dateObs in dataId:
455  dateObs = dataId[self.config.dateObs]
456  else:
457  dateObs = butler.queryMetadata('raw', [self.config.dateObs], dataId)[0]
458  if "T" not in dateObs:
459  dateObs = dateObs + "T12:00:00.0Z"
460  elif not dateObs.endswith("Z"):
461  dateObs += "Z"
462 
463  return dafBase.DateTime(dateObs, timescale).get(dafBase.DateTime.MJD)
464 
465  def getFilter(self, butler, dataId):
466  """Determine the filter from a data identifier"""
467  filt = butler.queryMetadata('raw', [self.config.filter], dataId)[0]
468  return filt
469 
470  def addMissingKeys(self, dataId, butler, missingKeys=None, calibName=None):
471  if calibName is None:
472  calibName = self.calibName
473 
474  if missingKeys is None:
475  missingKeys = set(butler.getKeys(calibName).keys()) - set(dataId.keys())
476 
477  for k in missingKeys:
478  try:
479  v = butler.queryMetadata('raw', [k], dataId) # n.b. --id refers to 'raw'
480  except Exception as e:
481  continue
482 
483  if len(v) == 0: # failed to lookup value
484  continue
485 
486  if len(v) == 1:
487  dataId[k] = v[0]
488  else:
489  raise RuntimeError("No unique lookup for %s: %s" % (k, v))
490 
491  def updateMetadata(self, calibImage, exposureTime, darkTime=None, **kwargs):
492  """!Update the metadata from the VisitInfo
493 
494  \param calibImage The image whose metadata is to be set
495  \param exposureTime The exposure time for the image
496  \param darkTime The time since the last read (default: exposureTime)
497  """
498 
499  if darkTime is None:
500  darkTime = exposureTime # avoid warning messages when using calibration products
501 
502  visitInfo = afwImage.makeVisitInfo(exposureTime=exposureTime, darkTime=darkTime, **kwargs)
503  md = calibImage.getMetadata()
504 
505  afwImage.setVisitInfoMetadata(md, visitInfo)
506 
507  def scatterProcess(self, pool, ccdIdLists):
508  """!Scatter the processing among the nodes
509 
510  We scatter each CCD independently (exposures aren't grouped together),
511  to make full use of all available processors. This necessitates piecing
512  everything back together in the same format as ccdIdLists afterwards.
513 
514  Only the master node executes this method.
515 
516  @param pool Process pool
517  @param ccdIdLists Dict of data identifier lists for each CCD name
518  @return Dict of lists of returned data for each CCD name
519  """
520  dataIdList = sum(ccdIdLists.values(), [])
521  self.log.info("Scatter processing")
522 
523  resultList = pool.map(self.process, dataIdList)
524 
525  # Piece everything back together
526  data = dict((ccdName, [None] * len(expList))
527  for ccdName, expList in ccdIdLists.items())
528  indices = dict(sum([[(tuple(dataId.values()) if dataId is not None else None, (ccdName, expNum))
529  for expNum, dataId in enumerate(expList)]
530  for ccdName, expList in ccdIdLists.items()], []))
531  for dataId, result in zip(dataIdList, resultList):
532  if dataId is None:
533  continue
534  ccdName, expNum = indices[tuple(dataId.values())]
535  data[ccdName][expNum] = result
536 
537  return data
538 
539  def process(self, cache, ccdId, outputName="postISRCCD"):
540  """!Process a CCD, specified by a data identifier
541 
542  After processing, optionally returns a result (produced by
543  the 'processResult' method) calculated from the processed
544  exposure. These results will be gathered by the master node,
545  and is a means for coordinated scaling of all CCDs for flats,
546  etc.
547 
548  Only slave nodes execute this method.
549 
550  @param cache Process pool cache
551  @param ccdId Data identifier for CCD
552  @param outputName Output dataset name for butler
553  @return result from 'processResult'
554  """
555  if ccdId is None:
556  self.log.warn("Null identifier received on %s" % NODE)
557  return None
558  sensorRef = getDataRef(cache.butler, ccdId)
559  if self.config.clobber or not sensorRef.datasetExists(outputName):
560  self.log.info("Processing %s on %s" % (ccdId, NODE))
561  try:
562  exposure = self.processSingle(sensorRef)
563  except Exception as e:
564  self.log.warn("Unable to process %s: %s" % (ccdId, e))
565  raise
566  return None
567  self.processWrite(sensorRef, exposure)
568  else:
569  self.log.info(
570  "Using previously persisted processed exposure for %s" % (sensorRef.dataId,))
571  exposure = sensorRef.get(outputName, immediate=True)
572  return self.processResult(exposure)
573 
574  def processSingle(self, dataRef):
575  """Process a single CCD, specified by a data reference
576 
577  Generally, this simply means doing ISR.
578 
579  Only slave nodes execute this method.
580  """
581  return self.isr.runDataRef(dataRef).exposure
582 
583  def processWrite(self, dataRef, exposure, outputName="postISRCCD"):
584  """!Write the processed CCD
585 
586  We need to write these out because we can't hold them all in
587  memory at once.
588 
589  Only slave nodes execute this method.
590 
591  @param dataRef Data reference
592  @param exposure CCD exposure to write
593  @param outputName Output dataset name for butler.
594  """
595  dataRef.put(exposure, outputName)
596 
597  def processResult(self, exposure):
598  """Extract processing results from a processed exposure
599 
600  This method generates what is gathered by the master node.
601  This can be a background measurement or similar for scaling
602  flat-fields. It must be picklable!
603 
604  Only slave nodes execute this method.
605  """
606  return None
607 
608  def scale(self, ccdIdLists, data):
609  """!Determine scaling across CCDs and exposures
610 
611  This is necessary mainly for flats, so as to determine a
612  consistent scaling across the entire focal plane. This
613  implementation is simply a placeholder.
614 
615  Only the master node executes this method.
616 
617  @param ccdIdLists Dict of data identifier lists for each CCD tuple
618  @param data Dict of lists of returned data for each CCD tuple
619  @return dict of Struct(ccdScale: scaling for CCD,
620  expScales: scaling for each exposure
621  ) for each CCD tuple
622  """
623  self.log.info("Scale on %s" % NODE)
624  return dict((name, Struct(ccdScale=None, expScales=[None] * len(ccdIdLists[name])))
625  for name in ccdIdLists)
626 
627  def scatterCombine(self, pool, outputId, ccdIdLists, scales):
628  """!Scatter the combination of exposures across multiple nodes
629 
630  In this case, we can only scatter across as many nodes as
631  there are CCDs.
632 
633  Only the master node executes this method.
634 
635  @param pool Process pool
636  @param outputId Output identifier (exposure part only)
637  @param ccdIdLists Dict of data identifier lists for each CCD name
638  @param scales Dict of structs with scales, for each CCD name
639  """
640  self.log.info("Scatter combination")
641  data = [Struct(ccdName=ccdName, ccdIdList=ccdIdLists[ccdName], scales=scales[ccdName]) for
642  ccdName in ccdIdLists]
643  pool.map(self.combine, data, outputId)
644 
645  def combine(self, cache, struct, outputId):
646  """!Combine multiple exposures of a particular CCD and write the output
647 
648  Only the slave nodes execute this method.
649 
650  @param cache Process pool cache
651  @param struct Parameters for the combination, which has the following components:
652  * ccdName Name tuple for CCD
653  * ccdIdList List of data identifiers for combination
654  * scales Scales to apply (expScales are scalings for each exposure,
655  ccdScale is final scale for combined image)
656  @param outputId Data identifier for combined image (exposure part only)
657  """
658  # Check if we need to look up any keys that aren't in the output dataId
659  fullOutputId = {k: struct.ccdName[i] for i, k in enumerate(self.config.ccdKeys)}
660  fullOutputId.update(outputId)
661  self.addMissingKeys(fullOutputId, cache.butler)
662  fullOutputId.update(outputId) # must be after the call to queryMetadata
663  outputId = fullOutputId
664  del fullOutputId
665 
666  dataRefList = [getDataRef(cache.butler, dataId) if dataId is not None else None for
667  dataId in struct.ccdIdList]
668  self.log.info("Combining %s on %s" % (outputId, NODE))
669  calib = self.combination.run(dataRefList, expScales=struct.scales.expScales,
670  finalScale=struct.scales.ccdScale)
671 
672  if not hasattr(calib, "getMetadata"):
673  if hasattr(calib, "getVariance"):
674  calib = afwImage.makeExposure(calib)
675  else:
676  calib = afwImage.DecoratedImageF(calib.getImage()) # n.b. hardwires "F" for the output type
677 
678  self.updateMetadata(calib, self.exposureTime)
679 
680  self.recordCalibInputs(cache.butler, calib,
681  struct.ccdIdList, outputId)
682 
683  self.interpolateNans(calib)
684 
685  self.write(cache.butler, calib, outputId)
686 
687  def recordCalibInputs(self, butler, calib, dataIdList, outputId):
688  """!Record metadata including the inputs and creation details
689 
690  This metadata will go into the FITS header.
691 
692  @param butler Data butler
693  @param calib Combined calib exposure.
694  @param dataIdList List of data identifiers for calibration inputs
695  @param outputId Data identifier for output
696  """
697  header = calib.getMetadata()
698  header.add("OBSTYPE", self.calibName) # Used by ingestCalibs.py
699 
700  # date, time, host, and root
701  now = time.localtime()
702  header.add("CALIB_CREATION_DATE", time.strftime("%Y-%m-%d", now))
703  header.add("CALIB_CREATION_TIME", time.strftime("%X %Z", now))
704  # add date-obs as its absence upsets ExposureInfo; use the mean date that the calibs were taken
705  header.add("DATE-OBS", "%sT00:00:00.00" % outputId[self.config.dateCalib])
706 
707  # Inputs
708  visits = [str(dictToTuple(dataId, self.config.visitKeys)) for dataId in dataIdList if
709  dataId is not None]
710  for i, v in enumerate(sorted(set(visits))):
711  header.add("CALIB_INPUT_%d" % (i,), v)
712 
713  header.add("CALIB_ID", " ".join("%s=%s" % (key, value)
714  for key, value in outputId.items()))
715  checksum(calib, header)
716 
717  def interpolateNans(self, image):
718  """Interpolate over NANs in the combined image
719 
720  NANs can result from masked areas on the CCD. We don't want them getting
721  into our science images, so we replace them with the median of the image.
722  """
723  if hasattr(image, "getMaskedImage"): # Deal with Exposure vs Image
724  self.interpolateNans(image.getMaskedImage().getVariance())
725  image = image.getMaskedImage().getImage()
726  if hasattr(image, "getImage"): # Deal with DecoratedImage or MaskedImage vs Image
727  image = image.getImage()
728  array = image.getArray()
729  bad = np.isnan(array)
730  array[bad] = np.median(array[np.logical_not(bad)])
731 
732  def write(self, butler, exposure, dataId):
733  """!Write the final combined calib
734 
735  Only the slave nodes execute this method
736 
737  @param butler Data butler
738  @param exposure CCD exposure to write
739  @param dataId Data identifier for output
740  """
741  self.log.info("Writing %s on %s" % (dataId, NODE))
742  butler.put(exposure, self.calibName, dataId)
743 
744 
746  """Configuration for bias construction.
747 
748  No changes required compared to the base class, but
749  subclassed for distinction.
750  """
751  pass
752 
753 
754 class BiasTask(CalibTask):
755  """Bias construction"""
756  ConfigClass = BiasConfig
757  _DefaultName = "bias"
758  calibName = "bias"
759  filterName = "NONE" # Sets this filter name in the output
760  exposureTime = 0.0 # sets this exposureTime in the output
761 
762  @classmethod
763  def applyOverrides(cls, config):
764  """Overrides to apply for bias construction"""
765  config.isr.doBias = False
766  config.isr.doDark = False
767  config.isr.doFlat = False
768  config.isr.doFringe = False
769 
770 
772  """Configuration for dark construction"""
773  doRepair = Field(dtype=bool, default=True, doc="Repair artifacts?")
774  psfFwhm = Field(dtype=float, default=3.0, doc="Repair PSF FWHM (pixels)")
775  psfSize = Field(dtype=int, default=21, doc="Repair PSF size (pixels)")
776  crGrow = Field(dtype=int, default=2, doc="Grow radius for CR (pixels)")
777  repair = ConfigurableField(
778  target=RepairTask, doc="Task to repair artifacts")
779 
780  def setDefaults(self):
781  CalibConfig.setDefaults(self)
782  self.combination.mask.append("CR")
783 
784 
786  """Dark construction
787 
788  The only major difference from the base class is a cosmic-ray
789  identification stage, and dividing each image by the dark time
790  to generate images of the dark rate.
791  """
792  ConfigClass = DarkConfig
793  _DefaultName = "dark"
794  calibName = "dark"
795  filterName = "NONE" # Sets this filter name in the output
796 
797  def __init__(self, *args, **kwargs):
798  CalibTask.__init__(self, *args, **kwargs)
799  self.makeSubtask("repair")
800 
801  @classmethod
802  def applyOverrides(cls, config):
803  """Overrides to apply for dark construction"""
804  config.isr.doDark = False
805  config.isr.doFlat = False
806  config.isr.doFringe = False
807 
808  def processSingle(self, sensorRef):
809  """Process a single CCD
810 
811  Besides the regular ISR, also masks cosmic-rays and divides each
812  processed image by the dark time to generate images of the dark rate.
813  The dark time is provided by the 'getDarkTime' method.
814  """
815  exposure = CalibTask.processSingle(self, sensorRef)
816 
817  if self.config.doRepair:
818  psf = measAlg.DoubleGaussianPsf(self.config.psfSize, self.config.psfSize,
819  self.config.psfFwhm/(2*math.sqrt(2*math.log(2))))
820  exposure.setPsf(psf)
821  self.repair.run(exposure, keepCRs=False)
822  if self.config.crGrow > 0:
823  mask = exposure.getMaskedImage().getMask().clone()
824  mask &= mask.getPlaneBitMask("CR")
825  fpSet = afwDet.FootprintSet(
826  mask, afwDet.Threshold(0.5))
827  fpSet = afwDet.FootprintSet(fpSet, self.config.crGrow, True)
828  fpSet.setMask(exposure.getMaskedImage().getMask(), "CR")
829 
830  mi = exposure.getMaskedImage()
831  mi /= self.getDarkTime(exposure)
832  return exposure
833 
834  def getDarkTime(self, exposure):
835  """Retrieve the dark time for an exposure"""
836  darkTime = exposure.getInfo().getVisitInfo().getDarkTime()
837  if not np.isfinite(darkTime):
838  raise RuntimeError("Non-finite darkTime")
839  return darkTime
840 
841 
843  """Configuration for flat construction"""
844  iterations = Field(dtype=int, default=10,
845  doc="Number of iterations for scale determination")
846  stats = ConfigurableField(target=CalibStatsTask,
847  doc="Background statistics configuration")
848 
849 
851  """Flat construction
852 
853  The principal change from the base class involves gathering the background
854  values from each image and using them to determine the scalings for the final
855  combination.
856  """
857  ConfigClass = FlatConfig
858  _DefaultName = "flat"
859  calibName = "flat"
860 
861  @classmethod
862  def applyOverrides(cls, config):
863  """Overrides for flat construction"""
864  config.isr.doFlat = False
865  config.isr.doFringe = False
866 
867  def __init__(self, *args, **kwargs):
868  CalibTask.__init__(self, *args, **kwargs)
869  self.makeSubtask("stats")
870 
871  def processResult(self, exposure):
872  return self.stats.run(exposure)
873 
874  def scale(self, ccdIdLists, data):
875  """Determine the scalings for the final combination
876 
877  We have a matrix B_ij = C_i E_j, where C_i is the relative scaling
878  of one CCD to all the others in an exposure, and E_j is the scaling
879  of the exposure. We convert everything to logarithms so we can work
880  with a linear system. We determine the C_i and E_j from B_ij by iteration,
881  under the additional constraint that the average CCD scale is unity.
882 
883  This algorithm comes from Eugene Magnier and Pan-STARRS.
884  """
885  assert len(ccdIdLists.values()) > 0, "No successful CCDs"
886  lengths = set([len(expList) for expList in ccdIdLists.values()])
887  assert len(
888  lengths) == 1, "Number of successful exposures for each CCD differs"
889  assert tuple(lengths)[0] > 0, "No successful exposures"
890  # Format background measurements into a matrix
891  indices = dict((name, i) for i, name in enumerate(ccdIdLists))
892  bgMatrix = np.array([[0.0] * len(expList)
893  for expList in ccdIdLists.values()])
894  for name in ccdIdLists:
895  i = indices[name]
896  bgMatrix[i] = [
897  d if d is not None else np.nan for d in data[name]]
898 
899  numpyPrint = np.get_printoptions()
900  np.set_printoptions(threshold=np.inf)
901  self.log.info("Input backgrounds: %s" % bgMatrix)
902 
903  # Flat-field scaling
904  numCcds = len(ccdIdLists)
905  numExps = bgMatrix.shape[1]
906  # log(Background) for each exposure/component
907  bgMatrix = np.log(bgMatrix)
908  bgMatrix = np.ma.masked_array(bgMatrix, np.isnan(bgMatrix))
909  # Initial guess at log(scale) for each component
910  compScales = np.zeros(numCcds)
911  expScales = np.array(
912  [(bgMatrix[:, i0] - compScales).mean() for i0 in range(numExps)])
913 
914  for iterate in range(self.config.iterations):
915  compScales = np.array(
916  [(bgMatrix[i1, :] - expScales).mean() for i1 in range(numCcds)])
917  expScales = np.array(
918  [(bgMatrix[:, i2] - compScales).mean() for i2 in range(numExps)])
919 
920  avgScale = np.average(np.exp(compScales))
921  compScales -= np.log(avgScale)
922  self.log.debug("Iteration %d exposure scales: %s",
923  iterate, np.exp(expScales))
924  self.log.debug("Iteration %d component scales: %s",
925  iterate, np.exp(compScales))
926 
927  expScales = np.array(
928  [(bgMatrix[:, i3] - compScales).mean() for i3 in range(numExps)])
929 
930  if np.any(np.isnan(expScales)):
931  raise RuntimeError("Bad exposure scales: %s --> %s" %
932  (bgMatrix, expScales))
933 
934  expScales = np.exp(expScales)
935  compScales = np.exp(compScales)
936 
937  self.log.info("Exposure scales: %s" % expScales)
938  self.log.info("Component relative scaling: %s" % compScales)
939  np.set_printoptions(**numpyPrint)
940 
941  return dict((ccdName, Struct(ccdScale=compScales[indices[ccdName]], expScales=expScales))
942  for ccdName in ccdIdLists)
943 
944 
946  """Configuration for fringe construction"""
947  stats = ConfigurableField(target=CalibStatsTask,
948  doc="Background statistics configuration")
949  subtractBackground = ConfigurableField(target=measAlg.SubtractBackgroundTask,
950  doc="Background configuration")
951  detection = ConfigurableField(
952  target=measAlg.SourceDetectionTask, doc="Detection configuration")
953  detectSigma = Field(dtype=float, default=1.0,
954  doc="Detection PSF gaussian sigma")
955 
956 
958  """Fringe construction task
959 
960  The principal change from the base class is that the images are
961  background-subtracted and rescaled by the background.
962 
963  XXX This is probably not right for a straight-up combination, as we
964  are currently doing, since the fringe amplitudes need not scale with
965  the continuum.
966 
967  XXX Would like to have this do PCA and generate multiple images, but
968  that will take a bit of work with the persistence code.
969  """
970  ConfigClass = FringeConfig
971  _DefaultName = "fringe"
972  calibName = "fringe"
973 
974  @classmethod
975  def applyOverrides(cls, config):
976  """Overrides for fringe construction"""
977  config.isr.doFringe = False
978 
979  def __init__(self, *args, **kwargs):
980  CalibTask.__init__(self, *args, **kwargs)
981  self.makeSubtask("detection")
982  self.makeSubtask("stats")
983  self.makeSubtask("subtractBackground")
984 
985  def processSingle(self, sensorRef):
986  """Subtract the background and normalise by the background level"""
987  exposure = CalibTask.processSingle(self, sensorRef)
988  bgLevel = self.stats.run(exposure)
989  self.subtractBackground.run(exposure)
990  mi = exposure.getMaskedImage()
991  mi /= bgLevel
992  footprintSets = self.detection.detectFootprints(
993  exposure, sigma=self.config.detectSigma)
994  mask = exposure.getMaskedImage().getMask()
995  detected = 1 << mask.addMaskPlane("DETECTED")
996  for fpSet in (footprintSets.positive, footprintSets.negative):
997  if fpSet is not None:
998  afwDet.setMaskFromFootprintList(
999  mask, fpSet.getFootprints(), detected)
1000  return exposure
def processWrite(self, dataRef, exposure, outputName="postISRCCD")
Write the processed CCD.
def scatterCombine(self, pool, outputId, ccdIdLists, scales)
Scatter the combination of exposures across multiple nodes.
def run(self, exposureOrImage)
Measure a particular statistic on an image (of some sort).
def checksum(obj, header=None, sumType="MD5")
Calculate a checksum of an object.
Definition: checksum.py:29
def getCcdIdListFromExposures(expRefList, level="sensor", ccdKeys=["ccd"])
Determine a list of CCDs from exposure references.
def run(self, expRefList, butler, calibId)
Construct a calib from a list of exposure references.
def __call__(self, parser, namespace, values, option_string)
def dictToTuple(dict_, keys)
Return a tuple of specific values from a dict.
def getDataRef(butler, dataId, datasetType="raw")
Definition: utils.py:17
def updateMetadata(self, calibImage, exposureTime, darkTime=None, kwargs)
Update the metadata from the VisitInfo.
def combine(self, target, imageList, stats)
Combine multiple images.
def getMjd(self, butler, dataId, timescale=dafBase.DateTime.UTC)
def process(self, cache, ccdId, outputName="postISRCCD")
Process a CCD, specified by a data identifier.
def combine(self, cache, struct, outputId)
Combine multiple exposures of a particular CCD and write the output.
def write(self, butler, exposure, dataId)
Write the final combined calib.
def recordCalibInputs(self, butler, calib, dataIdList, outputId)
Record metadata including the inputs and creation details.
def run(self, sensorRefList, expScales=None, finalScale=None, inputName="postISRCCD")
Combine calib images for a single sensor.
def getOutputId(self, expRefList, calibId)
Generate the data identifier for the output calib.
def scale(self, ccdIdLists, data)
Determine scaling across CCDs and exposures.
def addMissingKeys(self, dataId, butler, missingKeys=None, calibName=None)
def getDimensions(self, sensorRefList, inputName="postISRCCD")
Base class for constructing calibs.
def scatterProcess(self, pool, ccdIdLists)
Scatter the processing among the nodes.
def batchWallTime(cls, time, parsedCmd, numCores)