lsst.pipe.drivers  13.0-17-g3ce57e5+33
 All Classes Namespaces Files Functions Variables Pages
constructCalibs.py
Go to the documentation of this file.
1 from __future__ import absolute_import, division, print_function
2 import sys
3 import math
4 import time
5 import argparse
6 import traceback
7 
8 import numpy as np
9 from builtins import zip
10 from builtins import range
11 
12 from lsst.pex.config import Config, ConfigurableField, Field, ListField
13 from lsst.pipe.base import Task, Struct, TaskRunner, ArgumentParser
14 import lsst.daf.base as dafBase
15 import lsst.afw.math as afwMath
16 import lsst.afw.geom as afwGeom
17 import lsst.afw.detection as afwDet
18 import lsst.afw.image as afwImage
19 from lsst.afw.image import VisitInfo
20 import lsst.meas.algorithms as measAlg
21 from lsst.pipe.tasks.repair import RepairTask
22 from lsst.ip.isr import IsrTask
23 
24 from lsst.ctrl.pool.parallel import BatchPoolTask
25 from lsst.ctrl.pool.pool import Pool, NODE
26 
27 from .checksum import checksum
28 from .utils import getDataRef
29 
30 
31 class CalibStatsConfig(Config):
32  """Parameters controlling the measurement of background statistics"""
33  stat = Field(doc="Statistic to use to estimate background (from lsst.afw.math)", dtype=int,
34  default=int(afwMath.MEANCLIP))
35  clip = Field(doc="Clipping threshold for background",
36  dtype=float, default=3.0)
37  nIter = Field(doc="Clipping iterations for background",
38  dtype=int, default=3)
39  mask = ListField(doc="Mask planes to reject",
40  dtype=str, default=["DETECTED", "BAD"])
41 
42 
43 class CalibStatsTask(Task):
44  """Measure statistics on the background
45 
46  This can be useful for scaling the background, e.g., for flats and fringe frames.
47  """
48  ConfigClass = CalibStatsConfig
49 
50  def run(self, exposureOrImage):
51  """!Measure a particular statistic on an image (of some sort).
52 
53  @param exposureOrImage Exposure, MaskedImage or Image.
54  @return Value of desired statistic
55  """
56  stats = afwMath.StatisticsControl(self.config.clip, self.config.nIter,
57  afwImage.Mask.getPlaneBitMask(self.config.mask))
58  try:
59  image = exposureOrImage.getMaskedImage()
60  except:
61  try:
62  image = exposureOrImage.getImage()
63  except:
64  image = exposureOrImage
65 
66  return afwMath.makeStatistics(image, self.config.stat, stats).getValue()
67 
68 
69 class CalibCombineConfig(Config):
70  """Configuration for combining calib images"""
71  rows = Field(doc="Number of rows to read at a time",
72  dtype=int, default=512)
73  mask = ListField(doc="Mask planes to respect", dtype=str,
74  default=["SAT", "DETECTED", "INTRP"])
75  combine = Field(doc="Statistic to use for combination (from lsst.afw.math)", dtype=int,
76  default=int(afwMath.MEANCLIP))
77  clip = Field(doc="Clipping threshold for combination",
78  dtype=float, default=3.0)
79  nIter = Field(doc="Clipping iterations for combination",
80  dtype=int, default=3)
81  stats = ConfigurableField(target=CalibStatsTask,
82  doc="Background statistics configuration")
83 
84 
85 class CalibCombineTask(Task):
86  """Task to combine calib images"""
87  ConfigClass = CalibCombineConfig
88 
89  def __init__(self, *args, **kwargs):
90  Task.__init__(self, *args, **kwargs)
91  self.makeSubtask("stats")
92 
93  def run(self, sensorRefList, expScales=None, finalScale=None, inputName="postISRCCD"):
94  """!Combine calib images for a single sensor
95 
96  @param sensorRefList List of data references to combine (for a single sensor)
97  @param expScales List of scales to apply for each exposure
98  @param finalScale Desired scale for final combined image
99  @param inputName Data set name for inputs
100  @return combined image
101  """
102  width, height = self.getDimensions(sensorRefList)
103  stats = afwMath.StatisticsControl(self.config.clip, self.config.nIter,
104  afwImage.Mask.getPlaneBitMask(self.config.mask))
105 
106  # Combine images
107  combined = afwImage.MaskedImageF(width, height)
108  numImages = len(sensorRefList)
109  imageList = [None]*numImages
110  for start in range(0, height, self.config.rows):
111  rows = min(self.config.rows, height - start)
112  box = afwGeom.Box2I(afwGeom.Point2I(0, start),
113  afwGeom.Extent2I(width, rows))
114  subCombined = combined.Factory(combined, box)
115 
116  for i, sensorRef in enumerate(sensorRefList):
117  if sensorRef is None:
118  imageList[i] = None
119  continue
120  exposure = sensorRef.get(inputName + "_sub", bbox=box)
121  if expScales is not None:
122  self.applyScale(exposure, expScales[i])
123  imageList[i] = exposure.getMaskedImage()
124 
125  self.combine(subCombined, imageList, stats)
126 
127  if finalScale is not None:
128  background = self.stats.run(combined)
129  self.log.info("%s: Measured background of stack is %f; adjusting to %f" %
130  (NODE, background, finalScale))
131  combined *= finalScale / background
132 
133  return combined
134 
135  def getDimensions(self, sensorRefList, inputName="postISRCCD"):
136  """Get dimensions of the inputs"""
137  dimList = []
138  for sensorRef in sensorRefList:
139  if sensorRef is None:
140  continue
141  md = sensorRef.get(inputName + "_md")
142  dimList.append(afwGeom.Extent2I(
143  md.get("NAXIS1"), md.get("NAXIS2")))
144  return getSize(dimList)
145 
146  def applyScale(self, exposure, scale=None):
147  """Apply scale to input exposure
148 
149  This implementation applies a flux scaling: the input exposure is
150  divided by the provided scale.
151  """
152  if scale is not None:
153  mi = exposure.getMaskedImage()
154  mi /= scale
155 
156  def combine(self, target, imageList, stats):
157  """!Combine multiple images
158 
159  @param target Target image to receive the combined pixels
160  @param imageList List of input images
161  @param stats Statistics control
162  """
163  images = [img for img in imageList if img is not None]
164  afwMath.statisticsStack(target, images, afwMath.Property(self.config.combine), stats)
165 
166 
167 def getSize(dimList):
168  """Determine a consistent size, given a list of image sizes"""
169  dim = set((w, h) for w, h in dimList)
170  dim.discard(None)
171  if len(dim) != 1:
172  raise RuntimeError("Inconsistent dimensions: %s" % dim)
173  return dim.pop()
174 
175 
176 def dictToTuple(dict_, keys):
177  """!Return a tuple of specific values from a dict
178 
179  This provides a hashable representation of the dict from certain keywords.
180  This can be useful for creating e.g., a tuple of the values in the DataId
181  that identify the CCD.
182 
183  @param dict_ dict to parse
184  @param keys keys to extract (order is important)
185  @return tuple of values
186  """
187  return tuple(dict_[k] for k in keys)
188 
189 
190 def getCcdIdListFromExposures(expRefList, level="sensor", ccdKeys=["ccd"]):
191  """!Determine a list of CCDs from exposure references
192 
193  This essentially inverts the exposure-level references (which
194  provides a list of CCDs for each exposure), by providing
195  a dataId list for each CCD. Consider an input list of exposures
196  [e1, e2, e3], and each exposure has CCDs c1 and c2. Then this
197  function returns:
198 
199  {(c1,): [e1c1, e2c1, e3c1], (c2,): [e1c2, e2c2, e3c2]}
200 
201  This is a dict whose keys are tuples of the identifying values of a
202  CCD (usually just the CCD number) and the values are lists of dataIds
203  for that CCD in each exposure. A missing dataId is given the value
204  None.
205 
206  @param expRefList List of data references for exposures
207  @param level Level for the butler to generate CCDs
208  @param ccdKeys DataId keywords that identify a CCD
209  @return dict of data identifier lists for each CCD
210  """
211  expIdList = [[ccdRef.dataId for ccdRef in expRef.subItems(
212  level)] for expRef in expRefList]
213 
214  # Determine what additional keys make a CCD from an exposure
215  ccdKeys = set(ccdKeys) # Set of keywords in the dataId that identify a CCD
216  ccdNames = set() # Set of tuples which are values for each of the CCDs in an exposure
217  for ccdIdList in expIdList:
218  for ccdId in ccdIdList:
219  name = dictToTuple(ccdId, ccdKeys)
220  ccdNames.add(name)
221 
222  # Turn the list of CCDs for each exposure into a list of exposures for
223  # each CCD
224  ccdLists = {}
225  for n, ccdIdList in enumerate(expIdList):
226  for ccdId in ccdIdList:
227  name = dictToTuple(ccdId, ccdKeys)
228  if name not in ccdLists:
229  ccdLists[name] = []
230  ccdLists[name].append(ccdId)
231 
232  return ccdLists
233 
234 
235 class CalibIdAction(argparse.Action):
236  """Split name=value pairs and put the result in a dict"""
237 
238  def __call__(self, parser, namespace, values, option_string):
239  output = getattr(namespace, self.dest, {})
240  for nameValue in values:
241  name, sep, valueStr = nameValue.partition("=")
242  if not valueStr:
243  parser.error("%s value %s must be in form name=value" %
244  (option_string, nameValue))
245  output[name] = valueStr
246  setattr(namespace, self.dest, output)
247 
248 
249 class CalibArgumentParser(ArgumentParser):
250  """ArgumentParser for calibration construction"""
251 
252  def __init__(self, calibName, *args, **kwargs):
253  """Add a --calibId argument to the standard pipe_base argument parser"""
254  ArgumentParser.__init__(self, *args, **kwargs)
255  self.calibName = calibName
256  self.add_id_argument("--id", datasetType="raw",
257  help="input identifiers, e.g., --id visit=123 ccd=4")
258  self.add_argument("--calibId", nargs="*", action=CalibIdAction, default={},
259  help="identifiers for calib, e.g., --calibId version=1",
260  metavar="KEY=VALUE1[^VALUE2[^VALUE3...]")
261 
262  def parse_args(self, *args, **kwargs):
263  """Parse arguments
264 
265  Checks that the "--calibId" provided works.
266  """
267  namespace = ArgumentParser.parse_args(self, *args, **kwargs)
268 
269  keys = namespace.butler.getKeys(self.calibName)
270  parsed = {}
271  for name, value in namespace.calibId.items():
272  if name not in keys:
273  self.error(
274  "%s is not a relevant calib identifier key (%s)" % (name, keys))
275  parsed[name] = keys[name](value)
276  namespace.calibId = parsed
277 
278  return namespace
279 
280 
281 class CalibConfig(Config):
282  """Configuration for constructing calibs"""
283  clobber = Field(dtype=bool, default=True,
284  doc="Clobber existing processed images?")
285  isr = ConfigurableField(target=IsrTask, doc="ISR configuration")
286  dateObs = Field(dtype=str, default="dateObs",
287  doc="Key for observation date in exposure registry")
288  dateCalib = Field(dtype=str, default="calibDate",
289  doc="Key for calib date in calib registry")
290  filter = Field(dtype=str, default="filter",
291  doc="Key for filter name in exposure/calib registries")
292  combination = ConfigurableField(
293  target=CalibCombineTask, doc="Calib combination configuration")
294  ccdKeys = ListField(dtype=str, default=[
295  "ccd"], doc="DataId keywords specifying a CCD")
296  visitKeys = ListField(dtype=str, default=[
297  "visit"], doc="DataId keywords specifying a visit")
298  calibKeys = ListField(dtype=str, default=[],
299  doc="DataId keywords specifying a calibration")
300 
301  def setDefaults(self):
302  self.isr.doWrite = False
303 
304 
305 class CalibTaskRunner(TaskRunner):
306  """Get parsed values into the CalibTask.run"""
307  @staticmethod
308  def getTargetList(parsedCmd, **kwargs):
309  return [dict(expRefList=parsedCmd.id.refList, butler=parsedCmd.butler, calibId=parsedCmd.calibId)]
310 
311  def __call__(self, args):
312  """Call the Task with the kwargs from getTargetList"""
313  task = self.TaskClass(config=self.config, log=self.log)
314  exitStatus = 0 # exit status for the shell
315  if self.doRaise:
316  result = task.run(**args)
317  else:
318  try:
319  result = task.run(**args)
320  except Exception as e:
321  exitStatus = 1 # n.b. The shell exit value is the number of dataRefs returning
322  # non-zero, so the actual value used here is lost
323  task.log.fatal("Failed: %s" % e)
324  traceback.print_exc(file=sys.stderr)
325 
326  if self.doReturnResults:
327  return Struct(
328  exitStatus=exitStatus,
329  args=args,
330  metadata=task.metadata,
331  result=result,
332  )
333  else:
334  return Struct(
335  exitStatus=exitStatus,
336  )
337 
338 class CalibTask(BatchPoolTask):
339  """!Base class for constructing calibs.
340 
341  This should be subclassed for each of the required calib types.
342  The subclass should be sure to define the following class variables:
343  * _DefaultName: default name of the task, used by CmdLineTask
344  * calibName: name of the calibration data set in the butler
345  The subclass may optionally set:
346  * filterName: filter name to give the resultant calib
347  """
348  ConfigClass = CalibConfig
349  RunnerClass = CalibTaskRunner
350  filterName = None
351  calibName = None
352  exposureTime = 1.0 # sets this exposureTime in the output
353 
354  def __init__(self, *args, **kwargs):
355  """Constructor"""
356  BatchPoolTask.__init__(self, *args, **kwargs)
357  self.makeSubtask("isr")
358  self.makeSubtask("combination")
359 
360  @classmethod
361  def batchWallTime(cls, time, parsedCmd, numCores):
362  numCcds = len(parsedCmd.butler.get("camera"))
363  numExps = len(cls.RunnerClass.getTargetList(
364  parsedCmd)[0]['expRefList'])
365  numCycles = int(numCcds/float(numCores) + 0.5)
366  return time*numExps*numCycles
367 
368  @classmethod
369  def _makeArgumentParser(cls, *args, **kwargs):
370  kwargs.pop("doBatch", False)
371  return CalibArgumentParser(calibName=cls.calibName, name=cls._DefaultName, *args, **kwargs)
372 
373  def run(self, expRefList, butler, calibId):
374  """!Construct a calib from a list of exposure references
375 
376  This is the entry point, called by the TaskRunner.__call__
377 
378  Only the master node executes this method.
379 
380  @param expRefList List of data references at the exposure level
381  @param butler Data butler
382  @param calibId Identifier dict for calib
383  """
384  for expRef in expRefList:
385  self.addMissingKeys(expRef.dataId, butler, self.config.ccdKeys, 'raw')
386 
387  outputId = self.getOutputId(expRefList, calibId)
388  ccdIdLists = getCcdIdListFromExposures(
389  expRefList, level="sensor", ccdKeys=self.config.ccdKeys)
390 
391  # Ensure we can generate filenames for each output
392  outputIdItemList = list(outputId.items())
393  for ccdName in ccdIdLists:
394  dataId = dict([(k, ccdName[i]) for i, k in enumerate(self.config.ccdKeys)])
395  dataId.update(outputIdItemList)
396  self.addMissingKeys(dataId, butler)
397  dataId.update(outputIdItemList)
398 
399  try:
400  butler.get(self.calibName + "_filename", dataId)
401  except Exception as e:
402  raise RuntimeError(
403  "Unable to determine output filename \"%s_filename\" from %s: %s" %
404  (self.calibName, dataId, e))
405 
406  pool = Pool()
407  pool.storeSet(butler=butler)
408 
409  # Scatter: process CCDs independently
410  data = self.scatterProcess(pool, ccdIdLists)
411 
412  # Gather: determine scalings
413  scales = self.scale(ccdIdLists, data)
414 
415  # Scatter: combine
416  self.scatterCombine(pool, outputId, ccdIdLists, scales)
417 
418  def getOutputId(self, expRefList, calibId):
419  """!Generate the data identifier for the output calib
420 
421  The mean date and the common filter are included, using keywords
422  from the configuration. The CCD-specific part is not included
423  in the data identifier.
424 
425  @param expRefList List of data references at exposure level
426  @param calibId Data identifier elements for the calib provided by the user
427  @return data identifier
428  """
429  midTime = 0
430  filterName = None
431  for expRef in expRefList:
432  butler = expRef.getButler()
433  dataId = expRef.dataId
434 
435  midTime += self.getMjd(butler, dataId)
436  thisFilter = self.getFilter(
437  butler, dataId) if self.filterName is None else self.filterName
438  if filterName is None:
439  filterName = thisFilter
440  elif filterName != thisFilter:
441  raise RuntimeError("Filter mismatch for %s: %s vs %s" % (
442  dataId, thisFilter, filterName))
443 
444  midTime /= len(expRefList)
445  date = str(dafBase.DateTime(
446  midTime, dafBase.DateTime.MJD).toPython().date())
447 
448  outputId = {self.config.filter: filterName,
449  self.config.dateCalib: date}
450  outputId.update(calibId)
451  return outputId
452 
453  def getMjd(self, butler, dataId, timescale=dafBase.DateTime.UTC):
454  """Determine the Modified Julian Date (MJD; in TAI) from a data identifier"""
455  if self.config.dateObs in dataId:
456  dateObs = dataId[self.config.dateObs]
457  else:
458  dateObs = butler.queryMetadata('raw', [self.config.dateObs], dataId)[0]
459  if "T" not in dateObs:
460  dateObs = dateObs + "T12:00:00.0Z"
461  elif not dateObs.endswith("Z"):
462  dateObs += "Z"
463 
464  return dafBase.DateTime(dateObs, timescale).get(dafBase.DateTime.MJD)
465 
466  def getFilter(self, butler, dataId):
467  """Determine the filter from a data identifier"""
468  filt = butler.queryMetadata('raw', [self.config.filter], dataId)[0]
469  return filt
470 
471  def addMissingKeys(self, dataId, butler, missingKeys=None, calibName=None):
472  if calibName is None:
473  calibName = self.calibName
474 
475  if missingKeys is None:
476  missingKeys = set(butler.getKeys(calibName).keys()) - set(dataId.keys())
477 
478  for k in missingKeys:
479  try:
480  v = butler.queryMetadata('raw', [k], dataId) # n.b. --id refers to 'raw'
481  except Exception as e:
482  continue
483 
484  if len(v) == 0: # failed to lookup value
485  continue
486 
487  if len(v) == 1:
488  dataId[k] = v[0]
489  else:
490  raise RuntimeError("No unique lookup for %s: %s" % (k, v))
491 
492  def updateMetadata(self, calibImage, exposureTime, darkTime=None, **kwargs):
493  """!Update the metadata from the VisitInfo
494 
495  \param calibImage The image whose metadata is to be set
496  \param exposureTime The exposure time for the image
497  \param darkTime The time since the last read (default: exposureTime)
498  """
499 
500  if darkTime is None:
501  darkTime = exposureTime # avoid warning messages when using calibration products
502 
503  visitInfo = afwImage.makeVisitInfo(exposureTime=exposureTime, darkTime=darkTime, **kwargs)
504  md = calibImage.getMetadata()
505 
506  afwImage.setVisitInfoMetadata(md, visitInfo)
507 
508  def scatterProcess(self, pool, ccdIdLists):
509  """!Scatter the processing among the nodes
510 
511  We scatter each CCD independently (exposures aren't grouped together),
512  to make full use of all available processors. This necessitates piecing
513  everything back together in the same format as ccdIdLists afterwards.
514 
515  Only the master node executes this method.
516 
517  @param pool Process pool
518  @param ccdIdLists Dict of data identifier lists for each CCD name
519  @return Dict of lists of returned data for each CCD name
520  """
521  dataIdList = sum(ccdIdLists.values(), [])
522  self.log.info("Scatter processing")
523 
524  resultList = pool.map(self.process, dataIdList)
525 
526  # Piece everything back together
527  data = dict((ccdName, [None] * len(expList))
528  for ccdName, expList in ccdIdLists.items())
529  indices = dict(sum([[(tuple(dataId.values()) if dataId is not None else None, (ccdName, expNum))
530  for expNum, dataId in enumerate(expList)]
531  for ccdName, expList in ccdIdLists.items()], []))
532  for dataId, result in zip(dataIdList, resultList):
533  if dataId is None:
534  continue
535  ccdName, expNum = indices[tuple(dataId.values())]
536  data[ccdName][expNum] = result
537 
538  return data
539 
540  def process(self, cache, ccdId, outputName="postISRCCD"):
541  """!Process a CCD, specified by a data identifier
542 
543  After processing, optionally returns a result (produced by
544  the 'processResult' method) calculated from the processed
545  exposure. These results will be gathered by the master node,
546  and is a means for coordinated scaling of all CCDs for flats,
547  etc.
548 
549  Only slave nodes execute this method.
550 
551  @param cache Process pool cache
552  @param ccdId Data identifier for CCD
553  @param outputName Output dataset name for butler
554  @return result from 'processResult'
555  """
556  if ccdId is None:
557  self.log.warn("Null identifier received on %s" % NODE)
558  return None
559  sensorRef = getDataRef(cache.butler, ccdId)
560  if self.config.clobber or not sensorRef.datasetExists(outputName):
561  self.log.info("Processing %s on %s" % (ccdId, NODE))
562  try:
563  exposure = self.processSingle(sensorRef)
564  except Exception as e:
565  self.log.warn("Unable to process %s: %s" % (ccdId, e))
566  raise
567  return None
568  self.processWrite(sensorRef, exposure)
569  else:
570  self.log.info(
571  "Using previously persisted processed exposure for %s" % (sensorRef.dataId,))
572  exposure = sensorRef.get(outputName, immediate=True)
573  return self.processResult(exposure)
574 
575  def processSingle(self, dataRef):
576  """Process a single CCD, specified by a data reference
577 
578  Generally, this simply means doing ISR.
579 
580  Only slave nodes execute this method.
581  """
582  return self.isr.runDataRef(dataRef).exposure
583 
584  def processWrite(self, dataRef, exposure, outputName="postISRCCD"):
585  """!Write the processed CCD
586 
587  We need to write these out because we can't hold them all in
588  memory at once.
589 
590  Only slave nodes execute this method.
591 
592  @param dataRef Data reference
593  @param exposure CCD exposure to write
594  @param outputName Output dataset name for butler.
595  """
596  dataRef.put(exposure, outputName)
597 
598  def processResult(self, exposure):
599  """Extract processing results from a processed exposure
600 
601  This method generates what is gathered by the master node.
602  This can be a background measurement or similar for scaling
603  flat-fields. It must be picklable!
604 
605  Only slave nodes execute this method.
606  """
607  return None
608 
609  def scale(self, ccdIdLists, data):
610  """!Determine scaling across CCDs and exposures
611 
612  This is necessary mainly for flats, so as to determine a
613  consistent scaling across the entire focal plane. This
614  implementation is simply a placeholder.
615 
616  Only the master node executes this method.
617 
618  @param ccdIdLists Dict of data identifier lists for each CCD tuple
619  @param data Dict of lists of returned data for each CCD tuple
620  @return dict of Struct(ccdScale: scaling for CCD,
621  expScales: scaling for each exposure
622  ) for each CCD tuple
623  """
624  self.log.info("Scale on %s" % NODE)
625  return dict((name, Struct(ccdScale=None, expScales=[None] * len(ccdIdLists[name])))
626  for name in ccdIdLists)
627 
628  def scatterCombine(self, pool, outputId, ccdIdLists, scales):
629  """!Scatter the combination of exposures across multiple nodes
630 
631  In this case, we can only scatter across as many nodes as
632  there are CCDs.
633 
634  Only the master node executes this method.
635 
636  @param pool Process pool
637  @param outputId Output identifier (exposure part only)
638  @param ccdIdLists Dict of data identifier lists for each CCD name
639  @param scales Dict of structs with scales, for each CCD name
640  """
641  self.log.info("Scatter combination")
642  data = [Struct(ccdName=ccdName, ccdIdList=ccdIdLists[ccdName], scales=scales[ccdName]) for
643  ccdName in ccdIdLists]
644  pool.map(self.combine, data, outputId)
645 
646  def combine(self, cache, struct, outputId):
647  """!Combine multiple exposures of a particular CCD and write the output
648 
649  Only the slave nodes execute this method.
650 
651  @param cache Process pool cache
652  @param struct Parameters for the combination, which has the following components:
653  * ccdName Name tuple for CCD
654  * ccdIdList List of data identifiers for combination
655  * scales Scales to apply (expScales are scalings for each exposure,
656  ccdScale is final scale for combined image)
657  @param outputId Data identifier for combined image (exposure part only)
658  """
659  # Check if we need to look up any keys that aren't in the output dataId
660  fullOutputId = {k: struct.ccdName[i] for i, k in enumerate(self.config.ccdKeys)}
661  fullOutputId.update(outputId)
662  self.addMissingKeys(fullOutputId, cache.butler)
663  fullOutputId.update(outputId) # must be after the call to queryMetadata
664  outputId = fullOutputId
665  del fullOutputId
666 
667  dataRefList = [getDataRef(cache.butler, dataId) if dataId is not None else None for
668  dataId in struct.ccdIdList]
669  self.log.info("Combining %s on %s" % (outputId, NODE))
670  calib = self.combination.run(dataRefList, expScales=struct.scales.expScales,
671  finalScale=struct.scales.ccdScale)
672 
673  if not hasattr(calib, "getMetadata"):
674  if hasattr(calib, "getVariance"):
675  calib = afwImage.makeExposure(calib)
676  else:
677  calib = afwImage.DecoratedImageF(calib.getImage()) # n.b. hardwires "F" for the output type
678 
679  self.updateMetadata(calib, self.exposureTime)
680 
681  self.recordCalibInputs(cache.butler, calib,
682  struct.ccdIdList, outputId)
683 
684  self.interpolateNans(calib)
685 
686  self.write(cache.butler, calib, outputId)
687 
688  def recordCalibInputs(self, butler, calib, dataIdList, outputId):
689  """!Record metadata including the inputs and creation details
690 
691  This metadata will go into the FITS header.
692 
693  @param butler Data butler
694  @param calib Combined calib exposure.
695  @param dataIdList List of data identifiers for calibration inputs
696  @param outputId Data identifier for output
697  """
698  header = calib.getMetadata()
699  header.add("OBSTYPE", self.calibName) # Used by ingestCalibs.py
700 
701  # date, time, host, and root
702  now = time.localtime()
703  header.add("CALIB_CREATION_DATE", time.strftime("%Y-%m-%d", now))
704  header.add("CALIB_CREATION_TIME", time.strftime("%X %Z", now))
705  # add date-obs as its absence upsets ExposureInfo; use the mean date that the calibs were taken
706  header.add("DATE-OBS", "%sT00:00:00.00" % outputId[self.config.dateCalib])
707 
708  # Inputs
709  visits = [str(dictToTuple(dataId, self.config.visitKeys)) for dataId in dataIdList if
710  dataId is not None]
711  for i, v in enumerate(sorted(set(visits))):
712  header.add("CALIB_INPUT_%d" % (i,), v)
713 
714  header.add("CALIB_ID", " ".join("%s=%s" % (key, value)
715  for key, value in outputId.items()))
716  checksum(calib, header)
717 
718  def interpolateNans(self, image):
719  """Interpolate over NANs in the combined image
720 
721  NANs can result from masked areas on the CCD. We don't want them getting
722  into our science images, so we replace them with the median of the image.
723  """
724  if hasattr(image, "getMaskedImage"): # Deal with Exposure vs Image
725  self.interpolateNans(image.getMaskedImage().getVariance())
726  image = image.getMaskedImage().getImage()
727  if hasattr(image, "getImage"): # Deal with DecoratedImage or MaskedImage vs Image
728  image = image.getImage()
729  array = image.getArray()
730  bad = np.isnan(array)
731  array[bad] = np.median(array[np.logical_not(bad)])
732 
733  def write(self, butler, exposure, dataId):
734  """!Write the final combined calib
735 
736  Only the slave nodes execute this method
737 
738  @param butler Data butler
739  @param exposure CCD exposure to write
740  @param dataId Data identifier for output
741  """
742  self.log.info("Writing %s on %s" % (dataId, NODE))
743  butler.put(exposure, self.calibName, dataId)
744 
745 
747  """Configuration for bias construction.
748 
749  No changes required compared to the base class, but
750  subclassed for distinction.
751  """
752  pass
753 
754 
755 class BiasTask(CalibTask):
756  """Bias construction"""
757  ConfigClass = BiasConfig
758  _DefaultName = "bias"
759  calibName = "bias"
760  filterName = "NONE" # Sets this filter name in the output
761  exposureTime = 0.0 # sets this exposureTime in the output
762 
763  @classmethod
764  def applyOverrides(cls, config):
765  """Overrides to apply for bias construction"""
766  config.isr.doBias = False
767  config.isr.doDark = False
768  config.isr.doFlat = False
769  config.isr.doFringe = False
770 
771 
773  """Configuration for dark construction"""
774  doRepair = Field(dtype=bool, default=True, doc="Repair artifacts?")
775  psfFwhm = Field(dtype=float, default=3.0, doc="Repair PSF FWHM (pixels)")
776  psfSize = Field(dtype=int, default=21, doc="Repair PSF size (pixels)")
777  crGrow = Field(dtype=int, default=2, doc="Grow radius for CR (pixels)")
778  repair = ConfigurableField(
779  target=RepairTask, doc="Task to repair artifacts")
780 
781  def setDefaults(self):
782  CalibConfig.setDefaults(self)
783  self.combination.mask.append("CR")
784 
785 
787  """Dark construction
788 
789  The only major difference from the base class is a cosmic-ray
790  identification stage, and dividing each image by the dark time
791  to generate images of the dark rate.
792  """
793  ConfigClass = DarkConfig
794  _DefaultName = "dark"
795  calibName = "dark"
796  filterName = "NONE" # Sets this filter name in the output
797 
798  def __init__(self, *args, **kwargs):
799  CalibTask.__init__(self, *args, **kwargs)
800  self.makeSubtask("repair")
801 
802  @classmethod
803  def applyOverrides(cls, config):
804  """Overrides to apply for dark construction"""
805  config.isr.doDark = False
806  config.isr.doFlat = False
807  config.isr.doFringe = False
808 
809  def processSingle(self, sensorRef):
810  """Process a single CCD
811 
812  Besides the regular ISR, also masks cosmic-rays and divides each
813  processed image by the dark time to generate images of the dark rate.
814  The dark time is provided by the 'getDarkTime' method.
815  """
816  exposure = CalibTask.processSingle(self, sensorRef)
817 
818  if self.config.doRepair:
819  psf = measAlg.DoubleGaussianPsf(self.config.psfSize, self.config.psfSize,
820  self.config.psfFwhm/(2*math.sqrt(2*math.log(2))))
821  exposure.setPsf(psf)
822  self.repair.run(exposure, keepCRs=False)
823  if self.config.crGrow > 0:
824  mask = exposure.getMaskedImage().getMask().clone()
825  mask &= mask.getPlaneBitMask("CR")
826  fpSet = afwDet.FootprintSet(
827  mask, afwDet.Threshold(0.5))
828  fpSet = afwDet.FootprintSet(fpSet, self.config.crGrow, True)
829  fpSet.setMask(exposure.getMaskedImage().getMask(), "CR")
830 
831  mi = exposure.getMaskedImage()
832  mi /= self.getDarkTime(exposure)
833  return exposure
834 
835  def getDarkTime(self, exposure):
836  """Retrieve the dark time for an exposure"""
837  darkTime = exposure.getInfo().getVisitInfo().getDarkTime()
838  if not np.isfinite(darkTime):
839  raise RuntimeError("Non-finite darkTime")
840  return darkTime
841 
842 
844  """Configuration for flat construction"""
845  iterations = Field(dtype=int, default=10,
846  doc="Number of iterations for scale determination")
847  stats = ConfigurableField(target=CalibStatsTask,
848  doc="Background statistics configuration")
849 
850 
852  """Flat construction
853 
854  The principal change from the base class involves gathering the background
855  values from each image and using them to determine the scalings for the final
856  combination.
857  """
858  ConfigClass = FlatConfig
859  _DefaultName = "flat"
860  calibName = "flat"
861 
862  @classmethod
863  def applyOverrides(cls, config):
864  """Overrides for flat construction"""
865  config.isr.doFlat = False
866  config.isr.doFringe = False
867 
868  def __init__(self, *args, **kwargs):
869  CalibTask.__init__(self, *args, **kwargs)
870  self.makeSubtask("stats")
871 
872  def processResult(self, exposure):
873  return self.stats.run(exposure)
874 
875  def scale(self, ccdIdLists, data):
876  """Determine the scalings for the final combination
877 
878  We have a matrix B_ij = C_i E_j, where C_i is the relative scaling
879  of one CCD to all the others in an exposure, and E_j is the scaling
880  of the exposure. We convert everything to logarithms so we can work
881  with a linear system. We determine the C_i and E_j from B_ij by iteration,
882  under the additional constraint that the average CCD scale is unity.
883 
884  This algorithm comes from Eugene Magnier and Pan-STARRS.
885  """
886  assert len(ccdIdLists.values()) > 0, "No successful CCDs"
887  lengths = set([len(expList) for expList in ccdIdLists.values()])
888  assert len(
889  lengths) == 1, "Number of successful exposures for each CCD differs"
890  assert tuple(lengths)[0] > 0, "No successful exposures"
891  # Format background measurements into a matrix
892  indices = dict((name, i) for i, name in enumerate(ccdIdLists))
893  bgMatrix = np.array([[0.0] * len(expList)
894  for expList in ccdIdLists.values()])
895  for name in ccdIdLists:
896  i = indices[name]
897  bgMatrix[i] = [
898  d if d is not None else np.nan for d in data[name]]
899 
900  numpyPrint = np.get_printoptions()
901  np.set_printoptions(threshold='nan')
902  self.log.info("Input backgrounds: %s" % bgMatrix)
903 
904  # Flat-field scaling
905  numCcds = len(ccdIdLists)
906  numExps = bgMatrix.shape[1]
907  # log(Background) for each exposure/component
908  bgMatrix = np.log(bgMatrix)
909  bgMatrix = np.ma.masked_array(bgMatrix, np.isnan(bgMatrix))
910  # Initial guess at log(scale) for each component
911  compScales = np.zeros(numCcds)
912  expScales = np.array(
913  [(bgMatrix[:, i0] - compScales).mean() for i0 in range(numExps)])
914 
915  for iterate in range(self.config.iterations):
916  compScales = np.array(
917  [(bgMatrix[i1, :] - expScales).mean() for i1 in range(numCcds)])
918  expScales = np.array(
919  [(bgMatrix[:, i2] - compScales).mean() for i2 in range(numExps)])
920 
921  avgScale = np.average(np.exp(compScales))
922  compScales -= np.log(avgScale)
923  self.log.debug("Iteration %d exposure scales: %s",
924  iterate, np.exp(expScales))
925  self.log.debug("Iteration %d component scales: %s",
926  iterate, np.exp(compScales))
927 
928  expScales = np.array(
929  [(bgMatrix[:, i3] - compScales).mean() for i3 in range(numExps)])
930 
931  if np.any(np.isnan(expScales)):
932  raise RuntimeError("Bad exposure scales: %s --> %s" %
933  (bgMatrix, expScales))
934 
935  expScales = np.exp(expScales)
936  compScales = np.exp(compScales)
937 
938  self.log.info("Exposure scales: %s" % expScales)
939  self.log.info("Component relative scaling: %s" % compScales)
940  np.set_printoptions(**numpyPrint)
941 
942  return dict((ccdName, Struct(ccdScale=compScales[indices[ccdName]], expScales=expScales))
943  for ccdName in ccdIdLists)
944 
945 
947  """Configuration for fringe construction"""
948  stats = ConfigurableField(target=CalibStatsTask,
949  doc="Background statistics configuration")
950  subtractBackground = ConfigurableField(target=measAlg.SubtractBackgroundTask,
951  doc="Background configuration")
952  detection = ConfigurableField(
953  target=measAlg.SourceDetectionTask, doc="Detection configuration")
954  detectSigma = Field(dtype=float, default=1.0,
955  doc="Detection PSF gaussian sigma")
956 
957 
959  """Fringe construction task
960 
961  The principal change from the base class is that the images are
962  background-subtracted and rescaled by the background.
963 
964  XXX This is probably not right for a straight-up combination, as we
965  are currently doing, since the fringe amplitudes need not scale with
966  the continuum.
967 
968  XXX Would like to have this do PCA and generate multiple images, but
969  that will take a bit of work with the persistence code.
970  """
971  ConfigClass = FringeConfig
972  _DefaultName = "fringe"
973  calibName = "fringe"
974 
975  @classmethod
976  def applyOverrides(cls, config):
977  """Overrides for fringe construction"""
978  config.isr.doFringe = False
979 
980  def __init__(self, *args, **kwargs):
981  CalibTask.__init__(self, *args, **kwargs)
982  self.makeSubtask("detection")
983  self.makeSubtask("stats")
984  self.makeSubtask("subtractBackground")
985 
986  def processSingle(self, sensorRef):
987  """Subtract the background and normalise by the background level"""
988  exposure = CalibTask.processSingle(self, sensorRef)
989  bgLevel = self.stats.run(exposure)
990  self.subtractBackground.run(exposure)
991  mi = exposure.getMaskedImage()
992  mi /= bgLevel
993  footprintSets = self.detection.detectFootprints(
994  exposure, sigma=self.config.detectSigma)
995  mask = exposure.getMaskedImage().getMask()
996  detected = 1 << mask.addMaskPlane("DETECTED")
997  for fpSet in (footprintSets.positive, footprintSets.negative):
998  if fpSet is not None:
999  afwDet.setMaskFromFootprintList(
1000  mask, fpSet.getFootprints(), detected)
1001  return exposure
def run
Measure a particular statistic on an image (of some sort).
def getCcdIdListFromExposures
Determine a list of CCDs from exposure references.
def scatterProcess
Scatter the processing among the nodes.
def checksum
Calculate a checksum of an object.
Definition: checksum.py:29
def recordCalibInputs
Record metadata including the inputs and creation details.
def updateMetadata
Update the metadata from the VisitInfo.
def getOutputId
Generate the data identifier for the output calib.
def write
Write the final combined calib.
def run
Construct a calib from a list of exposure references.
def combine
Combine multiple exposures of a particular CCD and write the output.
def scale
Determine scaling across CCDs and exposures.
def scatterCombine
Scatter the combination of exposures across multiple nodes.
def run
Combine calib images for a single sensor.
def process
Process a CCD, specified by a data identifier.
def dictToTuple
Return a tuple of specific values from a dict.
def processWrite
Write the processed CCD.
Base class for constructing calibs.