lsst.pipe.drivers  13.0-12-gda22aa7
 All Classes Namespaces Files Functions Variables Pages
constructCalibs.py
Go to the documentation of this file.
1 from __future__ import absolute_import, division, print_function
2 import sys
3 import math
4 import time
5 import argparse
6 import traceback
7 
8 import numpy as np
9 from builtins import zip
10 from builtins import range
11 
12 from lsst.pex.config import Config, ConfigurableField, Field, ListField
13 from lsst.pipe.base import Task, Struct, TaskRunner, ArgumentParser
14 import lsst.daf.base as dafBase
15 import lsst.afw.math as afwMath
16 import lsst.afw.geom as afwGeom
17 import lsst.afw.detection as afwDet
18 import lsst.afw.image as afwImage
19 from lsst.afw.image import VisitInfo
20 import lsst.meas.algorithms as measAlg
21 from lsst.pipe.tasks.repair import RepairTask
22 from lsst.ip.isr import IsrTask
23 
24 from lsst.ctrl.pool.parallel import BatchPoolTask
25 from lsst.ctrl.pool.pool import Pool, NODE
26 
27 from .checksum import checksum
28 from .utils import getDataRef
29 
30 
31 class CalibStatsConfig(Config):
32  """Parameters controlling the measurement of background statistics"""
33  stat = Field(doc="Statistic to use to estimate background (from lsst.afw.math)", dtype=int,
34  default=int(afwMath.MEANCLIP))
35  clip = Field(doc="Clipping threshold for background",
36  dtype=float, default=3.0)
37  nIter = Field(doc="Clipping iterations for background",
38  dtype=int, default=3)
39  mask = ListField(doc="Mask planes to reject",
40  dtype=str, default=["DETECTED", "BAD"])
41 
42 
43 class CalibStatsTask(Task):
44  """Measure statistics on the background
45 
46  This can be useful for scaling the background, e.g., for flats and fringe frames.
47  """
48  ConfigClass = CalibStatsConfig
49 
50  def run(self, exposureOrImage):
51  """!Measure a particular statistic on an image (of some sort).
52 
53  @param exposureOrImage Exposure, MaskedImage or Image.
54  @return Value of desired statistic
55  """
56  stats = afwMath.StatisticsControl(self.config.clip, self.config.nIter,
57  afwImage.MaskU.getPlaneBitMask(self.config.mask))
58  try:
59  image = exposureOrImage.getMaskedImage()
60  except:
61  try:
62  image = exposureOrImage.getImage()
63  except:
64  image = exposureOrImage
65 
66  return afwMath.makeStatistics(image, self.config.stat, stats).getValue()
67 
68 
69 class CalibCombineConfig(Config):
70  """Configuration for combining calib images"""
71  rows = Field(doc="Number of rows to read at a time",
72  dtype=int, default=512)
73  mask = ListField(doc="Mask planes to respect", dtype=str,
74  default=["SAT", "DETECTED", "INTRP"])
75  combine = Field(doc="Statistic to use for combination (from lsst.afw.math)", dtype=int,
76  default=int(afwMath.MEANCLIP))
77  clip = Field(doc="Clipping threshold for combination",
78  dtype=float, default=3.0)
79  nIter = Field(doc="Clipping iterations for combination",
80  dtype=int, default=3)
81  stats = ConfigurableField(target=CalibStatsTask,
82  doc="Background statistics configuration")
83 
84 
85 class CalibCombineTask(Task):
86  """Task to combine calib images"""
87  ConfigClass = CalibCombineConfig
88 
89  def __init__(self, *args, **kwargs):
90  Task.__init__(self, *args, **kwargs)
91  self.makeSubtask("stats")
92 
93  def run(self, sensorRefList, expScales=None, finalScale=None, inputName="postISRCCD"):
94  """!Combine calib images for a single sensor
95 
96  @param sensorRefList List of data references to combine (for a single sensor)
97  @param expScales List of scales to apply for each exposure
98  @param finalScale Desired scale for final combined image
99  @param inputName Data set name for inputs
100  @return combined image
101  """
102  width, height = self.getDimensions(sensorRefList)
103  stats = afwMath.StatisticsControl(self.config.clip, self.config.nIter,
104  afwImage.MaskU.getPlaneBitMask(self.config.mask))
105 
106  # Combine images
107  combined = afwImage.MaskedImageF(width, height)
108  numImages = len(sensorRefList)
109  imageList = [None]*numImages
110  for start in range(0, height, self.config.rows):
111  rows = min(self.config.rows, height - start)
112  box = afwGeom.Box2I(afwGeom.Point2I(0, start),
113  afwGeom.Extent2I(width, rows))
114  subCombined = combined.Factory(combined, box)
115 
116  for i, sensorRef in enumerate(sensorRefList):
117  if sensorRef is None:
118  imageList[i] = None
119  continue
120  exposure = sensorRef.get(inputName + "_sub", bbox=box)
121  if expScales is not None:
122  self.applyScale(exposure, expScales[i])
123  imageList[i] = exposure.getMaskedImage()
124 
125  self.combine(subCombined, imageList, stats)
126 
127  if finalScale is not None:
128  background = self.stats.run(combined)
129  self.log.info("%s: Measured background of stack is %f; adjusting to %f" %
130  (NODE, background, finalScale))
131  combined *= finalScale / background
132 
133  return combined
134 
135  def getDimensions(self, sensorRefList, inputName="postISRCCD"):
136  """Get dimensions of the inputs"""
137  dimList = []
138  for sensorRef in sensorRefList:
139  if sensorRef is None:
140  continue
141  md = sensorRef.get(inputName + "_md")
142  dimList.append(afwGeom.Extent2I(
143  md.get("NAXIS1"), md.get("NAXIS2")))
144  return getSize(dimList)
145 
146  def applyScale(self, exposure, scale=None):
147  """Apply scale to input exposure
148 
149  This implementation applies a flux scaling: the input exposure is
150  divided by the provided scale.
151  """
152  if scale is not None:
153  mi = exposure.getMaskedImage()
154  mi /= scale
155 
156  def combine(self, target, imageList, stats):
157  """!Combine multiple images
158 
159  @param target Target image to receive the combined pixels
160  @param imageList List of input images
161  @param stats Statistics control
162  """
163  images = [img for img in imageList if img is not None]
164  afwMath.statisticsStack(target, images, afwMath.Property(self.config.combine), stats)
165 
166 
167 def getSize(dimList):
168  """Determine a consistent size, given a list of image sizes"""
169  dim = set((w, h) for w, h in dimList)
170  dim.discard(None)
171  if len(dim) != 1:
172  raise RuntimeError("Inconsistent dimensions: %s" % dim)
173  return dim.pop()
174 
175 
176 def dictToTuple(dict_, keys):
177  """!Return a tuple of specific values from a dict
178 
179  This provides a hashable representation of the dict from certain keywords.
180  This can be useful for creating e.g., a tuple of the values in the DataId
181  that identify the CCD.
182 
183  @param dict_ dict to parse
184  @param keys keys to extract (order is important)
185  @return tuple of values
186  """
187  return tuple(dict_[k] for k in keys)
188 
189 
190 def getCcdIdListFromExposures(expRefList, level="sensor", ccdKeys=["ccd"]):
191  """!Determine a list of CCDs from exposure references
192 
193  This essentially inverts the exposure-level references (which
194  provides a list of CCDs for each exposure), by providing
195  a dataId list for each CCD. Consider an input list of exposures
196  [e1, e2, e3], and each exposure has CCDs c1 and c2. Then this
197  function returns:
198 
199  {(c1,): [e1c1, e2c1, e3c1], (c2,): [e1c2, e2c2, e3c2]}
200 
201  This is a dict whose keys are tuples of the identifying values of a
202  CCD (usually just the CCD number) and the values are lists of dataIds
203  for that CCD in each exposure. A missing dataId is given the value
204  None.
205 
206  @param expRefList List of data references for exposures
207  @param level Level for the butler to generate CCDs
208  @param ccdKeys DataId keywords that identify a CCD
209  @return dict of data identifier lists for each CCD
210  """
211  expIdList = [[ccdRef.dataId for ccdRef in expRef.subItems(
212  level)] for expRef in expRefList]
213 
214  # Determine what additional keys make a CCD from an exposure
215  ccdKeys = set(ccdKeys) # Set of keywords in the dataId that identify a CCD
216  ccdNames = set() # Set of tuples which are values for each of the CCDs in an exposure
217  for ccdIdList in expIdList:
218  for ccdId in ccdIdList:
219  name = dictToTuple(ccdId, ccdKeys)
220  ccdNames.add(name)
221 
222  # Turn the list of CCDs for each exposure into a list of exposures for
223  # each CCD
224  ccdLists = {}
225  for n, ccdIdList in enumerate(expIdList):
226  for ccdId in ccdIdList:
227  name = dictToTuple(ccdId, ccdKeys)
228  if name not in ccdLists:
229  ccdLists[name] = []
230  ccdLists[name].append(ccdId)
231 
232  return ccdLists
233 
234 
235 class CalibIdAction(argparse.Action):
236  """Split name=value pairs and put the result in a dict"""
237 
238  def __call__(self, parser, namespace, values, option_string):
239  output = getattr(namespace, self.dest, {})
240  for nameValue in values:
241  name, sep, valueStr = nameValue.partition("=")
242  if not valueStr:
243  parser.error("%s value %s must be in form name=value" %
244  (option_string, nameValue))
245  output[name] = valueStr
246  setattr(namespace, self.dest, output)
247 
248 
249 class CalibArgumentParser(ArgumentParser):
250  """ArgumentParser for calibration construction"""
251 
252  def __init__(self, calibName, *args, **kwargs):
253  """Add a --calibId argument to the standard pipe_base argument parser"""
254  ArgumentParser.__init__(self, *args, **kwargs)
255  self.calibName = calibName
256  self.add_id_argument("--id", datasetType="raw",
257  help="input identifiers, e.g., --id visit=123 ccd=4")
258  self.add_argument("--calibId", nargs="*", action=CalibIdAction, default={},
259  help="identifiers for calib, e.g., --calibId version=1",
260  metavar="KEY=VALUE1[^VALUE2[^VALUE3...]")
261 
262  def parse_args(self, *args, **kwargs):
263  """Parse arguments
264 
265  Checks that the "--calibId" provided works.
266  """
267  namespace = ArgumentParser.parse_args(self, *args, **kwargs)
268 
269  keys = namespace.butler.getKeys(self.calibName)
270  parsed = {}
271  for name, value in namespace.calibId.items():
272  if name not in keys:
273  self.error(
274  "%s is not a relevant calib identifier key (%s)" % (name, keys))
275  parsed[name] = keys[name](value)
276  namespace.calibId = parsed
277 
278  return namespace
279 
280 
281 class CalibConfig(Config):
282  """Configuration for constructing calibs"""
283  clobber = Field(dtype=bool, default=True,
284  doc="Clobber existing processed images?")
285  isr = ConfigurableField(target=IsrTask, doc="ISR configuration")
286  dateObs = Field(dtype=str, default="dateObs",
287  doc="Key for observation date in exposure registry")
288  dateCalib = Field(dtype=str, default="calibDate",
289  doc="Key for calib date in calib registry")
290  filter = Field(dtype=str, default="filter",
291  doc="Key for filter name in exposure/calib registries")
292  combination = ConfigurableField(
293  target=CalibCombineTask, doc="Calib combination configuration")
294  ccdKeys = ListField(dtype=str, default=[
295  "ccd"], doc="DataId keywords specifying a CCD")
296  visitKeys = ListField(dtype=str, default=[
297  "visit"], doc="DataId keywords specifying a visit")
298  calibKeys = ListField(dtype=str, default=[],
299  doc="DataId keywords specifying a calibration")
300 
301  def setDefaults(self):
302  self.isr.doWrite = False
303 
304 
305 class CalibTaskRunner(TaskRunner):
306  """Get parsed values into the CalibTask.run"""
307  @staticmethod
308  def getTargetList(parsedCmd, **kwargs):
309  return [dict(expRefList=parsedCmd.id.refList, butler=parsedCmd.butler, calibId=parsedCmd.calibId)]
310 
311  def __call__(self, args):
312  """Call the Task with the kwargs from getTargetList"""
313  task = self.TaskClass(config=self.config, log=self.log)
314  if self.doRaise:
315  result = task.run(**args)
316  else:
317  try:
318  result = task.run(**args)
319  except Exception as e:
320  task.log.fatal("Failed: %s" % e)
321  traceback.print_exc(file=sys.stderr)
322 
323  if self.doReturnResults:
324  return Struct(
325  args=args,
326  metadata=task.metadata,
327  result=result,
328  )
329 
330 
331 class CalibTask(BatchPoolTask):
332  """!Base class for constructing calibs.
333 
334  This should be subclassed for each of the required calib types.
335  The subclass should be sure to define the following class variables:
336  * _DefaultName: default name of the task, used by CmdLineTask
337  * calibName: name of the calibration data set in the butler
338  The subclass may optionally set:
339  * filterName: filter name to give the resultant calib
340  """
341  ConfigClass = CalibConfig
342  RunnerClass = CalibTaskRunner
343  filterName = None
344  calibName = None
345  exposureTime = 1.0 # sets this exposureTime in the output
346 
347  def __init__(self, *args, **kwargs):
348  """Constructor"""
349  BatchPoolTask.__init__(self, *args, **kwargs)
350  self.makeSubtask("isr")
351  self.makeSubtask("combination")
352 
353  @classmethod
354  def batchWallTime(cls, time, parsedCmd, numCores):
355  numCcds = len(parsedCmd.butler.get("camera"))
356  numExps = len(cls.RunnerClass.getTargetList(
357  parsedCmd)[0]['expRefList'])
358  numCycles = int(numCcds/float(numCores) + 0.5)
359  return time*numExps*numCycles
360 
361  @classmethod
362  def _makeArgumentParser(cls, *args, **kwargs):
363  kwargs.pop("doBatch", False)
364  return CalibArgumentParser(calibName=cls.calibName, name=cls._DefaultName, *args, **kwargs)
365 
366  def run(self, expRefList, butler, calibId):
367  """!Construct a calib from a list of exposure references
368 
369  This is the entry point, called by the TaskRunner.__call__
370 
371  Only the master node executes this method.
372 
373  @param expRefList List of data references at the exposure level
374  @param butler Data butler
375  @param calibId Identifier dict for calib
376  """
377  for expRef in expRefList:
378  self.addMissingKeys(expRef.dataId, butler, self.config.ccdKeys, 'raw')
379 
380  outputId = self.getOutputId(expRefList, calibId)
381  ccdIdLists = getCcdIdListFromExposures(
382  expRefList, level="sensor", ccdKeys=self.config.ccdKeys)
383 
384  # Ensure we can generate filenames for each output
385  outputIdItemList = list(outputId.items())
386  for ccdName in ccdIdLists:
387  dataId = dict([(k, ccdName[i]) for i, k in enumerate(self.config.ccdKeys)])
388  self.addMissingKeys(dataId, butler)
389  dataId.update(outputIdItemList)
390 
391  try:
392  butler.get(self.calibName + "_filename", dataId)
393  except Exception as e:
394  raise RuntimeError(
395  "Unable to determine output filename \"%s_filename\" from %s: %s" %
396  (self.calibName, dataId, e))
397 
398  pool = Pool()
399  pool.storeSet(butler=butler)
400 
401  # Scatter: process CCDs independently
402  data = self.scatterProcess(pool, ccdIdLists)
403 
404  # Gather: determine scalings
405  scales = self.scale(ccdIdLists, data)
406 
407  # Scatter: combine
408  self.scatterCombine(pool, outputId, ccdIdLists, scales)
409 
410  def getOutputId(self, expRefList, calibId):
411  """!Generate the data identifier for the output calib
412 
413  The mean date and the common filter are included, using keywords
414  from the configuration. The CCD-specific part is not included
415  in the data identifier.
416 
417  @param expRefList List of data references at exposure level
418  @param calibId Data identifier elements for the calib provided by the user
419  @return data identifier
420  """
421  midTime = 0
422  filterName = None
423  for expRef in expRefList:
424  butler = expRef.getButler()
425  dataId = expRef.dataId
426 
427  midTime += self.getMjd(butler, dataId)
428  thisFilter = self.getFilter(
429  butler, dataId) if self.filterName is None else self.filterName
430  if filterName is None:
431  filterName = thisFilter
432  elif filterName != thisFilter:
433  raise RuntimeError("Filter mismatch for %s: %s vs %s" % (
434  dataId, thisFilter, filterName))
435 
436  midTime /= len(expRefList)
437  date = str(dafBase.DateTime(
438  midTime, dafBase.DateTime.MJD).toPython().date())
439 
440  outputId = {self.config.filter: filterName,
441  self.config.dateCalib: date}
442  outputId.update(calibId)
443  return outputId
444 
445  def getMjd(self, butler, dataId, timescale=dafBase.DateTime.UTC):
446  """Determine the Modified Julian Date (MJD; in TAI) from a data identifier"""
447  if self.config.dateObs in dataId:
448  dateObs = dataId[self.config.dateObs]
449  else:
450  dateObs = butler.queryMetadata('raw', [self.config.dateObs], dataId)[0]
451  if "T" not in dateObs:
452  dateObs = dateObs + "T12:00:00.0Z"
453  elif not dateObs.endswith("Z"):
454  dateObs += "Z"
455 
456  return dafBase.DateTime(dateObs, timescale).get(dafBase.DateTime.MJD)
457 
458  def getFilter(self, butler, dataId):
459  """Determine the filter from a data identifier"""
460  filt = butler.queryMetadata('raw', [self.config.filter], dataId)[0]
461  return filt
462 
463  def addMissingKeys(self, dataId, butler, missingKeys=None, calibName=None):
464  if calibName is None:
465  calibName = self.calibName
466 
467  if missingKeys is None:
468  missingKeys = set(butler.getKeys(calibName).keys()) - set(dataId.keys())
469 
470  for k in missingKeys:
471  try:
472  v = butler.queryMetadata('raw', [k], dataId) # n.b. --id refers to 'raw'
473  except Exception as e:
474  continue
475 
476  if len(v) == 0: # failed to lookup value
477  continue
478 
479  if len(v) == 1:
480  dataId[k] = v[0]
481  else:
482  raise RuntimeError("No unique lookup for %s: %s" % (k, v))
483 
484  def updateMetadata(self, calibImage, exposureTime, darkTime=None, **kwargs):
485  """!Update the metadata from the VisitInfo
486 
487  \param calibImage The image whose metadata is to be set
488  \param exposureTime The exposure time for the image
489  \param darkTime The time since the last read (default: exposureTime)
490  """
491 
492  if darkTime is None:
493  darkTime = exposureTime # avoid warning messages when using calibration products
494 
495  visitInfo = afwImage.makeVisitInfo(exposureTime=exposureTime, darkTime=darkTime, **kwargs)
496  md = calibImage.getMetadata()
497 
498  try:
499  afwImage.setVisitInfoMetadata(md, visitInfo)
500  except Exception as e:
501  #
502  # should not be needed with pybind11
503  #
504  # Convert a PropertySet to a PropertyList
505  #
506  _md = dafBase.PropertyList()
507  _md.combine(md)
508  md = _md
509  calibImage.setMetadata(md)
510 
511  afwImage.setVisitInfoMetadata(md, visitInfo)
512 
513  def scatterProcess(self, pool, ccdIdLists):
514  """!Scatter the processing among the nodes
515 
516  We scatter each CCD independently (exposures aren't grouped together),
517  to make full use of all available processors. This necessitates piecing
518  everything back together in the same format as ccdIdLists afterwards.
519 
520  Only the master node executes this method.
521 
522  @param pool Process pool
523  @param ccdIdLists Dict of data identifier lists for each CCD name
524  @return Dict of lists of returned data for each CCD name
525  """
526  dataIdList = sum(ccdIdLists.values(), [])
527  self.log.info("Scatter processing")
528 
529  resultList = pool.map(self.process, dataIdList)
530 
531  # Piece everything back together
532  data = dict((ccdName, [None] * len(expList))
533  for ccdName, expList in ccdIdLists.items())
534  indices = dict(sum([[(tuple(dataId.values()) if dataId is not None else None, (ccdName, expNum))
535  for expNum, dataId in enumerate(expList)]
536  for ccdName, expList in ccdIdLists.items()], []))
537  for dataId, result in zip(dataIdList, resultList):
538  if dataId is None:
539  continue
540  ccdName, expNum = indices[tuple(dataId.values())]
541  data[ccdName][expNum] = result
542 
543  return data
544 
545  def process(self, cache, ccdId, outputName="postISRCCD"):
546  """!Process a CCD, specified by a data identifier
547 
548  After processing, optionally returns a result (produced by
549  the 'processResult' method) calculated from the processed
550  exposure. These results will be gathered by the master node,
551  and is a means for coordinated scaling of all CCDs for flats,
552  etc.
553 
554  Only slave nodes execute this method.
555 
556  @param cache Process pool cache
557  @param ccdId Data identifier for CCD
558  @param outputName Output dataset name for butler
559  @return result from 'processResult'
560  """
561  if ccdId is None:
562  self.log.warn("Null identifier received on %s" % NODE)
563  return None
564  sensorRef = getDataRef(cache.butler, ccdId)
565  if self.config.clobber or not sensorRef.datasetExists(outputName):
566  self.log.info("Processing %s on %s" % (ccdId, NODE))
567  try:
568  exposure = self.processSingle(sensorRef)
569  except Exception as e:
570  self.log.warn("Unable to process %s: %s" % (ccdId, e))
571  raise
572  return None
573  self.processWrite(sensorRef, exposure)
574  else:
575  self.log.info(
576  "Using previously persisted processed exposure for %s" % (sensorRef.dataId,))
577  exposure = sensorRef.get(outputName, immediate=True)
578  return self.processResult(exposure)
579 
580  def processSingle(self, dataRef):
581  """Process a single CCD, specified by a data reference
582 
583  Generally, this simply means doing ISR.
584 
585  Only slave nodes execute this method.
586  """
587  return self.isr.runDataRef(dataRef).exposure
588 
589  def processWrite(self, dataRef, exposure, outputName="postISRCCD"):
590  """!Write the processed CCD
591 
592  We need to write these out because we can't hold them all in
593  memory at once.
594 
595  Only slave nodes execute this method.
596 
597  @param dataRef Data reference
598  @param exposure CCD exposure to write
599  @param outputName Output dataset name for butler.
600  """
601  dataRef.put(exposure, outputName)
602 
603  def processResult(self, exposure):
604  """Extract processing results from a processed exposure
605 
606  This method generates what is gathered by the master node.
607  This can be a background measurement or similar for scaling
608  flat-fields. It must be picklable!
609 
610  Only slave nodes execute this method.
611  """
612  return None
613 
614  def scale(self, ccdIdLists, data):
615  """!Determine scaling across CCDs and exposures
616 
617  This is necessary mainly for flats, so as to determine a
618  consistent scaling across the entire focal plane. This
619  implementation is simply a placeholder.
620 
621  Only the master node executes this method.
622 
623  @param ccdIdLists Dict of data identifier lists for each CCD tuple
624  @param data Dict of lists of returned data for each CCD tuple
625  @return dict of Struct(ccdScale: scaling for CCD,
626  expScales: scaling for each exposure
627  ) for each CCD tuple
628  """
629  self.log.info("Scale on %s" % NODE)
630  return dict((name, Struct(ccdScale=None, expScales=[None] * len(ccdIdLists[name])))
631  for name in ccdIdLists)
632 
633  def scatterCombine(self, pool, outputId, ccdIdLists, scales):
634  """!Scatter the combination of exposures across multiple nodes
635 
636  In this case, we can only scatter across as many nodes as
637  there are CCDs.
638 
639  Only the master node executes this method.
640 
641  @param pool Process pool
642  @param outputId Output identifier (exposure part only)
643  @param ccdIdLists Dict of data identifier lists for each CCD name
644  @param scales Dict of structs with scales, for each CCD name
645  """
646  self.log.info("Scatter combination")
647  data = [Struct(ccdName=ccdName, ccdIdList=ccdIdLists[ccdName], scales=scales[ccdName]) for
648  ccdName in ccdIdLists]
649  pool.map(self.combine, data, outputId)
650 
651  def combine(self, cache, struct, outputId):
652  """!Combine multiple exposures of a particular CCD and write the output
653 
654  Only the slave nodes execute this method.
655 
656  @param cache Process pool cache
657  @param struct Parameters for the combination, which has the following components:
658  * ccdName Name tuple for CCD
659  * ccdIdList List of data identifiers for combination
660  * scales Scales to apply (expScales are scalings for each exposure,
661  ccdScale is final scale for combined image)
662  @param outputId Data identifier for combined image (exposure part only)
663  """
664  # Check if we need to look up any keys that aren't in the output dataId
665  fullOutputId = {k: struct.ccdName[i] for i, k in enumerate(self.config.ccdKeys)}
666  self.addMissingKeys(fullOutputId, cache.butler)
667  fullOutputId.update(outputId) # must be after the call to queryMetadata
668  outputId = fullOutputId
669  del fullOutputId
670 
671  dataRefList = [getDataRef(cache.butler, dataId) if dataId is not None else None for
672  dataId in struct.ccdIdList]
673  self.log.info("Combining %s on %s" % (outputId, NODE))
674  calib = self.combination.run(dataRefList, expScales=struct.scales.expScales,
675  finalScale=struct.scales.ccdScale)
676 
677  if not hasattr(calib, "getMetadata"):
678  if hasattr(calib, "getVariance"):
679  calib = afwImage.makeExposure(calib)
680  else:
681  calib = afwImage.DecoratedImageF(calib.getImage()) # n.b. hardwires "F" for the output type
682 
683  self.updateMetadata(calib, self.exposureTime)
684 
685  self.recordCalibInputs(cache.butler, calib,
686  struct.ccdIdList, outputId)
687 
688  self.interpolateNans(calib)
689 
690  self.write(cache.butler, calib, outputId)
691 
692  def recordCalibInputs(self, butler, calib, dataIdList, outputId):
693  """!Record metadata including the inputs and creation details
694 
695  This metadata will go into the FITS header.
696 
697  @param butler Data butler
698  @param calib Combined calib exposure.
699  @param dataIdList List of data identifiers for calibration inputs
700  @param outputId Data identifier for output
701  """
702  header = calib.getMetadata()
703  header.add("OBSTYPE", self.calibName) # Used by ingestCalibs.py
704 
705  # date, time, host, and root
706  now = time.localtime()
707  header.add("CALIB_CREATION_DATE", time.strftime("%Y-%m-%d", now))
708  header.add("CALIB_CREATION_TIME", time.strftime("%X %Z", now))
709 
710  # Inputs
711  visits = [str(dictToTuple(dataId, self.config.visitKeys)) for dataId in dataIdList if
712  dataId is not None]
713  for i, v in enumerate(sorted(set(visits))):
714  header.add("CALIB_INPUT_%d" % (i,), v)
715 
716  header.add("CALIB_ID", " ".join("%s=%s" % (key, value)
717  for key, value in outputId.items()))
718  checksum(calib, header)
719 
720  def interpolateNans(self, image):
721  """Interpolate over NANs in the combined image
722 
723  NANs can result from masked areas on the CCD. We don't want them getting
724  into our science images, so we replace them with the median of the image.
725  """
726  if hasattr(image, "getMaskedImage"): # Deal with Exposure vs Image
727  self.interpolateNans(image.getMaskedImage().getVariance())
728  image = image.getMaskedImage().getImage()
729  if hasattr(image, "getImage"): # Deal with DecoratedImage or MaskedImage vs Image
730  image = image.getImage()
731  array = image.getArray()
732  bad = np.isnan(array)
733  array[bad] = np.median(array[np.logical_not(bad)])
734 
735  def write(self, butler, exposure, dataId):
736  """!Write the final combined calib
737 
738  Only the slave nodes execute this method
739 
740  @param butler Data butler
741  @param exposure CCD exposure to write
742  @param dataId Data identifier for output
743  """
744  self.log.info("Writing %s on %s" % (dataId, NODE))
745  butler.put(exposure, self.calibName, dataId)
746 
747 
749  """Configuration for bias construction.
750 
751  No changes required compared to the base class, but
752  subclassed for distinction.
753  """
754  pass
755 
756 
757 class BiasTask(CalibTask):
758  """Bias construction"""
759  ConfigClass = BiasConfig
760  _DefaultName = "bias"
761  calibName = "bias"
762  filterName = "NONE" # Sets this filter name in the output
763  exposureTime = 0.0 # sets this exposureTime in the output
764 
765  @classmethod
766  def applyOverrides(cls, config):
767  """Overrides to apply for bias construction"""
768  config.isr.doBias = False
769  config.isr.doDark = False
770  config.isr.doFlat = False
771  config.isr.doFringe = False
772 
773 
775  """Configuration for dark construction"""
776  doRepair = Field(dtype=bool, default=True, doc="Repair artifacts?")
777  psfFwhm = Field(dtype=float, default=3.0, doc="Repair PSF FWHM (pixels)")
778  psfSize = Field(dtype=int, default=21, doc="Repair PSF size (pixels)")
779  crGrow = Field(dtype=int, default=2, doc="Grow radius for CR (pixels)")
780  repair = ConfigurableField(
781  target=RepairTask, doc="Task to repair artifacts")
782 
783  def setDefaults(self):
784  CalibConfig.setDefaults(self)
785  self.combination.mask.append("CR")
786 
787 
789  """Dark construction
790 
791  The only major difference from the base class is a cosmic-ray
792  identification stage, and dividing each image by the dark time
793  to generate images of the dark rate.
794  """
795  ConfigClass = DarkConfig
796  _DefaultName = "dark"
797  calibName = "dark"
798  filterName = "NONE" # Sets this filter name in the output
799 
800  def __init__(self, *args, **kwargs):
801  CalibTask.__init__(self, *args, **kwargs)
802  self.makeSubtask("repair")
803 
804  @classmethod
805  def applyOverrides(cls, config):
806  """Overrides to apply for dark construction"""
807  config.isr.doDark = False
808  config.isr.doFlat = False
809  config.isr.doFringe = False
810 
811  def processSingle(self, sensorRef):
812  """Process a single CCD
813 
814  Besides the regular ISR, also masks cosmic-rays and divides each
815  processed image by the dark time to generate images of the dark rate.
816  The dark time is provided by the 'getDarkTime' method.
817  """
818  exposure = CalibTask.processSingle(self, sensorRef)
819 
820  if self.config.doRepair:
821  psf = measAlg.DoubleGaussianPsf(self.config.psfSize, self.config.psfSize,
822  self.config.psfFwhm/(2*math.sqrt(2*math.log(2))))
823  exposure.setPsf(psf)
824  self.repair.run(exposure, keepCRs=False)
825  if self.config.crGrow > 0:
826  mask = exposure.getMaskedImage().getMask().clone()
827  mask &= mask.getPlaneBitMask("CR")
828  fpSet = afwDet.FootprintSet(
829  mask, afwDet.Threshold(0.5))
830  fpSet = afwDet.FootprintSet(fpSet, self.config.crGrow, True)
831  fpSet.setMask(exposure.getMaskedImage().getMask(), "CR")
832 
833  mi = exposure.getMaskedImage()
834  mi /= self.getDarkTime(exposure)
835  return exposure
836 
837  def getDarkTime(self, exposure):
838  """Retrieve the dark time for an exposure"""
839  darkTime = exposure.getInfo().getVisitInfo().getDarkTime()
840  if not np.isfinite(darkTime):
841  raise RuntimeError("Non-finite darkTime")
842  return darkTime
843 
844 
846  """Configuration for flat construction"""
847  iterations = Field(dtype=int, default=10,
848  doc="Number of iterations for scale determination")
849  stats = ConfigurableField(target=CalibStatsTask,
850  doc="Background statistics configuration")
851 
852 
854  """Flat construction
855 
856  The principal change from the base class involves gathering the background
857  values from each image and using them to determine the scalings for the final
858  combination.
859  """
860  ConfigClass = FlatConfig
861  _DefaultName = "flat"
862  calibName = "flat"
863 
864  @classmethod
865  def applyOverrides(cls, config):
866  """Overrides for flat construction"""
867  config.isr.doFlat = False
868  config.isr.doFringe = False
869 
870  def __init__(self, *args, **kwargs):
871  CalibTask.__init__(self, *args, **kwargs)
872  self.makeSubtask("stats")
873 
874  def processResult(self, exposure):
875  return self.stats.run(exposure)
876 
877  def scale(self, ccdIdLists, data):
878  """Determine the scalings for the final combination
879 
880  We have a matrix B_ij = C_i E_j, where C_i is the relative scaling
881  of one CCD to all the others in an exposure, and E_j is the scaling
882  of the exposure. We convert everything to logarithms so we can work
883  with a linear system. We determine the C_i and E_j from B_ij by iteration,
884  under the additional constraint that the average CCD scale is unity.
885 
886  This algorithm comes from Eugene Magnier and Pan-STARRS.
887  """
888  assert len(ccdIdLists.values()) > 0, "No successful CCDs"
889  lengths = set([len(expList) for expList in ccdIdLists.values()])
890  assert len(
891  lengths) == 1, "Number of successful exposures for each CCD differs"
892  assert tuple(lengths)[0] > 0, "No successful exposures"
893  # Format background measurements into a matrix
894  indices = dict((name, i) for i, name in enumerate(ccdIdLists))
895  bgMatrix = np.array([[0.0] * len(expList)
896  for expList in ccdIdLists.values()])
897  for name in ccdIdLists:
898  i = indices[name]
899  bgMatrix[i] = [
900  d if d is not None else np.nan for d in data[name]]
901 
902  numpyPrint = np.get_printoptions()
903  np.set_printoptions(threshold='nan')
904  self.log.info("Input backgrounds: %s" % bgMatrix)
905 
906  # Flat-field scaling
907  numCcds = len(ccdIdLists)
908  numExps = bgMatrix.shape[1]
909  # log(Background) for each exposure/component
910  bgMatrix = np.log(bgMatrix)
911  bgMatrix = np.ma.masked_array(bgMatrix, np.isnan(bgMatrix))
912  # Initial guess at log(scale) for each component
913  compScales = np.zeros(numCcds)
914  expScales = np.array(
915  [(bgMatrix[:, i0] - compScales).mean() for i0 in range(numExps)])
916 
917  for iterate in range(self.config.iterations):
918  compScales = np.array(
919  [(bgMatrix[i1, :] - expScales).mean() for i1 in range(numCcds)])
920  expScales = np.array(
921  [(bgMatrix[:, i2] - compScales).mean() for i2 in range(numExps)])
922 
923  avgScale = np.average(np.exp(compScales))
924  compScales -= np.log(avgScale)
925  self.log.debug("Iteration %d exposure scales: %s",
926  iterate, np.exp(expScales))
927  self.log.debug("Iteration %d component scales: %s",
928  iterate, np.exp(compScales))
929 
930  expScales = np.array(
931  [(bgMatrix[:, i3] - compScales).mean() for i3 in range(numExps)])
932 
933  if np.any(np.isnan(expScales)):
934  raise RuntimeError("Bad exposure scales: %s --> %s" %
935  (bgMatrix, expScales))
936 
937  expScales = np.exp(expScales)
938  compScales = np.exp(compScales)
939 
940  self.log.info("Exposure scales: %s" % expScales)
941  self.log.info("Component relative scaling: %s" % compScales)
942  np.set_printoptions(**numpyPrint)
943 
944  return dict((ccdName, Struct(ccdScale=compScales[indices[ccdName]], expScales=expScales))
945  for ccdName in ccdIdLists)
946 
947 
949  """Configuration for fringe construction"""
950  stats = ConfigurableField(target=CalibStatsTask,
951  doc="Background statistics configuration")
952  subtractBackground = ConfigurableField(target=measAlg.SubtractBackgroundTask,
953  doc="Background configuration")
954  detection = ConfigurableField(
955  target=measAlg.SourceDetectionTask, doc="Detection configuration")
956  detectSigma = Field(dtype=float, default=1.0,
957  doc="Detection PSF gaussian sigma")
958 
959 
961  """Fringe construction task
962 
963  The principal change from the base class is that the images are
964  background-subtracted and rescaled by the background.
965 
966  XXX This is probably not right for a straight-up combination, as we
967  are currently doing, since the fringe amplitudes need not scale with
968  the continuum.
969 
970  XXX Would like to have this do PCA and generate multiple images, but
971  that will take a bit of work with the persistence code.
972  """
973  ConfigClass = FringeConfig
974  _DefaultName = "fringe"
975  calibName = "fringe"
976 
977  @classmethod
978  def applyOverrides(cls, config):
979  """Overrides for fringe construction"""
980  config.isr.doFringe = False
981 
982  def __init__(self, *args, **kwargs):
983  CalibTask.__init__(self, *args, **kwargs)
984  self.makeSubtask("detection")
985  self.makeSubtask("stats")
986  self.makeSubtask("subtractBackground")
987 
988  def processSingle(self, sensorRef):
989  """Subtract the background and normalise by the background level"""
990  exposure = CalibTask.processSingle(self, sensorRef)
991  bgLevel = self.stats.run(exposure)
992  self.subtractBackground.run(exposure)
993  mi = exposure.getMaskedImage()
994  mi /= bgLevel
995  footprintSets = self.detection.detectFootprints(
996  exposure, sigma=self.config.detectSigma)
997  mask = exposure.getMaskedImage().getMask()
998  detected = 1 << mask.addMaskPlane("DETECTED")
999  for fpSet in (footprintSets.positive, footprintSets.negative):
1000  if fpSet is not None:
1001  afwDet.setMaskFromFootprintList(
1002  mask, fpSet.getFootprints(), detected)
1003  return exposure
def run
Measure a particular statistic on an image (of some sort).
def getCcdIdListFromExposures
Determine a list of CCDs from exposure references.
def scatterProcess
Scatter the processing among the nodes.
def checksum
Calculate a checksum of an object.
Definition: checksum.py:29
def recordCalibInputs
Record metadata including the inputs and creation details.
def updateMetadata
Update the metadata from the VisitInfo.
def getOutputId
Generate the data identifier for the output calib.
def write
Write the final combined calib.
def run
Construct a calib from a list of exposure references.
def combine
Combine multiple exposures of a particular CCD and write the output.
def scale
Determine scaling across CCDs and exposures.
def scatterCombine
Scatter the combination of exposures across multiple nodes.
def run
Combine calib images for a single sensor.
def process
Process a CCD, specified by a data identifier.
def dictToTuple
Return a tuple of specific values from a dict.
def processWrite
Write the processed CCD.
Base class for constructing calibs.