lsst.pipe.drivers  13.0-10-geaaafb9
 All Classes Namespaces Files Functions Variables Pages
constructCalibs.py
Go to the documentation of this file.
1 from __future__ import absolute_import, division, print_function
2 import sys
3 import math
4 import time
5 import argparse
6 import traceback
7 
8 import numpy as np
9 from builtins import zip
10 from builtins import range
11 
12 from lsst.pex.config import Config, ConfigurableField, Field, ListField
13 from lsst.pipe.base import Task, Struct, TaskRunner, ArgumentParser
14 import lsst.daf.base as dafBase
15 import lsst.afw.math as afwMath
16 import lsst.afw.geom as afwGeom
17 import lsst.afw.detection as afwDet
18 import lsst.afw.image as afwImage
19 import lsst.meas.algorithms as measAlg
20 from lsst.pipe.tasks.repair import RepairTask
21 from lsst.ip.isr import IsrTask
22 
23 from lsst.ctrl.pool.parallel import BatchPoolTask
24 from lsst.ctrl.pool.pool import Pool, NODE
25 
26 from .checksum import checksum
27 from .utils import getDataRef
28 
29 
30 class CalibStatsConfig(Config):
31  """Parameters controlling the measurement of background statistics"""
32  stat = Field(doc="Statistic to use to estimate background (from lsst.afw.math)", dtype=int,
33  default=int(afwMath.MEANCLIP))
34  clip = Field(doc="Clipping threshold for background",
35  dtype=float, default=3.0)
36  nIter = Field(doc="Clipping iterations for background",
37  dtype=int, default=3)
38  mask = ListField(doc="Mask planes to reject",
39  dtype=str, default=["DETECTED", "BAD"])
40 
41 
42 class CalibStatsTask(Task):
43  """Measure statistics on the background
44 
45  This can be useful for scaling the background, e.g., for flats and fringe frames.
46  """
47  ConfigClass = CalibStatsConfig
48 
49  def run(self, exposureOrImage):
50  """!Measure a particular statistic on an image (of some sort).
51 
52  @param exposureOrImage Exposure, MaskedImage or Image.
53  @return Value of desired statistic
54  """
55  stats = afwMath.StatisticsControl(self.config.clip, self.config.nIter,
56  afwImage.MaskU.getPlaneBitMask(self.config.mask))
57  try:
58  image = exposureOrImage.getMaskedImage()
59  except:
60  try:
61  image = exposureOrImage.getImage()
62  except:
63  image = exposureOrImage
64 
65  return afwMath.makeStatistics(image, self.config.stat, stats).getValue()
66 
67 
68 class CalibCombineConfig(Config):
69  """Configuration for combining calib images"""
70  rows = Field(doc="Number of rows to read at a time",
71  dtype=int, default=512)
72  mask = ListField(doc="Mask planes to respect", dtype=str,
73  default=["SAT", "DETECTED", "INTRP"])
74  combine = Field(doc="Statistic to use for combination (from lsst.afw.math)", dtype=int,
75  default=int(afwMath.MEANCLIP))
76  clip = Field(doc="Clipping threshold for combination",
77  dtype=float, default=3.0)
78  nIter = Field(doc="Clipping iterations for combination",
79  dtype=int, default=3)
80  stats = ConfigurableField(target=CalibStatsTask,
81  doc="Background statistics configuration")
82 
83 
84 class CalibCombineTask(Task):
85  """Task to combine calib images"""
86  ConfigClass = CalibCombineConfig
87 
88  def __init__(self, *args, **kwargs):
89  Task.__init__(self, *args, **kwargs)
90  self.makeSubtask("stats")
91 
92  def run(self, sensorRefList, expScales=None, finalScale=None, inputName="postISRCCD"):
93  """!Combine calib images for a single sensor
94 
95  @param sensorRefList List of data references to combine (for a single sensor)
96  @param expScales List of scales to apply for each exposure
97  @param finalScale Desired scale for final combined image
98  @param inputName Data set name for inputs
99  @return combined image
100  """
101  width, height = self.getDimensions(sensorRefList)
102  maskVal = 0
103  for mask in self.config.mask:
104  maskVal |= afwImage.MaskU.getPlaneBitMask(mask)
105  stats = afwMath.StatisticsControl(
106  self.config.clip, self.config.nIter, maskVal)
107 
108  # Combine images
109  combined = afwImage.MaskedImageF(width, height)
110  numImages = len(sensorRefList)
111  imageList = [None]*numImages
112  for start in range(0, height, self.config.rows):
113  rows = min(self.config.rows, height - start)
114  box = afwGeom.Box2I(afwGeom.Point2I(0, start),
115  afwGeom.Extent2I(width, rows))
116  subCombined = combined.Factory(combined, box)
117 
118  for i, sensorRef in enumerate(sensorRefList):
119  if sensorRef is None:
120  imageList[i] = None
121  continue
122  exposure = sensorRef.get(inputName + "_sub", bbox=box)
123  if expScales is not None:
124  self.applyScale(exposure, expScales[i])
125  imageList[i] = exposure.getMaskedImage()
126 
127  self.combine(subCombined, imageList, stats)
128 
129  if finalScale is not None:
130  background = self.stats.run(combined)
131  self.log.info("%s: Measured background of stack is %f; adjusting to %f" %
132  (NODE, background, finalScale))
133  combined *= finalScale / background
134 
135  return afwImage.DecoratedImageF(combined.getImage())
136 
137  def getDimensions(self, sensorRefList, inputName="postISRCCD"):
138  """Get dimensions of the inputs"""
139  dimList = []
140  for sensorRef in sensorRefList:
141  if sensorRef is None:
142  continue
143  md = sensorRef.get(inputName + "_md")
144  dimList.append(afwGeom.Extent2I(
145  md.get("NAXIS1"), md.get("NAXIS2")))
146  return getSize(dimList)
147 
148  def applyScale(self, exposure, scale=None):
149  """Apply scale to input exposure
150 
151  This implementation applies a flux scaling: the input exposure is
152  divided by the provided scale.
153  """
154  if scale is not None:
155  mi = exposure.getMaskedImage()
156  mi /= scale
157 
158  def combine(self, target, imageList, stats):
159  """!Combine multiple images
160 
161  @param target Target image to receive the combined pixels
162  @param imageList List of input images
163  @param stats Statistics control
164  """
165  images = [img for img in imageList if img is not None]
166  afwMath.statisticsStack(target, images, afwMath.Property(self.config.combine), stats)
167 
168 
169 def getSize(dimList):
170  """Determine a consistent size, given a list of image sizes"""
171  dim = set((w, h) for w, h in dimList)
172  dim.discard(None)
173  if len(dim) != 1:
174  raise RuntimeError("Inconsistent dimensions: %s" % dim)
175  return dim.pop()
176 
177 
178 def dictToTuple(dict_, keys):
179  """!Return a tuple of specific values from a dict
180 
181  This provides a hashable representation of the dict from certain keywords.
182  This can be useful for creating e.g., a tuple of the values in the DataId
183  that identify the CCD.
184 
185  @param dict_ dict to parse
186  @param keys keys to extract (order is important)
187  @return tuple of values
188  """
189  return tuple(dict_[k] for k in keys)
190 
191 
192 def getCcdIdListFromExposures(expRefList, level="sensor", ccdKeys=["ccd"]):
193  """!Determine a list of CCDs from exposure references
194 
195  This essentially inverts the exposure-level references (which
196  provides a list of CCDs for each exposure), by providing
197  a dataId list for each CCD. Consider an input list of exposures
198  [e1, e2, e3], and each exposure has CCDs c1 and c2. Then this
199  function returns:
200 
201  {(c1,): [e1c1, e2c1, e3c1], (c2,): [e1c2, e2c2, e3c2]}
202 
203  This is a dict whose keys are tuples of the identifying values of a
204  CCD (usually just the CCD number) and the values are lists of dataIds
205  for that CCD in each exposure. A missing dataId is given the value
206  None.
207 
208  @param expRefList List of data references for exposures
209  @param level Level for the butler to generate CCDs
210  @param ccdKeys DataId keywords that identify a CCD
211  @return dict of data identifier lists for each CCD
212  """
213  expIdList = [[ccdRef.dataId for ccdRef in expRef.subItems(
214  level)] for expRef in expRefList]
215 
216  # Determine what additional keys make a CCD from an exposure
217  ccdKeys = set(ccdKeys) # Set of keywords in the dataId that identify a CCD
218  ccdNames = set() # Set of tuples which are values for each of the CCDs in an exposure
219  for ccdIdList in expIdList:
220  for ccdId in ccdIdList:
221  name = dictToTuple(ccdId, ccdKeys)
222  ccdNames.add(name)
223 
224  # Turn the list of CCDs for each exposure into a list of exposures for
225  # each CCD
226  ccdLists = {}
227  for n, ccdIdList in enumerate(expIdList):
228  for ccdId in ccdIdList:
229  name = dictToTuple(ccdId, ccdKeys)
230  if name not in ccdLists:
231  ccdLists[name] = []
232  ccdLists[name].append(ccdId)
233 
234  return ccdLists
235 
236 
237 class CalibIdAction(argparse.Action):
238  """Split name=value pairs and put the result in a dict"""
239 
240  def __call__(self, parser, namespace, values, option_string):
241  output = getattr(namespace, self.dest, {})
242  for nameValue in values:
243  name, sep, valueStr = nameValue.partition("=")
244  if not valueStr:
245  parser.error("%s value %s must be in form name=value" %
246  (option_string, nameValue))
247  output[name] = valueStr
248  setattr(namespace, self.dest, output)
249 
250 
251 class CalibArgumentParser(ArgumentParser):
252  """ArgumentParser for calibration construction"""
253 
254  def __init__(self, calibName, *args, **kwargs):
255  """Add a --calibId argument to the standard pipe_base argument parser"""
256  ArgumentParser.__init__(self, *args, **kwargs)
257  self.calibName = calibName
258  self.add_id_argument("--id", datasetType="raw",
259  help="input identifiers, e.g., --id visit=123 ccd=4")
260  self.add_argument("--calibId", nargs="*", action=CalibIdAction, default={},
261  help="identifiers for calib, e.g., --calibId version=1",
262  metavar="KEY=VALUE1[^VALUE2[^VALUE3...]")
263 
264  def parse_args(self, *args, **kwargs):
265  """Parse arguments
266 
267  Checks that the "--calibId" provided works.
268  """
269  namespace = ArgumentParser.parse_args(self, *args, **kwargs)
270 
271  keys = namespace.butler.getKeys(self.calibName)
272  parsed = {}
273  for name, value in namespace.calibId.items():
274  if name not in keys:
275  self.error(
276  "%s is not a relevant calib identifier key (%s)" % (name, keys))
277  parsed[name] = keys[name](value)
278  namespace.calibId = parsed
279 
280  return namespace
281 
282 
283 class CalibConfig(Config):
284  """Configuration for constructing calibs"""
285  clobber = Field(dtype=bool, default=True,
286  doc="Clobber existing processed images?")
287  isr = ConfigurableField(target=IsrTask, doc="ISR configuration")
288  dateObs = Field(dtype=str, default="dateObs",
289  doc="Key for observation date in exposure registry")
290  dateCalib = Field(dtype=str, default="calibDate",
291  doc="Key for calib date in calib registry")
292  filter = Field(dtype=str, default="filter",
293  doc="Key for filter name in exposure/calib registries")
294  combination = ConfigurableField(
295  target=CalibCombineTask, doc="Calib combination configuration")
296  ccdKeys = ListField(dtype=str, default=[
297  "ccd"], doc="DataId keywords specifying a CCD")
298  visitKeys = ListField(dtype=str, default=[
299  "visit"], doc="DataId keywords specifying a visit")
300  calibKeys = ListField(dtype=str, default=[],
301  doc="DataId keywords specifying a calibration")
302 
303  def setDefaults(self):
304  self.isr.doWrite = False
305 
306 
307 class CalibTaskRunner(TaskRunner):
308  """Get parsed values into the CalibTask.run"""
309  @staticmethod
310  def getTargetList(parsedCmd, **kwargs):
311  return [dict(expRefList=parsedCmd.id.refList, butler=parsedCmd.butler, calibId=parsedCmd.calibId)]
312 
313  def __call__(self, args):
314  """Call the Task with the kwargs from getTargetList"""
315  task = self.TaskClass(config=self.config, log=self.log)
316  if self.doRaise:
317  result = task.run(**args)
318  else:
319  try:
320  result = task.run(**args)
321  except Exception as e:
322  task.log.fatal("Failed: %s" % e)
323  traceback.print_exc(file=sys.stderr)
324 
325  if self.doReturnResults:
326  return Struct(
327  args=args,
328  metadata=task.metadata,
329  result=result,
330  )
331 
332 
333 class CalibTask(BatchPoolTask):
334  """!Base class for constructing calibs.
335 
336  This should be subclassed for each of the required calib types.
337  The subclass should be sure to define the following class variables:
338  * _DefaultName: default name of the task, used by CmdLineTask
339  * calibName: name of the calibration data set in the butler
340  The subclass may optionally set:
341  * filterName: filter name to give the resultant calib
342  """
343  ConfigClass = CalibConfig
344  RunnerClass = CalibTaskRunner
345  filterName = None
346  calibName = None
347 
348  def __init__(self, *args, **kwargs):
349  """Constructor"""
350  BatchPoolTask.__init__(self, *args, **kwargs)
351  self.makeSubtask("isr")
352  self.makeSubtask("combination")
353 
354  @classmethod
355  def batchWallTime(cls, time, parsedCmd, numCores):
356  numCcds = len(parsedCmd.butler.get("camera"))
357  numExps = len(cls.RunnerClass.getTargetList(
358  parsedCmd)[0]['expRefList'])
359  numCycles = int(numCcds/float(numCores) + 0.5)
360  return time*numExps*numCycles
361 
362  @classmethod
363  def _makeArgumentParser(cls, *args, **kwargs):
364  kwargs.pop("doBatch", False)
365  return CalibArgumentParser(calibName=cls.calibName, name=cls._DefaultName, *args, **kwargs)
366 
367  def run(self, expRefList, butler, calibId):
368  """!Construct a calib from a list of exposure references
369 
370  This is the entry point, called by the TaskRunner.__call__
371 
372  Only the master node executes this method.
373 
374  @param expRefList List of data references at the exposure level
375  @param butler Data butler
376  @param calibId Identifier dict for calib
377  """
378  for expRef in expRefList:
379  self.addMissingKeys(expRef.dataId, butler, self.config.ccdKeys, 'raw')
380 
381  outputId = self.getOutputId(expRefList, calibId)
382  ccdIdLists = getCcdIdListFromExposures(
383  expRefList, level="sensor", ccdKeys=self.config.ccdKeys)
384 
385  # Ensure we can generate filenames for each output
386  outputIdItemList = list(outputId.items())
387  for ccdName in ccdIdLists:
388  dataId = dict([(k, ccdName[i]) for i, k in enumerate(self.config.ccdKeys)])
389  self.addMissingKeys(dataId, butler)
390  dataId.update(outputIdItemList)
391 
392  try:
393  butler.get(self.calibName + "_filename", dataId)
394  except Exception as e:
395  raise RuntimeError(
396  "Unable to determine output filename \"%s_filename\" from %s: %s" %
397  (self.calibName, dataId, e))
398 
399  pool = Pool()
400  pool.storeSet(butler=butler)
401 
402  # Scatter: process CCDs independently
403  data = self.scatterProcess(pool, ccdIdLists)
404 
405  # Gather: determine scalings
406  scales = self.scale(ccdIdLists, data)
407 
408  # Scatter: combine
409  self.scatterCombine(pool, outputId, ccdIdLists, scales)
410 
411  def getOutputId(self, expRefList, calibId):
412  """!Generate the data identifier for the output calib
413 
414  The mean date and the common filter are included, using keywords
415  from the configuration. The CCD-specific part is not included
416  in the data identifier.
417 
418  @param expRefList List of data references at exposure level
419  @param calibId Data identifier elements for the calib provided by the user
420  @return data identifier
421  """
422  midTime = 0
423  filterName = None
424  for expRef in expRefList:
425  butler = expRef.getButler()
426  dataId = expRef.dataId
427 
428  midTime += self.getMjd(butler, dataId)
429  thisFilter = self.getFilter(
430  butler, dataId) if self.filterName is None else self.filterName
431  if filterName is None:
432  filterName = thisFilter
433  elif filterName != thisFilter:
434  raise RuntimeError("Filter mismatch for %s: %s vs %s" % (
435  dataId, thisFilter, filterName))
436 
437  midTime /= len(expRefList)
438  date = str(dafBase.DateTime(
439  midTime, dafBase.DateTime.MJD).toPython().date())
440 
441  outputId = {self.config.filter: filterName,
442  self.config.dateCalib: date}
443  outputId.update(calibId)
444  return outputId
445 
446  def getMjd(self, butler, dataId, timescale=dafBase.DateTime.UTC):
447  """Determine the Modified Julian Date (MJD; in TAI) from a data identifier"""
448  if self.config.dateObs in dataId:
449  dateObs = dataId[self.config.dateObs]
450  else:
451  dateObs = butler.queryMetadata('raw', [self.config.dateObs], dataId)[0]
452  if "T" not in dateObs:
453  dateObs = dateObs + "T12:00:00.0Z"
454  elif not dateObs.endswith("Z"):
455  dateObs += "Z"
456 
457  return dafBase.DateTime(dateObs, timescale).get(dafBase.DateTime.MJD)
458 
459  def getFilter(self, butler, dataId):
460  """Determine the filter from a data identifier"""
461  filt = butler.queryMetadata('raw', [self.config.filter], dataId)[0]
462  return filt
463 
464  def addMissingKeys(self, dataId, butler, missingKeys=None, calibName=None):
465  if calibName is None:
466  calibName = self.calibName
467 
468  if missingKeys is None:
469  missingKeys = set(butler.getKeys(calibName).keys()) - set(dataId.keys())
470 
471  for k in missingKeys:
472  try:
473  v = butler.queryMetadata('raw', [k], dataId) # n.b. --id refers to 'raw'
474  except Exception as e:
475  continue
476 
477  if len(v) == 0: # failed to lookup value
478  continue
479 
480  if len(v) == 1:
481  dataId[k] = v[0]
482  else:
483  raise RuntimeError("No unique lookup for %s: %s" % (k, v))
484 
485  def scatterProcess(self, pool, ccdIdLists):
486  """!Scatter the processing among the nodes
487 
488  We scatter each CCD independently (exposures aren't grouped together),
489  to make full use of all available processors. This necessitates piecing
490  everything back together in the same format as ccdIdLists afterwards.
491 
492  Only the master node executes this method.
493 
494  @param pool Process pool
495  @param ccdIdLists Dict of data identifier lists for each CCD name
496  @return Dict of lists of returned data for each CCD name
497  """
498  dataIdList = sum(ccdIdLists.values(), [])
499  self.log.info("Scatter processing")
500 
501  resultList = pool.map(self.process, dataIdList)
502 
503  # Piece everything back together
504  data = dict((ccdName, [None] * len(expList))
505  for ccdName, expList in ccdIdLists.items())
506  indices = dict(sum([[(tuple(dataId.values()) if dataId is not None else None, (ccdName, expNum))
507  for expNum, dataId in enumerate(expList)]
508  for ccdName, expList in ccdIdLists.items()], []))
509  for dataId, result in zip(dataIdList, resultList):
510  if dataId is None:
511  continue
512  ccdName, expNum = indices[tuple(dataId.values())]
513  data[ccdName][expNum] = result
514 
515  return data
516 
517  def process(self, cache, ccdId, outputName="postISRCCD"):
518  """!Process a CCD, specified by a data identifier
519 
520  After processing, optionally returns a result (produced by
521  the 'processResult' method) calculated from the processed
522  exposure. These results will be gathered by the master node,
523  and is a means for coordinated scaling of all CCDs for flats,
524  etc.
525 
526  Only slave nodes execute this method.
527 
528  @param cache Process pool cache
529  @param ccdId Data identifier for CCD
530  @param outputName Output dataset name for butler
531  @return result from 'processResult'
532  """
533  if ccdId is None:
534  self.log.warn("Null identifier received on %s" % NODE)
535  return None
536  sensorRef = getDataRef(cache.butler, ccdId)
537  if self.config.clobber or not sensorRef.datasetExists(outputName):
538  self.log.info("Processing %s on %s" % (ccdId, NODE))
539  try:
540  exposure = self.processSingle(sensorRef)
541  except Exception as e:
542  self.log.warn("Unable to process %s: %s" % (ccdId, e))
543  raise
544  return None
545  self.processWrite(sensorRef, exposure)
546  else:
547  self.log.info(
548  "Using previously persisted processed exposure for %s" % (sensorRef.dataId,))
549  exposure = sensorRef.get(outputName, immediate=True)
550  return self.processResult(exposure)
551 
552  def processSingle(self, dataRef):
553  """Process a single CCD, specified by a data reference
554 
555  Generally, this simply means doing ISR.
556 
557  Only slave nodes execute this method.
558  """
559  return self.isr.runDataRef(dataRef).exposure
560 
561  def processWrite(self, dataRef, exposure, outputName="postISRCCD"):
562  """!Write the processed CCD
563 
564  We need to write these out because we can't hold them all in
565  memory at once.
566 
567  Only slave nodes execute this method.
568 
569  @param dataRef Data reference
570  @param exposure CCD exposure to write
571  @param outputName Output dataset name for butler.
572  """
573  dataRef.put(exposure, outputName)
574 
575  def processResult(self, exposure):
576  """Extract processing results from a processed exposure
577 
578  This method generates what is gathered by the master node.
579  This can be a background measurement or similar for scaling
580  flat-fields. It must be picklable!
581 
582  Only slave nodes execute this method.
583  """
584  return None
585 
586  def scale(self, ccdIdLists, data):
587  """!Determine scaling across CCDs and exposures
588 
589  This is necessary mainly for flats, so as to determine a
590  consistent scaling across the entire focal plane. This
591  implementation is simply a placeholder.
592 
593  Only the master node executes this method.
594 
595  @param ccdIdLists Dict of data identifier lists for each CCD tuple
596  @param data Dict of lists of returned data for each CCD tuple
597  @return dict of Struct(ccdScale: scaling for CCD,
598  expScales: scaling for each exposure
599  ) for each CCD tuple
600  """
601  self.log.info("Scale on %s" % NODE)
602  return dict((name, Struct(ccdScale=None, expScales=[None] * len(ccdIdLists[name])))
603  for name in ccdIdLists)
604 
605  def scatterCombine(self, pool, outputId, ccdIdLists, scales):
606  """!Scatter the combination of exposures across multiple nodes
607 
608  In this case, we can only scatter across as many nodes as
609  there are CCDs.
610 
611  Only the master node executes this method.
612 
613  @param pool Process pool
614  @param outputId Output identifier (exposure part only)
615  @param ccdIdLists Dict of data identifier lists for each CCD name
616  @param scales Dict of structs with scales, for each CCD name
617  """
618  self.log.info("Scatter combination")
619  data = [Struct(ccdName=ccdName, ccdIdList=ccdIdLists[ccdName], scales=scales[ccdName]) for
620  ccdName in ccdIdLists]
621  pool.map(self.combine, data, outputId)
622 
623  def combine(self, cache, struct, outputId):
624  """!Combine multiple exposures of a particular CCD and write the output
625 
626  Only the slave nodes execute this method.
627 
628  @param cache Process pool cache
629  @param struct Parameters for the combination, which has the following components:
630  * ccdName Name tuple for CCD
631  * ccdIdList List of data identifiers for combination
632  * scales Scales to apply (expScales are scalings for each exposure,
633  ccdScale is final scale for combined image)
634  @param outputId Data identifier for combined image (exposure part only)
635  """
636  # Check if we need to look up any keys that aren't in the output dataId
637  fullOutputId = {k: struct.ccdName[i] for i, k in enumerate(self.config.ccdKeys)}
638  self.addMissingKeys(fullOutputId, cache.butler)
639  fullOutputId.update(outputId) # must be after the call to queryMetadata
640  outputId = fullOutputId
641  del fullOutputId
642 
643  dataRefList = [getDataRef(cache.butler, dataId) if dataId is not None else None for
644  dataId in struct.ccdIdList]
645  self.log.info("Combining %s on %s" % (outputId, NODE))
646  calib = self.combination.run(dataRefList, expScales=struct.scales.expScales,
647  finalScale=struct.scales.ccdScale)
648 
649  self.recordCalibInputs(cache.butler, calib,
650  struct.ccdIdList, outputId)
651 
652  self.interpolateNans(calib)
653 
654  self.write(cache.butler, calib, outputId)
655 
656  def recordCalibInputs(self, butler, calib, dataIdList, outputId):
657  """!Record metadata including the inputs and creation details
658 
659  This metadata will go into the FITS header.
660 
661  @param butler Data butler
662  @param calib Combined calib exposure.
663  @param dataIdList List of data identifiers for calibration inputs
664  @param outputId Data identifier for output
665  """
666  header = calib.getMetadata()
667  header.add("OBSTYPE", self.calibName) # Used by ingestCalibs.py
668 
669  # date, time, host, and root
670  now = time.localtime()
671  header.add("CALIB_CREATION_DATE", time.strftime("%Y-%m-%d", now))
672  header.add("CALIB_CREATION_TIME", time.strftime("%X %Z", now))
673 
674  # Inputs
675  visits = [str(dictToTuple(dataId, self.config.visitKeys)) for dataId in dataIdList if
676  dataId is not None]
677  for i, v in enumerate(sorted(set(visits))):
678  header.add("CALIB_INPUT_%d" % (i,), v)
679 
680  header.add("CALIB_ID", " ".join("%s=%s" % (key, value)
681  for key, value in outputId.items()))
682  checksum(calib, header)
683 
684  def interpolateNans(self, image):
685  """Interpolate over NANs in the combined image
686 
687  NANs can result from masked areas on the CCD. We don't want them getting
688  into our science images, so we replace them with the median of the image.
689  """
690  if hasattr(image, "getMaskedImage"): # Deal with Exposure vs Image
691  self.interpolateNans(image.getMaskedImage().getVariance())
692  image = image.getMaskedImage().getImage()
693  if hasattr(image, "getImage"): # Deal with DecoratedImage or MaskedImage vs Image
694  image = image.getImage()
695  array = image.getArray()
696  bad = np.isnan(array)
697  array[bad] = np.median(array[np.logical_not(bad)])
698 
699  def write(self, butler, exposure, dataId):
700  """!Write the final combined calib
701 
702  Only the slave nodes execute this method
703 
704  @param butler Data butler
705  @param exposure CCD exposure to write
706  @param dataId Data identifier for output
707  """
708  self.log.info("Writing %s on %s" % (dataId, NODE))
709  butler.put(exposure, self.calibName, dataId)
710 
711 
713  """Configuration for bias construction.
714 
715  No changes required compared to the base class, but
716  subclassed for distinction.
717  """
718  pass
719 
720 
721 class BiasTask(CalibTask):
722  """Bias construction"""
723  ConfigClass = BiasConfig
724  _DefaultName = "bias"
725  calibName = "bias"
726  filterName = "NONE" # Sets this filter name in the output
727 
728  @classmethod
729  def applyOverrides(cls, config):
730  """Overrides to apply for bias construction"""
731  config.isr.doBias = False
732  config.isr.doDark = False
733  config.isr.doFlat = False
734  config.isr.doFringe = False
735 
736 
738  """Task to combine dark images"""
739  def run(*args, **kwargs):
740  combined = CalibCombineTask.run(*args, **kwargs)
741 
742  # Update the metadata
743  visitInfo = afwImage.VisitInfo(exposureTime=1.0, darkTime=1.0)
744  md = combined.getMetadata()
745  afwImage.setVisitInfoMetadata(md, visitInfo)
746 
747  return combined
748 
749 
751  """Configuration for dark construction"""
752  doRepair = Field(dtype=bool, default=True, doc="Repair artifacts?")
753  psfFwhm = Field(dtype=float, default=3.0, doc="Repair PSF FWHM (pixels)")
754  psfSize = Field(dtype=int, default=21, doc="Repair PSF size (pixels)")
755  crGrow = Field(dtype=int, default=2, doc="Grow radius for CR (pixels)")
756  repair = ConfigurableField(
757  target=RepairTask, doc="Task to repair artifacts")
758 
759  def setDefaults(self):
760  CalibConfig.setDefaults(self)
761  self.combination.retarget(DarkCombineTask)
762  self.combination.mask.append("CR")
763 
764 
766  """Dark construction
767 
768  The only major difference from the base class is a cosmic-ray
769  identification stage, and dividing each image by the dark time
770  to generate images of the dark rate.
771  """
772  ConfigClass = DarkConfig
773  _DefaultName = "dark"
774  calibName = "dark"
775  filterName = "NONE" # Sets this filter name in the output
776 
777  def __init__(self, *args, **kwargs):
778  CalibTask.__init__(self, *args, **kwargs)
779  self.makeSubtask("repair")
780 
781  @classmethod
782  def applyOverrides(cls, config):
783  """Overrides to apply for dark construction"""
784  config.isr.doDark = False
785  config.isr.doFlat = False
786  config.isr.doFringe = False
787 
788  def processSingle(self, sensorRef):
789  """Process a single CCD
790 
791  Besides the regular ISR, also masks cosmic-rays and divides each
792  processed image by the dark time to generate images of the dark rate.
793  The dark time is provided by the 'getDarkTime' method.
794  """
795  exposure = CalibTask.processSingle(self, sensorRef)
796 
797  if self.config.doRepair:
798  psf = measAlg.DoubleGaussianPsf(self.config.psfSize, self.config.psfSize,
799  self.config.psfFwhm/(2*math.sqrt(2*math.log(2))))
800  exposure.setPsf(psf)
801  self.repair.run(exposure, keepCRs=False)
802  if self.config.crGrow > 0:
803  mask = exposure.getMaskedImage().getMask().clone()
804  mask &= mask.getPlaneBitMask("CR")
805  fpSet = afwDet.FootprintSet(
806  mask, afwDet.Threshold(0.5))
807  fpSet = afwDet.FootprintSet(fpSet, self.config.crGrow, True)
808  fpSet.setMask(exposure.getMaskedImage().getMask(), "CR")
809 
810  mi = exposure.getMaskedImage()
811  mi /= self.getDarkTime(exposure)
812  return exposure
813 
814  def getDarkTime(self, exposure):
815  """Retrieve the dark time for an exposure"""
816  darkTime = exposure.getInfo().getVisitInfo().getDarkTime()
817  if not np.isfinite(darkTime):
818  raise RuntimeError("Non-finite darkTime")
819  return darkTime
820 
821 
823  """Configuration for flat construction"""
824  iterations = Field(dtype=int, default=10,
825  doc="Number of iterations for scale determination")
826  stats = ConfigurableField(target=CalibStatsTask,
827  doc="Background statistics configuration")
828 
829 
831  """Flat construction
832 
833  The principal change from the base class involves gathering the background
834  values from each image and using them to determine the scalings for the final
835  combination.
836  """
837  ConfigClass = FlatConfig
838  _DefaultName = "flat"
839  calibName = "flat"
840 
841  @classmethod
842  def applyOverrides(cls, config):
843  """Overrides for flat construction"""
844  config.isr.doFlat = False
845  config.isr.doFringe = False
846 
847  def __init__(self, *args, **kwargs):
848  CalibTask.__init__(self, *args, **kwargs)
849  self.makeSubtask("stats")
850 
851  def processResult(self, exposure):
852  return self.stats.run(exposure)
853 
854  def scale(self, ccdIdLists, data):
855  """Determine the scalings for the final combination
856 
857  We have a matrix B_ij = C_i E_j, where C_i is the relative scaling
858  of one CCD to all the others in an exposure, and E_j is the scaling
859  of the exposure. We convert everything to logarithms so we can work
860  with a linear system. We determine the C_i and E_j from B_ij by iteration,
861  under the additional constraint that the average CCD scale is unity.
862 
863  This algorithm comes from Eugene Magnier and Pan-STARRS.
864  """
865  assert len(ccdIdLists.values()) > 0, "No successful CCDs"
866  lengths = set([len(expList) for expList in ccdIdLists.values()])
867  assert len(
868  lengths) == 1, "Number of successful exposures for each CCD differs"
869  assert tuple(lengths)[0] > 0, "No successful exposures"
870  # Format background measurements into a matrix
871  indices = dict((name, i) for i, name in enumerate(ccdIdLists))
872  bgMatrix = np.array([[0.0] * len(expList)
873  for expList in ccdIdLists.values()])
874  for name in ccdIdLists:
875  i = indices[name]
876  bgMatrix[i] = [
877  d if d is not None else np.nan for d in data[name]]
878 
879  numpyPrint = np.get_printoptions()
880  np.set_printoptions(threshold='nan')
881  self.log.info("Input backgrounds: %s" % bgMatrix)
882 
883  # Flat-field scaling
884  numCcds = len(ccdIdLists)
885  numExps = bgMatrix.shape[1]
886  # log(Background) for each exposure/component
887  bgMatrix = np.log(bgMatrix)
888  bgMatrix = np.ma.masked_array(bgMatrix, np.isnan(bgMatrix))
889  # Initial guess at log(scale) for each component
890  compScales = np.zeros(numCcds)
891  expScales = np.array(
892  [(bgMatrix[:, i0] - compScales).mean() for i0 in range(numExps)])
893 
894  for iterate in range(self.config.iterations):
895  compScales = np.array(
896  [(bgMatrix[i1, :] - expScales).mean() for i1 in range(numCcds)])
897  expScales = np.array(
898  [(bgMatrix[:, i2] - compScales).mean() for i2 in range(numExps)])
899 
900  avgScale = np.average(np.exp(compScales))
901  compScales -= np.log(avgScale)
902  self.log.debug("Iteration %d exposure scales: %s",
903  iterate, np.exp(expScales))
904  self.log.debug("Iteration %d component scales: %s",
905  iterate, np.exp(compScales))
906 
907  expScales = np.array(
908  [(bgMatrix[:, i3] - compScales).mean() for i3 in range(numExps)])
909 
910  if np.any(np.isnan(expScales)):
911  raise RuntimeError("Bad exposure scales: %s --> %s" %
912  (bgMatrix, expScales))
913 
914  expScales = np.exp(expScales)
915  compScales = np.exp(compScales)
916 
917  self.log.info("Exposure scales: %s" % expScales)
918  self.log.info("Component relative scaling: %s" % compScales)
919  np.set_printoptions(**numpyPrint)
920 
921  return dict((ccdName, Struct(ccdScale=compScales[indices[ccdName]], expScales=expScales))
922  for ccdName in ccdIdLists)
923 
924 
926  """Configuration for fringe construction"""
927  stats = ConfigurableField(target=CalibStatsTask,
928  doc="Background statistics configuration")
929  subtractBackground = ConfigurableField(target=measAlg.SubtractBackgroundTask,
930  doc="Background configuration")
931  detection = ConfigurableField(
932  target=measAlg.SourceDetectionTask, doc="Detection configuration")
933  detectSigma = Field(dtype=float, default=1.0,
934  doc="Detection PSF gaussian sigma")
935 
936 
938  """Fringe construction task
939 
940  The principal change from the base class is that the images are
941  background-subtracted and rescaled by the background.
942 
943  XXX This is probably not right for a straight-up combination, as we
944  are currently doing, since the fringe amplitudes need not scale with
945  the continuum.
946 
947  XXX Would like to have this do PCA and generate multiple images, but
948  that will take a bit of work with the persistence code.
949  """
950  ConfigClass = FringeConfig
951  _DefaultName = "fringe"
952  calibName = "fringe"
953 
954  @classmethod
955  def applyOverrides(cls, config):
956  """Overrides for fringe construction"""
957  config.isr.doFringe = False
958 
959  def __init__(self, *args, **kwargs):
960  CalibTask.__init__(self, *args, **kwargs)
961  self.makeSubtask("detection")
962  self.makeSubtask("stats")
963  self.makeSubtask("subtractBackground")
964 
965  def processSingle(self, sensorRef):
966  """Subtract the background and normalise by the background level"""
967  exposure = CalibTask.processSingle(self, sensorRef)
968  bgLevel = self.stats.run(exposure)
969  self.subtractBackground.run(exposure)
970  mi = exposure.getMaskedImage()
971  mi /= bgLevel
972  footprintSets = self.detection.detectFootprints(
973  exposure, sigma=self.config.detectSigma)
974  mask = exposure.getMaskedImage().getMask()
975  detected = 1 << mask.addMaskPlane("DETECTED")
976  for fpSet in (footprintSets.positive, footprintSets.negative):
977  if fpSet is not None:
978  afwDet.setMaskFromFootprintList(
979  mask, fpSet.getFootprints(), detected)
980  return exposure
def run
Measure a particular statistic on an image (of some sort).
def getCcdIdListFromExposures
Determine a list of CCDs from exposure references.
def scatterProcess
Scatter the processing among the nodes.
def checksum
Calculate a checksum of an object.
Definition: checksum.py:29
def recordCalibInputs
Record metadata including the inputs and creation details.
def getOutputId
Generate the data identifier for the output calib.
def write
Write the final combined calib.
def run
Construct a calib from a list of exposure references.
def combine
Combine multiple exposures of a particular CCD and write the output.
def scale
Determine scaling across CCDs and exposures.
def scatterCombine
Scatter the combination of exposures across multiple nodes.
def run
Combine calib images for a single sensor.
def process
Process a CCD, specified by a data identifier.
def dictToTuple
Return a tuple of specific values from a dict.
def processWrite
Write the processed CCD.
Base class for constructing calibs.