lsst.cp.pipe  20.0.0-8-gea2affd+00f470feff
utils.py
Go to the documentation of this file.
1 # This file is part of cp_pipe.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (https://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <https://www.gnu.org/licenses/>.
21 #
22 
23 __all__ = ['PairedVisitListTaskRunner', 'SingleVisitListTaskRunner',
24  'NonexistentDatasetTaskDataIdContainer', 'parseCmdlineNumberString',
25  'countMaskedPixels', 'checkExpLengthEqual']
26 
27 import re
28 import numpy as np
29 from scipy.optimize import leastsq
30 import numpy.polynomial.polynomial as poly
31 
32 import lsst.pipe.base as pipeBase
33 import lsst.ip.isr as ipIsr
34 from lsst.ip.isr import isrMock
35 import lsst.log
36 
37 import galsim
38 
39 
40 def calculateWeightedReducedChi2(measured, model, weightsMeasured, nData, nParsModel):
41  """Calculate weighted reduced chi2.
42 
43  Parameters
44  ----------
45 
46  measured : `list`
47  List with measured data.
48 
49  model : `list`
50  List with modeled data.
51 
52  weightsMeasured : `list`
53  List with weights for the measured data.
54 
55  nData : `int`
56  Number of data points.
57 
58  nParsModel : `int`
59  Number of parameters in the model.
60 
61  Returns
62  -------
63 
64  redWeightedChi2 : `float`
65  Reduced weighted chi2.
66  """
67 
68  wRes = (measured - model)*weightsMeasured
69  return ((wRes*wRes).sum())/(nData-nParsModel)
70 
71 
72 def makeMockFlats(expTime, gain=1.0, readNoiseElectrons=5, fluxElectrons=1000,
73  randomSeedFlat1=1984, randomSeedFlat2=666, powerLawBfParams=[]):
74  """Create a pair or mock flats with isrMock.
75 
76  Parameters
77  ----------
78  expTime : `float`
79  Exposure time of the flats.
80 
81  gain : `float`, optional
82  Gain, in e/ADU.
83 
84  readNoiseElectrons : `float`, optional
85  Read noise rms, in electrons.
86 
87  fluxElectrons : `float`, optional
88  Flux of flats, in electrons per second.
89 
90  randomSeedFlat1 : `int`, optional
91  Random seed for the normal distrubutions for the mean signal and noise (flat1).
92 
93  randomSeedFlat2 : `int`, optional
94  Random seed for the normal distrubutions for the mean signal and noise (flat2).
95 
96  powerLawBfParams : `list`, optional
97  Parameters for `galsim.cdmodel.PowerLawCD` to simulate the brightter-fatter effect.
98 
99  Returns
100  -------
101 
102  flatExp1 : `lsst.afw.image.exposure.exposure.ExposureF`
103  First exposure of flat field pair.
104 
105  flatExp2 : `lsst.afw.image.exposure.exposure.ExposureF`
106  Second exposure of flat field pair.
107 
108  Notes
109  -----
110  The parameters of `galsim.cdmodel.PowerLawCD` are `n, r0, t0, rx, tx, r, t, alpha`. For more
111  information about their meaning, see the Galsim documentation
112  https://galsim-developers.github.io/GalSim/_build/html/_modules/galsim/cdmodel.html
113  and Gruen+15 (1501.02802).
114 
115  Example: galsim.cdmodel.PowerLawCD(8, 1.1e-7, 1.1e-7, 1.0e-8, 1.0e-8, 1.0e-9, 1.0e-9, 2.0)
116  """
117  flatFlux = fluxElectrons # e/s
118  flatMean = flatFlux*expTime # e
119  readNoise = readNoiseElectrons # e
120 
121  mockImageConfig = isrMock.IsrMock.ConfigClass()
122 
123  mockImageConfig.flatDrop = 0.99999
124  mockImageConfig.isTrimmed = True
125 
126  flatExp1 = isrMock.FlatMock(config=mockImageConfig).run()
127  flatExp2 = flatExp1.clone()
128  (shapeY, shapeX) = flatExp1.getDimensions()
129  flatWidth = np.sqrt(flatMean)
130 
131  rng1 = np.random.RandomState(randomSeedFlat1)
132  flatData1 = rng1.normal(flatMean, flatWidth, (shapeX, shapeY)) + rng1.normal(0.0, readNoise,
133  (shapeX, shapeY))
134  rng2 = np.random.RandomState(randomSeedFlat2)
135  flatData2 = rng2.normal(flatMean, flatWidth, (shapeX, shapeY)) + rng2.normal(0.0, readNoise,
136  (shapeX, shapeY))
137  # Simulate BF with power law model in galsim
138  if len(powerLawBfParams):
139  if not len(powerLawBfParams) == 8:
140  raise RuntimeError("Wrong number of parameters for `galsim.cdmodel.PowerLawCD`. " +
141  f"Expected 8; passed {len(powerLawBfParams)}.")
142  cd = galsim.cdmodel.PowerLawCD(*powerLawBfParams)
143  tempFlatData1 = galsim.Image(flatData1)
144  temp2FlatData1 = cd.applyForward(tempFlatData1)
145 
146  tempFlatData2 = galsim.Image(flatData2)
147  temp2FlatData2 = cd.applyForward(tempFlatData2)
148 
149  flatExp1.image.array[:] = temp2FlatData1.array/gain # ADU
150  flatExp2.image.array[:] = temp2FlatData2.array/gain # ADU
151  else:
152  flatExp1.image.array[:] = flatData1/gain # ADU
153  flatExp2.image.array[:] = flatData2/gain # ADU
154 
155  return flatExp1, flatExp2
156 
157 
158 def countMaskedPixels(maskedIm, maskPlane):
159  """Count the number of pixels in a given mask plane."""
160  maskBit = maskedIm.mask.getPlaneBitMask(maskPlane)
161  nPix = np.where(np.bitwise_and(maskedIm.mask.array, maskBit))[0].flatten().size
162  return nPix
163 
164 
165 class PairedVisitListTaskRunner(pipeBase.TaskRunner):
166  """Subclass of TaskRunner for handling intrinsically paired visits.
167 
168  This transforms the processed arguments generated by the ArgumentParser
169  into the arguments expected by tasks which take visit pairs for their
170  run() methods.
171 
172  Such tasks' run() methods tend to take two arguments,
173  one of which is the dataRef (as usual), and the other is the list
174  of visit-pairs, in the form of a list of tuples.
175  This list is supplied on the command line as documented,
176  and this class parses that, and passes the parsed version
177  to the run() method.
178 
179  See pipeBase.TaskRunner for more information.
180  """
181 
182  @staticmethod
183  def getTargetList(parsedCmd, **kwargs):
184  """Parse the visit list and pass through explicitly."""
185  visitPairs = []
186  for visitStringPair in parsedCmd.visitPairs:
187  visitStrings = visitStringPair.split(",")
188  if len(visitStrings) != 2:
189  raise RuntimeError("Found {} visits in {} instead of 2".format(len(visitStrings),
190  visitStringPair))
191  try:
192  visits = [int(visit) for visit in visitStrings]
193  except Exception:
194  raise RuntimeError("Could not parse {} as two integer visit numbers".format(visitStringPair))
195  visitPairs.append(visits)
196 
197  return pipeBase.TaskRunner.getTargetList(parsedCmd, visitPairs=visitPairs, **kwargs)
198 
199 
200 def parseCmdlineNumberString(inputString):
201  """Parse command line numerical expression sytax and return as list of int
202 
203  Take an input of the form "'1..5:2^123..126'" as a string, and return
204  a list of ints as [1, 3, 5, 123, 124, 125, 126]
205  """
206  outList = []
207  for subString in inputString.split("^"):
208  mat = re.search(r"^(\d+)\.\.(\d+)(?::(\d+))?$", subString)
209  if mat:
210  v1 = int(mat.group(1))
211  v2 = int(mat.group(2))
212  v3 = mat.group(3)
213  v3 = int(v3) if v3 else 1
214  for v in range(v1, v2 + 1, v3):
215  outList.append(int(v))
216  else:
217  outList.append(int(subString))
218  return outList
219 
220 
221 class SingleVisitListTaskRunner(pipeBase.TaskRunner):
222  """Subclass of TaskRunner for tasks requiring a list of visits per dataRef.
223 
224  This transforms the processed arguments generated by the ArgumentParser
225  into the arguments expected by tasks which require a list of visits
226  to be supplied for each dataRef, as is common in `lsst.cp.pipe` code.
227 
228  Such tasks' run() methods tend to take two arguments,
229  one of which is the dataRef (as usual), and the other is the list
230  of visits.
231  This list is supplied on the command line as documented,
232  and this class parses that, and passes the parsed version
233  to the run() method.
234 
235  See `lsst.pipe.base.TaskRunner` for more information.
236  """
237 
238  @staticmethod
239  def getTargetList(parsedCmd, **kwargs):
240  """Parse the visit list and pass through explicitly."""
241  # if this has been pre-parsed and therefore doesn't have length of one
242  # then something has gone wrong, so execution should stop here.
243  assert len(parsedCmd.visitList) == 1, 'visitList parsing assumptions violated'
244  visits = parseCmdlineNumberString(parsedCmd.visitList[0])
245 
246  return pipeBase.TaskRunner.getTargetList(parsedCmd, visitList=visits, **kwargs)
247 
248 
249 class NonexistentDatasetTaskDataIdContainer(pipeBase.DataIdContainer):
250  """A DataIdContainer for the tasks for which the output does
251  not yet exist."""
252 
253  def makeDataRefList(self, namespace):
254  """Compute refList based on idList.
255 
256  This method must be defined as the dataset does not exist before this
257  task is run.
258 
259  Parameters
260  ----------
261  namespace
262  Results of parsing the command-line.
263 
264  Notes
265  -----
266  Not called if ``add_id_argument`` called
267  with ``doMakeDataRefList=False``.
268  Note that this is almost a copy-and-paste of the vanilla
269  implementation, but without checking if the datasets already exist,
270  as this task exists to make them.
271  """
272  if self.datasetType is None:
273  raise RuntimeError("Must call setDatasetType first")
274  butler = namespace.butler
275  for dataId in self.idList:
276  refList = list(butler.subset(datasetType=self.datasetType, level=self.level, dataId=dataId))
277  # exclude nonexistent data
278  # this is a recursive test, e.g. for the sake of "raw" data
279  if not refList:
280  namespace.log.warn("No data found for dataId=%s", dataId)
281  continue
282  self.refList += refList
283 
284 
285 def fitLeastSq(initialParams, dataX, dataY, function):
286  """Do a fit and estimate the parameter errors using using scipy.optimize.leastq.
287 
288  optimize.leastsq returns the fractional covariance matrix. To estimate the
289  standard deviation of the fit parameters, multiply the entries of this matrix
290  by the unweighted reduced chi squared and take the square root of the diagonal elements.
291 
292  Parameters
293  ----------
294  initialParams : `list` of `float`
295  initial values for fit parameters. For ptcFitType=POLYNOMIAL, its length
296  determines the degree of the polynomial.
297 
298  dataX : `numpy.array` of `float`
299  Data in the abscissa axis.
300 
301  dataY : `numpy.array` of `float`
302  Data in the ordinate axis.
303 
304  function : callable object (function)
305  Function to fit the data with.
306 
307  Return
308  ------
309  pFitSingleLeastSquares : `list` of `float`
310  List with fitted parameters.
311 
312  pErrSingleLeastSquares : `list` of `float`
313  List with errors for fitted parameters.
314 
315  reducedChiSqSingleLeastSquares : `float`
316  Unweighted reduced chi squared
317  """
318 
319  def errFunc(p, x, y):
320  return function(p, x) - y
321 
322  pFit, pCov, infoDict, errMessage, success = leastsq(errFunc, initialParams,
323  args=(dataX, dataY), full_output=1, epsfcn=0.0001)
324 
325  if (len(dataY) > len(initialParams)) and pCov is not None:
326  reducedChiSq = (errFunc(pFit, dataX, dataY)**2).sum()/(len(dataY)-len(initialParams))
327  pCov *= reducedChiSq
328  else:
329  pCov = np.zeros((len(initialParams), len(initialParams)))
330  pCov[:, :] = np.inf
331  reducedChiSq = np.inf
332 
333  errorVec = []
334  for i in range(len(pFit)):
335  errorVec.append(np.fabs(pCov[i][i])**0.5)
336 
337  pFitSingleLeastSquares = pFit
338  pErrSingleLeastSquares = np.array(errorVec)
339 
340  return pFitSingleLeastSquares, pErrSingleLeastSquares, reducedChiSq
341 
342 
343 def fitBootstrap(initialParams, dataX, dataY, function, confidenceSigma=1.):
344  """Do a fit using least squares and bootstrap to estimate parameter errors.
345 
346  The bootstrap error bars are calculated by fitting 100 random data sets.
347 
348  Parameters
349  ----------
350  initialParams : `list` of `float`
351  initial values for fit parameters. For ptcFitType=POLYNOMIAL, its length
352  determines the degree of the polynomial.
353 
354  dataX : `numpy.array` of `float`
355  Data in the abscissa axis.
356 
357  dataY : `numpy.array` of `float`
358  Data in the ordinate axis.
359 
360  function : callable object (function)
361  Function to fit the data with.
362 
363  confidenceSigma : `float`
364  Number of sigmas that determine confidence interval for the bootstrap errors.
365 
366  Return
367  ------
368  pFitBootstrap : `list` of `float`
369  List with fitted parameters.
370 
371  pErrBootstrap : `list` of `float`
372  List with errors for fitted parameters.
373 
374  reducedChiSqBootstrap : `float`
375  Reduced chi squared.
376  """
377 
378  def errFunc(p, x, y):
379  return function(p, x) - y
380 
381  # Fit first time
382  pFit, _ = leastsq(errFunc, initialParams, args=(dataX, dataY), full_output=0)
383 
384  # Get the stdev of the residuals
385  residuals = errFunc(pFit, dataX, dataY)
386  sigmaErrTotal = np.std(residuals)
387 
388  # 100 random data sets are generated and fitted
389  pars = []
390  for i in range(100):
391  randomDelta = np.random.normal(0., sigmaErrTotal, len(dataY))
392  randomDataY = dataY + randomDelta
393  randomFit, _ = leastsq(errFunc, initialParams,
394  args=(dataX, randomDataY), full_output=0)
395  pars.append(randomFit)
396  pars = np.array(pars)
397  meanPfit = np.mean(pars, 0)
398 
399  # confidence interval for parameter estimates
400  nSigma = confidenceSigma
401  errPfit = nSigma*np.std(pars, 0)
402  pFitBootstrap = meanPfit
403  pErrBootstrap = errPfit
404 
405  reducedChiSq = (errFunc(pFitBootstrap, dataX, dataY)**2).sum()/(len(dataY)-len(initialParams))
406  return pFitBootstrap, pErrBootstrap, reducedChiSq
407 
408 
409 def funcPolynomial(pars, x):
410  """Polynomial function definition
411  Parameters
412  ----------
413  params : `list`
414  Polynomial coefficients. Its length determines the polynomial order.
415 
416  x : `numpy.array`
417  Signal mu (ADU).
418 
419  Returns
420  -------
421  C_00 (variance) in ADU^2.
422  """
423  return poly.polyval(x, [*pars]) # C_00
424 
425 
426 def funcAstier(pars, x):
427  """Single brighter-fatter parameter model for PTC; Equation 16 of Astier+19.
428 
429  Parameters
430  ----------
431  params : `list`
432  Parameters of the model: a00 (brightter-fatter), gain (e/ADU), and noise (e^2).
433 
434  x : `numpy.array`
435  Signal mu (ADU).
436 
437  Returns
438  -------
439  C_00 (variance) in ADU^2.
440  """
441  a00, gain, noise = pars
442  return 0.5/(a00*gain*gain)*(np.exp(2*a00*x*gain)-1) + noise/(gain*gain) # C_00
443 
444 
445 def checkExpLengthEqual(exp1, exp2, v1=None, v2=None, raiseWithMessage=False):
446  """Check the exposure lengths of two exposures are equal.
447 
448  Parameters:
449  -----------
450  exp1 : `lsst.afw.image.exposure.ExposureF`
451  First exposure to check
452  exp2 : `lsst.afw.image.exposure.ExposureF`
453  Second exposure to check
454  v1 : `int` or `str`, optional
455  First visit of the visit pair
456  v2 : `int` or `str`, optional
457  Second visit of the visit pair
458  raiseWithMessage : `bool`
459  If True, instead of returning a bool, raise a RuntimeError if exposure
460  times are not equal, with a message about which visits mismatch if the
461  information is available.
462 
463  Raises:
464  -------
465  RuntimeError
466  Raised if the exposure lengths of the two exposures are not equal
467  """
468  expTime1 = exp1.getInfo().getVisitInfo().getExposureTime()
469  expTime2 = exp2.getInfo().getVisitInfo().getExposureTime()
470  if expTime1 != expTime2:
471  if raiseWithMessage:
472  msg = "Exposure lengths for visit pairs must be equal. " + \
473  "Found %s and %s" % (expTime1, expTime2)
474  if v1 and v2:
475  msg += " for visit pair %s, %s" % (v1, v2)
476  raise RuntimeError(msg)
477  else:
478  return False
479  return True
480 
481 
482 def validateIsrConfig(isrTask, mandatory=None, forbidden=None, desirable=None, undesirable=None,
483  checkTrim=True, logName=None):
484  """Check that appropriate ISR settings have been selected for the task.
485 
486  Note that this checks that the task itself is configured correctly rather
487  than checking a config.
488 
489  Parameters
490  ----------
491  isrTask : `lsst.ip.isr.IsrTask`
492  The task whose config is to be validated
493 
494  mandatory : `iterable` of `str`
495  isr steps that must be set to True. Raises if False or missing
496 
497  forbidden : `iterable` of `str`
498  isr steps that must be set to False. Raises if True, warns if missing
499 
500  desirable : `iterable` of `str`
501  isr steps that should probably be set to True. Warns is False, info if
502  missing
503 
504  undesirable : `iterable` of `str`
505  isr steps that should probably be set to False. Warns is True, info if
506  missing
507 
508  checkTrim : `bool`
509  Check to ensure the isrTask's assembly subtask is trimming the images.
510  This is a separate config as it is very ugly to do this within the
511  normal configuration lists as it is an option of a sub task.
512 
513  Raises
514  ------
515  RuntimeError
516  Raised if ``mandatory`` config parameters are False,
517  or if ``forbidden`` parameters are True.
518 
519  TypeError
520  Raised if parameter ``isrTask`` is an invalid type.
521 
522  Notes
523  -----
524  Logs warnings using an isrValidation logger for desirable/undesirable
525  options that are of the wrong polarity or if keys are missing.
526  """
527  if not isinstance(isrTask, ipIsr.IsrTask):
528  raise TypeError(f'Must supply an instance of lsst.ip.isr.IsrTask not {type(isrTask)}')
529 
530  configDict = isrTask.config.toDict()
531 
532  if logName and isinstance(logName, str):
533  log = lsst.log.getLogger(logName)
534  else:
535  log = lsst.log.getLogger("isrValidation")
536 
537  if mandatory:
538  for configParam in mandatory:
539  if configParam not in configDict:
540  raise RuntimeError(f"Mandatory parameter {configParam} not found in the isr configuration.")
541  if configDict[configParam] is False:
542  raise RuntimeError(f"Must set config.isr.{configParam} to True for this task.")
543 
544  if forbidden:
545  for configParam in forbidden:
546  if configParam not in configDict:
547  log.warn(f"Failed to find forbidden key {configParam} in the isr config. The keys in the"
548  " forbidden list should each have an associated Field in IsrConfig:"
549  " check that there is not a typo in this case.")
550  continue
551  if configDict[configParam] is True:
552  raise RuntimeError(f"Must set config.isr.{configParam} to False for this task.")
553 
554  if desirable:
555  for configParam in desirable:
556  if configParam not in configDict:
557  log.info(f"Failed to find key {configParam} in the isr config. You probably want" +
558  " to set the equivalent for your obs_package to True.")
559  continue
560  if configDict[configParam] is False:
561  log.warn(f"Found config.isr.{configParam} set to False for this task." +
562  " The cp_pipe Config recommends setting this to True.")
563  if undesirable:
564  for configParam in undesirable:
565  if configParam not in configDict:
566  log.info(f"Failed to find key {configParam} in the isr config. You probably want" +
567  " to set the equivalent for your obs_package to False.")
568  continue
569  if configDict[configParam] is True:
570  log.warn(f"Found config.isr.{configParam} set to True for this task." +
571  " The cp_pipe Config recommends setting this to False.")
572 
573  if checkTrim: # subtask setting, seems non-trivial to combine with above lists
574  if not isrTask.assembleCcd.config.doTrim:
575  raise RuntimeError("Must trim when assembling CCDs. Set config.isr.assembleCcd.doTrim to True")
lsst.cp.pipe.utils.validateIsrConfig
def validateIsrConfig(isrTask, mandatory=None, forbidden=None, desirable=None, undesirable=None, checkTrim=True, logName=None)
Definition: utils.py:482
lsst.cp.pipe.utils.NonexistentDatasetTaskDataIdContainer.makeDataRefList
def makeDataRefList(self, namespace)
Definition: utils.py:253
lsst.cp.pipe.utils.fitBootstrap
def fitBootstrap(initialParams, dataX, dataY, function, confidenceSigma=1.)
Definition: utils.py:343
lsst.cp.pipe.utils.calculateWeightedReducedChi2
def calculateWeightedReducedChi2(measured, model, weightsMeasured, nData, nParsModel)
Definition: utils.py:40
lsst.cp.pipe.utils.funcPolynomial
def funcPolynomial(pars, x)
Definition: utils.py:409
lsst.cp.pipe.utils.PairedVisitListTaskRunner.getTargetList
def getTargetList(parsedCmd, **kwargs)
Definition: utils.py:183
lsst.cp.pipe.utils.fitLeastSq
def fitLeastSq(initialParams, dataX, dataY, function)
Definition: utils.py:285
lsst.cp.pipe.utils.countMaskedPixels
def countMaskedPixels(maskedIm, maskPlane)
Definition: utils.py:158
lsst.cp.pipe.utils.SingleVisitListTaskRunner.getTargetList
def getTargetList(parsedCmd, **kwargs)
Definition: utils.py:239
lsst.cp.pipe.utils.PairedVisitListTaskRunner
Definition: utils.py:165
lsst.cp.pipe.utils.funcAstier
def funcAstier(pars, x)
Definition: utils.py:426
lsst.cp.pipe.utils.SingleVisitListTaskRunner
Definition: utils.py:221
lsst.cp.pipe.utils.checkExpLengthEqual
def checkExpLengthEqual(exp1, exp2, v1=None, v2=None, raiseWithMessage=False)
Definition: utils.py:445
lsst.cp.pipe.utils.parseCmdlineNumberString
def parseCmdlineNumberString(inputString)
Definition: utils.py:200
lsst::ip::isr
lsst.cp.pipe.utils.makeMockFlats
def makeMockFlats(expTime, gain=1.0, readNoiseElectrons=5, fluxElectrons=1000, randomSeedFlat1=1984, randomSeedFlat2=666, powerLawBfParams=[])
Definition: utils.py:72
lsst::pipe::base
lsst::log
lsst.cp.pipe.utils.NonexistentDatasetTaskDataIdContainer
Definition: utils.py:249