Coverage for python/lsst/cp/pipe/utils.py : 10%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of cp_pipe.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
21#
23__all__ = ['PairedVisitListTaskRunner', 'SingleVisitListTaskRunner',
24 'NonexistentDatasetTaskDataIdContainer', 'parseCmdlineNumberString',
25 'countMaskedPixels', 'checkExpLengthEqual', 'ddict2dict']
27import re
28import numpy as np
29from scipy.optimize import leastsq
30import numpy.polynomial.polynomial as poly
32import lsst.pipe.base as pipeBase
33import lsst.ip.isr as ipIsr
34from lsst.ip.isr import isrMock
35import lsst.log
37import galsim
40def calculateWeightedReducedChi2(measured, model, weightsMeasured, nData, nParsModel):
41 """Calculate weighted reduced chi2.
43 Parameters
44 ----------
46 measured : `list`
47 List with measured data.
49 model : `list`
50 List with modeled data.
52 weightsMeasured : `list`
53 List with weights for the measured data.
55 nData : `int`
56 Number of data points.
58 nParsModel : `int`
59 Number of parameters in the model.
61 Returns
62 -------
64 redWeightedChi2 : `float`
65 Reduced weighted chi2.
66 """
68 wRes = (measured - model)*weightsMeasured
69 return ((wRes*wRes).sum())/(nData-nParsModel)
72def makeMockFlats(expTime, gain=1.0, readNoiseElectrons=5, fluxElectrons=1000,
73 randomSeedFlat1=1984, randomSeedFlat2=666, powerLawBfParams=[]):
74 """Create a pair or mock flats with isrMock.
76 Parameters
77 ----------
78 expTime : `float`
79 Exposure time of the flats.
81 gain : `float`, optional
82 Gain, in e/ADU.
84 readNoiseElectrons : `float`, optional
85 Read noise rms, in electrons.
87 fluxElectrons : `float`, optional
88 Flux of flats, in electrons per second.
90 randomSeedFlat1 : `int`, optional
91 Random seed for the normal distrubutions for the mean signal and noise (flat1).
93 randomSeedFlat2 : `int`, optional
94 Random seed for the normal distrubutions for the mean signal and noise (flat2).
96 powerLawBfParams : `list`, optional
97 Parameters for `galsim.cdmodel.PowerLawCD` to simulate the brightter-fatter effect.
99 Returns
100 -------
102 flatExp1 : `lsst.afw.image.exposure.exposure.ExposureF`
103 First exposure of flat field pair.
105 flatExp2 : `lsst.afw.image.exposure.exposure.ExposureF`
106 Second exposure of flat field pair.
108 Notes
109 -----
110 The parameters of `galsim.cdmodel.PowerLawCD` are `n, r0, t0, rx, tx, r, t, alpha`. For more
111 information about their meaning, see the Galsim documentation
112 https://galsim-developers.github.io/GalSim/_build/html/_modules/galsim/cdmodel.html
113 and Gruen+15 (1501.02802).
115 Example: galsim.cdmodel.PowerLawCD(8, 1.1e-7, 1.1e-7, 1.0e-8, 1.0e-8, 1.0e-9, 1.0e-9, 2.0)
116 """
117 flatFlux = fluxElectrons # e/s
118 flatMean = flatFlux*expTime # e
119 readNoise = readNoiseElectrons # e
121 mockImageConfig = isrMock.IsrMock.ConfigClass()
123 mockImageConfig.flatDrop = 0.99999
124 mockImageConfig.isTrimmed = True
126 flatExp1 = isrMock.FlatMock(config=mockImageConfig).run()
127 flatExp2 = flatExp1.clone()
128 (shapeY, shapeX) = flatExp1.getDimensions()
129 flatWidth = np.sqrt(flatMean)
131 rng1 = np.random.RandomState(randomSeedFlat1)
132 flatData1 = rng1.normal(flatMean, flatWidth, (shapeX, shapeY)) + rng1.normal(0.0, readNoise,
133 (shapeX, shapeY))
134 rng2 = np.random.RandomState(randomSeedFlat2)
135 flatData2 = rng2.normal(flatMean, flatWidth, (shapeX, shapeY)) + rng2.normal(0.0, readNoise,
136 (shapeX, shapeY))
137 # Simulate BF with power law model in galsim
138 if len(powerLawBfParams):
139 if not len(powerLawBfParams) == 8:
140 raise RuntimeError("Wrong number of parameters for `galsim.cdmodel.PowerLawCD`. " +
141 f"Expected 8; passed {len(powerLawBfParams)}.")
142 cd = galsim.cdmodel.PowerLawCD(*powerLawBfParams)
143 tempFlatData1 = galsim.Image(flatData1)
144 temp2FlatData1 = cd.applyForward(tempFlatData1)
146 tempFlatData2 = galsim.Image(flatData2)
147 temp2FlatData2 = cd.applyForward(tempFlatData2)
149 flatExp1.image.array[:] = temp2FlatData1.array/gain # ADU
150 flatExp2.image.array[:] = temp2FlatData2.array/gain # ADU
151 else:
152 flatExp1.image.array[:] = flatData1/gain # ADU
153 flatExp2.image.array[:] = flatData2/gain # ADU
155 return flatExp1, flatExp2
158def countMaskedPixels(maskedIm, maskPlane):
159 """Count the number of pixels in a given mask plane."""
160 maskBit = maskedIm.mask.getPlaneBitMask(maskPlane)
161 nPix = np.where(np.bitwise_and(maskedIm.mask.array, maskBit))[0].flatten().size
162 return nPix
165class PairedVisitListTaskRunner(pipeBase.TaskRunner):
166 """Subclass of TaskRunner for handling intrinsically paired visits.
168 This transforms the processed arguments generated by the ArgumentParser
169 into the arguments expected by tasks which take visit pairs for their
170 run() methods.
172 Such tasks' run() methods tend to take two arguments,
173 one of which is the dataRef (as usual), and the other is the list
174 of visit-pairs, in the form of a list of tuples.
175 This list is supplied on the command line as documented,
176 and this class parses that, and passes the parsed version
177 to the run() method.
179 See pipeBase.TaskRunner for more information.
180 """
182 @staticmethod
183 def getTargetList(parsedCmd, **kwargs):
184 """Parse the visit list and pass through explicitly."""
185 visitPairs = []
186 for visitStringPair in parsedCmd.visitPairs:
187 visitStrings = visitStringPair.split(",")
188 if len(visitStrings) != 2:
189 raise RuntimeError("Found {} visits in {} instead of 2".format(len(visitStrings),
190 visitStringPair))
191 try:
192 visits = [int(visit) for visit in visitStrings]
193 except Exception:
194 raise RuntimeError("Could not parse {} as two integer visit numbers".format(visitStringPair))
195 visitPairs.append(visits)
197 return pipeBase.TaskRunner.getTargetList(parsedCmd, visitPairs=visitPairs, **kwargs)
200def parseCmdlineNumberString(inputString):
201 """Parse command line numerical expression sytax and return as list of int
203 Take an input of the form "'1..5:2^123..126'" as a string, and return
204 a list of ints as [1, 3, 5, 123, 124, 125, 126]
205 """
206 outList = []
207 for subString in inputString.split("^"):
208 mat = re.search(r"^(\d+)\.\.(\d+)(?::(\d+))?$", subString)
209 if mat:
210 v1 = int(mat.group(1))
211 v2 = int(mat.group(2))
212 v3 = mat.group(3)
213 v3 = int(v3) if v3 else 1
214 for v in range(v1, v2 + 1, v3):
215 outList.append(int(v))
216 else:
217 outList.append(int(subString))
218 return outList
221class SingleVisitListTaskRunner(pipeBase.TaskRunner):
222 """Subclass of TaskRunner for tasks requiring a list of visits per dataRef.
224 This transforms the processed arguments generated by the ArgumentParser
225 into the arguments expected by tasks which require a list of visits
226 to be supplied for each dataRef, as is common in `lsst.cp.pipe` code.
228 Such tasks' run() methods tend to take two arguments,
229 one of which is the dataRef (as usual), and the other is the list
230 of visits.
231 This list is supplied on the command line as documented,
232 and this class parses that, and passes the parsed version
233 to the run() method.
235 See `lsst.pipe.base.TaskRunner` for more information.
236 """
238 @staticmethod
239 def getTargetList(parsedCmd, **kwargs):
240 """Parse the visit list and pass through explicitly."""
241 # if this has been pre-parsed and therefore doesn't have length of one
242 # then something has gone wrong, so execution should stop here.
243 assert len(parsedCmd.visitList) == 1, 'visitList parsing assumptions violated'
244 visits = parseCmdlineNumberString(parsedCmd.visitList[0])
246 return pipeBase.TaskRunner.getTargetList(parsedCmd, visitList=visits, **kwargs)
249class NonexistentDatasetTaskDataIdContainer(pipeBase.DataIdContainer):
250 """A DataIdContainer for the tasks for which the output does
251 not yet exist."""
253 def makeDataRefList(self, namespace):
254 """Compute refList based on idList.
256 This method must be defined as the dataset does not exist before this
257 task is run.
259 Parameters
260 ----------
261 namespace
262 Results of parsing the command-line.
264 Notes
265 -----
266 Not called if ``add_id_argument`` called
267 with ``doMakeDataRefList=False``.
268 Note that this is almost a copy-and-paste of the vanilla
269 implementation, but without checking if the datasets already exist,
270 as this task exists to make them.
271 """
272 if self.datasetType is None:
273 raise RuntimeError("Must call setDatasetType first")
274 butler = namespace.butler
275 for dataId in self.idList:
276 refList = list(butler.subset(datasetType=self.datasetType, level=self.level, dataId=dataId))
277 # exclude nonexistent data
278 # this is a recursive test, e.g. for the sake of "raw" data
279 if not refList:
280 namespace.log.warn("No data found for dataId=%s", dataId)
281 continue
282 self.refList += refList
285def irlsFit(initialParams, dataX, dataY, function, weightsY=None):
286 """Iteratively reweighted least squares fit.
288 This uses the `lsst.cp.pipe.utils.fitLeastSq`, but applies
289 weights based on the Cauchy distribution to the fitter. See
290 e.g. Holland and Welsch, 1977, doi:10.1080/03610927708827533
292 Parameters
293 ----------
294 initialParams : `list` [`float`]
295 Starting parameters.
296 dataX : `numpy.array` [`float`]
297 Abscissa data.
298 dataY : `numpy.array` [`float`]
299 Ordinate data.
300 function : callable
301 Function to fit.
302 weightsY : `numpy.array` [`float`]
303 Weights to apply to the data.
305 Returns
306 -------
307 polyFit : `list` [`float`]
308 Final best fit parameters.
309 polyFitErr : `list` [`float`]
310 Final errors on fit parameters.
311 chiSq : `float`
312 Reduced chi squared.
313 weightsY : `list` [`float`]
314 Final weights used for each point.
316 """
317 if not weightsY:
318 weightsY = np.ones_like(dataX)
320 polyFit, polyFitErr, chiSq = fitLeastSq(initialParams, dataX, dataY, function, weightsY=weightsY)
321 for iteration in range(10):
322 # Use Cauchy weights
323 resid = np.abs(dataY - function(polyFit, dataX)) / np.sqrt(dataY)
324 weightsY = 1.0 / (1.0 + np.sqrt(resid / 2.385))
325 polyFit, polyFitErr, chiSq = fitLeastSq(initialParams, dataX, dataY, function, weightsY=weightsY)
327 return polyFit, polyFitErr, chiSq, weightsY
330def fitLeastSq(initialParams, dataX, dataY, function, weightsY=None):
331 """Do a fit and estimate the parameter errors using using scipy.optimize.leastq.
333 optimize.leastsq returns the fractional covariance matrix. To estimate the
334 standard deviation of the fit parameters, multiply the entries of this matrix
335 by the unweighted reduced chi squared and take the square root of the diagonal elements.
337 Parameters
338 ----------
339 initialParams : `list` of `float`
340 initial values for fit parameters. For ptcFitType=POLYNOMIAL, its length
341 determines the degree of the polynomial.
343 dataX : `numpy.array` of `float`
344 Data in the abscissa axis.
346 dataY : `numpy.array` of `float`
347 Data in the ordinate axis.
349 function : callable object (function)
350 Function to fit the data with.
352 weightsY : `numpy.array` of `float`
353 Weights of the data in the ordinate axis.
355 Return
356 ------
357 pFitSingleLeastSquares : `list` of `float`
358 List with fitted parameters.
360 pErrSingleLeastSquares : `list` of `float`
361 List with errors for fitted parameters.
363 reducedChiSqSingleLeastSquares : `float`
364 Reduced chi squared, unweighted if weightsY is not provided.
365 """
366 if weightsY is None:
367 weightsY = np.ones(len(dataX))
369 def errFunc(p, x, y, weightsY=None):
370 if weightsY is None:
371 weightsY = np.ones(len(x))
372 return (function(p, x) - y)*weightsY
374 pFit, pCov, infoDict, errMessage, success = leastsq(errFunc, initialParams,
375 args=(dataX, dataY, weightsY), full_output=1,
376 epsfcn=0.0001)
378 if (len(dataY) > len(initialParams)) and pCov is not None:
379 reducedChiSq = calculateWeightedReducedChi2(dataY, function(pFit, dataX), weightsY, len(dataY),
380 len(initialParams))
381 pCov *= reducedChiSq
382 else:
383 pCov = np.zeros((len(initialParams), len(initialParams)))
384 pCov[:, :] = np.nan
385 reducedChiSq = np.nan
387 errorVec = []
388 for i in range(len(pFit)):
389 errorVec.append(np.fabs(pCov[i][i])**0.5)
391 pFitSingleLeastSquares = pFit
392 pErrSingleLeastSquares = np.array(errorVec)
394 return pFitSingleLeastSquares, pErrSingleLeastSquares, reducedChiSq
397def fitBootstrap(initialParams, dataX, dataY, function, weightsY=None, confidenceSigma=1.):
398 """Do a fit using least squares and bootstrap to estimate parameter errors.
400 The bootstrap error bars are calculated by fitting 100 random data sets.
402 Parameters
403 ----------
404 initialParams : `list` of `float`
405 initial values for fit parameters. For ptcFitType=POLYNOMIAL, its length
406 determines the degree of the polynomial.
408 dataX : `numpy.array` of `float`
409 Data in the abscissa axis.
411 dataY : `numpy.array` of `float`
412 Data in the ordinate axis.
414 function : callable object (function)
415 Function to fit the data with.
417 weightsY : `numpy.array` of `float`, optional.
418 Weights of the data in the ordinate axis.
420 confidenceSigma : `float`, optional.
421 Number of sigmas that determine confidence interval for the bootstrap errors.
423 Return
424 ------
425 pFitBootstrap : `list` of `float`
426 List with fitted parameters.
428 pErrBootstrap : `list` of `float`
429 List with errors for fitted parameters.
431 reducedChiSqBootstrap : `float`
432 Reduced chi squared, unweighted if weightsY is not provided.
433 """
434 if weightsY is None:
435 weightsY = np.ones(len(dataX))
437 def errFunc(p, x, y, weightsY):
438 if weightsY is None:
439 weightsY = np.ones(len(x))
440 return (function(p, x) - y)*weightsY
442 # Fit first time
443 pFit, _ = leastsq(errFunc, initialParams, args=(dataX, dataY, weightsY), full_output=0)
445 # Get the stdev of the residuals
446 residuals = errFunc(pFit, dataX, dataY, weightsY)
447 # 100 random data sets are generated and fitted
448 pars = []
449 for i in range(100):
450 randomDelta = np.random.normal(0., np.fabs(residuals), len(dataY))
451 randomDataY = dataY + randomDelta
452 randomFit, _ = leastsq(errFunc, initialParams,
453 args=(dataX, randomDataY, weightsY), full_output=0)
454 pars.append(randomFit)
455 pars = np.array(pars)
456 meanPfit = np.mean(pars, 0)
458 # confidence interval for parameter estimates
459 errPfit = confidenceSigma*np.std(pars, 0)
460 pFitBootstrap = meanPfit
461 pErrBootstrap = errPfit
463 reducedChiSq = calculateWeightedReducedChi2(dataY, function(pFitBootstrap, dataX), weightsY, len(dataY),
464 len(initialParams))
465 return pFitBootstrap, pErrBootstrap, reducedChiSq
468def funcPolynomial(pars, x):
469 """Polynomial function definition
470 Parameters
471 ----------
472 params : `list`
473 Polynomial coefficients. Its length determines the polynomial order.
475 x : `numpy.array`
476 Abscisa array.
478 Returns
479 -------
480 Ordinate array after evaluating polynomial of order len(pars)-1 at `x`.
481 """
482 return poly.polyval(x, [*pars])
485def funcAstier(pars, x):
486 """Single brighter-fatter parameter model for PTC; Equation 16 of Astier+19.
488 Parameters
489 ----------
490 params : `list`
491 Parameters of the model: a00 (brightter-fatter), gain (e/ADU), and noise (e^2).
493 x : `numpy.array`
494 Signal mu (ADU).
496 Returns
497 -------
498 C_00 (variance) in ADU^2.
499 """
500 a00, gain, noise = pars
501 return 0.5/(a00*gain*gain)*(np.exp(2*a00*x*gain)-1) + noise/(gain*gain) # C_00
504def checkExpLengthEqual(exp1, exp2, v1=None, v2=None, raiseWithMessage=False):
505 """Check the exposure lengths of two exposures are equal.
507 Parameters:
508 -----------
509 exp1 : `lsst.afw.image.exposure.ExposureF`
510 First exposure to check
511 exp2 : `lsst.afw.image.exposure.ExposureF`
512 Second exposure to check
513 v1 : `int` or `str`, optional
514 First visit of the visit pair
515 v2 : `int` or `str`, optional
516 Second visit of the visit pair
517 raiseWithMessage : `bool`
518 If True, instead of returning a bool, raise a RuntimeError if exposure
519 times are not equal, with a message about which visits mismatch if the
520 information is available.
522 Raises:
523 -------
524 RuntimeError
525 Raised if the exposure lengths of the two exposures are not equal
526 """
527 expTime1 = exp1.getInfo().getVisitInfo().getExposureTime()
528 expTime2 = exp2.getInfo().getVisitInfo().getExposureTime()
529 if expTime1 != expTime2:
530 if raiseWithMessage:
531 msg = "Exposure lengths for visit pairs must be equal. " + \
532 "Found %s and %s" % (expTime1, expTime2)
533 if v1 and v2:
534 msg += " for visit pair %s, %s" % (v1, v2)
535 raise RuntimeError(msg)
536 else:
537 return False
538 return True
541def validateIsrConfig(isrTask, mandatory=None, forbidden=None, desirable=None, undesirable=None,
542 checkTrim=True, logName=None):
543 """Check that appropriate ISR settings have been selected for the task.
545 Note that this checks that the task itself is configured correctly rather
546 than checking a config.
548 Parameters
549 ----------
550 isrTask : `lsst.ip.isr.IsrTask`
551 The task whose config is to be validated
553 mandatory : `iterable` of `str`
554 isr steps that must be set to True. Raises if False or missing
556 forbidden : `iterable` of `str`
557 isr steps that must be set to False. Raises if True, warns if missing
559 desirable : `iterable` of `str`
560 isr steps that should probably be set to True. Warns is False, info if
561 missing
563 undesirable : `iterable` of `str`
564 isr steps that should probably be set to False. Warns is True, info if
565 missing
567 checkTrim : `bool`
568 Check to ensure the isrTask's assembly subtask is trimming the images.
569 This is a separate config as it is very ugly to do this within the
570 normal configuration lists as it is an option of a sub task.
572 Raises
573 ------
574 RuntimeError
575 Raised if ``mandatory`` config parameters are False,
576 or if ``forbidden`` parameters are True.
578 TypeError
579 Raised if parameter ``isrTask`` is an invalid type.
581 Notes
582 -----
583 Logs warnings using an isrValidation logger for desirable/undesirable
584 options that are of the wrong polarity or if keys are missing.
585 """
586 if not isinstance(isrTask, ipIsr.IsrTask):
587 raise TypeError(f'Must supply an instance of lsst.ip.isr.IsrTask not {type(isrTask)}')
589 configDict = isrTask.config.toDict()
591 if logName and isinstance(logName, str):
592 log = lsst.log.getLogger(logName)
593 else:
594 log = lsst.log.getLogger("isrValidation")
596 if mandatory:
597 for configParam in mandatory:
598 if configParam not in configDict:
599 raise RuntimeError(f"Mandatory parameter {configParam} not found in the isr configuration.")
600 if configDict[configParam] is False:
601 raise RuntimeError(f"Must set config.isr.{configParam} to True for this task.")
603 if forbidden:
604 for configParam in forbidden:
605 if configParam not in configDict:
606 log.warn(f"Failed to find forbidden key {configParam} in the isr config. The keys in the"
607 " forbidden list should each have an associated Field in IsrConfig:"
608 " check that there is not a typo in this case.")
609 continue
610 if configDict[configParam] is True:
611 raise RuntimeError(f"Must set config.isr.{configParam} to False for this task.")
613 if desirable:
614 for configParam in desirable:
615 if configParam not in configDict:
616 log.info(f"Failed to find key {configParam} in the isr config. You probably want" +
617 " to set the equivalent for your obs_package to True.")
618 continue
619 if configDict[configParam] is False:
620 log.warn(f"Found config.isr.{configParam} set to False for this task." +
621 " The cp_pipe Config recommends setting this to True.")
622 if undesirable:
623 for configParam in undesirable:
624 if configParam not in configDict:
625 log.info(f"Failed to find key {configParam} in the isr config. You probably want" +
626 " to set the equivalent for your obs_package to False.")
627 continue
628 if configDict[configParam] is True:
629 log.warn(f"Found config.isr.{configParam} set to True for this task." +
630 " The cp_pipe Config recommends setting this to False.")
632 if checkTrim: # subtask setting, seems non-trivial to combine with above lists
633 if not isrTask.assembleCcd.config.doTrim:
634 raise RuntimeError("Must trim when assembling CCDs. Set config.isr.assembleCcd.doTrim to True")
637def ddict2dict(d):
638 """Convert nested default dictionaries to regular dictionaries.
640 This is needed to prevent yaml persistence issues.
642 Parameters
643 ----------
644 d : `defaultdict`
645 A possibly nested set of `defaultdict`.
647 Returns
648 -------
649 dict : `dict`
650 A possibly nested set of `dict`.
651 """
652 for k, v in d.items():
653 if isinstance(v, dict):
654 d[k] = ddict2dict(v)
655 return dict(d)