Coverage for python/lsst/cp/pipe/utils.py: 10%
252 statements
« prev ^ index » next coverage.py v7.2.1, created at 2023-03-12 11:11 +0000
« prev ^ index » next coverage.py v7.2.1, created at 2023-03-12 11:11 +0000
1# This file is part of cp_pipe.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
21#
23__all__ = ['PairedVisitListTaskRunner', 'SingleVisitListTaskRunner',
24 'NonexistentDatasetTaskDataIdContainer', 'parseCmdlineNumberString',
25 'countMaskedPixels', 'checkExpLengthEqual', 'ddict2dict']
27import re
28import numpy as np
29from scipy.optimize import leastsq
30import numpy.polynomial.polynomial as poly
31from scipy.stats import norm
33import lsst.pipe.base as pipeBase
34import lsst.ip.isr as ipIsr
35from lsst.ip.isr import isrMock
36import lsst.log
37import lsst.afw.image
39import galsim
42def sigmaClipCorrection(nSigClip):
43 """Correct measured sigma to account for clipping.
45 If we clip our input data and then measure sigma, then the
46 measured sigma is smaller than the true value because real
47 points beyond the clip threshold have been removed. This is a
48 small (1.5% at nSigClip=3) effect when nSigClip >~ 3, but the
49 default parameters for measure crosstalk use nSigClip=2.0.
50 This causes the measured sigma to be about 15% smaller than
51 real. This formula corrects the issue, for the symmetric case
52 (upper clip threshold equal to lower clip threshold).
54 Parameters
55 ----------
56 nSigClip : `float`
57 Number of sigma the measurement was clipped by.
59 Returns
60 -------
61 scaleFactor : `float`
62 Scale factor to increase the measured sigma by.
63 """
64 varFactor = 1.0 - (2 * nSigClip * norm.pdf(nSigClip)) / (norm.cdf(nSigClip) - norm.cdf(-nSigClip))
65 return 1.0 / np.sqrt(varFactor)
68def calculateWeightedReducedChi2(measured, model, weightsMeasured, nData, nParsModel):
69 """Calculate weighted reduced chi2.
71 Parameters
72 ----------
74 measured : `list`
75 List with measured data.
77 model : `list`
78 List with modeled data.
80 weightsMeasured : `list`
81 List with weights for the measured data.
83 nData : `int`
84 Number of data points.
86 nParsModel : `int`
87 Number of parameters in the model.
89 Returns
90 -------
92 redWeightedChi2 : `float`
93 Reduced weighted chi2.
94 """
95 wRes = (measured - model)*weightsMeasured
96 return ((wRes*wRes).sum())/(nData-nParsModel)
99def makeMockFlats(expTime, gain=1.0, readNoiseElectrons=5, fluxElectrons=1000,
100 randomSeedFlat1=1984, randomSeedFlat2=666, powerLawBfParams=[],
101 expId1=0, expId2=1):
102 """Create a pair or mock flats with isrMock.
104 Parameters
105 ----------
106 expTime : `float`
107 Exposure time of the flats.
109 gain : `float`, optional
110 Gain, in e/ADU.
112 readNoiseElectrons : `float`, optional
113 Read noise rms, in electrons.
115 fluxElectrons : `float`, optional
116 Flux of flats, in electrons per second.
118 randomSeedFlat1 : `int`, optional
119 Random seed for the normal distrubutions for the mean signal
120 and noise (flat1).
122 randomSeedFlat2 : `int`, optional
123 Random seed for the normal distrubutions for the mean signal
124 and noise (flat2).
126 powerLawBfParams : `list`, optional
127 Parameters for `galsim.cdmodel.PowerLawCD` to simulate the
128 brightter-fatter effect.
130 expId1 : `int`, optional
131 Exposure ID for first flat.
133 expId2 : `int`, optional
134 Exposure ID for second flat.
136 Returns
137 -------
139 flatExp1 : `lsst.afw.image.exposure.ExposureF`
140 First exposure of flat field pair.
142 flatExp2 : `lsst.afw.image.exposure.ExposureF`
143 Second exposure of flat field pair.
145 Notes
146 -----
147 The parameters of `galsim.cdmodel.PowerLawCD` are `n, r0, t0, rx,
148 tx, r, t, alpha`. For more information about their meaning, see
149 the Galsim documentation
150 https://galsim-developers.github.io/GalSim/_build/html/_modules/galsim/cdmodel.html # noqa: W505
151 and Gruen+15 (1501.02802).
153 Example: galsim.cdmodel.PowerLawCD(8, 1.1e-7, 1.1e-7, 1.0e-8,
154 1.0e-8, 1.0e-9, 1.0e-9, 2.0)
155 """
156 flatFlux = fluxElectrons # e/s
157 flatMean = flatFlux*expTime # e
158 readNoise = readNoiseElectrons # e
160 mockImageConfig = isrMock.IsrMock.ConfigClass()
162 mockImageConfig.flatDrop = 0.99999
163 mockImageConfig.isTrimmed = True
165 flatExp1 = isrMock.FlatMock(config=mockImageConfig).run()
166 flatExp2 = flatExp1.clone()
167 (shapeY, shapeX) = flatExp1.getDimensions()
168 flatWidth = np.sqrt(flatMean)
170 rng1 = np.random.RandomState(randomSeedFlat1)
171 flatData1 = rng1.normal(flatMean, flatWidth, (shapeX, shapeY)) + rng1.normal(0.0, readNoise,
172 (shapeX, shapeY))
173 rng2 = np.random.RandomState(randomSeedFlat2)
174 flatData2 = rng2.normal(flatMean, flatWidth, (shapeX, shapeY)) + rng2.normal(0.0, readNoise,
175 (shapeX, shapeY))
176 # Simulate BF with power law model in galsim
177 if len(powerLawBfParams):
178 if not len(powerLawBfParams) == 8:
179 raise RuntimeError("Wrong number of parameters for `galsim.cdmodel.PowerLawCD`. "
180 f"Expected 8; passed {len(powerLawBfParams)}.")
181 cd = galsim.cdmodel.PowerLawCD(*powerLawBfParams)
182 tempFlatData1 = galsim.Image(flatData1)
183 temp2FlatData1 = cd.applyForward(tempFlatData1)
185 tempFlatData2 = galsim.Image(flatData2)
186 temp2FlatData2 = cd.applyForward(tempFlatData2)
188 flatExp1.image.array[:] = temp2FlatData1.array/gain # ADU
189 flatExp2.image.array[:] = temp2FlatData2.array/gain # ADU
190 else:
191 flatExp1.image.array[:] = flatData1/gain # ADU
192 flatExp2.image.array[:] = flatData2/gain # ADU
194 visitInfoExp1 = lsst.afw.image.VisitInfo(exposureId=expId1, exposureTime=expTime)
195 visitInfoExp2 = lsst.afw.image.VisitInfo(exposureId=expId2, exposureTime=expTime)
197 flatExp1.getInfo().setVisitInfo(visitInfoExp1)
198 flatExp2.getInfo().setVisitInfo(visitInfoExp2)
200 return flatExp1, flatExp2
203def countMaskedPixels(maskedIm, maskPlane):
204 """Count the number of pixels in a given mask plane.
206 Parameters
207 ----------
208 maskedIm : `~lsst.afw.image.MaskedImage`
209 Masked image to examine.
210 maskPlane : `str`
211 Name of the mask plane to examine.
213 Returns
214 -------
215 nPix : `int`
216 Number of pixels in the requested mask plane.
217 """
218 maskBit = maskedIm.mask.getPlaneBitMask(maskPlane)
219 nPix = np.where(np.bitwise_and(maskedIm.mask.array, maskBit))[0].flatten().size
220 return nPix
223class PairedVisitListTaskRunner(pipeBase.TaskRunner):
224 """Subclass of TaskRunner for handling intrinsically paired visits.
226 This transforms the processed arguments generated by the ArgumentParser
227 into the arguments expected by tasks which take visit pairs for their
228 run() methods.
230 Such tasks' run() methods tend to take two arguments,
231 one of which is the dataRef (as usual), and the other is the list
232 of visit-pairs, in the form of a list of tuples.
233 This list is supplied on the command line as documented,
234 and this class parses that, and passes the parsed version
235 to the run() method.
237 See pipeBase.TaskRunner for more information.
238 """
240 @staticmethod
241 def getTargetList(parsedCmd, **kwargs):
242 """Parse the visit list and pass through explicitly."""
243 visitPairs = []
244 for visitStringPair in parsedCmd.visitPairs:
245 visitStrings = visitStringPair.split(",")
246 if len(visitStrings) != 2:
247 raise RuntimeError("Found {} visits in {} instead of 2".format(len(visitStrings),
248 visitStringPair))
249 try:
250 visits = [int(visit) for visit in visitStrings]
251 except Exception:
252 raise RuntimeError("Could not parse {} as two integer visit numbers".format(visitStringPair))
253 visitPairs.append(visits)
255 return pipeBase.TaskRunner.getTargetList(parsedCmd, visitPairs=visitPairs, **kwargs)
258def parseCmdlineNumberString(inputString):
259 """Parse command line numerical expression sytax and return as list of int
261 Take an input of the form "'1..5:2^123..126'" as a string, and return
262 a list of ints as [1, 3, 5, 123, 124, 125, 126]
264 Parameters
265 ----------
266 inputString : `str`
267 String to be parsed.
269 Returns
270 -------
271 outList : `list` [`int`]
272 List of integers identified in the string.
273 """
274 outList = []
275 for subString in inputString.split("^"):
276 mat = re.search(r"^(\d+)\.\.(\d+)(?::(\d+))?$", subString)
277 if mat:
278 v1 = int(mat.group(1))
279 v2 = int(mat.group(2))
280 v3 = mat.group(3)
281 v3 = int(v3) if v3 else 1
282 for v in range(v1, v2 + 1, v3):
283 outList.append(int(v))
284 else:
285 outList.append(int(subString))
286 return outList
289class SingleVisitListTaskRunner(pipeBase.TaskRunner):
290 """Subclass of TaskRunner for tasks requiring a list of visits per dataRef.
292 This transforms the processed arguments generated by the ArgumentParser
293 into the arguments expected by tasks which require a list of visits
294 to be supplied for each dataRef, as is common in `lsst.cp.pipe` code.
296 Such tasks' run() methods tend to take two arguments,
297 one of which is the dataRef (as usual), and the other is the list
298 of visits.
299 This list is supplied on the command line as documented,
300 and this class parses that, and passes the parsed version
301 to the run() method.
303 See `lsst.pipe.base.TaskRunner` for more information.
304 """
306 @staticmethod
307 def getTargetList(parsedCmd, **kwargs):
308 """Parse the visit list and pass through explicitly."""
309 # if this has been pre-parsed and therefore doesn't have length of one
310 # then something has gone wrong, so execution should stop here.
311 assert len(parsedCmd.visitList) == 1, 'visitList parsing assumptions violated'
312 visits = parseCmdlineNumberString(parsedCmd.visitList[0])
314 return pipeBase.TaskRunner.getTargetList(parsedCmd, visitList=visits, **kwargs)
317class NonexistentDatasetTaskDataIdContainer(pipeBase.DataIdContainer):
318 """A DataIdContainer for the tasks for which the output does
319 not yet exist."""
321 def makeDataRefList(self, namespace):
322 """Compute refList based on idList.
324 This method must be defined as the dataset does not exist before this
325 task is run.
327 Parameters
328 ----------
329 namespace
330 Results of parsing the command-line.
332 Notes
333 -----
334 Not called if ``add_id_argument`` called
335 with ``doMakeDataRefList=False``.
336 Note that this is almost a copy-and-paste of the vanilla
337 implementation, but without checking if the datasets already exist,
338 as this task exists to make them.
339 """
340 if self.datasetType is None:
341 raise RuntimeError("Must call setDatasetType first")
342 butler = namespace.butler
343 for dataId in self.idList:
344 refList = list(butler.subset(datasetType=self.datasetType, level=self.level, dataId=dataId))
345 # exclude nonexistent data
346 # this is a recursive test, e.g. for the sake of "raw" data
347 if not refList:
348 namespace.log.warn("No data found for dataId=%s", dataId)
349 continue
350 self.refList += refList
353def irlsFit(initialParams, dataX, dataY, function, weightsY=None, weightType='Cauchy'):
354 """Iteratively reweighted least squares fit.
356 This uses the `lsst.cp.pipe.utils.fitLeastSq`, but applies weights
357 based on the Cauchy distribution by default. Other weight options
358 are implemented. See e.g. Holland and Welsch, 1977,
359 doi:10.1080/03610927708827533
361 Parameters
362 ----------
363 initialParams : `list` [`float`]
364 Starting parameters.
365 dataX : `numpy.array`, (N,)
366 Abscissa data.
367 dataY : `numpy.array`, (N,)
368 Ordinate data.
369 function : callable
370 Function to fit.
371 weightsY : `numpy.array`, (N,)
372 Weights to apply to the data.
373 weightType : `str`, optional
374 Type of weighting to use. One of Cauchy, Anderson, bisquare,
375 box, Welsch, Huber, logistic, or Fair.
377 Returns
378 -------
379 polyFit : `list` [`float`]
380 Final best fit parameters.
381 polyFitErr : `list` [`float`]
382 Final errors on fit parameters.
383 chiSq : `float`
384 Reduced chi squared.
385 weightsY : `list` [`float`]
386 Final weights used for each point.
388 Raises
389 ------
390 RuntimeError :
391 Raised if an unknown weightType string is passed.
392 """
393 if not weightsY:
394 weightsY = np.ones_like(dataX)
396 polyFit, polyFitErr, chiSq = fitLeastSq(initialParams, dataX, dataY, function, weightsY=weightsY)
397 for iteration in range(10):
398 resid = np.abs(dataY - function(polyFit, dataX)) / np.sqrt(dataY)
399 if weightType == 'Cauchy':
400 # Use Cauchy weighting. This is a soft weight.
401 # At [2, 3, 5, 10] sigma, weights are [.59, .39, .19, .05].
402 Z = resid / 2.385
403 weightsY = 1.0 / (1.0 + np.square(Z))
404 elif weightType == 'Anderson':
405 # Anderson+1972 weighting. This is a hard weight.
406 # At [2, 3, 5, 10] sigma, weights are [.67, .35, 0.0, 0.0].
407 Z = resid / (1.339 * np.pi)
408 weightsY = np.where(Z < 1.0, np.sinc(Z), 0.0)
409 elif weightType == 'bisquare':
410 # Beaton and Tukey (1974) biweight. This is a hard weight.
411 # At [2, 3, 5, 10] sigma, weights are [.81, .59, 0.0, 0.0].
412 Z = resid / 4.685
413 weightsY = np.where(Z < 1.0, 1.0 - np.square(Z), 0.0)
414 elif weightType == 'box':
415 # Hinich and Talwar (1975). This is a hard weight.
416 # At [2, 3, 5, 10] sigma, weights are [1.0, 0.0, 0.0, 0.0].
417 weightsY = np.where(resid < 2.795, 1.0, 0.0)
418 elif weightType == 'Welsch':
419 # Dennis and Welsch (1976). This is a hard weight.
420 # At [2, 3, 5, 10] sigma, weights are [.64, .36, .06, 1e-5].
421 Z = resid / 2.985
422 weightsY = np.exp(-1.0 * np.square(Z))
423 elif weightType == 'Huber':
424 # Huber (1964) weighting. This is a soft weight.
425 # At [2, 3, 5, 10] sigma, weights are [.67, .45, .27, .13].
426 Z = resid / 1.345
427 weightsY = np.where(Z < 1.0, 1.0, 1 / Z)
428 elif weightType == 'logistic':
429 # Logistic weighting. This is a soft weight.
430 # At [2, 3, 5, 10] sigma, weights are [.56, .40, .24, .12].
431 Z = resid / 1.205
432 weightsY = np.tanh(Z) / Z
433 elif weightType == 'Fair':
434 # Fair (1974) weighting. This is a soft weight.
435 # At [2, 3, 5, 10] sigma, weights are [.41, .32, .22, .12].
436 Z = resid / 1.4
437 weightsY = (1.0 / (1.0 + (Z)))
438 else:
439 raise RuntimeError(f"Unknown weighting type: {weightType}")
440 polyFit, polyFitErr, chiSq = fitLeastSq(initialParams, dataX, dataY, function, weightsY=weightsY)
442 return polyFit, polyFitErr, chiSq, weightsY
445def fitLeastSq(initialParams, dataX, dataY, function, weightsY=None):
446 """Do a fit and estimate the parameter errors using using
447 scipy.optimize.leastq.
449 optimize.leastsq returns the fractional covariance matrix. To
450 estimate the standard deviation of the fit parameters, multiply
451 the entries of this matrix by the unweighted reduced chi squared
452 and take the square root of the diagonal elements.
454 Parameters
455 ----------
456 initialParams : `list` [`float`]
457 initial values for fit parameters. For ptcFitType=POLYNOMIAL,
458 its length determines the degree of the polynomial.
460 dataX : `numpy.array`, (N,)
461 Data in the abscissa axis.
463 dataY : `numpy.array`, (N,)
464 Data in the ordinate axis.
466 function : callable object (function)
467 Function to fit the data with.
469 weightsY : `numpy.array`, (N,)
470 Weights of the data in the ordinate axis.
472 Return
473 ------
474 pFitSingleLeastSquares : `list` [`float`]
475 List with fitted parameters.
477 pErrSingleLeastSquares : `list` [`float`]
478 List with errors for fitted parameters.
480 reducedChiSqSingleLeastSquares : `float`
481 Reduced chi squared, unweighted if weightsY is not provided.
482 """
483 if weightsY is None:
484 weightsY = np.ones(len(dataX))
486 def errFunc(p, x, y, weightsY=None):
487 if weightsY is None:
488 weightsY = np.ones(len(x))
489 return (function(p, x) - y)*weightsY
491 pFit, pCov, infoDict, errMessage, success = leastsq(errFunc, initialParams,
492 args=(dataX, dataY, weightsY), full_output=1,
493 epsfcn=0.0001)
495 if (len(dataY) > len(initialParams)) and pCov is not None:
496 reducedChiSq = calculateWeightedReducedChi2(dataY, function(pFit, dataX), weightsY, len(dataY),
497 len(initialParams))
498 pCov *= reducedChiSq
499 else:
500 pCov = np.zeros((len(initialParams), len(initialParams)))
501 pCov[:, :] = np.nan
502 reducedChiSq = np.nan
504 errorVec = []
505 for i in range(len(pFit)):
506 errorVec.append(np.fabs(pCov[i][i])**0.5)
508 pFitSingleLeastSquares = pFit
509 pErrSingleLeastSquares = np.array(errorVec)
511 return pFitSingleLeastSquares, pErrSingleLeastSquares, reducedChiSq
514def fitBootstrap(initialParams, dataX, dataY, function, weightsY=None, confidenceSigma=1.):
515 """Do a fit using least squares and bootstrap to estimate parameter errors.
517 The bootstrap error bars are calculated by fitting 100 random data sets.
519 Parameters
520 ----------
521 initialParams : `list` [`float`]
522 initial values for fit parameters. For ptcFitType=POLYNOMIAL,
523 its length determines the degree of the polynomial.
525 dataX : `numpy.array`, (N,)
526 Data in the abscissa axis.
528 dataY : `numpy.array`, (N,)
529 Data in the ordinate axis.
531 function : callable object (function)
532 Function to fit the data with.
534 weightsY : `numpy.array`, (N,), optional.
535 Weights of the data in the ordinate axis.
537 confidenceSigma : `float`, optional.
538 Number of sigmas that determine confidence interval for the
539 bootstrap errors.
541 Return
542 ------
543 pFitBootstrap : `list` [`float`]
544 List with fitted parameters.
546 pErrBootstrap : `list` [`float`]
547 List with errors for fitted parameters.
549 reducedChiSqBootstrap : `float`
550 Reduced chi squared, unweighted if weightsY is not provided.
551 """
552 if weightsY is None:
553 weightsY = np.ones(len(dataX))
555 def errFunc(p, x, y, weightsY):
556 if weightsY is None:
557 weightsY = np.ones(len(x))
558 return (function(p, x) - y)*weightsY
560 # Fit first time
561 pFit, _ = leastsq(errFunc, initialParams, args=(dataX, dataY, weightsY), full_output=0)
563 # Get the stdev of the residuals
564 residuals = errFunc(pFit, dataX, dataY, weightsY)
565 # 100 random data sets are generated and fitted
566 pars = []
567 for i in range(100):
568 randomDelta = np.random.normal(0., np.fabs(residuals), len(dataY))
569 randomDataY = dataY + randomDelta
570 randomFit, _ = leastsq(errFunc, initialParams,
571 args=(dataX, randomDataY, weightsY), full_output=0)
572 pars.append(randomFit)
573 pars = np.array(pars)
574 meanPfit = np.mean(pars, 0)
576 # confidence interval for parameter estimates
577 errPfit = confidenceSigma*np.std(pars, 0)
578 pFitBootstrap = meanPfit
579 pErrBootstrap = errPfit
581 reducedChiSq = calculateWeightedReducedChi2(dataY, function(pFitBootstrap, dataX), weightsY, len(dataY),
582 len(initialParams))
583 return pFitBootstrap, pErrBootstrap, reducedChiSq
586def funcPolynomial(pars, x):
587 """Polynomial function definition
588 Parameters
589 ----------
590 params : `list`
591 Polynomial coefficients. Its length determines the polynomial order.
593 x : `numpy.array`, (N,)
594 Abscisa array.
596 Returns
597 -------
598 y : `numpy.array`, (N,)
599 Ordinate array after evaluating polynomial of order
600 len(pars)-1 at `x`.
601 """
602 return poly.polyval(x, [*pars])
605def funcAstier(pars, x):
606 """Single brighter-fatter parameter model for PTC; Equation 16 of
607 Astier+19.
609 Parameters
610 ----------
611 params : `list`
612 Parameters of the model: a00 (brightter-fatter), gain (e/ADU),
613 and noise (e^2).
615 x : `numpy.array`, (N,)
616 Signal mu (ADU).
618 Returns
619 -------
620 y : `numpy.array`, (N,)
621 C_00 (variance) in ADU^2.
622 """
623 a00, gain, noise = pars
624 return 0.5/(a00*gain*gain)*(np.exp(2*a00*x*gain)-1) + noise/(gain*gain) # C_00
627def arrangeFlatsByExpTime(exposureList, exposureIdList):
628 """Arrange exposures by exposure time.
630 Parameters
631 ----------
632 exposureList : `list` [`lsst.afw.image.ExposureF`]
633 Input list of exposures.
635 exposureIdList : `list` [`int`]
636 List of exposure ids as obtained by dataId[`exposure`].
638 Returns
639 ------
640 flatsAtExpTime : `dict` [`float`,
641 `list`[(`lsst.afw.image.ExposureF`, `int`)]]
642 Dictionary that groups flat-field exposures (and their IDs) that have
643 the same exposure time (seconds).
644 """
645 flatsAtExpTime = {}
646 assert len(exposureList) == len(exposureIdList), "Different lengths for exp. list and exp. ID lists"
647 for exp, expId in zip(exposureList, exposureIdList):
648 expTime = exp.getInfo().getVisitInfo().getExposureTime()
649 listAtExpTime = flatsAtExpTime.setdefault(expTime, [])
650 listAtExpTime.append((exp, expId))
652 return flatsAtExpTime
655def arrangeFlatsByExpId(exposureList, exposureIdList):
656 """Arrange exposures by exposure ID.
658 There is no guarantee that this will properly group exposures, but
659 allows a sequence of flats that have different illumination
660 (despite having the same exposure time) to be processed.
662 Parameters
663 ----------
664 exposureList : `list`[`lsst.afw.image.ExposureF`]
665 Input list of exposures.
667 exposureIdList : `list`[`int`]
668 List of exposure ids as obtained by dataId[`exposure`].
670 Returns
671 ------
672 flatsAtExpId : `dict` [`float`,
673 `list`[(`lsst.afw.image.ExposureF`, `int`)]]
674 Dictionary that groups flat-field exposures (and their IDs)
675 sequentially by their exposure id.
677 Notes
678 -----
680 This algorithm sorts the input exposures by their exposure id, and
681 then assigns each pair of exposures (exp_j, exp_{j+1}) to pair k,
682 such that 2*k = j, where j is the python index of one of the
683 exposures (starting from zero). By checking for the IndexError
684 while appending, we can ensure that there will only ever be fully
685 populated pairs.
686 """
687 flatsAtExpId = {}
688 # sortedExposures = sorted(exposureList, key=lambda exp:
689 # exp.getInfo().getVisitInfo().getExposureId())
690 assert len(exposureList) == len(exposureIdList), "Different lengths for exp. list and exp. ID lists"
691 # Sort exposures by expIds, which are in the second list `exposureIdList`.
692 sortedExposures = sorted(zip(exposureList, exposureIdList), key=lambda pair: pair[1])
694 for jPair, expTuple in enumerate(sortedExposures):
695 if (jPair + 1) % 2:
696 kPair = jPair // 2
697 listAtExpId = flatsAtExpId.setdefault(kPair, [])
698 try:
699 listAtExpId.append(expTuple)
700 listAtExpId.append(sortedExposures[jPair + 1])
701 except IndexError:
702 pass
704 return flatsAtExpId
707def checkExpLengthEqual(exp1, exp2, v1=None, v2=None, raiseWithMessage=False):
708 """Check the exposure lengths of two exposures are equal.
710 Parameters
711 ----------
712 exp1 : `lsst.afw.image.Exposure`
713 First exposure to check
714 exp2 : `lsst.afw.image.Exposure`
715 Second exposure to check
716 v1 : `int` or `str`, optional
717 First visit of the visit pair
718 v2 : `int` or `str`, optional
719 Second visit of the visit pair
720 raiseWithMessage : `bool`
721 If True, instead of returning a bool, raise a RuntimeError if
722 exposure times are not equal, with a message about which
723 visits mismatch if the information is available.
725 Returns
726 -------
727 success : `bool`
728 This is true if the exposures have equal exposure times.
730 Raises
731 ------
732 RuntimeError
733 Raised if the exposure lengths of the two exposures are not equal
734 """
735 expTime1 = exp1.getInfo().getVisitInfo().getExposureTime()
736 expTime2 = exp2.getInfo().getVisitInfo().getExposureTime()
737 if expTime1 != expTime2:
738 if raiseWithMessage:
739 msg = "Exposure lengths for visit pairs must be equal. " + \
740 "Found %s and %s" % (expTime1, expTime2)
741 if v1 and v2:
742 msg += " for visit pair %s, %s" % (v1, v2)
743 raise RuntimeError(msg)
744 else:
745 return False
746 return True
749def validateIsrConfig(isrTask, mandatory=None, forbidden=None, desirable=None, undesirable=None,
750 checkTrim=True, logName=None):
751 """Check that appropriate ISR settings have been selected for the task.
753 Note that this checks that the task itself is configured correctly rather
754 than checking a config.
756 Parameters
757 ----------
758 isrTask : `lsst.ip.isr.IsrTask`
759 The task whose config is to be validated
761 mandatory : `iterable` [`str`]
762 isr steps that must be set to True. Raises if False or missing
764 forbidden : `iterable` [`str`]
765 isr steps that must be set to False. Raises if True, warns if missing
767 desirable : `iterable` [`str`]
768 isr steps that should probably be set to True. Warns is False,
769 info if missing
771 undesirable : `iterable` [`str`]
772 isr steps that should probably be set to False. Warns is True,
773 info if missing
775 checkTrim : `bool`
776 Check to ensure the isrTask's assembly subtask is trimming the
777 images. This is a separate config as it is very ugly to do
778 this within the normal configuration lists as it is an option
779 of a sub task.
781 Raises
782 ------
783 RuntimeError
784 Raised if ``mandatory`` config parameters are False,
785 or if ``forbidden`` parameters are True.
787 TypeError
788 Raised if parameter ``isrTask`` is an invalid type.
790 Notes
791 -----
792 Logs warnings using an isrValidation logger for desirable/undesirable
793 options that are of the wrong polarity or if keys are missing.
794 """
795 if not isinstance(isrTask, ipIsr.IsrTask):
796 raise TypeError(f'Must supply an instance of lsst.ip.isr.IsrTask not {type(isrTask)}')
798 configDict = isrTask.config.toDict()
800 if logName and isinstance(logName, str):
801 log = lsst.log.getLogger(logName)
802 else:
803 log = lsst.log.getLogger("isrValidation")
805 if mandatory:
806 for configParam in mandatory:
807 if configParam not in configDict:
808 raise RuntimeError(f"Mandatory parameter {configParam} not found in the isr configuration.")
809 if configDict[configParam] is False:
810 raise RuntimeError(f"Must set config.isr.{configParam} to True for this task.")
812 if forbidden:
813 for configParam in forbidden:
814 if configParam not in configDict:
815 log.warn(f"Failed to find forbidden key {configParam} in the isr config. The keys in the"
816 " forbidden list should each have an associated Field in IsrConfig:"
817 " check that there is not a typo in this case.")
818 continue
819 if configDict[configParam] is True:
820 raise RuntimeError(f"Must set config.isr.{configParam} to False for this task.")
822 if desirable:
823 for configParam in desirable:
824 if configParam not in configDict:
825 log.info(f"Failed to find key {configParam} in the isr config. You probably want"
826 " to set the equivalent for your obs_package to True.")
827 continue
828 if configDict[configParam] is False:
829 log.warn(f"Found config.isr.{configParam} set to False for this task."
830 " The cp_pipe Config recommends setting this to True.")
831 if undesirable:
832 for configParam in undesirable:
833 if configParam not in configDict:
834 log.info(f"Failed to find key {configParam} in the isr config. You probably want"
835 " to set the equivalent for your obs_package to False.")
836 continue
837 if configDict[configParam] is True:
838 log.warn(f"Found config.isr.{configParam} set to True for this task."
839 " The cp_pipe Config recommends setting this to False.")
841 if checkTrim: # subtask setting, seems non-trivial to combine with above lists
842 if not isrTask.assembleCcd.config.doTrim:
843 raise RuntimeError("Must trim when assembling CCDs. Set config.isr.assembleCcd.doTrim to True")
846def ddict2dict(d):
847 """Convert nested default dictionaries to regular dictionaries.
849 This is needed to prevent yaml persistence issues.
851 Parameters
852 ----------
853 d : `defaultdict`
854 A possibly nested set of `defaultdict`.
856 Returns
857 -------
858 dict : `dict`
859 A possibly nested set of `dict`.
860 """
861 for k, v in d.items():
862 if isinstance(v, dict):
863 d[k] = ddict2dict(v)
864 return dict(d)