Coverage for python/lsst/cp/pipe/utils.py : 10%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of cp_pipe.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
21#
23__all__ = ['PairedVisitListTaskRunner', 'SingleVisitListTaskRunner',
24 'NonexistentDatasetTaskDataIdContainer', 'parseCmdlineNumberString',
25 'countMaskedPixels', 'checkExpLengthEqual']
27import re
28import numpy as np
29from scipy.optimize import leastsq
30import numpy.polynomial.polynomial as poly
32import lsst.pipe.base as pipeBase
33import lsst.ip.isr as ipIsr
34import lsst.log
37def countMaskedPixels(maskedIm, maskPlane):
38 """Count the number of pixels in a given mask plane."""
39 maskBit = maskedIm.mask.getPlaneBitMask(maskPlane)
40 nPix = np.where(np.bitwise_and(maskedIm.mask.array, maskBit))[0].flatten().size
41 return nPix
44class PairedVisitListTaskRunner(pipeBase.TaskRunner):
45 """Subclass of TaskRunner for handling intrinsically paired visits.
47 This transforms the processed arguments generated by the ArgumentParser
48 into the arguments expected by tasks which take visit pairs for their
49 run() methods.
51 Such tasks' run() methods tend to take two arguments,
52 one of which is the dataRef (as usual), and the other is the list
53 of visit-pairs, in the form of a list of tuples.
54 This list is supplied on the command line as documented,
55 and this class parses that, and passes the parsed version
56 to the run() method.
58 See pipeBase.TaskRunner for more information.
59 """
61 @staticmethod
62 def getTargetList(parsedCmd, **kwargs):
63 """Parse the visit list and pass through explicitly."""
64 visitPairs = []
65 for visitStringPair in parsedCmd.visitPairs:
66 visitStrings = visitStringPair.split(",")
67 if len(visitStrings) != 2:
68 raise RuntimeError("Found {} visits in {} instead of 2".format(len(visitStrings),
69 visitStringPair))
70 try:
71 visits = [int(visit) for visit in visitStrings]
72 except Exception:
73 raise RuntimeError("Could not parse {} as two integer visit numbers".format(visitStringPair))
74 visitPairs.append(visits)
76 return pipeBase.TaskRunner.getTargetList(parsedCmd, visitPairs=visitPairs, **kwargs)
79def parseCmdlineNumberString(inputString):
80 """Parse command line numerical expression sytax and return as list of int
82 Take an input of the form "'1..5:2^123..126'" as a string, and return
83 a list of ints as [1, 3, 5, 123, 124, 125, 126]
84 """
85 outList = []
86 for subString in inputString.split("^"):
87 mat = re.search(r"^(\d+)\.\.(\d+)(?::(\d+))?$", subString)
88 if mat:
89 v1 = int(mat.group(1))
90 v2 = int(mat.group(2))
91 v3 = mat.group(3)
92 v3 = int(v3) if v3 else 1
93 for v in range(v1, v2 + 1, v3):
94 outList.append(int(v))
95 else:
96 outList.append(int(subString))
97 return outList
100class SingleVisitListTaskRunner(pipeBase.TaskRunner):
101 """Subclass of TaskRunner for tasks requiring a list of visits per dataRef.
103 This transforms the processed arguments generated by the ArgumentParser
104 into the arguments expected by tasks which require a list of visits
105 to be supplied for each dataRef, as is common in `lsst.cp.pipe` code.
107 Such tasks' run() methods tend to take two arguments,
108 one of which is the dataRef (as usual), and the other is the list
109 of visits.
110 This list is supplied on the command line as documented,
111 and this class parses that, and passes the parsed version
112 to the run() method.
114 See `lsst.pipe.base.TaskRunner` for more information.
115 """
117 @staticmethod
118 def getTargetList(parsedCmd, **kwargs):
119 """Parse the visit list and pass through explicitly."""
120 # if this has been pre-parsed and therefore doesn't have length of one
121 # then something has gone wrong, so execution should stop here.
122 assert len(parsedCmd.visitList) == 1, 'visitList parsing assumptions violated'
123 visits = parseCmdlineNumberString(parsedCmd.visitList[0])
125 return pipeBase.TaskRunner.getTargetList(parsedCmd, visitList=visits, **kwargs)
128class NonexistentDatasetTaskDataIdContainer(pipeBase.DataIdContainer):
129 """A DataIdContainer for the tasks for which the output does
130 not yet exist."""
132 def makeDataRefList(self, namespace):
133 """Compute refList based on idList.
135 This method must be defined as the dataset does not exist before this
136 task is run.
138 Parameters
139 ----------
140 namespace
141 Results of parsing the command-line.
143 Notes
144 -----
145 Not called if ``add_id_argument`` called
146 with ``doMakeDataRefList=False``.
147 Note that this is almost a copy-and-paste of the vanilla
148 implementation, but without checking if the datasets already exist,
149 as this task exists to make them.
150 """
151 if self.datasetType is None:
152 raise RuntimeError("Must call setDatasetType first")
153 butler = namespace.butler
154 for dataId in self.idList:
155 refList = list(butler.subset(datasetType=self.datasetType, level=self.level, dataId=dataId))
156 # exclude nonexistent data
157 # this is a recursive test, e.g. for the sake of "raw" data
158 if not refList:
159 namespace.log.warn("No data found for dataId=%s", dataId)
160 continue
161 self.refList += refList
164def fitLeastSq(initialParams, dataX, dataY, function):
165 """Do a fit and estimate the parameter errors using using scipy.optimize.leastq.
167 optimize.leastsq returns the fractional covariance matrix. To estimate the
168 standard deviation of the fit parameters, multiply the entries of this matrix
169 by the unweighted reduced chi squared and take the square root of the diagonal elements.
171 Parameters
172 ----------
173 initialParams : `list` of `float`
174 initial values for fit parameters. For ptcFitType=POLYNOMIAL, its length
175 determines the degree of the polynomial.
177 dataX : `numpy.array` of `float`
178 Data in the abscissa axis.
180 dataY : `numpy.array` of `float`
181 Data in the ordinate axis.
183 function : callable object (function)
184 Function to fit the data with.
186 Return
187 ------
188 pFitSingleLeastSquares : `list` of `float`
189 List with fitted parameters.
191 pErrSingleLeastSquares : `list` of `float`
192 List with errors for fitted parameters.
194 reducedChiSqSingleLeastSquares : `float`
195 Unweighted reduced chi squared
196 """
198 def errFunc(p, x, y):
199 return function(p, x) - y
201 pFit, pCov, infoDict, errMessage, success = leastsq(errFunc, initialParams,
202 args=(dataX, dataY), full_output=1, epsfcn=0.0001)
204 if (len(dataY) > len(initialParams)) and pCov is not None:
205 reducedChiSq = (errFunc(pFit, dataX, dataY)**2).sum()/(len(dataY)-len(initialParams))
206 pCov *= reducedChiSq
207 else:
208 pCov = np.zeros((len(initialParams), len(initialParams)))
209 pCov[:, :] = np.inf
210 reducedChiSq = np.inf
212 errorVec = []
213 for i in range(len(pFit)):
214 errorVec.append(np.fabs(pCov[i][i])**0.5)
216 pFitSingleLeastSquares = pFit
217 pErrSingleLeastSquares = np.array(errorVec)
219 return pFitSingleLeastSquares, pErrSingleLeastSquares, reducedChiSq
222def fitBootstrap(initialParams, dataX, dataY, function, confidenceSigma=1.):
223 """Do a fit using least squares and bootstrap to estimate parameter errors.
225 The bootstrap error bars are calculated by fitting 100 random data sets.
227 Parameters
228 ----------
229 initialParams : `list` of `float`
230 initial values for fit parameters. For ptcFitType=POLYNOMIAL, its length
231 determines the degree of the polynomial.
233 dataX : `numpy.array` of `float`
234 Data in the abscissa axis.
236 dataY : `numpy.array` of `float`
237 Data in the ordinate axis.
239 function : callable object (function)
240 Function to fit the data with.
242 confidenceSigma : `float`
243 Number of sigmas that determine confidence interval for the bootstrap errors.
245 Return
246 ------
247 pFitBootstrap : `list` of `float`
248 List with fitted parameters.
250 pErrBootstrap : `list` of `float`
251 List with errors for fitted parameters.
253 reducedChiSqBootstrap : `float`
254 Reduced chi squared.
255 """
257 def errFunc(p, x, y):
258 return function(p, x) - y
260 # Fit first time
261 pFit, _ = leastsq(errFunc, initialParams, args=(dataX, dataY), full_output=0)
263 # Get the stdev of the residuals
264 residuals = errFunc(pFit, dataX, dataY)
265 sigmaErrTotal = np.std(residuals)
267 # 100 random data sets are generated and fitted
268 pars = []
269 for i in range(100):
270 randomDelta = np.random.normal(0., sigmaErrTotal, len(dataY))
271 randomDataY = dataY + randomDelta
272 randomFit, _ = leastsq(errFunc, initialParams,
273 args=(dataX, randomDataY), full_output=0)
274 pars.append(randomFit)
275 pars = np.array(pars)
276 meanPfit = np.mean(pars, 0)
278 # confidence interval for parameter estimates
279 nSigma = confidenceSigma
280 errPfit = nSigma*np.std(pars, 0)
281 pFitBootstrap = meanPfit
282 pErrBootstrap = errPfit
284 reducedChiSq = (errFunc(pFitBootstrap, dataX, dataY)**2).sum()/(len(dataY)-len(initialParams))
285 return pFitBootstrap, pErrBootstrap, reducedChiSq
288def funcPolynomial(pars, x):
289 """Polynomial function definition
290 Parameters
291 ----------
292 params : `list`
293 Polynomial coefficients. Its length determines the polynomial order.
295 x : `numpy.array`
296 Signal mu (ADU).
298 Returns
299 -------
300 C_00 (variance) in ADU^2.
301 """
302 return poly.polyval(x, [*pars]) # C_00
305def funcAstier(pars, x):
306 """Single brighter-fatter parameter model for PTC; Equation 16 of Astier+19.
308 Parameters
309 ----------
310 params : `list`
311 Parameters of the model: a00 (brightter-fatter), gain (e/ADU), and noise (e^2).
313 x : `numpy.array`
314 Signal mu (ADU).
316 Returns
317 -------
318 C_00 (variance) in ADU^2.
319 """
320 a00, gain, noise = pars
321 return 0.5/(a00*gain*gain)*(np.exp(2*a00*x*gain)-1) + noise/(gain*gain) # C_00
324def checkExpLengthEqual(exp1, exp2, v1=None, v2=None, raiseWithMessage=False):
325 """Check the exposure lengths of two exposures are equal.
327 Parameters:
328 -----------
329 exp1 : `lsst.afw.image.exposure.ExposureF`
330 First exposure to check
331 exp2 : `lsst.afw.image.exposure.ExposureF`
332 Second exposure to check
333 v1 : `int` or `str`, optional
334 First visit of the visit pair
335 v2 : `int` or `str`, optional
336 Second visit of the visit pair
337 raiseWithMessage : `bool`
338 If True, instead of returning a bool, raise a RuntimeError if exposure
339 times are not equal, with a message about which visits mismatch if the
340 information is available.
342 Raises:
343 -------
344 RuntimeError
345 Raised if the exposure lengths of the two exposures are not equal
346 """
347 expTime1 = exp1.getInfo().getVisitInfo().getExposureTime()
348 expTime2 = exp2.getInfo().getVisitInfo().getExposureTime()
349 if expTime1 != expTime2:
350 if raiseWithMessage:
351 msg = "Exposure lengths for visit pairs must be equal. " + \
352 "Found %s and %s" % (expTime1, expTime2)
353 if v1 and v2:
354 msg += " for visit pair %s, %s" % (v1, v2)
355 raise RuntimeError(msg)
356 else:
357 return False
358 return True
361def validateIsrConfig(isrTask, mandatory=None, forbidden=None, desirable=None, undesirable=None,
362 checkTrim=True, logName=None):
363 """Check that appropriate ISR settings have been selected for the task.
365 Note that this checks that the task itself is configured correctly rather
366 than checking a config.
368 Parameters
369 ----------
370 isrTask : `lsst.ip.isr.IsrTask`
371 The task whose config is to be validated
373 mandatory : `iterable` of `str`
374 isr steps that must be set to True. Raises if False or missing
376 forbidden : `iterable` of `str`
377 isr steps that must be set to False. Raises if True, warns if missing
379 desirable : `iterable` of `str`
380 isr steps that should probably be set to True. Warns is False, info if
381 missing
383 undesirable : `iterable` of `str`
384 isr steps that should probably be set to False. Warns is True, info if
385 missing
387 checkTrim : `bool`
388 Check to ensure the isrTask's assembly subtask is trimming the images.
389 This is a separate config as it is very ugly to do this within the
390 normal configuration lists as it is an option of a sub task.
392 Raises
393 ------
394 RuntimeError
395 Raised if ``mandatory`` config parameters are False,
396 or if ``forbidden`` parameters are True.
398 TypeError
399 Raised if parameter ``isrTask`` is an invalid type.
401 Notes
402 -----
403 Logs warnings using an isrValidation logger for desirable/undesirable
404 options that are of the wrong polarity or if keys are missing.
405 """
406 if not isinstance(isrTask, ipIsr.IsrTask):
407 raise TypeError(f'Must supply an instance of lsst.ip.isr.IsrTask not {type(isrTask)}')
409 configDict = isrTask.config.toDict()
411 if logName and isinstance(logName, str):
412 log = lsst.log.getLogger(logName)
413 else:
414 log = lsst.log.getLogger("isrValidation")
416 if mandatory:
417 for configParam in mandatory:
418 if configParam not in configDict:
419 raise RuntimeError(f"Mandatory parameter {configParam} not found in the isr configuration.")
420 if configDict[configParam] is False:
421 raise RuntimeError(f"Must set config.isr.{configParam} to True for this task.")
423 if forbidden:
424 for configParam in forbidden:
425 if configParam not in configDict:
426 log.warn(f"Failed to find forbidden key {configParam} in the isr config. The keys in the"
427 " forbidden list should each have an associated Field in IsrConfig:"
428 " check that there is not a typo in this case.")
429 continue
430 if configDict[configParam] is True:
431 raise RuntimeError(f"Must set config.isr.{configParam} to False for this task.")
433 if desirable:
434 for configParam in desirable:
435 if configParam not in configDict:
436 log.info(f"Failed to find key {configParam} in the isr config. You probably want" +
437 " to set the equivalent for your obs_package to True.")
438 continue
439 if configDict[configParam] is False:
440 log.warn(f"Found config.isr.{configParam} set to False for this task." +
441 " The cp_pipe Config recommends setting this to True.")
442 if undesirable:
443 for configParam in undesirable:
444 if configParam not in configDict:
445 log.info(f"Failed to find key {configParam} in the isr config. You probably want" +
446 " to set the equivalent for your obs_package to False.")
447 continue
448 if configDict[configParam] is True:
449 log.warn(f"Found config.isr.{configParam} set to True for this task." +
450 " The cp_pipe Config recommends setting this to False.")
452 if checkTrim: # subtask setting, seems non-trivial to combine with above lists
453 if not isrTask.assembleCcd.config.doTrim:
454 raise RuntimeError("Must trim when assembling CCDs. Set config.isr.assembleCcd.doTrim to True")