Coverage for python/lsst/validate/drp/util.py: 21%
112 statements
« prev ^ index » next coverage.py v7.1.0, created at 2023-02-05 19:03 -0800
« prev ^ index » next coverage.py v7.1.0, created at 2023-02-05 19:03 -0800
1# LSST Data Management System
2# Copyright 2008-2016 AURA/LSST.
3#
4# This product includes software developed by the
5# LSST Project (http://www.lsst.org/).
6#
7# This program is free software: you can redistribute it and/or modify
8# it under the terms of the GNU General Public License as published by
9# the Free Software Foundation, either version 3 of the License, or
10# (at your option) any later version.
11#
12# This program is distributed in the hope that it will be useful,
13# but WITHOUT ANY WARRANTY; without even the implied warranty of
14# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15# GNU General Public License for more details.
16#
17# You should have received a copy of the LSST License Statement and
18# the GNU General Public License along with this program. If not,
19# see <https://www.lsstcorp.org/LegalNotices/>.
20"""Miscellaneous functions to support lsst.validate.drp."""
22import os
24import numpy as np
26import yaml
28import lsst.daf.persistence as dafPersist
29import lsst.pipe.base as pipeBase
30import lsst.geom as geom
33def ellipticity_from_cat(cat, slot_shape='slot_Shape'):
34 """Calculate the ellipticity of the Shapes in a catalog from the 2nd moments.
36 Parameters
37 ----------
38 cat : `lsst.afw.table.BaseCatalog`
39 A catalog with 'slot_Shape' defined and '_xx', '_xy', '_yy'
40 entries for the target of 'slot_Shape'.
41 E.g., 'slot_shape' defined as 'base_SdssShape'
42 And 'base_SdssShape_xx', 'base_SdssShape_xy', 'base_SdssShape_yy' defined.
43 slot_shape : str, optional
44 Specify what slot shape requested. Intended use is to get the PSF shape
45 estimates by specifying 'slot_shape=slot_PsfShape'
46 instead of the default 'slot_shape=slot_Shape'.
48 Returns
49 -------
50 e, e1, e2 : complex, float, float
51 Complex ellipticity, real part, imaginary part
52 """
53 i_xx, i_xy, i_yy = cat.get(slot_shape+'_xx'), cat.get(slot_shape+'_xy'), cat.get(slot_shape+'_yy')
54 return ellipticity(i_xx, i_xy, i_yy)
57def ellipticity_from_shape(shape):
58 """Calculate the ellipticty of shape from its moments.
60 Parameters
61 ----------
62 shape : `lsst.afw.geom.ellipses.Quadrupole`
63 The LSST generic shape object returned by psf.computeShape()
64 or source.getShape() for a specific source.
65 Imeplementation: just needs to have .getIxx, .getIxy, .getIyy methods
66 that each return a float describing the respective second moments.
68 Returns
69 -------
70 e, e1, e2 : complex, float, float
71 Complex ellipticity, real part, imaginary part
72 """
73 i_xx, i_xy, i_yy = shape.getIxx(), shape.getIxy(), shape.getIyy()
74 return ellipticity(i_xx, i_xy, i_yy)
77def ellipticity(i_xx, i_xy, i_yy):
78 """Calculate ellipticity from second moments.
80 Parameters
81 ----------
82 i_xx : float or `numpy.array`
83 i_xy : float or `numpy.array`
84 i_yy : float or `numpy.array`
86 Returns
87 -------
88 e, e1, e2 : (float, float, float) or (numpy.array, numpy.array, numpy.array)
89 Complex ellipticity, real component, imaginary component
90 """
91 e = (i_xx - i_yy + 2j*i_xy) / (i_xx + i_yy)
92 e1 = np.real(e)
93 e2 = np.imag(e)
94 return e, e1, e2
97def averageRaDec(ra, dec):
98 """Calculate average RA, Dec from input lists using spherical geometry.
100 Parameters
101 ----------
102 ra : `list` [`float`]
103 RA in [radians]
104 dec : `list` [`float`]
105 Dec in [radians]
107 Returns
108 -------
109 float, float
110 meanRa, meanDec -- Tuple of average RA, Dec [radians]
111 """
112 assert(len(ra) == len(dec))
114 angleRa = [geom.Angle(r, geom.radians) for r in ra]
115 angleDec = [geom.Angle(d, geom.radians) for d in dec]
116 coords = [geom.SpherePoint(ar, ad, geom.radians) for (ar, ad) in zip(angleRa, angleDec)]
118 meanRa, meanDec = geom.averageSpherePoint(coords)
120 return meanRa.asRadians(), meanDec.asRadians()
123def averageRaDecFromCat(cat):
124 """Calculate the average right ascension and declination from a catalog.
126 Convenience wrapper around averageRaDec
128 Parameters
129 ----------
130 cat : collection
131 Object with .get method for 'coord_ra', 'coord_dec' that returns radians.
133 Returns
134 -------
135 ra_mean : `float`
136 Mean RA in radians.
137 dec_mean : `float`
138 Mean Dec in radians.
139 """
140 return averageRaDec(cat.get('coord_ra'), cat.get('coord_dec'))
143def positionRms(ra_mean, dec_mean, ra, dec):
144 """Calculate the RMS between an array of coordinates and a reference (mean) position.
146 Parameters
147 ----------
148 ra_mean : `float`
149 Mean RA in radians.
150 dec_mean : `float`
151 Mean Dec in radians.
152 ra : `numpy.array` [`float`]
153 Array of RA in radians.
154 dec : `numpy.array` [`float`]
155 Array of Dec in radians.
157 Returns
158 -------
159 pos_rms : `float`
160 RMS scatter of positions in milliarcseconds.
162 Notes
163 -----
164 The RMS of a single-element array will be returned as 0.
165 The RMS of an empty array will be returned as NaN.
166 """
167 separations = sphDist(ra_mean, dec_mean, ra, dec)
168 # Note we don't want `np.std` of separations, which would give us the
169 # std around the average of separations.
170 # We've already taken out the average,
171 # so we want the sqrt of the mean of the squares.
172 pos_rms_rad = np.sqrt(np.mean(separations**2)) # radians
173 pos_rms_mas = geom.radToMas(pos_rms_rad) # milliarcsec
175 return pos_rms_mas
178def positionRmsFromCat(cat):
179 """Calculate the RMS for RA, Dec for a set of observations an object.
181 Parameters
182 ----------
183 cat : collection
184 Object with .get method for 'coord_ra', 'coord_dec' that returns radians.
186 Returns
187 -------
188 pos_rms : `float`
189 RMS scatter of positions in milliarcseconds.
190 """
191 ra_avg, dec_avg = averageRaDecFromCat(cat)
192 ra, dec = cat.get('coord_ra'), cat.get('coord_dec')
193 return positionRms(ra_avg, dec_avg, ra, dec)
196def sphDist(ra_mean, dec_mean, ra, dec):
197 """Calculate distance on the surface of a unit sphere.
199 Parameters
200 ----------
201 ra_mean : `float`
202 Mean RA in radians.
203 dec_mean : `float`
204 Mean Dec in radians.
205 ra : `numpy.array` [`float`]
206 Array of RA in radians.
207 dec : `numpy.array` [`float`]
208 Array of Dec in radians.
210 Notes
211 -----
212 Uses the Haversine formula to preserve accuracy at small angles.
214 Law of cosines approach doesn't work well for the typically very small
215 differences that we're looking at here.
216 """
217 # Haversine
218 dra = ra - ra_mean
219 ddec = dec - dec_mean
220 a = np.square(np.sin(ddec/2)) + \
221 np.cos(dec_mean)*np.cos(dec)*np.square(np.sin(dra/2))
222 dist = 2 * np.arcsin(np.sqrt(a))
224 # This is what the law of cosines would look like
225 # dist = np.arccos(np.sin(dec1)*np.sin(dec2) + np.cos(dec1)*np.cos(dec2)*np.cos(ra1 - ra2))
227 # This will also work, but must run separately for each element
228 # whereas the numpy version will run on either scalars or arrays:
229 # sp1 = geom.SpherePoint(ra1, dec1, geom.radians)
230 # sp2 = geom.SpherePoint(ra2, dec2, geom.radians)
231 # return sp1.separation(sp2).asRadians()
233 return dist
236def averageRaFromCat(cat):
237 """Compute the average right ascension from a catalog of measurements.
239 This function is used as an aggregate function to extract just RA
240 from lsst.validate.drp.matchreduce.build_matched_dataset
242 The actual computation involves both RA and Dec.
244 The intent is to use this for a set of measurements of the same source
245 but that's neither enforced nor required.
247 Parameters
248 ----------
249 cat : collection
250 Object with .get method for 'coord_ra', 'coord_dec' that returns radians.
252 Returns
253 -------
254 ra_mean : `float`
255 Mean RA in radians.
256 """
257 meanRa, meanDec = averageRaDecFromCat(cat)
258 return meanRa
261def averageDecFromCat(cat):
262 """Compute the average declination from a catalog of measurements.
264 This function is used as an aggregate function to extract just declination
265 from lsst.validate.drp.matchreduce.build_matched_dataset
267 The actual computation involves both RA and Dec.
269 The intent is to use this for a set of measurements of the same source
270 but that's neither enforced nor required.
272 Parameters
273 ----------
274 cat : collection
275 Object with .get method for 'coord_ra', 'coord_dec' that returns radians.
277 Returns
278 -------
279 dec_mean : `float`
280 Mean Dec in radians.
281 """
282 meanRa, meanDec = averageRaDecFromCat(cat)
283 return meanDec
286def medianEllipticityResidualsFromCat(cat):
287 """Compute the median ellipticty residuals from a catalog of measurements.
289 This function is used as an aggregate function to extract just declination
290 from lsst.validate.drp.matchreduce.build_matched_dataset
292 The intent is to use this for a set of measurements of the same source
293 but that's neither enforced nor required.
295 Parameters
296 ----------
297 cat : collection
298 Object with .get method for 'e1', 'e2' that returns radians.
300 Returns
301 -------
302 e1_median : `float`
303 Median real ellipticity residual.
304 e2_median : `float`
305 Median imaginary ellipticity residual.
306 """
307 e1_median = np.median(cat.get('e1') - cat.get('psf_e1'))
308 e2_median = np.median(cat.get('e2') - cat.get('psf_e2'))
309 return e1_median, e2_median
312def medianEllipticity1ResidualsFromCat(cat):
313 """Compute the median real ellipticty residuals from a catalog of measurements.
315 Parameters
316 ----------
317 cat : collection
318 Object with .get method for 'e1', 'psf_e1' that returns radians.
320 Returns
321 -------
322 e1_median : `float`
323 Median imaginary ellipticity residual.
324 """
325 e1_median = np.median(cat.get('e1') - cat.get('psf_e1'))
326 return e1_median
329def medianEllipticity2ResidualsFromCat(cat):
330 """Compute the median imaginary ellipticty residuals from a catalog of measurements.
332 Parameters
333 ----------
334 cat : collection
335 Object with .get method for 'e2', 'psf_e2' that returns radians.
337 Returns
338 -------
339 e2_median : `float`
340 Median imaginary ellipticity residual.
341 """
342 e2_median = np.median(cat.get('e2') - cat.get('psf_e2'))
343 return e2_median
346def getCcdKeyName(dataId):
347 """Return the key in a dataId that's referring to the CCD or moral equivalent.
349 Parameters
350 ----------
351 dataId : `dict`
352 A dictionary that will be searched for a key that matches
353 an entry in the hardcoded list of possible names for the CCD field.
355 Returns
356 -------
357 name : `str`
358 The name of the key.
360 Notes
361 -----
362 Motivation: Different camera mappings use different keys to indicate
363 the different amps/ccds in the same exposure. This function looks
364 through the reference dataId to locate a field that could be the one.
365 """
366 possibleCcdFieldNames = ['detector', 'ccd', 'ccdnum', 'camcol', 'sensor']
368 for name in possibleCcdFieldNames:
369 if name in dataId:
370 return name
371 else:
372 return 'ccd'
375def raftSensorToInt(visitId):
376 """Construct an int that encodes raft, sensor coordinates.
378 Parameters
379 ----------
380 visitId : `dict`
381 A dictionary containing raft and sensor keys.
383 Returns
384 -------
385 id : `int`
386 The integer id of the raft/sensor.
388 Examples
389 --------
390 >>> vId = {'filter': 'y', 'raft': '2,2', 'sensor': '1,2', 'visit': 307}
391 >>> raftSensorToInt(vId)
392 2212
393 """
394 def pair_to_int(tuple_string):
395 x, y = tuple_string.split(',')
396 return 10 * int(x) + 1 * int(y)
398 raft_int = pair_to_int(visitId['raft'])
399 sensor_int = pair_to_int(visitId['sensor'])
400 return 100*raft_int + sensor_int
403def repoNameToPrefix(repo):
404 """Generate a base prefix for plots based on the repo name.
406 Parameters
407 ----------
408 repo : `str`
409 The repo path.
411 Returns
412 -------
413 repo_base : `str`
414 The base prefix for the repo.
416 Examples
417 --------
418 >>> repoNameToPrefix('a/b/c')
419 'a_b_c'
420 >>> repoNameToPrefix('/bar/foo/')
421 'bar_foo'
422 >>> repoNameToPrefix('CFHT/output')
423 'CFHT_output'
424 >>> repoNameToPrefix('./CFHT/output')
425 'CFHT_output'
426 >>> repoNameToPrefix('.a/CFHT/output')
427 'a_CFHT_output'
428 >>> repoNameToPrefix('bar/foo.json')
429 'bar_foo'
430 """
432 repo_base, ext = os.path.splitext(repo)
433 return repo_base.lstrip('.').strip(os.sep).replace(os.sep, "_")
436def discoverDataIds(repo, **kwargs):
437 """Retrieve a list of all dataIds in a repo.
439 Parameters
440 ----------
441 repo : `str`
442 Path of a repository with 'src' entries.
444 Returns
445 -------
446 dataIds : `list`
447 dataIds in the butler that exist.
449 Notes
450 -----
451 May consider making this an iterator if large N becomes important.
452 However, will likely need to know things like, "all unique filters"
453 of a data set anyway, so would need to go through chain at least once.
454 """
455 butler = dafPersist.Butler(repo)
456 thisSubset = butler.subset(datasetType='src', **kwargs)
457 # This totally works, but would be better to do this as a TaskRunner?
458 dataIds = [dr.dataId for dr in thisSubset
459 if dr.datasetExists(datasetType='src') and dr.datasetExists(datasetType='calexp')]
460 # Make sure we have the filter information
461 for dId in dataIds:
462 response = butler.queryMetadata(datasetType='src', format=['filter'], dataId=dId)
463 filterForThisDataId = response[0]
464 dId['filter'] = filterForThisDataId
466 return dataIds
469def loadParameters(configFile):
470 """Load configuration parameters from a yaml file.
472 Parameters
473 ----------
474 configFile : `str`
475 YAML file that stores visit, filter, ccd,
476 good_mag_limit, medianAstromscatterRef, medianPhotoscatterRef, matchRef
477 and other parameters
479 Returns
480 -------
481 pipeStruct: `lsst.pipe.base.Struct`
482 Struct with configuration parameters.
483 """
484 with open(configFile, mode='r') as stream:
485 data = yaml.safe_load(stream)
487 return pipeBase.Struct(**data)
490def loadDataIdsAndParameters(configFile):
491 """Load data IDs, magnitude range, and expected metrics from a yaml file.
493 Parameters
494 ----------
495 configFile : `str`
496 YAML file that stores visit, filter, ccd,
497 and additional configuration parameters such as
498 brightSnrMin, medianAstromscatterRef, medianPhotoscatterRef, matchRef
500 Returns
501 -------
502 pipeStruct: `lsst.pipe.base.Struct`
503 Struct with attributes of dataIds - dict and configuration parameters.
504 """
505 parameters = loadParameters(configFile).getDict()
507 ccdKeyName = getCcdKeyName(parameters)
508 try:
509 dataIds = constructDataIds(parameters['filter'], parameters['visits'],
510 parameters[ccdKeyName], ccdKeyName)
511 for key in ['filter', 'visits', ccdKeyName]:
512 del parameters[key]
514 except KeyError:
515 # If the above parameters are not in the `parameters` dict,
516 # presumably because they were not in the configFile
517 # then we return no dataIds.
518 dataIds = []
520 return pipeBase.Struct(dataIds=dataIds, **parameters)
523def constructDataIds(filters, visits, ccds, ccdKeyName='ccd'):
524 """Returns a list of dataIds consisting of every combination of visit & ccd for each filter.
526 Parameters
527 ----------
528 filters : `str` or `list` [`str`]
529 If str, will be interpreted as one filter to be applied to all visits.
530 visits : `list` [`int`]
531 ccds : `list` [`int`]
532 ccdKeyName : `str`, optional
533 Name to distinguish different parts of a focal plane.
534 Generally 'ccd', but might be 'ccdnum', or 'amp', or 'ccdamp'.
535 Refer to your `obs_*/policy/*Mapper.paf`.
537 Returns
538 -------
539 dataIds : `list`
540 dataIDs suitable to be used with the LSST Butler.
542 Examples
543 --------
544 >>> dataIds = constructDataIds('r', [100, 200], [10, 11, 12])
545 >>> for dataId in dataIds: print(dataId)
546 {'filter': 'r', 'visit': 100, 'ccd': 10}
547 {'filter': 'r', 'visit': 100, 'ccd': 11}
548 {'filter': 'r', 'visit': 100, 'ccd': 12}
549 {'filter': 'r', 'visit': 200, 'ccd': 10}
550 {'filter': 'r', 'visit': 200, 'ccd': 11}
551 {'filter': 'r', 'visit': 200, 'ccd': 12}
552 """
553 if isinstance(filters, str):
554 filters = [filters for _ in visits]
556 assert len(filters) == len(visits)
557 dataIds = [{'filter': f, 'visit': v, ccdKeyName: c}
558 for (f, v) in zip(filters, visits)
559 for c in ccds]
561 return dataIds
564def loadRunList(configFile):
565 """Load run list from a YAML file.
567 Parameters
568 ----------
569 configFile : `str`
570 YAML file that stores visit, filter, ccd,
572 Returns
573 -------
574 runList : `list`
575 run list lines.
577 Examples
578 --------
579 An example YAML file would include entries of (for some CFHT data)
580 visits: [849375, 850587]
581 filter: 'r'
582 ccd: [12, 13, 14, 21, 22, 23]
583 or (for some DECam data)
584 visits: [176837, 176846]
585 filter: 'z'
586 ccdnum: [10, 11, 12, 13, 14, 15, 16, 17, 18]
588 Note 'ccd' for CFHT and 'ccdnum' for DECam. These entries will be used to build
589 dataIds, so these fields should be as the camera mapping defines them.
591 `visits` and `ccd` (or `ccdnum`) must be lists, even if there's only one element.
592 """
593 stream = open(configFile, mode='r')
594 data = yaml.safe_load(stream)
596 ccdKeyName = getCcdKeyName(data)
597 runList = constructRunList(data['visits'], data[ccdKeyName], ccdKeyName=ccdKeyName)
599 return runList
602def constructRunList(visits, ccds, ccdKeyName='ccd'):
603 """Construct a comprehensive runList for processCcd.py.
605 Parameters
606 ----------
607 visits : `list` of `int`
608 The desired visits.
609 ccds : `list` of `int`
610 The desired ccds.
612 Returns
613 -------
614 `list`
615 list of strings suitable to be used with the LSST Butler.
617 Examples
618 --------
619 >>> runList = constructRunList([100, 200], [10, 11, 12])
620 >>> print(runList)
621 ['--id visit=100 ccd=10^11^12', '--id visit=200 ccd=10^11^12']
622 >>> runList = constructRunList([100, 200], [10, 11, 12], ccdKeyName='ccdnum')
623 >>> print(runList)
624 ['--id visit=100 ccdnum=10^11^12', '--id visit=200 ccdnum=10^11^12']
626 Notes
627 -----
628 The LSST parsing convention is to use '^' as list separators
629 for arguments to `--id`. While surprising, this convention
630 allows for CCD names to include ','. E.g., 'R1,2'.
631 Currently ignores `filter` because `visit` should be unique w.r.t filter.
632 """
633 runList = ["--id visit=%d %s=%s" % (v, ccdKeyName, "^".join([str(c) for c in ccds]))
634 for v in visits]
636 return runList