Coverage for python/lsst/atmospec/utils.py: 10%
244 statements
« prev ^ index » next coverage.py v7.3.1, created at 2023-09-20 13:28 +0000
« prev ^ index » next coverage.py v7.3.1, created at 2023-09-20 13:28 +0000
1# This file is part of atmospec.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22__all__ = [
23 "argMaxNd",
24 "gainFromFlatPair",
25 "getAmpReadNoiseFromRawExp",
26 "getLinearStagePosition",
27 "getFilterAndDisperserFromExp",
28 "getSamplePoints",
29 "getTargetCentroidFromWcs",
30 "isDispersedDataId",
31 "isDispersedExp",
32 "isExposureTrimmed",
33 "makeGainFlat",
34 "rotateExposure",
35 "simbadLocationForTarget",
36 "vizierLocationForTarget",
37 "runNotebook",
38]
40import logging
41import numpy as np
42import sys
43import lsst.afw.math as afwMath
44import lsst.afw.image as afwImage
45from lsst.ctrl.mpexec import SimplePipelineExecutor
46import lsst.afw.geom as afwGeom
47import lsst.geom as geom
48import lsst.daf.butler as dafButler
49from astro_metadata_translator import ObservationInfo
50import lsst.pex.config as pexConfig
51from lsst.pipe.base import Pipeline
52from lsst.obs.lsst.translators.lsst import FILTER_DELIMITER
53from lsst.utils.iteration import ensure_iterable
55import astropy
56import astropy.units as u
57from astropy.coordinates import SkyCoord, Distance
60def makeGainFlat(exposure, gainDict, invertGains=False):
61 """Given an exposure, make a flat from the gains.
63 Construct an exposure where the image array data contains the gain of the
64 pixel, such that dividing by (or mutiplying by) the image will convert
65 an image from ADU to electrons.
67 Parameters
68 ----------
69 detectorExposure : `lsst.afw.image.exposure`
70 The template exposure for which the flat is to be made.
72 gainDict : `dict` of `float`
73 A dict of the amplifiers' gains, keyed by the amp names.
75 invertGains : `bool`
76 Gains are specified in inverted units and should be applied as such.
78 Returns
79 -------
80 gainFlat : `lsst.afw.image.exposure`
81 The gain flat
82 """
83 flat = exposure.clone()
84 detector = flat.getDetector()
85 ampNames = set(list(a.getName() for a in detector))
86 assert set(gainDict.keys()) == ampNames
88 for amp in detector:
89 bbox = amp.getBBox()
90 if invertGains:
91 flat[bbox].maskedImage.image.array[:, :] = 1./gainDict[amp.getName()]
92 else:
93 flat[bbox].maskedImage.image.array[:, :] = gainDict[amp.getName()]
94 flat.maskedImage.mask[:] = 0x0
95 flat.maskedImage.variance[:] = 0.0
97 return flat
100def argMaxNd(array):
101 """Get the index of the max value of an array.
103 If there are multiple occurences of the maximum value
104 just return the first.
105 """
106 return np.unravel_index(np.argmax(array, axis=None), array.shape)
109def getSamplePoints(start, stop, nSamples, includeEndpoints=False, integers=False):
110 """Get the locations of the coordinates to use to sample a range evenly
112 Divide a range up and return the coordinated to use in order to evenly
113 sample the range. If asking for integers, rounded values are returned,
114 rather than int-truncated ones.
116 If not including the endpoints, divide the (stop-start) range into nSamples
117 and return the midpoint of each section, thus leaving a sectionLength/2 gap
118 between the first/last samples and the range start/end.
120 If including the endpoints, the first and last points will be
121 start, stop respectively, and other points will be the endpoints of the
122 remaining nSamples-1 sections.
124 Visually, for a range:
126 |--*--|--*--|--*--|--*--| return * if not including end points, n=4
127 |-*-|-*-|-*-|-*-|-*-|-*-| return * if not including end points, n=6
129 *-----*-----*-----*-----* return * if we ARE including end points, n=4
130 *---*---*---*---*---*---* return * if we ARE including end points, n=6
131 """
133 if not includeEndpoints:
134 r = (stop-start)/(2*nSamples)
135 points = [((2*pointNum+1)*r) for pointNum in range(nSamples)]
136 else:
137 if nSamples <= 1:
138 raise RuntimeError('nSamples must be >= 2 if including endpoints')
139 if nSamples == 2:
140 points = [start, stop]
141 else:
142 r = (stop-start)/(nSamples-1)
143 points = [start]
144 points.extend([((pointNum)*r) for pointNum in range(1, nSamples)])
146 if integers:
147 return [int(x) for x in np.round(points)]
148 return points
151def isExposureTrimmed(exp):
152 det = exp.getDetector()
153 if exp.getDimensions() == det.getBBox().getDimensions():
154 return True
155 return False
158def getAmpReadNoiseFromRawExp(rawExp, ampNum, nOscanBorderPix=0):
159 """XXX doctring here
161 Trim identically in all direction for convenience"""
162 if isExposureTrimmed(rawExp):
163 raise RuntimeError('Got an assembled exposure instead of a raw one')
165 det = rawExp.getDetector()
167 amp = det[ampNum]
168 if nOscanBorderPix == 0:
169 noise = np.std(rawExp[amp.getRawHorizontalOverscanBBox()].image.array)
170 else:
171 b = nOscanBorderPix # line length limits :/
172 noise = np.std(rawExp[amp.getRawHorizontalOverscanBBox()].image.array[b:-b, b:-b])
173 return noise
176def gainFromFlatPair(flat1, flat2, correctionType=None, rawExpForNoiseCalc=None, overscanBorderSize=0):
177 """Calculate the gain from a pair of flats.
179 The basic premise is 1/g = <(I1 - I2)^2/(I1 + I2)>
180 Corrections for the variable QE and the read-noise are then made
181 following the derivation in Robert's forthcoming book, which gets
183 1/g = <(I1 - I2)^2/(I1 + I2)> - 1/mu(sigma^2 - 1/2g^2)
185 If you are lazy, see below for the solution.
186 https://www.wolframalpha.com/input/?i=solve+1%2Fy+%3D+c+-+(1%2Fm)*(s^2+-+1%2F(2y^2))+for+y
188 where mu is the average signal level, and sigma is the
189 amplifier's readnoise. The way the correction is applied depends on
190 the value supplied for correctionType.
192 correctionType is one of [None, 'simple' or 'full']
193 None : uses the 1/g = <(I1 - I2)^2/(I1 + I2)> formula
194 'simple' : uses the gain from the None method for the 1/2g^2 term
195 'full' : solves the full equation for g, discarding the non-physical
196 solution to the resulting quadratic
198 Parameters
199 ----------
200 flat1 : `lsst.afw.image.exposure`
201 The first of the postISR assembled, overscan-subtracted flat pairs
203 flat2 : `lsst.afw.image.exposure`
204 The second of the postISR assembled, overscan-subtracted flat pairs
206 correctionType : `str` or `None`
207 The correction applied, one of [None, 'simple', 'full']
209 rawExpForNoiseCalc : `lsst.afw.image.exposure`
210 A raw (un-assembled) image from which to measure the noise
212 overscanBorderSize : `int`
213 The number of pixels to crop from the overscan region in all directions
215 Returns
216 -------
217 gainDict : `dict`
218 Dictionary of the amplifier gains, keyed by ampName
219 """
220 if correctionType not in [None, 'simple', 'full']:
221 raise RuntimeError("Unknown correction type %s" % correctionType)
223 if correctionType is not None and rawExpForNoiseCalc is None:
224 raise RuntimeError("Must supply rawFlat if performing correction")
226 gains = {}
227 det = flat1.getDetector()
228 for ampNum, amp in enumerate(det):
229 i1 = flat1[amp.getBBox()].image.array
230 i2 = flat2[amp.getBBox()].image.array
231 const = np.mean((i1 - i2)**2 / (i1 + i2))
232 basicGain = 1. / const
234 if correctionType is None:
235 gains[amp.getName()] = basicGain
236 continue
238 mu = (np.mean(i1) + np.mean(i2)) / 2.
239 sigma = getAmpReadNoiseFromRawExp(rawExpForNoiseCalc, ampNum, overscanBorderSize)
241 if correctionType == 'simple':
242 simpleGain = 1/(const - (1/mu)*(sigma**2 - (1/2*basicGain**2)))
243 gains[amp.getName()] = simpleGain
245 elif correctionType == 'full':
246 root = np.sqrt(mu**2 - 2*mu*const + 2*sigma**2)
247 denom = (2*const*mu - 2*sigma**2)
249 positiveSolution = (root + mu)/denom
250 negativeSolution = (mu - root)/denom # noqa: F841 unused, but the other solution
252 gains[amp.getName()] = positiveSolution
254 return gains
257def rotateExposure(exp, nDegrees, kernelName='lanczos4', logger=None):
258 """Rotate an exposure by nDegrees clockwise.
260 Parameters
261 ----------
262 exp : `lsst.afw.image.exposure.Exposure`
263 The exposure to rotate
264 nDegrees : `float`
265 Number of degrees clockwise to rotate by
266 kernelName : `str`
267 Name of the warping kernel, used to instantiate the warper.
268 logger : `logging.Logger`
269 Logger for logging warnings
271 Returns
272 -------
273 rotatedExp : `lsst.afw.image.exposure.Exposure`
274 A copy of the input exposure, rotated by nDegrees
275 """
276 nDegrees = nDegrees % 360
278 if not logger:
279 logger = logging.getLogger(__name__)
281 wcs = exp.getWcs()
282 if not wcs:
283 logger.warning("Can't rotate exposure without a wcs - returning exp unrotated")
284 return exp.clone() # return a clone so it's always returning a copy as this is what default does
286 warper = afwMath.Warper(kernelName)
287 if isinstance(exp, afwImage.ExposureU):
288 # TODO: remove once this bug is fixed - DM-20258
289 logger.info('Converting ExposureU to ExposureF due to bug')
290 logger.info('Remove this workaround after DM-20258')
291 exp = afwImage.ExposureF(exp, deep=True)
293 affineRotTransform = geom.AffineTransform.makeRotation(nDegrees*geom.degrees)
294 transformP2toP2 = afwGeom.makeTransform(affineRotTransform)
295 rotatedWcs = afwGeom.makeModifiedWcs(transformP2toP2, wcs, False)
297 rotatedExp = warper.warpExposure(rotatedWcs, exp)
298 # rotatedExp.setXY0(geom.Point2I(0, 0)) # TODO: check no longer required
299 return rotatedExp
302def airMassFromRawMetadata(md):
303 """Calculate the visit's airmass from the raw header information.
305 Parameters
306 ----------
307 md : `Mapping`
308 The raw header.
310 Returns
311 -------
312 airmass : `float`
313 Returns the airmass, or 0.0 if the calculation fails.
314 Zero was chosen as it is an obviously unphysical value, but means
315 that calling code doesn't have to test if None, as numeric values can
316 be used more easily in place.
317 """
318 try:
319 obsInfo = ObservationInfo(md, subset={"boresight_airmass"})
320 except Exception:
321 return 0.0
322 return obsInfo.boresight_airmass
325def getTargetCentroidFromWcs(exp, target, doMotionCorrection=True, logger=None):
326 """Get the target's centroid, given an exposure with fitted WCS.
328 Parameters
329 ----------
330 exp : `lsst.afw.exposure.Exposure`
331 Exposure with fitted WCS.
333 target : `str`
334 The name of the target, e.g. 'HD 55852'
336 doMotionCorrection : `bool`, optional
337 Correct for proper motion and parallax if possible.
338 This requires the object is found in Vizier rather than Simbad.
339 If that is not possible, a warning is logged, and the uncorrected
340 centroid is returned.
342 Returns
343 -------
344 pixCoord : `tuple` of `float`, or `None`
345 The pixel (x, y) of the target's centroid, or None if the object
346 is not found.
347 """
348 if logger is None:
349 logger = logging.getLogger(__name__)
351 resultFrom = None
352 targetLocation = None
353 # try vizier, but it is slow, unreliable, and
354 # many objects are found but have no Hipparcos entries
355 try:
356 targetLocation = vizierLocationForTarget(exp, target, doMotionCorrection=doMotionCorrection)
357 resultFrom = 'vizier'
358 logger.info("Target location for %s retrieved from Vizier", target)
360 # fail over to simbad - it has ~every target, but no proper motions
361 except ValueError:
362 try:
363 logger.warning("Target %s not found in Vizier, failing over to try Simbad", target)
364 targetLocation = simbadLocationForTarget(target)
365 resultFrom = 'simbad'
366 logger.info("Target location for %s retrieved from Simbad", target)
367 except ValueError as inst: # simbad found zero or several results for target
368 logger.warning("%s", inst.args[0])
369 return None
371 if not targetLocation:
372 return None
374 if doMotionCorrection and resultFrom == 'simbad':
375 logger.warning("Failed to apply motion correction because %s was"
376 " only found in Simbad, not Vizier/Hipparcos", target)
378 pixCoord = exp.getWcs().skyToPixel(targetLocation)
379 return pixCoord
382def simbadLocationForTarget(target):
383 """Get the target location from Simbad.
385 Parameters
386 ----------
387 target : `str`
388 The name of the target, e.g. 'HD 55852'
390 Returns
391 -------
392 targetLocation : `lsst.geom.SpherePoint`
393 Nominal location of the target object, uncorrected for
394 proper motion and parallax.
396 Raises
397 ------
398 ValueError
399 If object not found, or if multiple entries for the object are found.
400 """
401 # do not import at the module level - tests crash due to a race
402 # condition with directory creation
403 from astroquery.simbad import Simbad
405 obj = Simbad.query_object(target)
406 if not obj:
407 raise ValueError(f"Failed to find {target} in simbad!")
408 if len(obj) != 1:
409 raise ValueError(f"Found {len(obj)} simbad entries for {target}!")
411 raStr = obj[0]['RA']
412 decStr = obj[0]['DEC']
413 skyLocation = SkyCoord(raStr, decStr, unit=(u.hourangle, u.degree), frame='icrs')
414 raRad, decRad = skyLocation.ra.rad, skyLocation.dec.rad
415 ra = geom.Angle(raRad)
416 dec = geom.Angle(decRad)
417 targetLocation = geom.SpherePoint(ra, dec)
418 return targetLocation
421def vizierLocationForTarget(exp, target, doMotionCorrection):
422 """Get the target location from Vizier optionally correction motion.
424 Parameters
425 ----------
426 target : `str`
427 The name of the target, e.g. 'HD 55852'
429 Returns
430 -------
431 targetLocation : `lsst.geom.SpherePoint` or `None`
432 Location of the target object, optionally corrected for
433 proper motion and parallax.
435 Raises
436 ------
437 ValueError
438 If object not found in Hipparcos2 via Vizier.
439 This is quite common, even for bright objects.
440 """
441 # do not import at the module level - tests crash due to a race
442 # condition with directory creation
443 from astroquery.vizier import Vizier
445 result = Vizier.query_object(target) # result is an empty table list for an unknown target
446 try:
447 star = result['I/311/hip2']
448 except TypeError: # if 'I/311/hip2' not in result (result doesn't allow easy checking without a try)
449 raise ValueError
451 epoch = "J1991.25"
452 coord = SkyCoord(ra=star[0]['RArad']*u.Unit(star['RArad'].unit),
453 dec=star[0]['DErad']*u.Unit(star['DErad'].unit),
454 obstime=epoch,
455 pm_ra_cosdec=star[0]['pmRA']*u.Unit(star['pmRA'].unit), # NB contains cosdec already
456 pm_dec=star[0]['pmDE']*u.Unit(star['pmDE'].unit),
457 distance=Distance(parallax=star[0]['Plx']*u.Unit(star['Plx'].unit)))
459 if doMotionCorrection:
460 expDate = exp.getInfo().getVisitInfo().getDate()
461 obsTime = astropy.time.Time(expDate.get(expDate.EPOCH), format='jyear', scale='tai')
462 newCoord = coord.apply_space_motion(new_obstime=obsTime)
463 else:
464 newCoord = coord
466 raRad, decRad = newCoord.ra.rad, newCoord.dec.rad
467 ra = geom.Angle(raRad)
468 dec = geom.Angle(decRad)
469 targetLocation = geom.SpherePoint(ra, dec)
470 return targetLocation
473def isDispersedExp(exp):
474 """Check if an exposure is dispersed.
476 Parameters
477 ----------
478 exp : `lsst.afw.image.Exposure`
479 The exposure.
481 Returns
482 -------
483 isDispersed : `bool`
484 Whether it is a dispersed image or not.
485 """
486 filterFullName = exp.filter.physicalLabel
487 if FILTER_DELIMITER not in filterFullName:
488 raise RuntimeError(f"Error parsing filter name {filterFullName}")
489 filt, grating = filterFullName.split(FILTER_DELIMITER)
490 if grating.upper().startswith('EMPTY'):
491 return False
492 return True
495def isDispersedDataId(dataId, butler):
496 """Check if a dataId corresponds to a dispersed image.
498 Parameters
499 ----------
500 dataId : `dict`
501 The dataId.
502 butler : `lsst.daf.butler.Butler`
503 The butler.
505 Returns
506 -------
507 isDispersed : `bool`
508 Whether it is a dispersed image or not.
509 """
510 if isinstance(butler, dafButler.Butler):
511 # TODO: DM-38265 Need to make this work with DataCoords
512 assert 'day_obs' in dataId or 'exposure.day_obs' in dataId, f'failed to find day_obs in {dataId}'
513 assert 'seq_num' in dataId or 'exposure.seq_num' in dataId, f'failed to find seq_num in {dataId}'
514 seq_num = dataId['seq_num'] if 'seq_num' in dataId else dataId['exposure.seq_num']
515 day_obs = dataId['day_obs'] if 'day_obs' in dataId else dataId['exposure.day_obs']
516 where = "exposure.day_obs=day_obs AND exposure.seq_num=seq_num"
517 expRecords = butler.registry.queryDimensionRecords("exposure", where=where,
518 bind={'day_obs': day_obs,
519 'seq_num': seq_num})
520 expRecords = set(expRecords)
521 assert len(expRecords) == 1, f'Found more than one exposure record for {dataId}'
522 filterFullName = expRecords.pop().physical_filter
523 else:
524 raise RuntimeError(f'Expected a butler, got {type(butler)}')
525 if FILTER_DELIMITER not in filterFullName:
526 raise RuntimeError(f"Error parsing filter name {filterFullName}")
527 filt, grating = filterFullName.split(FILTER_DELIMITER)
528 if grating.upper().startswith('EMPTY'):
529 return False
530 return True
533def getLinearStagePosition(exp):
534 """Get the linear stage position.
536 Parameters
537 ----------
538 exp : `lsst.afw.image.Exposure`
539 The exposure.
541 Returns
542 -------
543 position : `float`
544 The position of the linear stage, in mm.
545 """
546 md = exp.getMetadata()
547 linearStagePosition = 115 # this seems to be the rough zero-point for some reason
548 if 'LINSPOS' in md:
549 position = md['LINSPOS'] # linear stage position in mm from CCD, larger->further from CCD
550 if position is not None:
551 linearStagePosition += position
552 return linearStagePosition
555def getFilterAndDisperserFromExp(exp):
556 """Get the filter and disperser from an exposure.
558 Parameters
559 ----------
560 exp : `lsst.afw.image.Exposure`
561 The exposure.
563 Returns
564 -------
565 filter, disperser : `tuple` of `str`
566 The filter and the disperser names, as strings.
567 """
568 filterFullName = exp.getFilter().physicalLabel
569 if FILTER_DELIMITER not in filterFullName:
570 filt = filterFullName
571 grating = exp.getInfo().getMetadata()['GRATING']
572 else:
573 filt, grating = filterFullName.split(FILTER_DELIMITER)
574 return filt, grating
577def runNotebook(dataId,
578 outputCollection,
579 *,
580 extraInputCollections=None,
581 taskConfigs={},
582 configOptions={},
583 embargo=False):
584 """Run the ProcessStar pipeline for a single dataId, writing to the
585 specified output collection.
587 This is a convenience function to allow running single dataIds in notebooks
588 so that plots can be inspected easily. This is not designed for bulk data
589 reductions.
591 Parameters
592 ----------
593 dataId : `dict`
594 The dataId to run.
595 outputCollection : `str`, optional
596 Output collection name.
597 extraInputCollections : `list` of `str`
598 Any extra input collections to use when processing.
599 taskConfigs : `dict` [`lsst.pipe.base.PipelineTaskConfig`], optional
600 Dictionary of config config classes. The key of the ``taskConfigs``
601 dict is the relevant task label. The value of ``taskConfigs``
602 is a task config object to apply. See notes for ignored items.
603 configOptions : `dict` [`dict`], optional
604 Dictionary of individual config options. The key of the
605 ``configOptions`` dict is the relevant task label. The value
606 of ``configOptions`` is another dict that contains config
607 key/value overrides to apply.
608 embargo : `bool`, optional
609 Use the embargo repo?
611 Returns
612 -------
613 spectraction : `lsst.atmospec.spectraction.Spectraction`
614 The extracted spectraction object.
616 Notes
617 -----
618 Any ConfigurableInstances in supplied task config overrides will be
619 ignored. Currently (see DM-XXXXX) this causes a RecursionError.
620 """
621 def makeQuery(dataId):
622 dayObs = dataId['day_obs'] if 'day_obs' in dataId else dataId['exposure.day_obs']
623 seqNum = dataId['seq_num'] if 'seq_num' in dataId else dataId['exposure.seq_num']
624 queryString = (f"exposure.day_obs={dayObs} AND "
625 f"exposure.seq_num={seqNum} AND "
626 "instrument='LATISS'")
628 return queryString
629 repo = "LATISS" if not embargo else "/repo/embargo"
631 # TODO: use LATISS_DEFAULT_COLLECTIONS here?
632 inputs = ['LATISS/raw/all', 'refcats', 'LATISS/calib']
633 if extraInputCollections is not None:
634 extraInputCollections = ensure_iterable(extraInputCollections)
635 inputs.extend(extraInputCollections)
636 butler = SimplePipelineExecutor.prep_butler(repo,
637 inputs=inputs,
638 output=outputCollection)
640 pipeline = Pipeline.fromFile("${ATMOSPEC_DIR}/pipelines/processStar.yaml")
642 for taskName, configClass in taskConfigs.items():
643 for option, value in configClass.items():
644 # connections require special treatment
645 if isinstance(value, configClass.ConnectionsConfigClass):
646 for connectionOption, connectionValue in value.items():
647 pipeline.addConfigOverride(taskName,
648 f'{option}.{connectionOption}',
649 connectionValue)
650 # ConfigurableInstance has to be done with .retarget()
651 elif not isinstance(value, pexConfig.ConfigurableInstance):
652 pipeline.addConfigOverride(taskName, option, value)
654 for taskName, configDict in configOptions.items():
655 for option, value in configDict.items():
656 # ConfigurableInstance has to be done with .retarget()
657 if not isinstance(value, pexConfig.ConfigurableInstance):
658 pipeline.addConfigOverride(taskName, option, value)
660 query = makeQuery(dataId)
661 executor = SimplePipelineExecutor.from_pipeline(pipeline,
662 where=query,
663 root=repo,
664 butler=butler)
666 logging.basicConfig(level=logging.INFO, stream=sys.stdout)
667 quanta = executor.run()
669 # quanta is just a plain list, but the items know their names, so rather
670 # than just taking the third item and relying on that being the position in
671 # the pipeline we get the item by name
672 processStarQuantum = [q for q in quanta if q.taskName == 'lsst.atmospec.processStar.ProcessStarTask'][0]
673 dataRef = processStarQuantum.outputs['spectractorSpectrum'][0]
674 result = butler.get(dataRef)
675 return result