Coverage for python/lsst/atmospec/processStar.py: 27%
334 statements
« prev ^ index » next coverage.py v7.3.2, created at 2023-11-02 12:39 +0000
« prev ^ index » next coverage.py v7.3.2, created at 2023-11-02 12:39 +0000
1# This file is part of atmospec.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22__all__ = ['ProcessStarTask', 'ProcessStarTaskConfig']
24import os
25import shutil
26import numpy as np
27import matplotlib.pyplot as plt
29import lsstDebug
30import lsst.afw.image as afwImage
31import lsst.geom as geom
32from lsst.ip.isr import IsrTask
33import lsst.pex.config as pexConfig
34from lsst.pex.config import FieldValidationError
35import lsst.pipe.base as pipeBase
36import lsst.pipe.base.connectionTypes as cT
37from lsst.pipe.base.task import TaskError
39from lsst.utils import getPackageDir
40from lsst.pipe.tasks.characterizeImage import CharacterizeImageTask
41from lsst.meas.algorithms import ReferenceObjectLoader, MagnitudeLimit
42from lsst.meas.astrom import AstrometryTask, FitAffineWcsTask
44import lsst.afw.detection as afwDetect
46from .spectraction import SpectractorShim
47from .utils import getLinearStagePosition, isDispersedExp, getFilterAndDisperserFromExp
49COMMISSIONING = False # allows illegal things for on the mountain usage.
51# TODO:
52# Sort out read noise and gain
53# remove dummy image totally
54# talk to Jeremy about turning the image beforehand and giving new coords
55# deal with not having ambient temp
56# Gen3ification
57# astropy warning for units on save
58# but actually just remove all manual saves entirely, I think?
59# Make SED persistable
60# Move to QFM for star finding failover case
61# Remove old cruft functions
62# change spectractions run method to be ~all kwargs with *,...
65class ProcessStarTaskConnections(pipeBase.PipelineTaskConnections,
66 dimensions=("instrument", "visit", "detector")):
67 inputExp = cT.Input(
68 name="icExp",
69 doc="Image-characterize output exposure",
70 storageClass="ExposureF",
71 dimensions=("instrument", "visit", "detector"),
72 multiple=False,
73 )
74 inputCentroid = cT.Input(
75 name="atmospecCentroid",
76 doc="The main star centroid in yaml format.",
77 storageClass="StructuredDataDict",
78 dimensions=("instrument", "visit", "detector"),
79 multiple=False,
80 )
81 spectractorSpectrum = cT.Output(
82 name="spectractorSpectrum",
83 doc="The Spectractor output spectrum.",
84 storageClass="SpectractorSpectrum",
85 dimensions=("instrument", "visit", "detector"),
86 )
87 spectractorImage = cT.Output(
88 name="spectractorImage",
89 doc="The Spectractor output image.",
90 storageClass="SpectractorImage",
91 dimensions=("instrument", "visit", "detector"),
92 )
93 spectrumForwardModelFitParameters = cT.Output(
94 name="spectrumForwardModelFitParameters",
95 doc="The full forward model fit parameters.",
96 storageClass="SpectractorFitParameters",
97 dimensions=("instrument", "visit", "detector"),
98 )
99 spectrumLibradtranFitParameters = cT.Output(
100 name="spectrumLibradtranFitParameters",
101 doc="The fitted Spectractor atmospheric parameters from fitting the atmosphere with libradtran"
102 " on the spectrum.",
103 storageClass="SpectractorFitParameters",
104 dimensions=("instrument", "visit", "detector"),
105 )
106 spectrogramLibradtranFitParameters = cT.Output(
107 name="spectrogramLibradtranFitParameters",
108 doc="The fitted Spectractor atmospheric parameters from fitting the atmosphere with libradtran"
109 " directly on the spectrogram.",
110 storageClass="SpectractorFitParameters",
111 dimensions=("instrument", "visit", "detector"),
112 )
114 def __init__(self, *, config=None):
115 super().__init__(config=config)
116 if not config.doFullForwardModelDeconvolution:
117 self.outputs.remove("spectrumForwardModelFitParameters")
118 if not config.doFitAtmosphere:
119 self.outputs.remove("spectrumLibradtranFitParameters")
120 if not config.doFitAtmosphereOnSpectrogram:
121 self.outputs.remove("spectrogramLibradtranFitParameters")
124class ProcessStarTaskConfig(pipeBase.PipelineTaskConfig,
125 pipelineConnections=ProcessStarTaskConnections):
126 """Configuration parameters for ProcessStarTask."""
127 # Spectractor parameters:
128 targetCentroidMethod = pexConfig.ChoiceField(
129 dtype=str,
130 doc="Method to get target centroid. "
131 "SPECTRACTOR_FIT_TARGET_CENTROID internally.",
132 default="auto",
133 allowed={
134 # note that although this config option controls
135 # SPECTRACTOR_FIT_TARGET_CENTROID, it doesn't map there directly,
136 # because Spectractor only has the concepts of guess, fit and wcs,
137 # and it calls "exact" "guess" internally, so that's remapped.
138 "auto": "If the upstream astrometric fit succeeded, and therefore"
139 " the centroid is an exact one, use that as an ``exact`` value,"
140 " otherwise tell Spectractor to ``fit`` the centroid",
141 "exact": "Use a given input value as source of truth.",
142 "fit": "Fit a 2d Moffat model to the target.",
143 "WCS": "Use the target's catalog location and the image's wcs.",
144 }
145 )
146 rotationAngleMethod = pexConfig.ChoiceField(
147 dtype=str,
148 doc="Method used to get the image rotation angle. "
149 "SPECTRACTOR_COMPUTE_ROTATION_ANGLE internally.",
150 default="disperser",
151 allowed={
152 # XXX MFL: probably need to use setDefaults to set this based on
153 # the disperser. I think Ronchi gratings want hessian and the
154 # holograms want disperser.
155 "False": "Do not rotate the image.",
156 "disperser": "Use the disperser angle geometry as specified in the disperser definition.",
157 "hessian": "Compute the angle from the image using a Hessian transform.",
158 }
159 )
160 doDeconvolveSpectrum = pexConfig.Field(
161 dtype=bool,
162 doc="Deconvolve the spectrogram with a simple 2D PSF analysis? "
163 "SPECTRACTOR_DECONVOLUTION_PSF2D internally.",
164 default=True,
165 )
166 doFullForwardModelDeconvolution = pexConfig.Field(
167 dtype=bool,
168 doc="Deconvolve the spectrogram with full forward model? "
169 "SPECTRACTOR_DECONVOLUTION_FFM internally.",
170 default=True,
171 )
172 deconvolutionSigmaClip = pexConfig.Field(
173 dtype=float,
174 doc="Sigma clipping level for the deconvolution when fitting the full forward model? "
175 "SPECTRACTOR_DECONVOLUTION_SIGMA_CLIP internally.",
176 default=100,
177 )
178 doSubtractBackground = pexConfig.Field(
179 dtype=bool,
180 doc="Subtract the background with Spectractor? "
181 "SPECTRACTOR_BACKGROUND_SUBTRACTION internally.",
182 default=True,
183 )
184 rebin = pexConfig.Field(
185 dtype=int,
186 doc="Rebinning factor to use on the input image, in pixels. "
187 "CCD_REBIN internally.",
188 default=2, # TODO Change to 1 once speed issues are resolved
189 )
190 xWindow = pexConfig.Field(
191 dtype=int,
192 doc="Window x size to search for the target object. Ignored if targetCentroidMethod in ('exact, wcs')"
193 "XWINDOW internally.",
194 default=150,
195 )
196 yWindow = pexConfig.Field(
197 dtype=int,
198 doc="Window y size to search for the targeted object. Ignored if targetCentroidMethod in "
199 "('exact, wcs')"
200 "YWINDOW internally.",
201 default=150,
202 )
203 xWindowRotated = pexConfig.Field(
204 dtype=int,
205 doc="Window x size to search for the target object in the rotated image. "
206 "Ignored if rotationAngleMethod=False"
207 "XWINDOW_ROT internally.",
208 default=50,
209 )
210 yWindowRotated = pexConfig.Field(
211 dtype=int,
212 doc="Window y size to search for the target object in the rotated image. "
213 "Ignored if rotationAngleMethod=False"
214 "YWINDOW_ROT internally.",
215 default=50,
216 )
217 pixelShiftPrior = pexConfig.Field( 217 ↛ exitline 217 didn't jump to the function exit
218 dtype=float,
219 doc="Prior on the reliability of the centroid estimate in pixels. "
220 "PIXSHIFT_PRIOR internally.",
221 default=5,
222 check=lambda x: x > 0,
223 )
224 doFilterRotatedImage = pexConfig.Field(
225 dtype=bool,
226 doc="Apply a filter to the rotated image? If not True, this creates residuals and correlated noise. "
227 "ROT_PREFILTER internally.",
228 default=True,
229 )
230 imageRotationSplineOrder = pexConfig.Field(
231 dtype=int,
232 doc="Order of the spline used when rotating the image. "
233 "ROT_ORDER internally.",
234 default=5,
235 # XXX min value of 3 for allowed range, max 5
236 )
237 rotationAngleMin = pexConfig.Field(
238 dtype=float,
239 doc="In the Hessian analysis to compute the rotation angle, cut all angles below this, in degrees. "
240 "ROT_ANGLE_MIN internally.",
241 default=-10,
242 )
243 rotationAngleMax = pexConfig.Field(
244 dtype=float,
245 doc="In the Hessian analysis to compute rotation angle, cut all angles above this, in degrees. "
246 "ROT_ANGLE_MAX internally.",
247 default=10,
248 )
249 plotLineWidth = pexConfig.Field(
250 dtype=float,
251 doc="Line width parameter for plotting. "
252 "LINEWIDTH internally.",
253 default=2,
254 )
255 verbose = pexConfig.Field(
256 dtype=bool,
257 doc="Set verbose mode? "
258 "VERBOSE internally.",
259 default=True, # sets INFO level logging in Spectractor
260 )
261 spectractorDebugMode = pexConfig.Field(
262 dtype=bool,
263 doc="Set spectractor debug mode? "
264 "DEBUG internally.",
265 default=True,
266 )
267 spectractorDebugLogging = pexConfig.Field(
268 dtype=bool,
269 doc="Set spectractor debug logging? "
270 "DEBUG_LOGGING internally.",
271 default=False
272 )
273 doDisplay = pexConfig.Field(
274 dtype=bool,
275 doc="Display plots, for example when running in a notebook? "
276 "DISPLAY internally.",
277 default=True
278 )
279 lambdaMin = pexConfig.Field(
280 dtype=int,
281 doc="Minimum wavelength for spectral extraction (in nm). "
282 "LAMBDA_MIN internally.",
283 default=300
284 )
285 lambdaMax = pexConfig.Field(
286 dtype=int,
287 doc=" maximum wavelength for spectrum extraction (in nm). "
288 "LAMBDA_MAX internally.",
289 default=1100
290 )
291 lambdaStep = pexConfig.Field(
292 dtype=float,
293 doc="Step size for the wavelength array (in nm). "
294 "LAMBDA_STEP internally.",
295 default=1,
296 )
297 spectralOrder = pexConfig.ChoiceField(
298 dtype=int,
299 doc="The spectral order to extract. "
300 "SPEC_ORDER internally.",
301 default=1,
302 allowed={
303 1: "The first order spectrum in the positive y direction",
304 -1: "The first order spectrum in the negative y direction",
305 2: "The second order spectrum in the positive y direction",
306 -2: "The second order spectrum in the negative y direction",
307 }
308 )
309 signalWidth = pexConfig.Field( # TODO: change this to be set wrt the focus/seeing, i.e. FWHM from imChar
310 dtype=int,
311 doc="Half transverse width of the signal rectangular window in pixels. "
312 "PIXWIDTH_SIGNAL internally.",
313 default=40,
314 )
315 backgroundDistance = pexConfig.Field(
316 dtype=int,
317 doc="Distance from dispersion axis to analyse the background in pixels. "
318 "PIXDIST_BACKGROUND internally.",
319 default=140,
320 )
321 backgroundWidth = pexConfig.Field(
322 dtype=int,
323 doc="Transverse width of the background rectangular window in pixels. "
324 "PIXWIDTH_BACKGROUND internally.",
325 default=40,
326 )
327 backgroundBoxSize = pexConfig.Field(
328 dtype=int,
329 doc="Box size for sextractor evaluation of the background. "
330 "PIXWIDTH_BOXSIZE internally.",
331 default=20,
332 )
333 backgroundOrder = pexConfig.Field(
334 dtype=int,
335 doc="The order of the polynomial background to fit in the transverse direction. "
336 "BGD_ORDER internally.",
337 default=1,
338 )
339 psfType = pexConfig.ChoiceField(
340 dtype=str,
341 doc="The PSF model type to use. "
342 "PSF_TYPE internally.",
343 default="Moffat",
344 allowed={
345 "Moffat": "A Moffat function",
346 "MoffatGauss": "A Moffat plus a Gaussian"
347 }
348 )
349 psfPolynomialOrder = pexConfig.Field(
350 dtype=int,
351 doc="The order of the polynomials to model wavelength dependence of the PSF shape parameters. "
352 "PSF_POLY_ORDER internally.",
353 default=2
354 )
355 psfRegularization = pexConfig.Field(
356 dtype=float,
357 doc="Regularisation parameter for the chisq minimisation to extract the spectrum. "
358 "PSF_FIT_REG_PARAM internally.",
359 default=1,
360 # XXX allowed range strictly positive
361 )
362 psfTransverseStepSize = pexConfig.Field(
363 dtype=int,
364 doc="Step size in pixels for the first transverse PSF1D fit. "
365 "PSF_PIXEL_STEP_TRANSVERSE_FIT internally.",
366 default=10,
367 )
368 psfFwhmClip = pexConfig.Field(
369 dtype=float,
370 doc="PSF is not evaluated outside a region larger than max(signalWidth, psfFwhmClip*fwhm) pixels. "
371 "PSF_FWHM_CLIP internally.",
372 default=2,
373 )
374 calibBackgroundOrder = pexConfig.Field(
375 dtype=int,
376 doc="Order of the background polynomial to fit. "
377 "CALIB_BGD_ORDER internally.",
378 default=3,
379 )
380 calibPeakWidth = pexConfig.Field(
381 dtype=int,
382 doc="Half-range to look for local extrema in pixels around tabulated line values. "
383 "CALIB_PEAK_WIDTH internally.",
384 default=7
385 )
386 calibBackgroundWidth = pexConfig.Field(
387 dtype=int,
388 doc="Size of the peak sides to use to fit spectrum base line. "
389 "CALIB_BGD_WIDTH internally.",
390 default=15,
391 )
392 calibSavgolWindow = pexConfig.Field(
393 dtype=int,
394 doc="Window size for the savgol filter in pixels. "
395 "CALIB_SAVGOL_WINDOW internally.",
396 default=5,
397 )
398 calibSavgolOrder = pexConfig.Field(
399 dtype=int,
400 doc="Polynomial order for the savgol filter. "
401 "CALIB_SAVGOL_ORDER internally.",
402 default=2,
403 )
404 transmissionSystematicError = pexConfig.Field(
405 dtype=float,
406 doc="The systematic error on the instrumental transmission. OBS_TRANSMISSION_SYSTEMATICS internally",
407 default=0.005
408 )
409 instrumentTransmissionOverride = pexConfig.Field(
410 dtype=str,
411 doc="File to use for the full instrumental transmission. Must be located in the"
412 " $SPECTRACTOR_DIR/spectractor/simulation/AuxTelThroughput/ directory."
413 " OBS_FULL_INSTRUMENT_TRANSMISSON internally.",
414 default="multispectra_holo4_003_HD142331_AuxTel_throughput.txt"
415 )
416 offsetFromMainStar = pexConfig.Field(
417 dtype=int,
418 doc="Number of pixels from the main star's centroid to start extraction",
419 default=100
420 )
421 spectrumLengthPixels = pexConfig.Field(
422 dtype=int,
423 doc="Length of the spectrum in pixels",
424 default=5000
425 )
426 # ProcessStar own parameters
427 isr = pexConfig.ConfigurableField(
428 target=IsrTask,
429 doc="Task to perform instrumental signature removal",
430 )
431 charImage = pexConfig.ConfigurableField(
432 target=CharacterizeImageTask,
433 doc="""Task to characterize a science exposure:
434 - detect sources, usually at high S/N
435 - estimate the background, which is subtracted from the image and returned as field "background"
436 - estimate a PSF model, which is added to the exposure
437 - interpolate over defects and cosmic rays, updating the image, variance and mask planes
438 """,
439 )
440 doWrite = pexConfig.Field(
441 dtype=bool,
442 doc="Write out the results?",
443 default=True,
444 )
445 doFlat = pexConfig.Field(
446 dtype=bool,
447 doc="Flatfield the image?",
448 default=True
449 )
450 doCosmics = pexConfig.Field(
451 dtype=bool,
452 doc="Repair cosmic rays?",
453 default=True
454 )
455 doDisplayPlots = pexConfig.Field(
456 dtype=bool,
457 doc="Matplotlib show() the plots, so they show up in a notebook or X window",
458 default=False
459 )
460 doSavePlots = pexConfig.Field(
461 dtype=bool,
462 doc="Save matplotlib plots to output rerun?",
463 default=False
464 )
465 forceObjectName = pexConfig.Field(
466 dtype=str,
467 doc="A supplementary name for OBJECT. Will be forced to apply to ALL visits, so this should only"
468 " ONLY be used for immediate commissioning debug purposes. All long term fixes should be"
469 " supplied as header fix-up yaml files.",
470 default=""
471 )
472 referenceFilterOverride = pexConfig.Field(
473 dtype=str,
474 doc="Which filter in the reference catalog to match to?",
475 default="phot_g_mean"
476 )
477 # This is a post-processing function in Spectractor and therefore isn't
478 # controlled by its top-level function, and thus doesn't map to a
479 # spectractor.parameters ALL_CAPS config option
480 doFitAtmosphere = pexConfig.Field(
481 dtype=bool,
482 doc="Use uvspec to fit the atmosphere? Requires the binary to be available.",
483 default=False
484 )
485 # This is a post-processing function in Spectractor and therefore isn't
486 # controlled by its top-level function, and thus doesn't map to a
487 # spectractor.parameters ALL_CAPS config option
488 doFitAtmosphereOnSpectrogram = pexConfig.Field(
489 dtype=bool,
490 doc="Experimental option to use uvspec to fit the atmosphere directly on the spectrogram?"
491 " Requires the binary to be available.",
492 default=False
493 )
495 def setDefaults(self):
496 self.isr.doWrite = False
497 self.charImage.doWriteExposure = False
499 self.charImage.doApCorr = False
500 self.charImage.doMeasurePsf = False
501 self.charImage.repair.cosmicray.nCrPixelMax = 100000
502 self.charImage.repair.doCosmicRay = False
503 if self.charImage.doMeasurePsf:
504 self.charImage.measurePsf.starSelector['objectSize'].signalToNoiseMin = 10.0
505 self.charImage.measurePsf.starSelector['objectSize'].fluxMin = 5000.0
506 self.charImage.detection.includeThresholdMultiplier = 3
507 self.isr.overscan.fitType = 'MEDIAN_PER_ROW'
509 def validate(self):
510 super().validate()
511 uvspecPath = shutil.which('uvspec')
512 if uvspecPath is None and self.doFitAtmosphere is True:
513 raise FieldValidationError(self.__class__.doFitAtmosphere, self, "uvspec is not in the path,"
514 " but doFitAtmosphere is True.")
515 if uvspecPath is None and self.doFitAtmosphereOnSpectrogram is True:
516 raise FieldValidationError(self.__class__.doFitAtmosphereOnSpectrogram, self, "uvspec is not in"
517 " the path, but doFitAtmosphere is True.")
520class ProcessStarTask(pipeBase.PipelineTask):
521 """Task for the spectral extraction of single-star dispersed images.
523 For a full description of how this tasks works, see the run() method.
524 """
526 ConfigClass = ProcessStarTaskConfig
527 _DefaultName = "processStar"
529 def __init__(self, **kwargs):
530 # TODO: rename psfRefObjLoader to refObjLoader
531 super().__init__(**kwargs)
532 self.makeSubtask("isr")
533 self.makeSubtask("charImage", refObjLoader=None)
535 self.debug = lsstDebug.Info(__name__)
536 if self.debug.enabled:
537 self.log.info("Running with debug enabled...")
538 # If we're displaying, test it works and save displays for later.
539 # It's worth testing here as displays are flaky and sometimes
540 # can't be contacted, and given processing takes a while,
541 # it's a shame to fail late due to display issues.
542 if self.debug.display:
543 try:
544 import lsst.afw.display as afwDisp
545 afwDisp.setDefaultBackend(self.debug.displayBackend)
546 afwDisp.Display.delAllDisplays()
547 # pick an unlikely number to be safe xxx replace this
548 self.disp1 = afwDisp.Display(987, open=True)
550 im = afwImage.ImageF(2, 2)
551 im.array[:] = np.ones((2, 2))
552 self.disp1.mtv(im)
553 self.disp1.erase()
554 afwDisp.setDefaultMaskTransparency(90)
555 except NameError:
556 self.debug.display = False
557 self.log.warn('Failed to setup/connect to display! Debug display has been disabled')
559 if self.debug.notHeadless:
560 pass # other backend options can go here
561 else: # this stop windows popping up when plotting. When headless, use 'agg' backend too
562 plt.interactive(False)
564 self.config.validate()
565 self.config.freeze()
567 def findObjects(self, exp, nSigma=None, grow=0):
568 """Find the objects in a postISR exposure."""
569 nPixMin = self.config.mainStarNpixMin
570 if not nSigma:
571 nSigma = self.config.mainStarNsigma
572 if not grow:
573 grow = self.config.mainStarGrow
574 isotropic = self.config.mainStarGrowIsotropic
576 threshold = afwDetect.Threshold(nSigma, afwDetect.Threshold.STDEV)
577 footPrintSet = afwDetect.FootprintSet(exp.getMaskedImage(), threshold, "DETECTED", nPixMin)
578 if grow > 0:
579 footPrintSet = afwDetect.FootprintSet(footPrintSet, grow, isotropic)
580 return footPrintSet
582 def _getEllipticity(self, shape):
583 """Calculate the ellipticity given a quadrupole shape.
585 Parameters
586 ----------
587 shape : `lsst.afw.geom.ellipses.Quadrupole`
588 The quadrupole shape
590 Returns
591 -------
592 ellipticity : `float`
593 The magnitude of the ellipticity
594 """
595 ixx = shape.getIxx()
596 iyy = shape.getIyy()
597 ixy = shape.getIxy()
598 ePlus = (ixx - iyy) / (ixx + iyy)
599 eCross = 2*ixy / (ixx + iyy)
600 return (ePlus**2 + eCross**2)**0.5
602 def getRoundestObject(self, footPrintSet, parentExp, fluxCut=1e-15):
603 """Get the roundest object brighter than fluxCut from a footPrintSet.
605 Parameters
606 ----------
607 footPrintSet : `lsst.afw.detection.FootprintSet`
608 The set of footprints resulting from running detection on parentExp
610 parentExp : `lsst.afw.image.exposure`
611 The parent exposure for the footprint set.
613 fluxCut : `float`
614 The flux, below which, sources are rejected.
616 Returns
617 -------
618 source : `lsst.afw.detection.Footprint`
619 The winning footprint from the input footPrintSet
620 """
621 self.log.debug("ellipticity\tflux/1e6\tcentroid")
622 sourceDict = {}
623 for fp in footPrintSet.getFootprints():
624 shape = fp.getShape()
625 e = self._getEllipticity(shape)
626 flux = fp.getSpans().flatten(parentExp.image.array, parentExp.image.getXY0()).sum()
627 self.log.debug("%.4f\t%.2f\t%s"%(e, flux/1e6, str(fp.getCentroid())))
628 if flux > fluxCut:
629 sourceDict[e] = fp
631 return sourceDict[sorted(sourceDict.keys())[0]]
633 def getBrightestObject(self, footPrintSet, parentExp, roundnessCut=1e9):
634 """Get the brightest object rounder than the cut from a footPrintSet.
636 Parameters
637 ----------
638 footPrintSet : `lsst.afw.detection.FootprintSet`
639 The set of footprints resulting from running detection on parentExp
641 parentExp : `lsst.afw.image.exposure`
642 The parent exposure for the footprint set.
644 roundnessCut : `float`
645 The ellipticity, above which, sources are rejected.
647 Returns
648 -------
649 source : `lsst.afw.detection.Footprint`
650 The winning footprint from the input footPrintSet
651 """
652 self.log.debug("ellipticity\tflux\tcentroid")
653 sourceDict = {}
654 for fp in footPrintSet.getFootprints():
655 shape = fp.getShape()
656 e = self._getEllipticity(shape)
657 flux = fp.getSpans().flatten(parentExp.image.array, parentExp.image.getXY0()).sum()
658 self.log.debug("%.4f\t%.2f\t%s"%(e, flux/1e6, str(fp.getCentroid())))
659 if e < roundnessCut:
660 sourceDict[flux] = fp
662 return sourceDict[sorted(sourceDict.keys())[-1]]
664 def findMainSource(self, exp):
665 """Return the x,y of the brightest or roundest object in an exposure.
667 Given a postISR exposure, run source detection on it, and return the
668 centroid of the main star. Depending on the task configuration, this
669 will either be the roundest object above a certain flux cutoff, or
670 the brightest object which is rounder than some ellipticity cutoff.
672 Parameters
673 ----------
674 exp : `afw.image.Exposure`
675 The postISR exposure in which to find the main star
677 Returns
678 -------
679 x, y : `tuple` of `float`
680 The centroid of the main star in the image
682 Notes
683 -----
684 Behavior of this method is controlled by many task config params
685 including, for the detection stage:
686 config.mainStarNpixMin
687 config.mainStarNsigma
688 config.mainStarGrow
689 config.mainStarGrowIsotropic
691 And post-detection, for selecting the main source:
692 config.mainSourceFindingMethod
693 config.mainStarFluxCut
694 config.mainStarRoundnessCut
695 """
696 # TODO: probably replace all this with QFM
697 fpSet = self.findObjects(exp)
698 if self.config.mainSourceFindingMethod == 'ROUNDEST':
699 source = self.getRoundestObject(fpSet, exp, fluxCut=self.config.mainStarFluxCut)
700 elif self.config.mainSourceFindingMethod == 'BRIGHTEST':
701 source = self.getBrightestObject(fpSet, exp,
702 roundnessCut=self.config.mainStarRoundnessCut)
703 else:
704 # should be impossible as this is a choice field, but still
705 raise RuntimeError("Invalid source finding method "
706 f"selected: {self.config.mainSourceFindingMethod}")
707 return source.getCentroid()
709 def updateMetadata(self, exp, **kwargs):
710 """Update an exposure's metadata with set items from the visit info.
712 Spectractor expects many items, like the hour angle and airmass, to be
713 in the metadata, so pull them out of the visit info etc and put them
714 into the main metadata. Also updates the metadata with any supplied
715 kwargs.
717 Parameters
718 ----------
719 exp : `lsst.afw.image.Exposure`
720 The exposure to update.
721 **kwargs : `dict`
722 The items to add.
723 """
724 md = exp.getMetadata()
725 vi = exp.getInfo().getVisitInfo()
727 ha = vi.getBoresightHourAngle().asDegrees()
728 airmass = vi.getBoresightAirmass()
730 md['HA'] = ha
731 md.setComment('HA', 'Hour angle of observation start')
733 md['AIRMASS'] = airmass
734 md.setComment('AIRMASS', 'Airmass at observation start')
736 if 'centroid' in kwargs:
737 centroid = kwargs['centroid']
738 else:
739 centroid = (None, None)
741 md['OBJECTX'] = centroid[0]
742 md.setComment('OBJECTX', 'x pixel coordinate of object centroid')
744 md['OBJECTY'] = centroid[1]
745 md.setComment('OBJECTY', 'y pixel coordinate of object centroid')
747 exp.setMetadata(md)
749 def runQuantum(self, butlerQC, inputRefs, outputRefs):
750 inputs = butlerQC.get(inputRefs)
752 inputs['dataIdDict'] = inputRefs.inputExp.dataId.byName()
754 outputs = self.run(**inputs)
755 butlerQC.put(outputs, outputRefs)
757 def getNormalizedTargetName(self, target):
758 """Normalize the name of the target.
760 All targets which start with 'spec:' are converted to the name of the
761 star without the leading 'spec:'. Any objects with mappings defined in
762 data/nameMappings.txt are converted to the mapped name.
764 Parameters
765 ----------
766 target : `str`
767 The name of the target.
769 Returns
770 -------
771 normalizedTarget : `str`
772 The normalized name of the target.
773 """
774 target = target.replace('spec:', '')
776 nameMappingsFile = os.path.join(getPackageDir('atmospec'), 'data', 'nameMappings.txt')
777 names, mappedNames = np.loadtxt(nameMappingsFile, dtype=str, unpack=True)
778 assert len(names) == len(mappedNames)
779 conversions = {name: mapped for name, mapped in zip(names, mappedNames)}
781 if target in conversions.keys():
782 converted = conversions[target]
783 self.log.info(f"Converted target name {target} to {converted}")
784 return converted
785 return target
787 def _getSpectractorTargetSetting(self, inputCentroid):
788 """Calculate the value to set SPECTRACTOR_FIT_TARGET_CENTROID to.
790 Parameters
791 ----------
792 inputCentroid : `dict`
793 The `atmospecCentroid` dict, as received in the task input data.
795 Returns
796 -------
797 centroidMethod : `str`
798 The value to set SPECTRACTOR_FIT_TARGET_CENTROID to.
799 """
801 # if mode is auto and the astrometry worked then it's an exact
802 # centroid, and otherwise we fit, as per docs on this option.
803 if self.config.targetCentroidMethod == 'auto':
804 if inputCentroid['astrometricMatch'] is True:
805 self.log.info("Auto centroid is using exact centroid for target from the astrometry")
806 return 'guess' # this means exact
807 else:
808 self.log.info("Auto centroid is using FIT in Spectractor to get the target centroid")
809 return 'fit' # this means exact
811 # this is just renaming the config parameter because guess sounds like
812 # an instruction, and really we're saying to take this as given.
813 if self.config.targetCentroidMethod == 'exact':
814 return 'guess'
816 # all other options fall through
817 return self.config.targetCentroidMethod
819 def run(self, *, inputExp, inputCentroid, dataIdDict):
820 if not isDispersedExp(inputExp):
821 raise RuntimeError(f"Exposure is not a dispersed image {dataIdDict}")
822 starNames = self.loadStarNames()
824 overrideDict = {
825 # normal config parameters
826 'SPECTRACTOR_FIT_TARGET_CENTROID': self._getSpectractorTargetSetting(inputCentroid),
827 'SPECTRACTOR_COMPUTE_ROTATION_ANGLE': self.config.rotationAngleMethod,
828 'SPECTRACTOR_DECONVOLUTION_PSF2D': self.config.doDeconvolveSpectrum,
829 'SPECTRACTOR_DECONVOLUTION_FFM': self.config.doFullForwardModelDeconvolution,
830 'SPECTRACTOR_DECONVOLUTION_SIGMA_CLIP': self.config.deconvolutionSigmaClip,
831 'SPECTRACTOR_BACKGROUND_SUBTRACTION': self.config.doSubtractBackground,
832 'CCD_REBIN': self.config.rebin,
833 'XWINDOW': self.config.xWindow,
834 'YWINDOW': self.config.yWindow,
835 'XWINDOW_ROT': self.config.xWindowRotated,
836 'YWINDOW_ROT': self.config.yWindowRotated,
837 'PIXSHIFT_PRIOR': self.config.pixelShiftPrior,
838 'ROT_PREFILTER': self.config.doFilterRotatedImage,
839 'ROT_ORDER': self.config.imageRotationSplineOrder,
840 'ROT_ANGLE_MIN': self.config.rotationAngleMin,
841 'ROT_ANGLE_MAX': self.config.rotationAngleMax,
842 'LINEWIDTH': self.config.plotLineWidth,
843 'VERBOSE': self.config.verbose,
844 'DEBUG': self.config.spectractorDebugMode,
845 'DEBUG_LOGGING': self.config.spectractorDebugLogging,
846 'DISPLAY': self.config.doDisplay,
847 'LAMBDA_MIN': self.config.lambdaMin,
848 'LAMBDA_MAX': self.config.lambdaMax,
849 'LAMBDA_STEP': self.config.lambdaStep,
850 'SPEC_ORDER': self.config.spectralOrder,
851 'PIXWIDTH_SIGNAL': self.config.signalWidth,
852 'PIXDIST_BACKGROUND': self.config.backgroundDistance,
853 'PIXWIDTH_BACKGROUND': self.config.backgroundWidth,
854 'PIXWIDTH_BOXSIZE': self.config.backgroundBoxSize,
855 'BGD_ORDER': self.config.backgroundOrder,
856 'PSF_TYPE': self.config.psfType,
857 'PSF_POLY_ORDER': self.config.psfPolynomialOrder,
858 'PSF_FIT_REG_PARAM': self.config.psfRegularization,
859 'PSF_PIXEL_STEP_TRANSVERSE_FIT': self.config.psfTransverseStepSize,
860 'PSF_FWHM_CLIP': self.config.psfFwhmClip,
861 'CALIB_BGD_ORDER': self.config.calibBackgroundOrder,
862 'CALIB_PEAK_WIDTH': self.config.calibPeakWidth,
863 'CALIB_BGD_WIDTH': self.config.calibBackgroundWidth,
864 'CALIB_SAVGOL_WINDOW': self.config.calibSavgolWindow,
865 'CALIB_SAVGOL_ORDER': self.config.calibSavgolOrder,
866 'OBS_TRANSMISSION_SYSTEMATICS': self.config.transmissionSystematicError,
867 'OBS_FULL_INSTRUMENT_TRANSMISSON': self.config.instrumentTransmissionOverride,
869 # Hard-coded parameters
870 'OBS_NAME': 'AUXTEL',
871 'CCD_IMSIZE': 4000, # short axis - we trim the CCD to square
872 'CCD_MAXADU': 170000, # XXX need to set this from camera value
873 'CCD_GAIN': 1.1, # set programatically later, this is default nominal value
874 'OBS_NAME': 'AUXTEL',
875 'OBS_ALTITUDE': 2.66299616375123, # XXX get this from / check with utils value
876 'OBS_LATITUDE': -30.2446389756252, # XXX get this from / check with utils value
877 'OBS_EPOCH': "J2000.0",
878 'OBS_CAMERA_DEC_FLIP_SIGN': 1,
879 'OBS_CAMERA_RA_FLIP_SIGN': 1,
880 'OBS_SURFACE': 9636,
881 'PAPER': False,
882 'SAVE': False,
883 'DISTANCE2CCD_ERR': 0.4,
885 # Parameters set programatically
886 'LAMBDAS': np.arange(self.config.lambdaMin,
887 self.config.lambdaMax,
888 self.config.lambdaStep),
889 'CALIB_BGD_NPARAMS': self.config.calibBackgroundOrder + 1,
891 # Parameters set elsewhere
892 # OBS_CAMERA_ROTATION
893 # DISTANCE2CCD
894 }
896 supplementDict = {'CALLING_CODE': 'LSST_DM',
897 'STAR_NAMES': starNames}
899 # anything that changes between dataRefs!
900 resetParameters = {}
901 # TODO: look at what to do with config option doSavePlots
903 # TODO: think if this is the right place for this
904 # probably wants to go in spectraction.py really
905 linearStagePosition = getLinearStagePosition(inputExp)
906 _, grating = getFilterAndDisperserFromExp(inputExp)
907 if grating == 'holo4_003':
908 # the hologram is sealed with a 4 mm window and this is how
909 # spectractor handles this, so while it's quite ugly, do this to
910 # keep the behaviour the same for now.
911 linearStagePosition += 4 # hologram is sealed with a 4 mm window
912 overrideDict['DISTANCE2CCD'] = linearStagePosition
914 target = inputExp.visitInfo.object
915 target = self.getNormalizedTargetName(target)
916 if self.config.forceObjectName:
917 self.log.info(f"Forcing target name from {target} to {self.config.forceObjectName}")
918 target = self.config.forceObjectName
920 if target in ['FlatField position', 'Park position', 'Test', 'NOTSET']:
921 raise ValueError(f"OBJECT set to {target} - this is not a celestial object!")
923 packageDir = getPackageDir('atmospec')
924 configFilename = os.path.join(packageDir, 'config', 'auxtel.ini')
926 spectractor = SpectractorShim(configFile=configFilename,
927 paramOverrides=overrideDict,
928 supplementaryParameters=supplementDict,
929 resetParameters=resetParameters)
931 if 'astrometricMatch' in inputCentroid:
932 centroid = inputCentroid['centroid']
933 else: # it's a raw tuple
934 centroid = inputCentroid # TODO: put this support in the docstring
936 spectraction = spectractor.run(inputExp, *centroid, target,
937 self.config.doFitAtmosphere,
938 self.config.doFitAtmosphereOnSpectrogram)
940 self.log.info("Finished processing %s" % (dataIdDict))
942 return pipeBase.Struct(
943 spectractorSpectrum=spectraction.spectrum,
944 spectractorImage=spectraction.image,
945 spectrumForwardModelFitParameters=spectraction.spectrumForwardModelFitParameters,
946 spectrumLibradtranFitParameters=spectraction.spectrumLibradtranFitParameters,
947 spectrogramLibradtranFitParameters=spectraction.spectrogramLibradtranFitParameters
948 )
950 def runAstrometry(self, butler, exp, icSrc):
951 refObjLoaderConfig = ReferenceObjectLoader.ConfigClass()
952 refObjLoaderConfig.pixelMargin = 1000
953 # TODO: needs to be an Input Connection
954 refObjLoader = ReferenceObjectLoader(config=refObjLoaderConfig)
956 astromConfig = AstrometryTask.ConfigClass()
957 astromConfig.wcsFitter.retarget(FitAffineWcsTask)
958 astromConfig.referenceSelector.doMagLimit = True
959 magLimit = MagnitudeLimit()
960 magLimit.minimum = 1
961 magLimit.maximum = 15
962 astromConfig.referenceSelector.magLimit = magLimit
963 astromConfig.referenceSelector.magLimit.fluxField = "phot_g_mean_flux"
964 astromConfig.matcher.maxRotationDeg = 5.99
965 astromConfig.matcher.maxOffsetPix = 3000
966 astromConfig.sourceSelector['matcher'].minSnr = 10
967 solver = AstrometryTask(config=astromConfig, refObjLoader=refObjLoader)
969 # TODO: Change this to doing this the proper way
970 referenceFilterName = self.config.referenceFilterOverride
971 referenceFilterLabel = afwImage.FilterLabel(physical=referenceFilterName, band=referenceFilterName)
972 originalFilterLabel = exp.getFilter() # there's a better way of doing this with the task I think
973 exp.setFilter(referenceFilterLabel)
975 try:
976 astromResult = solver.run(sourceCat=icSrc, exposure=exp)
977 exp.setFilter(originalFilterLabel)
978 except (RuntimeError, TaskError):
979 self.log.warn("Solver failed to run completely")
980 exp.setFilter(originalFilterLabel)
981 return None
983 scatter = astromResult.scatterOnSky.asArcseconds()
984 if scatter < 1:
985 return astromResult
986 else:
987 self.log.warn("Failed to find an acceptable match")
988 return None
990 def pause(self):
991 if self.debug.pauseOnDisplay:
992 input("Press return to continue...")
993 return
995 def loadStarNames(self):
996 """Get the objects which should be treated as stars which do not begin
997 with HD.
999 Spectractor treats all objects which start HD as stars, and all which
1000 don't as calibration objects, e.g. arc lamps or planetary nebulae.
1001 Adding items to data/starNames.txt will cause them to be treated as
1002 regular stars.
1004 Returns
1005 -------
1006 starNames : `list` of `str`
1007 The list of all objects to be treated as stars despite not starting
1008 with HD.
1009 """
1010 starNameFile = os.path.join(getPackageDir('atmospec'), 'data', 'starNames.txt')
1011 with open(starNameFile, 'r') as f:
1012 lines = f.readlines()
1013 return [line.strip() for line in lines]
1015 def flatfield(self, exp, disp):
1016 """Placeholder for wavelength dependent flatfielding: TODO: DM-18141
1018 Will probably need a dataRef, as it will need to be retrieving flats
1019 over a range. Also, it will be somewhat complex, so probably needs
1020 moving to its own task"""
1021 self.log.warn("Flatfielding not yet implemented")
1022 return exp
1024 def repairCosmics(self, exp, disp):
1025 self.log.warn("Cosmic ray repair not yet implemented")
1026 return exp
1028 def measureSpectrum(self, exp, sourceCentroid, spectrumBBox, dispersionRelation):
1029 """Perform the spectral extraction, given a source location and exp."""
1031 self.extraction.initialise(exp, sourceCentroid, spectrumBBox, dispersionRelation)
1033 # xxx this method currently doesn't return an object - fix this
1034 spectrum = self.extraction.getFluxBasic()
1036 return spectrum
1038 def calcSpectrumBBox(self, exp, centroid, aperture, order='+1'):
1039 """Calculate the bbox for the spectrum, given the centroid.
1041 XXX Longer explanation here, inc. parameters
1042 TODO: Add support for order = "both"
1043 """
1044 extent = self.config.spectrumLengthPixels
1045 halfWidth = aperture//2
1046 translate_y = self.config.offsetFromMainStar
1047 sourceX = centroid[0]
1048 sourceY = centroid[1]
1050 if order == '-1':
1051 translate_y = - extent - self.config.offsetFromMainStar
1053 xStart = sourceX - halfWidth
1054 xEnd = sourceX + halfWidth - 1
1055 yStart = sourceY + translate_y
1056 yEnd = yStart + extent - 1
1058 xEnd = min(xEnd, exp.getWidth()-1)
1059 yEnd = min(yEnd, exp.getHeight()-1)
1060 yStart = max(yStart, 0)
1061 xStart = max(xStart, 0)
1062 assert (xEnd > xStart) and (yEnd > yStart)
1064 self.log.debug('(xStart, xEnd) = (%s, %s)'%(xStart, xEnd))
1065 self.log.debug('(yStart, yEnd) = (%s, %s)'%(yStart, yEnd))
1067 bbox = geom.Box2I(geom.Point2I(xStart, yStart), geom.Point2I(xEnd, yEnd))
1068 return bbox