Coverage for python/lsst/atmospec/processStar.py: 23%
270 statements
« prev ^ index » next coverage.py v6.5.0, created at 2022-10-05 12:34 -0700
1# This file is part of atmospec.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22__all__ = ['ProcessStarTask', 'ProcessStarTaskConfig']
24import os
25import numpy as np
26import matplotlib.pyplot as plt
28import lsstDebug
29import lsst.afw.image as afwImage
30import lsst.geom as geom
31from lsst.ip.isr import IsrTask
32import lsst.pex.config as pexConfig
33import lsst.pipe.base as pipeBase
34import lsst.pipe.base.connectionTypes as cT
35from lsst.pipe.base.task import TaskError
37from lsst.utils import getPackageDir
38from lsst.pipe.tasks.characterizeImage import CharacterizeImageTask
39from lsst.meas.algorithms import ReferenceObjectLoader, MagnitudeLimit
40from lsst.meas.astrom import AstrometryTask, FitAffineWcsTask
42import lsst.afw.detection as afwDetect
44from .spectraction import SpectractorShim
45from .utils import getLinearStagePosition
47COMMISSIONING = False # allows illegal things for on the mountain usage.
49# TODO:
50# Sort out read noise and gain
51# remove dummy image totally
52# talk to Jeremy about turning the image beforehand and giving new coords
53# deal with not having ambient temp
54# Gen3ification
55# astropy warning for units on save
56# but actually just remove all manual saves entirely, I think?
57# Make SED persistable
58# Move to QFM for star finding failover case
59# Remove old cruft functions
60# change spectractions run method to be ~all kwargs with *,...
63class ProcessStarTaskConnections(pipeBase.PipelineTaskConnections,
64 dimensions=("instrument", "visit", "detector")):
65 inputExp = cT.Input(
66 name="icExp",
67 doc="Image-characterize output exposure",
68 storageClass="ExposureF",
69 dimensions=("instrument", "visit", "detector"),
70 multiple=False,
71 )
72 inputCentroid = cT.Input(
73 name="atmospecCentroid",
74 doc="The main star centroid in yaml format.",
75 storageClass="StructuredDataDict",
76 dimensions=("instrument", "visit", "detector"),
77 multiple=False,
78 )
79 spectractorSpectrum = cT.Output(
80 name="spectractorSpectrum",
81 doc="The Spectractor output spectrum.",
82 storageClass="SpectractorSpectrum",
83 dimensions=("instrument", "visit", "detector"),
84 )
85 spectractorImage = cT.Output(
86 name="spectractorImage",
87 doc="The Spectractor output image.",
88 storageClass="SpectractorImage",
89 dimensions=("instrument", "visit", "detector"),
90 )
91 spectraction = cT.Output(
92 name="spectraction",
93 doc="The Spectractor output image.",
94 storageClass="Spectraction",
95 dimensions=("instrument", "visit", "detector"),
96 )
98 def __init__(self, *, config=None):
99 super().__init__(config=config)
102class ProcessStarTaskConfig(pipeBase.PipelineTaskConfig,
103 pipelineConnections=ProcessStarTaskConnections):
104 """Configuration parameters for ProcessStarTask."""
106 isr = pexConfig.ConfigurableField(
107 target=IsrTask,
108 doc="Task to perform instrumental signature removal",
109 )
110 charImage = pexConfig.ConfigurableField(
111 target=CharacterizeImageTask,
112 doc="""Task to characterize a science exposure:
113 - detect sources, usually at high S/N
114 - estimate the background, which is subtracted from the image and returned as field "background"
115 - estimate a PSF model, which is added to the exposure
116 - interpolate over defects and cosmic rays, updating the image, variance and mask planes
117 """,
118 )
119 doWrite = pexConfig.Field(
120 dtype=bool,
121 doc="Write out the results?",
122 default=True,
123 )
124 mainSourceFindingMethod = pexConfig.ChoiceField(
125 doc="Which attribute to prioritize when selecting the main source object",
126 dtype=str,
127 default="BRIGHTEST",
128 allowed={
129 "BRIGHTEST": "Select the brightest object with roundness > roundnessCut",
130 "ROUNDEST": "Select the roundest object with brightness > fluxCut",
131 }
132 )
133 mainStarRoundnessCut = pexConfig.Field(
134 dtype=float,
135 doc="Value of ellipticity above which to reject the brightest object."
136 " Ignored if mainSourceFindingMethod == BRIGHTEST",
137 default=0.2
138 )
139 mainStarFluxCut = pexConfig.Field(
140 dtype=float,
141 doc="Object flux below which to reject the roundest object."
142 " Ignored if mainSourceFindingMethod == ROUNDEST",
143 default=1e7
144 )
145 mainStarNpixMin = pexConfig.Field(
146 dtype=int,
147 doc="Minimum number of pixels for object detection of main star",
148 default=10
149 )
150 mainStarNsigma = pexConfig.Field(
151 dtype=int,
152 doc="nSigma for detection of main star",
153 default=200 # the m=0 is very bright indeed, and we don't want to detect much spectrum
154 )
155 mainStarGrow = pexConfig.Field(
156 dtype=int,
157 doc="Number of pixels to grow by when detecting main star. This"
158 " encourages the spectrum to merge into one footprint, but too much"
159 " makes everything round, compromising mainStarRoundnessCut's"
160 " effectiveness",
161 default=5
162 )
163 mainStarGrowIsotropic = pexConfig.Field(
164 dtype=bool,
165 doc="Grow main star's footprint isotropically?",
166 default=False
167 )
168 aperture = pexConfig.Field(
169 dtype=int,
170 doc="Width of the aperture to use in pixels",
171 default=250
172 )
173 spectrumLengthPixels = pexConfig.Field(
174 dtype=int,
175 doc="Length of the spectrum in pixels",
176 default=5000
177 )
178 offsetFromMainStar = pexConfig.Field(
179 dtype=int,
180 doc="Number of pixels from the main star's centroid to start extraction",
181 default=100
182 )
183 dispersionDirection = pexConfig.ChoiceField(
184 doc="Direction along which the image is dispersed",
185 dtype=str,
186 default="y",
187 allowed={
188 "x": "Dispersion along the serial direction",
189 "y": "Dispersion along the parallel direction",
190 }
191 )
192 spectralOrder = pexConfig.ChoiceField(
193 doc="Direction along which the image is dispersed",
194 dtype=str,
195 default="+1",
196 allowed={
197 "+1": "Use the m+1 spectrum",
198 "-1": "Use the m-1 spectrum",
199 "both": "Use both spectra",
200 }
201 )
202 binning = pexConfig.Field(
203 dtype=int,
204 doc="Bin the input image by this factor",
205 default=4
206 )
207 doFlat = pexConfig.Field(
208 dtype=bool,
209 doc="Flatfield the image?",
210 default=True
211 )
212 doCosmics = pexConfig.Field(
213 dtype=bool,
214 doc="Repair cosmic rays?",
215 default=True
216 )
217 doDisplayPlots = pexConfig.Field(
218 dtype=bool,
219 doc="Matplotlib show() the plots, so they show up in a notebook or X window",
220 default=False
221 )
222 doSavePlots = pexConfig.Field(
223 dtype=bool,
224 doc="Save matplotlib plots to output rerun?",
225 default=False
226 )
227 spectractorDebugMode = pexConfig.Field(
228 dtype=bool,
229 doc="Set debug mode for Spectractor",
230 default=True
231 )
232 spectractorDebugLogging = pexConfig.Field( # TODO: tie this to the task debug level?
233 dtype=bool,
234 doc="Set debug logging for Spectractor",
235 default=False
236 )
237 forceObjectName = pexConfig.Field(
238 dtype=str,
239 doc="A supplementary name for OBJECT. Will be forced to apply to ALL visits, so this should only"
240 " ONLY be used for immediate commissioning debug purposes. All long term fixes should be"
241 " supplied as header fix-up yaml files.",
242 default=""
243 )
244 referenceFilterOverride = pexConfig.Field(
245 dtype=str,
246 doc="Which filter in the reference catalog to match to?",
247 default="phot_g_mean"
248 )
250 def setDefaults(self):
251 self.isr.doWrite = False
252 self.charImage.doWriteExposure = False
254 self.charImage.doApCorr = False
255 self.charImage.doMeasurePsf = False
256 self.charImage.repair.cosmicray.nCrPixelMax = 100000
257 self.charImage.repair.doCosmicRay = False
258 if self.charImage.doMeasurePsf:
259 self.charImage.measurePsf.starSelector['objectSize'].signalToNoiseMin = 10.0
260 self.charImage.measurePsf.starSelector['objectSize'].fluxMin = 5000.0
261 self.charImage.detection.includeThresholdMultiplier = 3
262 self.isr.overscan.fitType = 'MEDIAN_PER_ROW'
265class ProcessStarTask(pipeBase.PipelineTask):
266 """Task for the spectral extraction of single-star dispersed images.
268 For a full description of how this tasks works, see the run() method.
269 """
271 ConfigClass = ProcessStarTaskConfig
272 _DefaultName = "processStar"
274 def __init__(self, *, butler=None, **kwargs):
275 # TODO: rename psfRefObjLoader to refObjLoader
276 super().__init__(**kwargs)
277 self.makeSubtask("isr")
278 self.makeSubtask("charImage", butler=butler, refObjLoader=None)
280 self.debug = lsstDebug.Info(__name__)
281 if self.debug.enabled:
282 self.log.info("Running with debug enabled...")
283 # If we're displaying, test it works and save displays for later.
284 # It's worth testing here as displays are flaky and sometimes
285 # can't be contacted, and given processing takes a while,
286 # it's a shame to fail late due to display issues.
287 if self.debug.display:
288 try:
289 import lsst.afw.display as afwDisp
290 afwDisp.setDefaultBackend(self.debug.displayBackend)
291 afwDisp.Display.delAllDisplays()
292 # pick an unlikely number to be safe xxx replace this
293 self.disp1 = afwDisp.Display(987, open=True)
295 im = afwImage.ImageF(2, 2)
296 im.array[:] = np.ones((2, 2))
297 self.disp1.mtv(im)
298 self.disp1.erase()
299 afwDisp.setDefaultMaskTransparency(90)
300 except NameError:
301 self.debug.display = False
302 self.log.warn('Failed to setup/connect to display! Debug display has been disabled')
304 if self.debug.notHeadless:
305 pass # other backend options can go here
306 else: # this stop windows popping up when plotting. When headless, use 'agg' backend too
307 plt.interactive(False)
309 self.config.validate()
310 self.config.freeze()
312 def findObjects(self, exp, nSigma=None, grow=0):
313 """Find the objects in a postISR exposure."""
314 nPixMin = self.config.mainStarNpixMin
315 if not nSigma:
316 nSigma = self.config.mainStarNsigma
317 if not grow:
318 grow = self.config.mainStarGrow
319 isotropic = self.config.mainStarGrowIsotropic
321 threshold = afwDetect.Threshold(nSigma, afwDetect.Threshold.STDEV)
322 footPrintSet = afwDetect.FootprintSet(exp.getMaskedImage(), threshold, "DETECTED", nPixMin)
323 if grow > 0:
324 footPrintSet = afwDetect.FootprintSet(footPrintSet, grow, isotropic)
325 return footPrintSet
327 def _getEllipticity(self, shape):
328 """Calculate the ellipticity given a quadrupole shape.
330 Parameters
331 ----------
332 shape : `lsst.afw.geom.ellipses.Quadrupole`
333 The quadrupole shape
335 Returns
336 -------
337 ellipticity : `float`
338 The magnitude of the ellipticity
339 """
340 ixx = shape.getIxx()
341 iyy = shape.getIyy()
342 ixy = shape.getIxy()
343 ePlus = (ixx - iyy) / (ixx + iyy)
344 eCross = 2*ixy / (ixx + iyy)
345 return (ePlus**2 + eCross**2)**0.5
347 def getRoundestObject(self, footPrintSet, parentExp, fluxCut=1e-15):
348 """Get the roundest object brighter than fluxCut from a footPrintSet.
350 Parameters
351 ----------
352 footPrintSet : `lsst.afw.detection.FootprintSet`
353 The set of footprints resulting from running detection on parentExp
355 parentExp : `lsst.afw.image.exposure`
356 The parent exposure for the footprint set.
358 fluxCut : `float`
359 The flux, below which, sources are rejected.
361 Returns
362 -------
363 source : `lsst.afw.detection.Footprint`
364 The winning footprint from the input footPrintSet
365 """
366 self.log.debug("ellipticity\tflux/1e6\tcentroid")
367 sourceDict = {}
368 for fp in footPrintSet.getFootprints():
369 shape = fp.getShape()
370 e = self._getEllipticity(shape)
371 flux = fp.getSpans().flatten(parentExp.image.array, parentExp.image.getXY0()).sum()
372 self.log.debug("%.4f\t%.2f\t%s"%(e, flux/1e6, str(fp.getCentroid())))
373 if flux > fluxCut:
374 sourceDict[e] = fp
376 return sourceDict[sorted(sourceDict.keys())[0]]
378 def getBrightestObject(self, footPrintSet, parentExp, roundnessCut=1e9):
379 """Get the brightest object rounder than the cut from a footPrintSet.
381 Parameters
382 ----------
383 footPrintSet : `lsst.afw.detection.FootprintSet`
384 The set of footprints resulting from running detection on parentExp
386 parentExp : `lsst.afw.image.exposure`
387 The parent exposure for the footprint set.
389 roundnessCut : `float`
390 The ellipticity, above which, sources are rejected.
392 Returns
393 -------
394 source : `lsst.afw.detection.Footprint`
395 The winning footprint from the input footPrintSet
396 """
397 self.log.debug("ellipticity\tflux\tcentroid")
398 sourceDict = {}
399 for fp in footPrintSet.getFootprints():
400 shape = fp.getShape()
401 e = self._getEllipticity(shape)
402 flux = fp.getSpans().flatten(parentExp.image.array, parentExp.image.getXY0()).sum()
403 self.log.debug("%.4f\t%.2f\t%s"%(e, flux/1e6, str(fp.getCentroid())))
404 if e < roundnessCut:
405 sourceDict[flux] = fp
407 return sourceDict[sorted(sourceDict.keys())[-1]]
409 def findMainSource(self, exp):
410 """Return the x,y of the brightest or roundest object in an exposure.
412 Given a postISR exposure, run source detection on it, and return the
413 centroid of the main star. Depending on the task configuration, this
414 will either be the roundest object above a certain flux cutoff, or
415 the brightest object which is rounder than some ellipticity cutoff.
417 Parameters
418 ----------
419 exp : `afw.image.Exposure`
420 The postISR exposure in which to find the main star
422 Returns
423 -------
424 x, y : `tuple` of `float`
425 The centroid of the main star in the image
427 Notes
428 -----
429 Behavior of this method is controlled by many task config params
430 including, for the detection stage:
431 config.mainStarNpixMin
432 config.mainStarNsigma
433 config.mainStarGrow
434 config.mainStarGrowIsotropic
436 And post-detection, for selecting the main source:
437 config.mainSourceFindingMethod
438 config.mainStarFluxCut
439 config.mainStarRoundnessCut
440 """
441 # TODO: probably replace all this with QFM
442 fpSet = self.findObjects(exp)
443 if self.config.mainSourceFindingMethod == 'ROUNDEST':
444 source = self.getRoundestObject(fpSet, exp, fluxCut=self.config.mainStarFluxCut)
445 elif self.config.mainSourceFindingMethod == 'BRIGHTEST':
446 source = self.getBrightestObject(fpSet, exp,
447 roundnessCut=self.config.mainStarRoundnessCut)
448 else:
449 # should be impossible as this is a choice field, but still
450 raise RuntimeError("Invalid source finding method "
451 f"selected: {self.config.mainSourceFindingMethod}")
452 return source.getCentroid()
454 def updateMetadata(self, exp, **kwargs):
455 md = exp.getMetadata()
456 vi = exp.getInfo().getVisitInfo()
458 ha = vi.getBoresightHourAngle().asDegrees()
459 airmass = vi.getBoresightAirmass()
461 md['HA'] = ha
462 md.setComment('HA', 'Hour angle of observation start')
464 md['AIRMASS'] = airmass
465 md.setComment('AIRMASS', 'Airmass at observation start')
467 if 'centroid' in kwargs:
468 centroid = kwargs['centroid']
469 else:
470 centroid = (None, None)
472 md['OBJECTX'] = centroid[0]
473 md.setComment('OBJECTX', 'x pixel coordinate of object centroid')
475 md['OBJECTY'] = centroid[1]
476 md.setComment('OBJECTY', 'y pixel coordinate of object centroid')
478 exp.setMetadata(md)
480 def runQuantum(self, butlerQC, inputRefs, outputRefs):
481 inputs = butlerQC.get(inputRefs)
483 inputs['dataIdDict'] = inputRefs.inputExp.dataId.byName()
485 outputs = self.run(**inputs)
486 butlerQC.put(outputs, outputRefs)
488 def run(self, *, inputExp, inputCentroid, dataIdDict):
489 starNames = self.loadStarNames()
491 overrideDict = {'SAVE': False,
492 'OBS_NAME': 'AUXTEL',
493 'DEBUG': self.config.spectractorDebugMode,
494 'DEBUG_LOGGING': self.config.spectractorDebugLogging,
495 'DISPLAY': self.config.doDisplayPlots,
496 'CCD_REBIN': self.config.binning,
497 'VERBOSE': 0,
498 # 'CCD_IMSIZE': 4000}
499 }
500 supplementDict = {'CALLING_CODE': 'LSST_DM',
501 'STAR_NAMES': starNames}
503 # anything that changes between dataRefs!
504 resetParameters = {}
505 # TODO: look at what to do with config option doSavePlots
507 # TODO: think if this is the right place for this
508 # probably wants to go in spectraction.py really
509 linearStagePosition = getLinearStagePosition(inputExp)
510 overrideDict['DISTANCE2CCD'] = linearStagePosition
512 target = inputExp.getMetadata()['OBJECT']
513 if self.config.forceObjectName:
514 self.log.info(f"Forcing target name from {target} to {self.config.forceObjectName}")
515 target = self.config.forceObjectName
517 if target in ['FlatField position', 'Park position', 'Test', 'NOTSET']:
518 raise ValueError(f"OBJECT set to {target} - this is not a celestial object!")
520 packageDir = getPackageDir('atmospec')
521 configFilename = os.path.join(packageDir, 'config', 'auxtel.ini')
523 spectractor = SpectractorShim(configFile=configFilename,
524 paramOverrides=overrideDict,
525 supplementaryParameters=supplementDict,
526 resetParameters=resetParameters)
528 if 'astrometricMatch' in inputCentroid:
529 centroid = inputCentroid['centroid']
530 else: # it's a raw tuple
531 centroid = inputCentroid # TODO: put this support in the docstring
533 spectraction = spectractor.run(inputExp, *centroid, target)
535 self.log.info("Finished processing %s" % (dataIdDict))
537 self.makeResultPickleable(spectraction)
539 return pipeBase.Struct(spectractorSpectrum=spectraction.spectrum,
540 spectractorImage=spectraction.image,
541 spectraction=spectraction)
543 def makeResultPickleable(self, result):
544 """Remove unpicklable components from the output"""
545 result.image.target.build_sed = None
546 result.spectrum.target.build_sed = None
547 result.image.target.sed = None
548 result.spectrum.disperser.load_files = None
549 result.image.disperser.load_files = None
551 result.spectrum.disperser.N_fit = None
552 result.spectrum.disperser.N_interp = None
553 result.spectrum.disperser.ratio_order_2over1 = None
554 result.spectrum.disperser.theta = None
556 def runAstrometry(self, butler, exp, icSrc):
557 refObjLoaderConfig = ReferenceObjectLoader.ConfigClass()
558 refObjLoaderConfig.pixelMargin = 1000
559 # TODO: needs to be an Input Connection
560 refObjLoader = ReferenceObjectLoader(config=refObjLoaderConfig)
562 astromConfig = AstrometryTask.ConfigClass()
563 astromConfig.wcsFitter.retarget(FitAffineWcsTask)
564 astromConfig.referenceSelector.doMagLimit = True
565 magLimit = MagnitudeLimit()
566 magLimit.minimum = 1
567 magLimit.maximum = 15
568 astromConfig.referenceSelector.magLimit = magLimit
569 astromConfig.referenceSelector.magLimit.fluxField = "phot_g_mean_flux"
570 astromConfig.matcher.maxRotationDeg = 5.99
571 astromConfig.matcher.maxOffsetPix = 3000
572 astromConfig.sourceSelector['matcher'].minSnr = 10
573 solver = AstrometryTask(config=astromConfig, refObjLoader=refObjLoader)
575 # TODO: Change this to doing this the proper way
576 referenceFilterName = self.config.referenceFilterOverride
577 referenceFilterLabel = afwImage.FilterLabel(physical=referenceFilterName, band=referenceFilterName)
578 originalFilterLabel = exp.getFilter() # there's a better way of doing this with the task I think
579 exp.setFilter(referenceFilterLabel)
581 try:
582 astromResult = solver.run(sourceCat=icSrc, exposure=exp)
583 exp.setFilter(originalFilterLabel)
584 except (RuntimeError, TaskError):
585 self.log.warn("Solver failed to run completely")
586 exp.setFilter(originalFilterLabel)
587 return None
589 scatter = astromResult.scatterOnSky.asArcseconds()
590 if scatter < 1:
591 return astromResult
592 else:
593 self.log.warn("Failed to find an acceptable match")
594 return None
596 def pause(self):
597 if self.debug.pauseOnDisplay:
598 input("Press return to continue...")
599 return
601 def loadStarNames(self):
602 starNameFile = os.path.join(getPackageDir('atmospec'), 'data', 'starNames.txt')
603 with open(starNameFile, 'r') as f:
604 lines = f.readlines()
605 return [line.strip() for line in lines]
607 def flatfield(self, exp, disp):
608 """Placeholder for wavelength dependent flatfielding: TODO: DM-18141
610 Will probably need a dataRef, as it will need to be retrieving flats
611 over a range. Also, it will be somewhat complex, so probably needs
612 moving to its own task"""
613 self.log.warn("Flatfielding not yet implemented")
614 return exp
616 def repairCosmics(self, exp, disp):
617 self.log.warn("Cosmic ray repair not yet implemented")
618 return exp
620 def measureSpectrum(self, exp, sourceCentroid, spectrumBBox, dispersionRelation):
621 """Perform the spectral extraction, given a source location and exp."""
623 self.extraction.initialise(exp, sourceCentroid, spectrumBBox, dispersionRelation)
625 # xxx this method currently doesn't return an object - fix this
626 spectrum = self.extraction.getFluxBasic()
628 return spectrum
630 def calcSpectrumBBox(self, exp, centroid, aperture, order='+1'):
631 """Calculate the bbox for the spectrum, given the centroid.
633 XXX Longer explanation here, inc. parameters
634 TODO: Add support for order = "both"
635 """
636 extent = self.config.spectrumLengthPixels
637 halfWidth = aperture//2
638 translate_y = self.config.offsetFromMainStar
639 sourceX = centroid[0]
640 sourceY = centroid[1]
642 if(order == '-1'):
643 translate_y = - extent - self.config.offsetFromMainStar
645 xStart = sourceX - halfWidth
646 xEnd = sourceX + halfWidth - 1
647 yStart = sourceY + translate_y
648 yEnd = yStart + extent - 1
650 xEnd = min(xEnd, exp.getWidth()-1)
651 yEnd = min(yEnd, exp.getHeight()-1)
652 yStart = max(yStart, 0)
653 xStart = max(xStart, 0)
654 assert (xEnd > xStart) and (yEnd > yStart)
656 self.log.debug('(xStart, xEnd) = (%s, %s)'%(xStart, xEnd))
657 self.log.debug('(yStart, yEnd) = (%s, %s)'%(yStart, yEnd))
659 bbox = geom.Box2I(geom.Point2I(xStart, yStart), geom.Point2I(xEnd, yEnd))
660 return bbox
662 # def calcRidgeLine(self, footprint):
663 # ridgeLine = np.zeros(self.footprint.length)
664 # for
666 # return ridgeLine