Coverage for python/lsst/obs/lsst/lsstCamMapper.py : 22%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of obs_lsst.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the LSST License Statement and
20# the GNU General Public License along with this program. If not,
21# see <http://www.lsstcorp.org/LegalNotices/>.
22#
23"""The LsstCam Mapper.""" # necessary to suppress D100 flake8 warning.
25import os
26import warnings
28import lsst.log
29import lsst.geom
30import lsst.utils as utils
31import lsst.afw.image as afwImage
32from lsst.obs.base import CameraMapper, MakeRawVisitInfoViaObsInfo
33import lsst.obs.base.yamlCamera as yamlCamera
34import lsst.daf.persistence as dafPersist
35from astro_metadata_translator import ObservationInfo
36from .translators import LsstCamTranslator
37from ._fitsHeader import readRawFitsHeader
38from ._instrument import LsstCam
40from .filters import LSSTCAM_FILTER_DEFINITIONS
41from .assembly import attachRawWcsFromBoresight, fixAmpsAndAssemble
43__all__ = ["LsstCamMapper", "LsstCamMakeRawVisitInfo"]
46class LsstCamMakeRawVisitInfo(MakeRawVisitInfoViaObsInfo):
47 """Make a VisitInfo from the FITS header of a raw image."""
50class LsstCamRawVisitInfo(LsstCamMakeRawVisitInfo):
51 metadataTranslator = LsstCamTranslator
54def assemble_raw(dataId, componentInfo, cls):
55 """Called by the butler to construct the composite type "raw".
57 Note that we still need to define "_raw" and copy various fields over.
59 Parameters
60 ----------
61 dataId : `lsst.daf.persistence.dataId.DataId`
62 The data ID.
63 componentInfo : `dict`
64 dict containing the components, as defined by the composite definition
65 in the mapper policy.
66 cls : 'object'
67 Unused.
69 Returns
70 -------
71 exposure : `lsst.afw.image.Exposure`
72 The assembled exposure.
73 """
75 ampExps = componentInfo['raw_amp'].obj
76 exposure = fixAmpsAndAssemble(ampExps, str(dataId))
77 md = componentInfo['raw_hdu'].obj
78 exposure.setMetadata(md)
80 attachRawWcsFromBoresight(exposure, dataId)
82 return exposure
85class LsstCamBaseMapper(CameraMapper):
86 """The Base Mapper for all LSST-style instruments.
87 """
89 packageName = 'obs_lsst'
90 _cameraName = "lsstCam"
91 yamlFileList = ("lsstCamMapper.yaml",) # list of yaml files to load, keeping the first occurrence
92 #
93 # do not set MakeRawVisitInfoClass or translatorClass to anything other
94 # than None!
95 #
96 # assemble_raw relies on autodetect as in butler Gen2 it doesn't know
97 # its mapper and cannot use mapper.makeRawVisitInfo()
98 #
99 MakeRawVisitInfoClass = None
100 translatorClass = None
101 filterDefinitions = LSSTCAM_FILTER_DEFINITIONS
103 def __init__(self, inputPolicy=None, **kwargs):
104 #
105 # Merge the list of .yaml files
106 #
107 policy = None
108 for yamlFile in self.yamlFileList:
109 policyFile = dafPersist.Policy.defaultPolicyFile(self.packageName, yamlFile, "policy")
110 npolicy = dafPersist.Policy(policyFile)
112 if policy is None:
113 policy = npolicy
114 else:
115 policy.merge(npolicy)
116 #
117 # Look for the calibrations root "root/CALIB" if not supplied
118 #
119 if kwargs.get('root', None) and not kwargs.get('calibRoot', None):
120 calibSearch = [os.path.join(kwargs['root'], 'CALIB')]
121 if "repositoryCfg" in kwargs:
122 calibSearch += [os.path.join(cfg.root, 'CALIB') for cfg in kwargs["repositoryCfg"].parents if
123 hasattr(cfg, "root")]
124 calibSearch += [cfg.root for cfg in kwargs["repositoryCfg"].parents if hasattr(cfg, "root")]
125 for calibRoot in calibSearch:
126 if os.path.exists(os.path.join(calibRoot, "calibRegistry.sqlite3")):
127 kwargs['calibRoot'] = calibRoot
128 break
129 if not kwargs.get('calibRoot', None):
130 lsst.log.Log.getLogger("LsstCamMapper").warn("Unable to find valid calib root directory")
132 super().__init__(policy, os.path.dirname(policyFile), **kwargs)
133 #
134 # The composite objects don't seem to set these
135 #
136 for d in (self.mappings, self.exposures):
137 d['raw'] = d['_raw']
139 self.filterDefinitions.reset()
140 self.filterDefinitions.defineFilters()
142 LsstCamMapper._nbit_tract = 16
143 LsstCamMapper._nbit_patch = 5
144 LsstCamMapper._nbit_filter = 7
146 LsstCamMapper._nbit_id = 64 - (LsstCamMapper._nbit_tract + 2*LsstCamMapper._nbit_patch
147 + LsstCamMapper._nbit_filter)
148 #
149 # The BOT has many ND filters in a second filter wheel, resulting in
150 # more than 128 composite filters. However, we're never going to
151 # build coadds with the BOT. So let's ignore the qualifier after
152 # the ~ in filter names when we're calculating the number of filters
153 #
154 # Because the first filter wheel can be empty some of baseFilters are
155 # actually in the second wheel, but that's OK -- we still easily fit
156 # in 7 bits (5 would actually be enough)
158 with warnings.catch_warnings():
159 # surpress Filter warnings; we already know getNames is deprecated
160 warnings.simplefilter('ignore', category=FutureWarning)
162 baseFilters = set()
163 for n in afwImage.Filter.getNames():
164 i = n.find('~')
165 if i >= 0:
166 n = n[:i]
168 baseFilters.add(n)
170 nFilter = len(baseFilters)
171 if nFilter >= 2**LsstCamMapper._nbit_filter:
172 raise RuntimeError("You have more filters (%d) defined than fit into the %d bits allocated" %
173 (nFilter, LsstCamMapper._nbit_filter))
175 @classmethod
176 def getCameraName(cls):
177 return cls._cameraName
179 @classmethod
180 def _makeCamera(cls, policy=None, repositoryDir=None, cameraYamlFile=None):
181 """Make a camera describing the camera geometry.
183 policy : ignored
184 repositoryDir : ignored
185 cameraYamlFile : `str`
186 The full path to a yaml file to be passed to `yamlCamera.makeCamera`
188 Returns
189 -------
190 camera : `lsst.afw.cameraGeom.Camera`
191 Camera geometry.
192 """
193 if not cameraYamlFile:
194 cameraYamlFile = os.path.join(utils.getPackageDir(cls.packageName), "policy",
195 ("%s.yaml" % cls.getCameraName()))
197 return yamlCamera.makeCamera(cameraYamlFile)
199 def _getRegistryValue(self, dataId, k):
200 """Return a value from a dataId, or look it up in the registry if it
201 isn't present."""
202 if k in dataId:
203 return dataId[k]
204 else:
205 dataType = "bias" if "taiObs" in dataId else "raw"
207 try:
208 return self.queryMetadata(dataType, [k], dataId)[0][0]
209 except IndexError:
210 raise RuntimeError("Unable to lookup %s in \"%s\" registry for dataId %s" %
211 (k, dataType, dataId))
213 def _extractDetectorName(self, dataId):
214 if "channel" in dataId: # they specified a channel
215 dataId = dataId.copy()
216 del dataId["channel"] # Do not include in query
217 raftName = self._getRegistryValue(dataId, "raftName")
218 detectorName = self._getRegistryValue(dataId, "detectorName")
220 return "%s_%s" % (raftName, detectorName)
222 def _computeCcdExposureId(self, dataId):
223 """Compute the 64-bit (long) identifier for a CCD exposure.
225 Parameters
226 ----------
227 dataId : `dict`
228 Data identifier including dayObs and seqNum.
230 Returns
231 -------
232 id : `int`
233 Integer identifier for a CCD exposure.
234 """
235 try:
236 visit = self._getRegistryValue(dataId, "visit")
237 except Exception:
238 raise KeyError(f"Require a visit ID to calculate detector exposure ID. Got: {dataId}")
240 if "detector" in dataId:
241 detector = dataId["detector"]
242 else:
243 detector = self.translatorClass.compute_detector_num_from_name(dataId['raftName'],
244 dataId['detectorName'])
246 return self.translatorClass.compute_detector_exposure_id(visit, detector)
248 def bypass_ccdExposureId(self, datasetType, pythonType, location, dataId):
249 return self._computeCcdExposureId(dataId)
251 def bypass_ccdExposureId_bits(self, datasetType, pythonType, location, dataId):
252 """How many bits are required for the maximum exposure ID"""
253 # 52 for "C" controller and 51 for "O"
254 return 52 # max detector_exposure_id ~ 3050121299999250
256 def _computeCoaddExposureId(self, dataId, singleFilter):
257 """Compute the 64-bit (long) identifier for a coadd.
259 Parameters
260 ----------
261 dataId : `dict`
262 Data identifier with tract and patch.
263 singleFilter : `bool`
264 True means the desired ID is for a single-filter coadd, in which
265 case ``dataId`` must contain filter.
266 """
268 tract = int(dataId['tract'])
269 if tract < 0 or tract >= 2**LsstCamMapper._nbit_tract:
270 raise RuntimeError('tract not in range [0,%d)' % (2**LsstCamMapper._nbit_tract))
271 patchX, patchY = [int(patch) for patch in dataId['patch'].split(',')]
272 for p in (patchX, patchY):
273 if p < 0 or p >= 2**LsstCamMapper._nbit_patch:
274 raise RuntimeError('patch component not in range [0, %d)' % 2**LsstCamMapper._nbit_patch)
275 oid = (((tract << LsstCamMapper._nbit_patch) + patchX) << LsstCamMapper._nbit_patch) + patchY
276 if singleFilter:
277 if afwImage.Filter(dataId['filter']).getId() >= 2**LsstCamMapper._nbit_filter:
278 raise RuntimeError("Filter %s has too high an ID (%d) to fit in %d bits",
279 afwImage.Filter(dataId['filter']),
280 afwImage.Filter(dataId['filter']).getId(),
281 LsstCamMapper._nbit_filter)
283 return (oid << LsstCamMapper._nbit_filter) + afwImage.Filter(dataId['filter']).getId()
284 return oid
286 def bypass_deepCoaddId_bits(self, *args, **kwargs):
287 """The number of bits used up for patch ID bits."""
288 return 64 - LsstCamMapper._nbit_id
290 def bypass_deepCoaddId(self, datasetType, pythonType, location, dataId):
291 return self._computeCoaddExposureId(dataId, True)
293 def bypass_dcrCoaddId_bits(self, datasetType, pythonType, location, dataId):
294 return self.bypass_deepCoaddId_bits(datasetType, pythonType, location, dataId)
296 def bypass_dcrCoaddId(self, datasetType, pythonType, location, dataId):
297 return self.bypass_deepCoaddId(datasetType, pythonType, location, dataId)
299 def bypass_deepMergedCoaddId_bits(self, *args, **kwargs):
300 """The number of bits used up for patch ID bits."""
301 return 64 - LsstCamMapper._nbit_id
303 def bypass_deepMergedCoaddId(self, datasetType, pythonType, location, dataId):
304 return self._computeCoaddExposureId(dataId, False)
306 def bypass_dcrMergedCoaddId_bits(self, *args, **kwargs):
307 """The number of bits used up for patch ID bits."""
308 return self.bypass_deepMergedCoaddId_bits(*args, **kwargs)
310 def bypass_dcrMergedCoaddId(self, datasetType, pythonType, location, dataId):
311 return self.bypass_deepMergedCoaddId(datasetType, pythonType, location, dataId)
313 def query_raw_amp(self, format, dataId):
314 """Return a list of tuples of values of the fields specified in
315 format, in order.
317 Parameters
318 ----------
319 format : `list`
320 The desired set of keys.
321 dataId : `dict`
322 A possible-incomplete ``dataId``.
324 Returns
325 -------
326 fields : `list` of `tuple`
327 Values of the fields specified in ``format``.
329 Raises
330 ------
331 ValueError
332 The channel number requested in ``dataId`` is out of range.
333 """
334 # set number of possible channels, 1..nChannel
335 # The wave front chips are special, 4k x 2k with only 8 amps
337 if "detectorName" in dataId:
338 detectorName = dataId.get("detectorName")
339 elif "detector" in dataId:
340 detector = dataId.get("detector")
341 if detector in self.camera:
342 name = self.camera[detector].getName()
343 detectorName = name.split('_')[1]
344 else:
345 raise RuntimeError('Unable to find detector %s in camera' % detector)
346 else:
347 logger = lsst.log.Log.getLogger("LsstCamMapper")
348 logger.debug('Unable to lookup either "detectorName" or "detector" in the dataId')
349 detectorName = "unknown"
351 if detectorName in ["SW0", "SW1"]:
352 nChannel = 8
353 else:
354 nChannel = 16
356 if "channel" in dataId: # they specified a channel
357 dataId = dataId.copy()
358 channel = dataId.pop('channel') # Do not include in query below
359 if channel > nChannel or channel < 1:
360 raise ValueError(f"Requested channel is out of range 0 < {channel} <= {nChannel}")
361 channels = [channel]
362 else:
363 channels = range(1, nChannel + 1) # we want all possible channels
365 if "channel" in format: # they asked for a channel, but we mustn't query for it
366 format = list(format)
367 channelIndex = format.index('channel') # where channel values should go
368 format.pop(channelIndex)
369 else:
370 channelIndex = None
372 dids = [] # returned list of dataIds
373 for value in self.query_raw(format, dataId):
374 if channelIndex is None:
375 dids.append(value)
376 else:
377 for c in channels:
378 did = list(value)
379 did.insert(channelIndex, c)
380 dids.append(tuple(did))
382 return dids
383 #
384 # The composite type "raw" doesn't provide e.g. query_raw, so we defined
385 # type _raw in the .paf file with the same template, and forward requests
386 # as necessary
387 #
389 def query_raw(self, *args, **kwargs):
390 """Magic method that is called automatically if it exists.
392 This code redirects the call to the right place, necessary because of
393 leading underscore on ``_raw``.
394 """
395 return self.query__raw(*args, **kwargs)
397 def map_raw_md(self, *args, **kwargs):
398 """Magic method that is called automatically if it exists.
400 This code redirects the call to the right place, necessary because of
401 leading underscore on ``_raw``.
402 """
403 return self.map__raw_md(*args, **kwargs)
405 def map_raw_filename(self, *args, **kwargs):
406 """Magic method that is called automatically if it exists.
408 This code redirects the call to the right place, necessary because of
409 leading underscore on ``_raw``.
410 """
411 return self.map__raw_filename(*args, **kwargs)
413 def bypass_raw_filename(self, *args, **kwargs):
414 """Magic method that is called automatically if it exists.
416 This code redirects the call to the right place, necessary because of
417 leading underscore on ``_raw``.
418 """
419 return self.bypass__raw_filename(*args, **kwargs)
421 def map_raw_visitInfo(self, *args, **kwargs):
422 """Magic method that is called automatically if it exists.
424 This code redirects the call to the right place, necessary because of
425 leading underscore on ``_raw``.
426 """
427 return self.map__raw_visitInfo(*args, **kwargs)
429 def bypass_raw_md(self, datasetType, pythonType, location, dataId):
430 fileName = location.getLocationsWithRoot()[0]
431 md = readRawFitsHeader(fileName, translator_class=self.translatorClass)
432 return md
434 def bypass_raw_hdu(self, datasetType, pythonType, location, dataId):
435 # We need to override raw_hdu so that we can trap a request
436 # for the primary HDU and merge it with the default content.
437 fileName = location.getLocationsWithRoot()[0]
438 md = readRawFitsHeader(fileName, translator_class=self.translatorClass)
439 return md
441 def bypass_raw_visitInfo(self, datasetType, pythonType, location, dataId):
442 fileName = location.getLocationsWithRoot()[0]
443 md = readRawFitsHeader(fileName, translator_class=self.translatorClass)
444 makeVisitInfo = self.MakeRawVisitInfoClass(log=self.log)
445 return makeVisitInfo(md)
447 def std_raw_amp(self, item, dataId):
448 return self._standardizeExposure(self.exposures['raw_amp'], item, dataId,
449 trimmed=False, setVisitInfo=False,
450 filter=False) # Don't set the filter for an amp
452 def std_raw(self, item, dataId, filter=True):
453 """Standardize a raw dataset by converting it to an
454 `~lsst.afw.image.Exposure` instead of an `~lsst.afw.image.Image`."""
456 exp = self._standardizeExposure(self.exposures['raw'], item, dataId, trimmed=False,
457 setVisitInfo=False, # it's already set, and the metadata's stripped
458 filter=False)
460 if filter:
461 obsInfo = ObservationInfo(exp.getMetadata(), translator_class=self.translatorClass)
462 filt = afwImage.FilterLabel(physical=obsInfo.physical_filter)
463 exp.setFilterLabel(filt)
465 return exp
468class LsstCamMapper(LsstCamBaseMapper):
469 """The mapper for lsstCam."""
470 translatorClass = LsstCamTranslator
471 MakeRawVisitInfoClass = LsstCamRawVisitInfo
472 _cameraName = "lsstCam"
473 _gen3instrument = LsstCam