Coverage for python/lsst/obs/lsst/lsstCamMapper.py : 22%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of obs_lsst.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the LSST License Statement and
20# the GNU General Public License along with this program. If not,
21# see <http://www.lsstcorp.org/LegalNotices/>.
22#
23"""The LsstCam Mapper.""" # necessary to suppress D100 flake8 warning.
25import os
26import lsst.log
27import lsst.geom
28import lsst.utils as utils
29import lsst.afw.image as afwImage
30from lsst.obs.base import CameraMapper, MakeRawVisitInfoViaObsInfo
31import lsst.obs.base.yamlCamera as yamlCamera
32import lsst.daf.persistence as dafPersist
33from astro_metadata_translator import ObservationInfo
34from .translators import LsstCamTranslator
35from ._fitsHeader import readRawFitsHeader
36from ._instrument import LsstCam
38from .filters import LSSTCAM_FILTER_DEFINITIONS
39from .assembly import attachRawWcsFromBoresight, fixAmpsAndAssemble
41__all__ = ["LsstCamMapper", "LsstCamMakeRawVisitInfo"]
44class LsstCamMakeRawVisitInfo(MakeRawVisitInfoViaObsInfo):
45 """Make a VisitInfo from the FITS header of a raw image."""
48class LsstCamRawVisitInfo(LsstCamMakeRawVisitInfo):
49 metadataTranslator = LsstCamTranslator
52def assemble_raw(dataId, componentInfo, cls):
53 """Called by the butler to construct the composite type "raw".
55 Note that we still need to define "_raw" and copy various fields over.
57 Parameters
58 ----------
59 dataId : `lsst.daf.persistence.dataId.DataId`
60 The data ID.
61 componentInfo : `dict`
62 dict containing the components, as defined by the composite definition
63 in the mapper policy.
64 cls : 'object'
65 Unused.
67 Returns
68 -------
69 exposure : `lsst.afw.image.Exposure`
70 The assembled exposure.
71 """
73 ampExps = componentInfo['raw_amp'].obj
74 exposure = fixAmpsAndAssemble(ampExps, str(dataId))
75 md = componentInfo['raw_hdu'].obj
76 exposure.setMetadata(md)
78 attachRawWcsFromBoresight(exposure, dataId)
80 return exposure
83class LsstCamBaseMapper(CameraMapper):
84 """The Base Mapper for all LSST-style instruments.
85 """
87 packageName = 'obs_lsst'
88 _cameraName = "lsstCam"
89 yamlFileList = ("lsstCamMapper.yaml",) # list of yaml files to load, keeping the first occurrence
90 #
91 # do not set MakeRawVisitInfoClass or translatorClass to anything other
92 # than None!
93 #
94 # assemble_raw relies on autodetect as in butler Gen2 it doesn't know
95 # its mapper and cannot use mapper.makeRawVisitInfo()
96 #
97 MakeRawVisitInfoClass = None
98 translatorClass = None
99 filterDefinitions = LSSTCAM_FILTER_DEFINITIONS
101 def __init__(self, inputPolicy=None, **kwargs):
102 #
103 # Merge the list of .yaml files
104 #
105 policy = None
106 for yamlFile in self.yamlFileList:
107 policyFile = dafPersist.Policy.defaultPolicyFile(self.packageName, yamlFile, "policy")
108 npolicy = dafPersist.Policy(policyFile)
110 if policy is None:
111 policy = npolicy
112 else:
113 policy.merge(npolicy)
114 #
115 # Look for the calibrations root "root/CALIB" if not supplied
116 #
117 if kwargs.get('root', None) and not kwargs.get('calibRoot', None):
118 calibSearch = [os.path.join(kwargs['root'], 'CALIB')]
119 if "repositoryCfg" in kwargs:
120 calibSearch += [os.path.join(cfg.root, 'CALIB') for cfg in kwargs["repositoryCfg"].parents if
121 hasattr(cfg, "root")]
122 calibSearch += [cfg.root for cfg in kwargs["repositoryCfg"].parents if hasattr(cfg, "root")]
123 for calibRoot in calibSearch:
124 if os.path.exists(os.path.join(calibRoot, "calibRegistry.sqlite3")):
125 kwargs['calibRoot'] = calibRoot
126 break
127 if not kwargs.get('calibRoot', None):
128 lsst.log.Log.getLogger("LsstCamMapper").warn("Unable to find valid calib root directory")
130 super().__init__(policy, os.path.dirname(policyFile), **kwargs)
131 #
132 # The composite objects don't seem to set these
133 #
134 for d in (self.mappings, self.exposures):
135 d['raw'] = d['_raw']
137 self.filterDefinitions.reset()
138 self.filterDefinitions.defineFilters()
140 LsstCamMapper._nbit_tract = 16
141 LsstCamMapper._nbit_patch = 5
142 LsstCamMapper._nbit_filter = 7
144 LsstCamMapper._nbit_id = 64 - (LsstCamMapper._nbit_tract + 2*LsstCamMapper._nbit_patch
145 + LsstCamMapper._nbit_filter)
146 #
147 # The BOT has many ND filters in a second filter wheel, resulting in
148 # more than 128 composite filters. However, we're never going to
149 # build coadds with the BOT. So let's ignore the qualifier after
150 # the ~ in filter names when we're calculating the number of filters
151 #
152 # Because the first filter wheel can be empty some of baseFilters are
153 # actually in the second wheel, but that's OK -- we still easily fit
154 # in 7 bits (5 would actually be enough)
156 baseFilters = set()
157 for n in afwImage.Filter.getNames():
158 i = n.find('~')
159 if i >= 0:
160 n = n[:i]
162 baseFilters.add(n)
164 nFilter = len(baseFilters)
165 if nFilter >= 2**LsstCamMapper._nbit_filter:
166 raise RuntimeError("You have more filters (%d) defined than fit into the %d bits allocated" %
167 (nFilter, LsstCamMapper._nbit_filter))
169 @classmethod
170 def getCameraName(cls):
171 return cls._cameraName
173 @classmethod
174 def _makeCamera(cls, policy=None, repositoryDir=None, cameraYamlFile=None):
175 """Make a camera describing the camera geometry.
177 policy : ignored
178 repositoryDir : ignored
179 cameraYamlFile : `str`
180 The full path to a yaml file to be passed to `yamlCamera.makeCamera`
182 Returns
183 -------
184 camera : `lsst.afw.cameraGeom.Camera`
185 Camera geometry.
186 """
187 if not cameraYamlFile:
188 cameraYamlFile = os.path.join(utils.getPackageDir(cls.packageName), "policy",
189 ("%s.yaml" % cls.getCameraName()))
191 return yamlCamera.makeCamera(cameraYamlFile)
193 def _getRegistryValue(self, dataId, k):
194 """Return a value from a dataId, or look it up in the registry if it
195 isn't present."""
196 if k in dataId:
197 return dataId[k]
198 else:
199 dataType = "bias" if "taiObs" in dataId else "raw"
201 try:
202 return self.queryMetadata(dataType, [k], dataId)[0][0]
203 except IndexError:
204 raise RuntimeError("Unable to lookup %s in \"%s\" registry for dataId %s" %
205 (k, dataType, dataId))
207 def _extractDetectorName(self, dataId):
208 if "channel" in dataId: # they specified a channel
209 dataId = dataId.copy()
210 del dataId["channel"] # Do not include in query
211 raftName = self._getRegistryValue(dataId, "raftName")
212 detectorName = self._getRegistryValue(dataId, "detectorName")
214 return "%s_%s" % (raftName, detectorName)
216 def _computeCcdExposureId(self, dataId):
217 """Compute the 64-bit (long) identifier for a CCD exposure.
219 Parameters
220 ----------
221 dataId : `dict`
222 Data identifier including dayObs and seqNum.
224 Returns
225 -------
226 id : `int`
227 Integer identifier for a CCD exposure.
228 """
229 try:
230 visit = self._getRegistryValue(dataId, "visit")
231 except Exception:
232 raise KeyError(f"Require a visit ID to calculate detector exposure ID. Got: {dataId}")
234 if "detector" in dataId:
235 detector = dataId["detector"]
236 else:
237 detector = self.translatorClass.compute_detector_num_from_name(dataId['raftName'],
238 dataId['detectorName'])
240 return self.translatorClass.compute_detector_exposure_id(visit, detector)
242 def bypass_ccdExposureId(self, datasetType, pythonType, location, dataId):
243 return self._computeCcdExposureId(dataId)
245 def bypass_ccdExposureId_bits(self, datasetType, pythonType, location, dataId):
246 """How many bits are required for the maximum exposure ID"""
247 # 52 for "C" controller and 51 for "O"
248 return 52 # max detector_exposure_id ~ 3050121299999250
250 def _computeCoaddExposureId(self, dataId, singleFilter):
251 """Compute the 64-bit (long) identifier for a coadd.
253 Parameters
254 ----------
255 dataId : `dict`
256 Data identifier with tract and patch.
257 singleFilter : `bool`
258 True means the desired ID is for a single-filter coadd, in which
259 case ``dataId`` must contain filter.
260 """
262 tract = int(dataId['tract'])
263 if tract < 0 or tract >= 2**LsstCamMapper._nbit_tract:
264 raise RuntimeError('tract not in range [0,%d)' % (2**LsstCamMapper._nbit_tract))
265 patchX, patchY = [int(patch) for patch in dataId['patch'].split(',')]
266 for p in (patchX, patchY):
267 if p < 0 or p >= 2**LsstCamMapper._nbit_patch:
268 raise RuntimeError('patch component not in range [0, %d)' % 2**LsstCamMapper._nbit_patch)
269 oid = (((tract << LsstCamMapper._nbit_patch) + patchX) << LsstCamMapper._nbit_patch) + patchY
270 if singleFilter:
271 if afwImage.Filter(dataId['filter']).getId() >= 2**LsstCamMapper._nbit_filter:
272 raise RuntimeError("Filter %s has too high an ID (%d) to fit in %d bits",
273 afwImage.Filter(dataId['filter']),
274 afwImage.Filter(dataId['filter']).getId(),
275 LsstCamMapper._nbit_filter)
277 return (oid << LsstCamMapper._nbit_filter) + afwImage.Filter(dataId['filter']).getId()
278 return oid
280 def bypass_deepCoaddId_bits(self, *args, **kwargs):
281 """The number of bits used up for patch ID bits."""
282 return 64 - LsstCamMapper._nbit_id
284 def bypass_deepCoaddId(self, datasetType, pythonType, location, dataId):
285 return self._computeCoaddExposureId(dataId, True)
287 def bypass_dcrCoaddId_bits(self, datasetType, pythonType, location, dataId):
288 return self.bypass_deepCoaddId_bits(datasetType, pythonType, location, dataId)
290 def bypass_dcrCoaddId(self, datasetType, pythonType, location, dataId):
291 return self.bypass_deepCoaddId(datasetType, pythonType, location, dataId)
293 def bypass_deepMergedCoaddId_bits(self, *args, **kwargs):
294 """The number of bits used up for patch ID bits."""
295 return 64 - LsstCamMapper._nbit_id
297 def bypass_deepMergedCoaddId(self, datasetType, pythonType, location, dataId):
298 return self._computeCoaddExposureId(dataId, False)
300 def bypass_dcrMergedCoaddId_bits(self, *args, **kwargs):
301 """The number of bits used up for patch ID bits."""
302 return self.bypass_deepMergedCoaddId_bits(*args, **kwargs)
304 def bypass_dcrMergedCoaddId(self, datasetType, pythonType, location, dataId):
305 return self.bypass_deepMergedCoaddId(datasetType, pythonType, location, dataId)
307 def query_raw_amp(self, format, dataId):
308 """Return a list of tuples of values of the fields specified in
309 format, in order.
311 Parameters
312 ----------
313 format : `list`
314 The desired set of keys.
315 dataId : `dict`
316 A possible-incomplete ``dataId``.
318 Returns
319 -------
320 fields : `list` of `tuple`
321 Values of the fields specified in ``format``.
323 Raises
324 ------
325 ValueError
326 The channel number requested in ``dataId`` is out of range.
327 """
328 # set number of possible channels, 1..nChannel
329 # The wave front chips are special, 4k x 2k with only 8 amps
331 if "detectorName" in dataId:
332 detectorName = dataId.get("detectorName")
333 elif "detector" in dataId:
334 detector = dataId.get("detector")
335 if detector in self.camera:
336 name = self.camera[detector].getName()
337 detectorName = name.split('_')[1]
338 else:
339 raise RuntimeError('Unable to find detector %s in camera' % detector)
340 else:
341 logger = lsst.log.Log.getLogger("LsstCamMapper")
342 logger.debug('Unable to lookup either "detectorName" or "detector" in the dataId')
343 detectorName = "unknown"
345 if detectorName in ["SW0", "SW1"]:
346 nChannel = 8
347 else:
348 nChannel = 16
350 if "channel" in dataId: # they specified a channel
351 dataId = dataId.copy()
352 channel = dataId.pop('channel') # Do not include in query below
353 if channel > nChannel or channel < 1:
354 raise ValueError(f"Requested channel is out of range 0 < {channel} <= {nChannel}")
355 channels = [channel]
356 else:
357 channels = range(1, nChannel + 1) # we want all possible channels
359 if "channel" in format: # they asked for a channel, but we mustn't query for it
360 format = list(format)
361 channelIndex = format.index('channel') # where channel values should go
362 format.pop(channelIndex)
363 else:
364 channelIndex = None
366 dids = [] # returned list of dataIds
367 for value in self.query_raw(format, dataId):
368 if channelIndex is None:
369 dids.append(value)
370 else:
371 for c in channels:
372 did = list(value)
373 did.insert(channelIndex, c)
374 dids.append(tuple(did))
376 return dids
377 #
378 # The composite type "raw" doesn't provide e.g. query_raw, so we defined
379 # type _raw in the .paf file with the same template, and forward requests
380 # as necessary
381 #
383 def query_raw(self, *args, **kwargs):
384 """Magic method that is called automatically if it exists.
386 This code redirects the call to the right place, necessary because of
387 leading underscore on ``_raw``.
388 """
389 return self.query__raw(*args, **kwargs)
391 def map_raw_md(self, *args, **kwargs):
392 """Magic method that is called automatically if it exists.
394 This code redirects the call to the right place, necessary because of
395 leading underscore on ``_raw``.
396 """
397 return self.map__raw_md(*args, **kwargs)
399 def map_raw_filename(self, *args, **kwargs):
400 """Magic method that is called automatically if it exists.
402 This code redirects the call to the right place, necessary because of
403 leading underscore on ``_raw``.
404 """
405 return self.map__raw_filename(*args, **kwargs)
407 def bypass_raw_filename(self, *args, **kwargs):
408 """Magic method that is called automatically if it exists.
410 This code redirects the call to the right place, necessary because of
411 leading underscore on ``_raw``.
412 """
413 return self.bypass__raw_filename(*args, **kwargs)
415 def map_raw_visitInfo(self, *args, **kwargs):
416 """Magic method that is called automatically if it exists.
418 This code redirects the call to the right place, necessary because of
419 leading underscore on ``_raw``.
420 """
421 return self.map__raw_visitInfo(*args, **kwargs)
423 def bypass_raw_md(self, datasetType, pythonType, location, dataId):
424 fileName = location.getLocationsWithRoot()[0]
425 md = readRawFitsHeader(fileName, translator_class=self.translatorClass)
426 return md
428 def bypass_raw_hdu(self, datasetType, pythonType, location, dataId):
429 # We need to override raw_hdu so that we can trap a request
430 # for the primary HDU and merge it with the default content.
431 fileName = location.getLocationsWithRoot()[0]
432 md = readRawFitsHeader(fileName, translator_class=self.translatorClass)
433 return md
435 def bypass_raw_visitInfo(self, datasetType, pythonType, location, dataId):
436 fileName = location.getLocationsWithRoot()[0]
437 md = readRawFitsHeader(fileName, translator_class=self.translatorClass)
438 makeVisitInfo = self.MakeRawVisitInfoClass(log=self.log)
439 return makeVisitInfo(md)
441 def std_raw_amp(self, item, dataId):
442 return self._standardizeExposure(self.exposures['raw_amp'], item, dataId,
443 trimmed=False, setVisitInfo=False,
444 filter=False) # Don't set the filter for an amp
446 def std_raw(self, item, dataId, filter=True):
447 """Standardize a raw dataset by converting it to an
448 `~lsst.afw.image.Exposure` instead of an `~lsst.afw.image.Image`."""
450 exp = self._standardizeExposure(self.exposures['raw'], item, dataId, trimmed=False,
451 setVisitInfo=False, # it's already set, and the metadata's stripped
452 filter=False)
454 if filter:
455 obsInfo = ObservationInfo(exp.getMetadata(), translator_class=self.translatorClass)
456 filt = afwImage.FilterLabel(physical=obsInfo.physical_filter)
457 exp.setFilterLabel(filt)
459 return exp
462class LsstCamMapper(LsstCamBaseMapper):
463 """The mapper for lsstCam."""
464 translatorClass = LsstCamTranslator
465 MakeRawVisitInfoClass = LsstCamRawVisitInfo
466 _cameraName = "lsstCam"
467 _gen3instrument = LsstCam